def __next__(self): """Return the next row in the dataset iterator. Raises StopIteration if end of file is reached or file has been closed. Automatically closes any open file when end of iteration is reached for the first time. Returns ------- vizier.datastore.base.DatasetRow """ if self.is_open: # Catch exception to close any open file try: row = next(self.reader) if self.has_row_ids: row = DatasetRow(int(row[0]), row[1:]) else: row = DatasetRow(self.line_count, row) self.line_count += 1 return row except StopIteration as ex: self.close() raise ex raise StopIteration
def test_default_json_reader(self): """Test functionality of Json dataset reader.""" reader = DefaultJsonDatasetReader(JSON_FILE) with self.assertRaises(StopIteration): next(reader) count = 0 with reader.open() as r: for row in r: self.assertEqual(len(row.values), 3) self.assertEqual(row.identifier, count) count += 1 self.assertEqual(count, 2) with self.assertRaises(StopIteration): next(reader) # Create a new dataset and read it tmp_file = tempfile.mkstemp()[1] reader = DefaultJsonDatasetReader(tmp_file) values = ['A', 'B', 1, 2] rows = [ DatasetRow(0, values), DatasetRow(1, values), DatasetRow(2, values) ] reader.write(rows) count = 0 with reader.open() as reader: for row in reader: self.assertEqual(len(row.values), 4) self.assertEqual(row.identifier, count) count += 1 self.assertEqual(count, len(rows)) os.remove(tmp_file)
def filter_columns(self, identifier: str, columns: List[int], names: List[str], datastore: Datastore) -> VizualApiResult: """Dataset projection operator. Returns a copy of the dataset with the given identifier that contains only those columns listed in columns. The list of names contains optional new names for the filtered columns. A value of None in names indicates that the name of the corresponding column is not changed. Raises ValueError if no dataset with given identifier exists or if any of the filter columns are unknown. Parameters ---------- identifier: string Unique dataset identifier columns: list(int) List of column identifier for columns in the result. names: list(string) Optional new names for filtered columns. datastore : vizier.datastore.fs.base.FileSystemDatastore Datastore to retireve and update datasets Returns ------- vizier.engine.packages.vizual.api.VizualApiResult """ # Get dataset. Raise exception if dataset is unknown dataset = datastore.get_dataset(identifier) if dataset is None: raise ValueError('unknown dataset \'' + identifier + '\'') # The schema of the new dataset only contains the columns in the given # list. Keep track of their index positions to filter values. schema = list() val_filter = list() for i in range(len(columns)): col_idx = dataset.get_index(columns[i]) if col_idx is None: raise ValueError('unknown column identifier \'' + str(columns[i]) + '\'') col = dataset.columns[col_idx] if not names[i] is None: schema.append( DatasetColumn(identifier=col.identifier, name=names[i], data_type=col.data_type)) else: schema.append(col) val_filter.append(col_idx) # Create a list of projected rows rows = list() for row in dataset.fetch_rows(): values = list() for v_idx in val_filter: values.append(row.values[v_idx]) rows.append(DatasetRow(identifier=row.identifier, values=values)) # Store updated dataset to get new identifier ds = datastore.create_dataset(columns=schema, rows=rows, properties={}) return VizualApiResult(ds)
def load_dataset( self, f_handle: FileHandle, proposed_schema: List[Tuple[str, str]] = []) -> FileSystemDatasetHandle: """Create a new dataset from a given file. Raises ValueError if the given file could not be loaded as a dataset. Parameters ---------- f_handle : vizier.filestore.base.FileHandle Handle for an uploaded file Returns ------- vizier.datastore.fs.dataset.FileSystemDatasetHandle """ # The file handle might be None in which case an exception is raised if f_handle is None: raise ValueError('unknown file') # Expects a file in a supported tabular data format. if not f_handle.is_tabular: raise ValueError('cannot create dataset from file \'' + f_handle.name + '\'') # Open the file as a csv file. Expects that the first row contains the # column names. Read dataset schema and dataset rows into two separate # lists. columns: List[DatasetColumn] = [] rows: List[DatasetRow] = [] with f_handle.open() as csvfile: reader = csv.reader(csvfile, delimiter=f_handle.delimiter) for col_name in next(reader): columns.append( DatasetColumn(identifier=len(columns), name=col_name.strip())) for row in reader: values = [cast(v.strip()) for v in row] rows.append( DatasetRow(identifier=str(len(rows)), values=values)) # Get unique identifier and create subfolder for the new dataset identifier = get_unique_identifier() dataset_dir = self.get_dataset_dir(identifier) os.makedirs(dataset_dir) # Write rows to data file data_file = os.path.join(dataset_dir, DATA_FILE) DefaultJsonDatasetReader(data_file).write(rows) # Create dataset an write descriptor to file dataset = FileSystemDatasetHandle(identifier=identifier, columns=columns, data_file=data_file, row_count=len(rows), max_row_id=len(rows) - 1) dataset.to_file( descriptor_file=os.path.join(dataset_dir, DESCRIPTOR_FILE)) return dataset
def test_deduplicate_annotations(self): """Test removing duplicated annotations.""" store = FileSystemDatastore(STORE_DIR) ds = store.create_dataset( columns=[ DatasetColumn(identifier=0, name='A'), DatasetColumn(identifier=1, name='B') ], rows=[DatasetRow(identifier=0, values=['a', 'b'])], annotations=DatasetMetadata( cells=[ DatasetAnnotation(column_id=0, row_id=0, key='X', value=1), DatasetAnnotation(column_id=0, row_id=0, key='X', value=2), DatasetAnnotation(column_id=1, row_id=0, key='X', value=3), DatasetAnnotation(column_id=1, row_id=1, key='X', value=3), DatasetAnnotation(column_id=0, row_id=0, key='Y', value=1), DatasetAnnotation(column_id=0, row_id=0, key='X', value=1), DatasetAnnotation(column_id=0, row_id=0, key='X', value=2), DatasetAnnotation(column_id=1, row_id=0, key='X', value=3), DatasetAnnotation(column_id=1, row_id=1, key='X', value=3), ], columns=[ DatasetAnnotation(column_id=0, key='A', value='x'), DatasetAnnotation(column_id=1, key='A', value='x'), DatasetAnnotation(column_id=0, key='A', value='x'), DatasetAnnotation(column_id=1, key='A', value='x'), DatasetAnnotation(column_id=0, key='A', value='x'), DatasetAnnotation(column_id=1, key='A', value='x'), DatasetAnnotation(column_id=0, key='A', value='x'), DatasetAnnotation(column_id=1, key='A', value='x') ], rows=[ DatasetAnnotation(row_id=0, key='E', value=100), DatasetAnnotation(row_id=0, key='E', value=100) ] ) ) ds = store.get_dataset(ds.identifier) self.assertEqual(len(ds.annotations.cells), 4) self.assertEqual(len(ds.annotations.columns), 2) self.assertEqual(len(ds.annotations.rows), 1) annos = ds.annotations.for_cell(column_id=0, row_id=0) self.assertEqual(len(annos), 3) self.assertTrue(1 in [a.value for a in annos]) self.assertTrue(2 in [a.value for a in annos]) self.assertFalse(3 in [a.value for a in annos]) self.assertEqual(len(ds.annotations.find_all(values=annos, key='X')), 2) with self.assertRaises(ValueError): ds.annotations.find_one(values=annos, key='X') self.assertEqual(len(ds.annotations.for_column(column_id=0)), 1) self.assertEqual(len(ds.annotations.for_row(row_id=0)), 1) annotations = ds.annotations.filter(columns=[1]) self.assertEqual(len(annotations.cells), 1) self.assertEqual(len(annotations.columns), 1) self.assertEqual(len(annotations.rows), 1)
def update_cell(self, identifier: str, column_id: int, row_id: str, value: str, datastore: Datastore) -> VizualApiResult: """Update a cell in a given dataset. Raises ValueError if no dataset with given identifier exists or if the specified cell is outside of the current dataset ranges. Parameters ---------- identifier : string Unique dataset identifier column_id: int Unique column identifier for updated cell row_id: int Unique row identifier value: string New cell value datastore : vizier.datastore.fs.base.FileSystemDatastore Datastore to retireve and update datasets Returns ------- vizier.engine.packages.vizual.api.VizualApiResult """ # Get dataset. Raise exception if dataset is unknown dataset = datastore.get_dataset(identifier) if dataset is None: raise ValueError('unknown dataset \'' + identifier + '\'') # Get column index forst in case it raises an exception col_idx = dataset.get_index(column_id) if col_idx is None: raise ValueError('unknown column identifier \'' + str(column_id) + '\'') # Update the specified cell in the given data array rows = dataset.fetch_rows() row_index = -1 for i in range(len(rows)): if int(rows[i].identifier) == int(row_id): row_index = i break # Make sure that row refers a valid row in the dataset if row_index < 0: raise ValueError('invalid row identifier \'' + str(row_id) + '\'') r = rows[row_index] values = list(r.values) values[col_idx] = value rows[row_index] = DatasetRow(identifier=r.identifier, values=values) # Store updated dataset to get new identifier ds = datastore.create_dataset(columns=dataset.columns, rows=rows, properties={}) return VizualApiResult(ds)
def test_query_annotations(self): """Test retrieving annotations via the datastore.""" store = FileSystemDatastore(STORE_DIR) ds = store.create_dataset( columns=[ DatasetColumn(identifier=0, name='A'), DatasetColumn(identifier=1, name='B') ], rows=[DatasetRow(identifier=0, values=['a', 'b'])], properties=EXAMPLE_PROPERTIES ) properties = store.get_properties(ds.identifier) self.assertEqual(len(properties["columns"]), 2)
def DATASET_ROW(obj): """Convert dictionary into a dataset row object. Parameters ---------- obj: dict Default serialization of a dataset row Returns ------- vizier.datastore.dataset.DatasetRow """ return DatasetRow(identifier=obj[labels.ID], values=obj[labels.ROWVALUES])
def open(self): """Setup the reader by querying the database and creating an in-memory copy of the dataset rows. Returns ------- vizier.datastore.reader.MimirDatasetReader """ # Query the database to retrieve dataset rows if reader is not already # open if not self.is_open: # Query the database to get the list of rows. Sort rows according to # order in row_ids and return a InMemReader sql = base.get_select_query(self.table_name, columns=self.columns) if self.rowid != None: sql += ' WHERE ROWID() = ' + str(self.rowid) if self.is_range_query: if self.limit > 0: sql += ' LIMIT ' + str(self.limit) if self.offset > 0: sql += ' OFFSET ' + str(self.offset) rs = mimir.vistrailsQueryMimirJson(sql+ ';', True, False) #self.row_ids = rs['prov'] # Initialize mapping of column rdb names to index positions in # dataset rows self.col_map = dict() for i in range(len(rs['schema'])): col = rs['schema'][i] self.col_map[base.sanitize_column_name(col['name'])] = i rs_rows = rs['data'] row_ids = rs['prov'] annotation_flags = rs['colTaint'] self.rows = list() for row_index in range(len(rs_rows)): row = rs_rows[row_index] row_annotation_flags = annotation_flags[row_index] row_id = str(row_ids[row_index]) values = [None] * len(self.columns) annotation_flag_values = [None] * len(self.columns) for i in range(len(self.columns)): col = self.columns[i] col_index = self.col_map[col.name_in_rdb] values[i] = base.mimir_value_to_python(row[col_index], col) annotation_flag_values[i] = row_annotation_flags[col_index] self.rows.append(DatasetRow(row_id, values, annotation_flag_values)) self.read_index = 0 self.is_open = True return self
def empty_dataset( self, datastore: Datastore, filestore: Filestore, initial_columns: List[Tuple[str, str]] = [("''", "unnamed_column")] ) -> VizualApiResult: """Create (or load) a new dataset from a given file or Uri. It is guaranteed that either the file identifier or the url are not None but one of them will be None. The user name and password may only be given if an url is given. The resources refer to any resoures (e.g., file identifier) that have been generated by a previous execution of the respective task. This allows to associate an identifier with a downloaded file to avoid future downloads (unless the reload flag is True). Parameters ---------- datastore : vizier.datastore.fs.base.FileSystemDatastore Datastore to retireve and update datasets filestore: vizier.filestore.Filestore Filestore to retrieve uploaded datasets Returns ------- vizier.engine.packages.vizual.api.VizualApiResult """ assert (isinstance(datastore, MimirDatastore)) ds = datastore.create_dataset( columns=[ MimirDatasetColumn(identifier=id, name_in_dataset=col, data_type="varchar") for id, (default, col) in enumerate(initial_columns) ], rows=[ DatasetRow( identifier=str(id), values=[default for default, col in initial_columns]) for id in range(1, 2) ], human_readable_name="Empty Table", ) return VizualApiResult(dataset=ds)
def test_query_annotations(self): """Test retrieving annotations via the datastore.""" store = FileSystemDatastore(STORE_DIR) ds = store.create_dataset( columns=[ DatasetColumn(identifier=0, name='A'), DatasetColumn(identifier=1, name='B') ], rows=[DatasetRow(identifier=0, values=['a', 'b'])], annotations=DatasetMetadata( cells=[ DatasetAnnotation(column_id=0, row_id=0, key='X', value=1), DatasetAnnotation(column_id=0, row_id=0, key='X', value=2), DatasetAnnotation(column_id=1, row_id=0, key='X', value=3), DatasetAnnotation(column_id=0, row_id=0, key='Y', value=1) ], columns=[ DatasetAnnotation(column_id=0, key='A', value='x'), DatasetAnnotation(column_id=1, key='A', value='x') ], rows=[ DatasetAnnotation(row_id=0, key='E', value=100) ] ) ) annos = store.get_annotations(ds.identifier, column_id=1) self.assertEqual(len(annos.columns), 1) self.assertEqual(len(annos.rows), 0) self.assertEqual(len(annos.cells), 0) annos = store.get_annotations(ds.identifier, column_id=0) self.assertEqual(len(annos.columns), 1) self.assertEqual(len(annos.rows), 0) self.assertEqual(len(annos.cells), 0) annos = store.get_annotations(ds.identifier, row_id=0) self.assertEqual(len(annos.columns), 0) self.assertEqual(len(annos.rows), 1) self.assertEqual(len(annos.cells), 0) annos = store.get_annotations(ds.identifier, column_id=1, row_id=0) self.assertEqual(len(annos.columns), 0) self.assertEqual(len(annos.rows), 0) self.assertEqual(len(annos.cells), 1) annos = store.get_annotations(ds.identifier, column_id=0, row_id=0) self.assertEqual(len(annos.columns), 0) self.assertEqual(len(annos.rows), 0) self.assertEqual(len(annos.cells), 3)
def open(self) -> "MimirDatasetReader": """Setup the reader by querying the database and creating an in-memory copy of the dataset rows. Returns ------- vizier.datastore.reader.MimirDatasetReader """ # Query the database to retrieve dataset rows if reader is not already # open if not self.is_open: # Query the database to get the list of rows. Sort rows according to # order in row_ids and return a InMemReader rs = mimir.getTable( table=self.table_name, columns=[col.name_in_rdb for col in self.columns], offset_to_rowid=self.rowid, limit=self.limit if self.is_range_query else None, offset=self.offset if self.is_range_query else None, include_uncertainty=True) #self.row_ids = rs['prov'] # Initialize mapping of column rdb names to index positions in # dataset rows rs_rows = rs['data'] row_ids = rs['prov'] annotation_flags = rs['colTaint'] self.rows = list() for row_index in range(len(rs_rows)): row = rs_rows[row_index] row_annotation_flags = annotation_flags[row_index] row_id = str(row_ids[row_index]) values = [None] * len(self.columns) annotation_flag_values: List[bool] = [False] * len( self.columns) for i in range(len(self.columns)): col = self.columns[i] values[i] = base.mimir_value_to_python(row[i], col) annotation_flag_values[i] = not row_annotation_flags[i] self.rows.append( DatasetRow(row_id, values, annotation_flag_values)) self.read_index = 0 self.is_open = True return self
def __next__(self): """Return the next row in the dataset iterator. Raises StopIteration if end of file is reached or file has been closed. Automatically closes any open file when end of iteration is reached for the first time. Returns ------- vizier.datastore.base.DatasetRow """ if self.is_open: if self.read_index < len(self.rows): r_dict = self.rows[self.read_index] row = DatasetRow(identifier=r_dict[KEY_ROW_ID], values=r_dict[KEY_ROW_VALUES]) self.read_index += 1 return row raise StopIteration
def test_properties(self): """Test loading a dataset from file.""" store = FileSystemDatastore(STORE_DIR) ds = store.create_dataset( columns=[ DatasetColumn(identifier=0, name='A'), DatasetColumn(identifier=1, name='B') ], rows=[DatasetRow(identifier=0, values=[1, 2])], properties=EXAMPLE_PROPERTIES ) ds = store.get_dataset(ds.identifier) column_props = ds.properties['columns'] self.assertEqual(len(column_props), 2) self.assertTrue('A' in [prop['name'] for prop in column_props]) # Reload datastore store = FileSystemDatastore(STORE_DIR) ds = store.get_dataset(ds.identifier) column_props = ds.properties['columns'] self.assertEqual(len(column_props), 2)
def test_create_dataset(self): """Test loading a dataset from file.""" store = FileSystemDatastore(STORE_DIR) ds = store.create_dataset( columns=[ DatasetColumn(identifier=0, name='A'), DatasetColumn(identifier=1, name='B') ], rows=[DatasetRow(identifier=0, values=['a', 'b'])] ) ds = store.get_dataset(ds.identifier) column_ids = [col.identifier for col in ds.columns] self.assertEqual(len(ds.columns), 2) for id in [0, 1]: self.assertTrue(id in column_ids) column_names = [col.name for col in ds.columns] for name in ['A', 'B']: self.assertTrue(name in column_names) rows = ds.fetch_rows() self.assertEqual(len(rows), 1) self.assertEqual(rows[0].values, ['a', 'b']) self.assertEqual(len(ds.annotations.cells), 0) self.assertEqual(len(ds.annotations.columns), 0) self.assertEqual(len(ds.annotations.rows), 0) # Reload the datastore store = FileSystemDatastore(STORE_DIR) ds = store.get_dataset(ds.identifier) column_ids = [col.identifier for col in ds.columns] self.assertEqual(len(ds.columns), 2) for id in [0, 1]: self.assertTrue(id in column_ids) column_names = [col.name for col in ds.columns] for name in ['A', 'B']: self.assertTrue(name in column_names) rows = ds.fetch_rows() self.assertEqual(len(rows), 1) self.assertEqual(rows[0].values, ['a', 'b']) self.assertEqual(len(ds.annotations.cells), 0) self.assertEqual(len(ds.annotations.columns), 0) self.assertEqual(len(ds.annotations.rows), 0)
def insert_row(self, identifier: str, position: int, datastore: Datastore) -> VizualApiResult: """Insert row at given position in a dataset. Raises ValueError if no dataset with given identifier exists or if the specified row psotion isoutside the dataset bounds. Parameters ---------- identifier: string Unique dataset identifier position: int Index position at which the row will be inserted datastore : vizier.datastore.fs.base.FileSystemDatastore Datastore to retireve and update datasets Returns ------- vizier.engine.packages.vizual.api.VizualApiResult """ # Get dataset. Raise exception if dataset is unknown dataset = datastore.get_dataset(identifier) print('---------------' + str(dataset.__class__.__name__)) if dataset is None: raise ValueError('unknown dataset \'' + identifier + '\'') assert (isinstance(dataset, FileSystemDatasetHandle)) # Make sure that position is a valid row index in the new dataset if position < 0 or position > dataset.row_count: raise ValueError('invalid row index \'' + str(position) + '\'') # Create empty set of values rows = dataset.fetch_rows() rows.insert( position, DatasetRow(identifier=str(dataset.max_row_id() + 1), values=[None] * len(dataset.columns))) # Store updated dataset to get new identifier ds = datastore.create_dataset(columns=dataset.columns, rows=rows, properties={}) return VizualApiResult(ds)
def test_validate_dataset(self): """Test the validate dataset function.""" columns = [] rows = [] # Empty dataset max_col_id, max_row_id = validate_dataset(columns, rows) self.assertEqual(max_col_id, -1) self.assertEqual(max_row_id, -1) max_col_id, max_row_id = validate_dataset( columns=columns, rows=rows ) self.assertEqual(max_col_id, -1) self.assertEqual(max_row_id, -1) # Valid set of columns and rows columns = [DatasetColumn(0, 'A'), DatasetColumn(10, 'B')] rows = [DatasetRow(0, [1, 2]), DatasetRow(4, [None, 2]), DatasetRow(2, [0, 0])] max_col_id, max_row_id = validate_dataset(columns, rows) self.assertEqual(max_col_id, 10) self.assertEqual(max_row_id, 4) max_col_id, max_row_id = validate_dataset( columns=columns, rows=rows ) self.assertEqual(max_col_id, 10) self.assertEqual(max_row_id, 4) # Column errors with self.assertRaises(ValueError): validate_dataset(columns + [DatasetColumn()], []) with self.assertRaises(ValueError): validate_dataset(columns + [DatasetColumn(10, 'C')], []) # Row errors with self.assertRaises(ValueError): validate_dataset(columns, rows + [DatasetRow(1000, [0, 1, 3])]) with self.assertRaises(ValueError): validate_dataset(columns, rows + [DatasetRow(-1, [1, 3])]) with self.assertRaises(ValueError): validate_dataset(columns, rows + [DatasetRow(0, [1, 3])])
def create_dataset( self, columns: List[DatasetColumn], rows: List[DatasetRow], properties: Optional[Dict[str, Any]] = None, human_readable_name: str = "Untitled Dataset", backend_options: Optional[List[Tuple[str, str]]] = None, dependencies: Optional[List[str]] = None) -> DatasetDescriptor: """Create a new dataset in the datastore. Expects at least the list of columns and the rows for the dataset. Raises ValueError if (1) the column identifier are not unique, (2) the row identifier are not uniqe, (3) the number of columns and values in a row do not match, (4) any of the column or row identifier have a negative value, or (5) if the given column or row counter have value lower or equal to any of the column or row identifier. Parameters ---------- columns: list(vizier.datastore.dataset.DatasetColumn) List of columns. It is expected that each column has a unique identifier. rows: list(vizier.datastore.dataset.DatasetRow) List of dataset rows. properties: dict(string, ANY), optional Properties for dataset components Returns ------- vizier.datastore.dataset.DatasetDescriptor """ # Validate (i) that each column has a unique identifier, (ii) each row # has a unique identifier, and (iii) that every row has exactly one # value per column. properties = {} if properties is None else properties dependencies = [] if dependencies is None else dependencies identifiers = set( int(row.identifier) for row in rows if row.identifier is not None and int(row.identifier) >= 0) identifiers.add(0) max_row_id = max(identifiers) rows = [ DatasetRow(identifier=row.identifier if row.identifier is not None and int(row.identifier) >= 0 else str(idx + max_row_id), values=row.values, caveats=row.caveats) for idx, row in enumerate(rows) ] _, max_row_id = validate_dataset(columns=columns, rows=rows) # Get new identifier and create directory for new dataset identifier = get_unique_identifier() dataset_dir = self.get_dataset_dir(identifier) os.makedirs(dataset_dir) # Write rows to data file data_file = os.path.join(dataset_dir, DATA_FILE) DefaultJsonDatasetReader(data_file).write(rows) # Create dataset an write dataset file dataset = FileSystemDatasetHandle(identifier=identifier, columns=columns, data_file=data_file, row_count=len(rows), max_row_id=max_row_id, properties=properties) dataset.to_file( descriptor_file=os.path.join(dataset_dir, DESCRIPTOR_FILE)) # Write metadata file if annotations are given if properties is not None: dataset.write_properties_to_file( self.get_properties_filename(identifier)) # Return handle for new dataset return DatasetDescriptor(identifier=dataset.identifier, name=human_readable_name, columns=dataset.columns)
api = VizierApiClient(URLS) PROJECT_ID = api.create_project({"name": "Test Client Datastore"}).identifier at_exit(api.delete_project, PROJECT_ID) # We're just doing some unit testing on the fields specific to DatastoreClient, so # ignore complaints about instantiating an abstract class store = DatastoreClient( # type: ignore[abstract] urls=DatastoreClientUrlFactory(urls=URLS, project_id=PROJECT_ID)) ds = store.create_dataset(columns=[ DatasetColumn(identifier=0, name='Name'), DatasetColumn(identifier=1, name='Age', data_type="int") ], rows=[ DatasetRow(identifier=0, values=['Alice', 32]), DatasetRow(identifier=1, values=['Bob', 23]) ], properties={"example_property": "foo"}) # print(ds) # print([col.identifier for col in ds.columns]) # print([col.name for col in ds.columns]) dh = store.get_dataset(ds.identifier) assert dh is not None for row in dh.fetch_rows(): print([row.identifier] + row.values) caveats = dh.get_caveats() # print("\n".join(c.__repr__ for c in caveats))
def test_update_annotations(self): """Test updating annotations via the datastore.""" store = FileSystemDatastore(STORE_DIR) ds = store.create_dataset( columns=[ DatasetColumn(identifier=0, name='A'), DatasetColumn(identifier=1, name='B') ], rows=[DatasetRow(identifier=0, values=['a', 'b'])], annotations=DatasetMetadata( cells=[ DatasetAnnotation(column_id=0, row_id=0, key='X', value=1), DatasetAnnotation(column_id=0, row_id=0, key='X', value=2), DatasetAnnotation(column_id=1, row_id=0, key='X', value=3), DatasetAnnotation(column_id=0, row_id=0, key='Y', value=1) ], columns=[ DatasetAnnotation(column_id=0, key='A', value='x'), DatasetAnnotation(column_id=1, key='A', value='x') ], rows=[ DatasetAnnotation(row_id=0, key='E', value=100) ] ) ) # INSERT row annotatins store.update_annotation( ds.identifier, key='D', row_id=0, new_value=200 ) annos = store.get_annotations(ds.identifier, row_id=0) self.assertEqual(len(annos.rows), 2) for key in ['D', 'E']: self.assertTrue(key in [a.key for a in annos.rows]) for val in [100, 200]: self.assertTrue(val in [a.value for a in annos.rows]) # UPDATE column annotation store.update_annotation( ds.identifier, key='A', column_id=1, old_value='x', new_value='y' ) annos = store.get_annotations(ds.identifier, column_id=1) self.assertEqual(annos.columns[0].key, 'A') self.assertEqual(annos.columns[0].value, 'y') # DELETE cell annotation store.update_annotation( ds.identifier, key='X', column_id=0, row_id=0, old_value=2, ) annos = store.get_annotations(ds.identifier, column_id=0, row_id=0) self.assertEqual(len(annos.cells), 2) for a in annos.cells: self.assertNotEqual(a.value, 2) result = store.update_annotation( ds.identifier, key='X', column_id=1, row_id=0, old_value=3, ) self.assertTrue(result) annos = store.get_annotations(ds.identifier, column_id=1, row_id=0) self.assertEqual(len(annos.cells), 0)