def test_entity_view_add_annotation_columns(syn, project, schedule_for_cleanup): folder1 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', parent=project, annotations={ 'strAnno': 'str1', 'intAnno': 1, 'floatAnno': 1.1 })) folder2 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', parent=project, annotations={ 'dateAnno': datetime.now(), 'strAnno': 'str2', 'intAnno': 2 })) schedule_for_cleanup(folder1) schedule_for_cleanup(folder2) scopeIds = [utils.id_of(folder1), utils.id_of(folder2)] # This test is to ensure that user code which use the deprecated field `type` continue to work # TODO: remove this test case in Synapse Python client 2.0 entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project) syn.store(entity_view) # This test is to ensure that user code which use the deprecated field `type` continue to work # TODO: remove this test case in Synapse Python client 2.0 entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='file', includeEntityTypes=[EntityViewType.PROJECT], parent=project) syn.store(entity_view) entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, includeEntityTypes=[EntityViewType.PROJECT], parent=project) syn.store(entity_view)
def _view_setup(cls): # set up a file view folder = syn.store( Folder(name="PartialRowTestFolder" + str(uuid.uuid4()), parent=project)) syn.store( File("~/path/doesnt/matter", name="f1", parent=folder, synapseStore=False)) syn.store( File("~/path/doesnt/matter/again", name="f2", parent=folder, synapseStore=False)) cols = [ Column(name='foo', columnType='STRING', maximumSize=1000), Column(name='bar', columnType='STRING') ] return syn.store( EntityViewSchema(name='PartialRowTestViews' + str(uuid.uuid4()), columns=cols, addDefaultViewColumns=False, parent=project, scopes=[folder]))
def test_entity_view_add_annotation_columns(): folder1 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', parent=project, annotations={ 'strAnno': 'str1', 'intAnno': 1, 'floatAnno': 1.1 })) folder2 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', parent=project, annotations={ 'dateAnno': datetime.now(), 'strAnno': 'str2', 'intAnno': 2 })) schedule_for_cleanup(folder1) schedule_for_cleanup(folder2) scopeIds = [utils.id_of(folder1), utils.id_of(folder2)] entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project) syn.store(entity_view)
def test_table_file_view_csv_update_annotations__includeEntityEtag(): folder = syn.store( synapseclient.Folder(name="updateAnnoFolder" + str(uuid.uuid4()), parent=project)) anno1_name = "annotationColumn1" anno2_name = "annotationColumn2" initial_annotations = { anno1_name: "initial_value1", anno2_name: "initial_value2" } file_entity = syn.store( File(name= "test_table_file_view_csv_update_annotations__includeEntityEtag", path="~/fakepath", synapseStore=False, parent=folder, annotations=initial_annotations)) annotation_columns = [ Column(name=anno1_name, columnType='STRING'), Column(name=anno2_name, columnType='STRING') ] entity_view = syn.store( EntityViewSchema(name="TestEntityViewSchemaUpdateAnnotation" + str(uuid.uuid4()), parent=project, scopes=[folder], columns=annotation_columns)) query_str = "SELECT {anno1}, {anno2} FROM {proj_id}".format( anno1=anno1_name, anno2=anno2_name, proj_id=utils.id_of(entity_view)) #modify first annotation using rowset rowset_query_result = syn.tableQuery(query_str, resultsAs="rowset") rowset = rowset_query_result.asRowSet() rowset_changed_anno_value = "rowset_value_change" rowset.rows[0].values[0] = rowset_changed_anno_value syn.store(rowset) #modify second annotation using csv csv_query_result = syn.tableQuery(query_str, resultsAs="csv") dataframe = csv_query_result.asDataFrame() csv_changed_anno_value = "csv_value_change" dataframe.ix[0, anno2_name] = csv_changed_anno_value syn.store(Table(utils.id_of(entity_view), dataframe)) #check annotations in the file entity. Annotations may not be immediately updated so we wait in while loop expected_annotations = { anno1_name: [rowset_changed_anno_value], anno2_name: [csv_changed_anno_value] } start_time = time.time() while (expected_annotations != file_entity.annotations): assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC) time.sleep(2) file_entity = syn.get(file_entity, downloadFile=False)
def __init__(self, datasetId: str, synapse: Synapse, name: str = None, temporary: bool = True, parentId: str = None) -> None: """Create a file view scoped to a dataset folder. Args: datasetId (str): Synapse ID for a dataset folder/project. synapse (Synapse): Used for Synapse requests. name (str): Name of the file view (temporary or not). temporary (bool): Whether to delete the file view on exit of either a 'with' statement or Python entirely. parentId (str, optional): Synapse ID specifying where to store the file view. Defaults to datasetId. """ self.datasetId = datasetId self.synapse = synapse self.is_temporary = temporary if name is None: self.name = f"schematic annotation file view for {self.datasetId}" if self.is_temporary: uid = secrets.token_urlsafe(5) self.name = f"{self.name} - UID {uid}" # TODO: Allow a DCC admin to configure a "universal parent" # Such as a Synapse project writeable by everyone. self.parentId = datasetId if parentId is None else parentId # TODO: Create local sharing setting to hide from everyone else view_schema = EntityViewSchema( name=self.name, parent=self.parentId, scopes=self.datasetId, includeEntityTypes=[EntityViewType.FILE, EntityViewType.FOLDER], addDefaultViewColumns=False, addAnnotationColumns=True) # TODO: Handle failure due to insufficient permissions by # creating a temporary new project to store view self.view_schema = self.synapse.store(view_schema) # These are filled in after calling `self.query()` self.results = None self.table = None # Ensure deletion of the file view (last resort) if self.is_temporary: atexit.register(self.delete)
def _ensure_syt_view(self): """ Ensure the syt table/view exists for the project. """ try: # This will fail if the schema doesn't exist. This is a synapseclient bug. self._syt_view = self._synapse_client.get(EntityViewSchema( name=self.SYT_VIEW_NAME, parent=self._project), downloadFile=False) except: pass if self._syt_view == None: evs = EntityViewSchema(name=self.SYT_VIEW_NAME, parent=self._project, scopes=[self._project], properties={'viewTypeMask': 9}) # Delete the 'type' property so we can set our own viewTypeMask to Files and Folders. evs.pop('type') # Since we removed 'type' we have to manually populate the base columns. evs.addColumn(Column(name='id', columnType='ENTITYID')) evs.addColumn(Column(name='parentId', columnType='ENTITYID')) evs.addColumn(Column(name='projectId', columnType='ENTITYID')) evs.addColumn(Column(name='type', columnType='STRING')) evs.addColumn( Column(name='name', columnType='STRING', maximumSize=256)) evs.addColumn( Column(name=self.ANNO_CHECKED_OUT_BY_ID, columnType='STRING')) evs.addColumn( Column(name=self.ANNO_CHECKED_OUT_BY_NAME, columnType='STRING')) evs.addColumn( Column(name=self.ANNO_CHECKED_OUT_DATE, columnType='DATE')) self._syt_view = self._synapse_client.store(evs)
def get_or_create_view(self, **kwargs) -> EntityViewSchema: """Gets an existing view schema by name and parent or creates a new one. Args: Same arguments as synapseclient.EntityViewSchema Returns: A synapseclient.EntityViewSchema. """ view = EntityViewSchema(**kwargs) view = self._find_by_obj_or_create(view) self.logger.info('{} View {} ({})'.format(self._update_str, view.name, view.id)) return view
def test_create_and_update_file_view(syn, project, schedule_for_cleanup): # Create a folder folder = Folder(str(uuid.uuid4()), parent=project, description='creating a file-view') folder = syn.store(folder) # Create dummy file with annotations in our folder path = utils.make_bogus_data_file() file_annotations = dict(fileFormat='jpg', dataType='image', artist='Banksy', medium='print', title='Girl With Ballon') schedule_for_cleanup(path) a_file = File(path, parent=folder, annotations=file_annotations) a_file = syn.store(a_file) schedule_for_cleanup(a_file) # Add new columns for the annotations on this file and get their IDs my_added_cols = [ syn.store(Column(name=k, columnType="STRING")) for k in file_annotations.keys() ] my_added_cols_ids = [c['id'] for c in my_added_cols] view_default_ids = [ c['id'] for c in syn._get_default_view_columns( "entityview", EntityViewType.FILE.value) ] col_ids = my_added_cols_ids + view_default_ids scopeIds = [folder['id'].lstrip('syn')] # Create an empty entity-view with defined scope as folder entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=True, addAnnotationColumns=False, type='file', columns=my_added_cols, parent=project) entity_view = syn.store(entity_view) schedule_for_cleanup(entity_view) assert set(scopeIds) == set(entity_view.scopeIds) assert set(col_ids) == set(entity_view.columnIds) assert EntityViewType.FILE.value == entity_view.viewTypeMask # get the current view-schema view = syn.tableQuery("select * from %s" % entity_view.id) schedule_for_cleanup(view.filepath) view_dict = list( csv.DictReader(io.open(view.filepath, encoding="utf-8", newline=''))) # check that all of the annotations were retrieved from the view assert set(file_annotations.keys()).issubset(set(view_dict[0].keys())) updated_a_file = syn.get(a_file.id, downloadFile=False) # Check that the values are the same as what was set # Both in the view and on the entity itself for k, v in file_annotations.items(): assert view_dict[0][k] == v assert updated_a_file.annotations[k][0] == v # Make a change to the view and store view_dict[0]['fileFormat'] = 'PNG' with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as temp: schedule_for_cleanup(temp.name) temp_filename = temp.name with io.open(temp_filename, mode='w', encoding="utf-8", newline='') as temp_file: dw = csv.DictWriter(temp_file, fieldnames=view_dict[0].keys(), quoting=csv.QUOTE_NONNUMERIC, lineterminator=str(os.linesep)) dw.writeheader() dw.writerows(view_dict) temp_file.flush() syn.store(Table(entity_view.id, temp_filename)) new_view_dict = list( csv.DictReader(io.open(temp_filename, encoding="utf-8", newline=''))) assert new_view_dict[0]['fileFormat'] == 'PNG' # query for the change start_time = time.time() new_view_results = syn.tableQuery("select * from %s" % entity_view.id) schedule_for_cleanup(new_view_results.filepath) new_view_dict = list( csv.DictReader( io.open(new_view_results.filepath, encoding="utf-8", newline=''))) # query until change is seen. while new_view_dict[0]['fileFormat'] != 'PNG': # check timeout assert time.time() - start_time < QUERY_TIMEOUT_SEC # query again new_view_results = syn.tableQuery("select * from %s" % entity_view.id) new_view_dict = list( csv.DictReader( io.open(new_view_results.filepath, encoding="utf-8", newline=''))) # paranoid check assert new_view_dict[0]['fileFormat'] == 'PNG'
def partial_rowset_test_state(syn, project): cols = [ Column(name='foo', columnType='INTEGER'), Column(name='bar', columnType='INTEGER') ] table_schema = syn.store( Schema(name='PartialRowTest' + str(uuid.uuid4()), columns=cols, parent=project)) data = [[1, None], [None, 2]] syn.store(RowSet(schema=table_schema, rows=[Row(r) for r in data])) # set up a file view folder = syn.store( Folder(name="PartialRowTestFolder" + str(uuid.uuid4()), parent=project)) syn.store( File("~/path/doesnt/matter", name="f1", parent=folder, synapseStore=False)) syn.store( File("~/path/doesnt/matter/again", name="f2", parent=folder, synapseStore=False)) cols = [ Column(name='foo', columnType='INTEGER'), Column(name='bar', columnType='INTEGER') ] view_schema = syn.store( EntityViewSchema(name='PartialRowTestViews' + str(uuid.uuid4()), columns=cols, addDefaultViewColumns=False, parent=project, scopes=[folder])) table_changes = [{'foo': 4}, {'bar': 5}] view_changes = [{'bar': 6}, {'foo': 7}] expected_table_cells = pd.DataFrame({ 'foo': [4.0, float('NaN')], 'bar': [float('NaN'), 5.0] }) expected_view_cells = pd.DataFrame({ 'foo': [float('NaN'), 7.0], 'bar': [6.0, float('NaN')] }) class TestState: def __init__(self): self.syn = syn self.project = project self.table_schema = table_schema self.view_schema = view_schema self.table_changes = table_changes self.view_changes = view_changes self.expected_table_cells = expected_table_cells self.expected_view_cells = expected_view_cells return TestState()
def test_entity_view_add_annotation_columns(): folder1 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', parent=project, annotations={ 'strAnno': 'str1', 'intAnno': 1, 'floatAnno': 1.1 })) folder2 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', parent=project, annotations={ 'dateAnno': datetime.now(), 'strAnno': 'str2', 'intAnno': 2 })) schedule_for_cleanup(folder1) schedule_for_cleanup(folder2) scopeIds = [utils.id_of(folder1), utils.id_of(folder2)] entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project) assert_true(entity_view['addAnnotationColumns']) #For some reason this call is eventually consistent but not immediately consistent. so we just wait till the size returned is correct expected_column_types = { 'dateAnno': 'DATE', 'intAnno': 'INTEGER', 'strAnno': 'STRING', 'floatAnno': 'DOUBLE' } columns = syn._get_annotation_entity_view_columns(scopeIds, 'project') start_time = time.time() while len(columns) != len(expected_column_types): assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC) columns = syn._get_annotation_entity_view_columns(scopeIds, 'project') time.sleep(2) entity_view = syn.store(entity_view) assert_false(entity_view['addAnnotationColumns']) view_column_types = { column['name']: column['columnType'] for column in syn.getColumns(entity_view) } assert_dict_equal(expected_column_types, view_column_types) #add another annotation to the project and make sure that EntityViewSchema only adds one moe column folder1['anotherAnnotation'] = 'I need healing!' folder1 = syn.store(folder1) prev_columns = list(entity_view.columnIds) # sometimes annotation columns are not immediately updated so we wait for it to update in a loop start_time = time.time() while len(entity_view.columnIds) != len(prev_columns) + 1: assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC) entity_view.addAnnotationColumns = True entity_view = syn.store(entity_view) expected_column_types.update({'anotherAnnotation': 'STRING'}) view_column_types = { column['name']: column['columnType'] for column in syn.getColumns(entity_view) } assert_dict_equal(expected_column_types, view_column_types)
def test_entity_view_add_annotation_columns(): proj1 = syn.store( Project(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', annotations={ 'strAnno': 'str1', 'intAnno': 1, 'floatAnno': 1.1 })) proj2 = syn.store( Project(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', annotations={ 'dateAnno': datetime.now(), 'strAnno': 'str2', 'intAnno': 2 })) schedule_for_cleanup(proj1) schedule_for_cleanup(proj2) scopeIds = [utils.id_of(proj1), utils.id_of(proj2)] entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project) assert_true(entity_view['addAnnotationColumns']) #For some reason this call is eventually consistent but not immediately consistent. so we just wait till the size returned is correct expected_column_types = { 'dateAnno': 'DATE', 'intAnno': 'INTEGER', 'strAnno': 'STRING', 'floatAnno': 'DOUBLE', 'concreteType': 'STRING' } columns = syn._get_annotation_entity_view_columns(scopeIds, 'project') while len(columns) != len(expected_column_types): columns = syn._get_annotation_entity_view_columns(scopeIds, 'project') time.sleep(2) entity_view = syn.store(entity_view) assert_false(entity_view['addAnnotationColumns']) view_column_types = { column['name']: column['columnType'] for column in syn.getColumns(entity_view.columnIds) } assert_dict_equal(expected_column_types, view_column_types) #add another annotation to the project and make sure that EntityViewSchema only adds one moe column proj1['anotherAnnotation'] = 'I need healing!' proj1 = syn.store(proj1) entity_view.addAnnotationColumns = True entity_view = syn.store(entity_view) expected_column_types.update({'anotherAnnotation': 'STRING'}) view_column_types = { column['name']: column['columnType'] for column in syn.getColumns(entity_view.columnIds) } assert_dict_equal(expected_column_types, view_column_types)