def setup(self): self.project_entity = synapseclient.Project(name=str(uuid.uuid4()), id="syn1234") self.second_project = synapseclient.Project(name=str(uuid.uuid4()), id="syn2345") self.file_ent = synapseclient.File(name='File', parent=self.project_entity.id) self.file_ent.id = "syn3456"
def test_get_or_create_project__call(): """Makes sure correct parameters are called""" project_name = str(uuid.uuid1()) project = synapseclient.Project(name=project_name) returned = synapseclient.Project(name=project_name, id=str(uuid.uuid1())) with patch.object(CREATE_CLS, "_find_by_obj_or_create", return_value=returned) as patch_find_or_create: new_project = CREATE_CLS.get_or_create_project(name=project_name) assert new_project == returned patch_find_or_create.assert_called_once_with(project)
def archive_writeup(syn, evaluation, stat="VALIDATED", reArchive=False): """ Archive the submissions for the given evaluation queue and store them in the destination synapse folder. :param evaluation: a synapse evaluation queue or its ID :param query: a query that will return the desired submissions. At least the ID must be returned. Defaults to: 'select * from evaluation_[EVAL_ID] where status=="SCORED"' """ if type(evaluation) != synapseclient.Evaluation: evaluation = syn.getEvaluation(evaluation) print("\n\nArchiving", evaluation.id, evaluation.name) print("-" * 60) for sub, status in syn.getSubmissionBundles(evaluation, status=stat): # retrieve file into cache and copy it to destination checkIfArchived = filter(lambda x: x.get("key") == "archived", status.annotations['stringAnnos']) if len(list(checkIfArchived)) == 0 or reArchive: projectEntity = synapseclient.Project( 'Archived {} {} {} {}'.format( sub.name.replace("&", "+").replace("'", ""), int(round(time.time() * 1000)), sub.id, sub.entityId)) entity = syn.store(projectEntity) adminPriv = [ 'DELETE', 'DOWNLOAD', 'CREATE', 'READ', 'CHANGE_PERMISSIONS', 'UPDATE', 'MODERATE', 'CHANGE_SETTINGS' ] syn.setPermissions(entity, "3324230", adminPriv) synapseutils.copy(syn, sub.entityId, entity.id) archived = {"archived": entity.id} status = utils.update_single_submission_status(status, archived) syn.store(status)
def storeSubmissions(syn, evaluationQueue, synProject=None, filetype="csv"): """Store submissions on Synapse and submit to the evaluation queue. Arguments --------- syn : synapseclient.Synapse evaluationQueue : int Evaluation ID. synProject : str Synapse ID of project to store files to. filetype : str filename extension to append to filenames (default 'csv'). Returns ------- None """ if not synProject: logger.info("Creating new Synapse Project") user_profile = syn.getUserProfile() project_name = "{}_{}_testSubmissions".format( user_profile['userName'], evaluationQueue) synProject = sc.Project(project_name) synProject = syn.store(synProject).id logger.info("Synapse project created at {}".format(synProject)) for submission in glob.glob("{}/*.{}".format(TEST_SUBMISSION_PATH, filetype)): logger.info("Storing {} to {} and submitting to {}".format( submission, synProject, evaluationQueue)) synFile = sc.File(submission, parent=synProject) synFile = syn.store(synFile) syn.submit(evaluationQueue, synFile, name=os.path.basename(submission))
def test__create_synapse_resources_acl(self): """Test ACL gets created""" project_name = 'Test Configuration' project_config = [{ 'name': project_name, 'type': 'Project', 'acl': ['fake'] }] expected_config = [{ 'name': project_name, 'type': 'Project', 'id': 'syn12222', 'acl': ['fake'] }] project_ent = synapseclient.Project(id="syn12222") with patch.object(self.create_cls, "get_or_create_project", return_value=project_ent) as patch_create,\ patch.object(create, "_set_acl") as patch_set: client._create_synapse_resources(config_list=project_config, creation_cls=self.create_cls) patch_create.assert_called_once_with(name=project_name) assert project_config == expected_config patch_set.assert_called_once_with( syn=self.create_cls.syn, entity=project_ent, acl_config=['fake'])
async def _create(self): name = '_TEMP_{0}_TEMP_'.format(str(uuid.uuid4())) logging.info('Creating file view project: {0}'.format(name)) self.view_project = await SynapseProxy.storeAsync( syn.Project(name=name)) logging.info('Creating file view: {0}'.format(name)) cols = [ syn.Column(name=self.COL_ID, columnType='ENTITYID'), syn.Column(name=self.COL_DATAFILEHANDLEID, columnType='FILEHANDLEID'), syn.Column(name=self.COL_NAME, columnType='STRING', maximumSize=256) ] schema = syn.EntityViewSchema( name=name, columns=cols, properties=None, parent=self.view_project, scopes=[self.scope], includeEntityTypes=[syn.EntityViewType.FILE], addDefaultViewColumns=False, addAnnotationColumns=False) self.view = await SynapseProxy.storeAsync(schema)
def test_syncFromSynapse(): """This function tests recursive download as defined in syncFromSynapse most of the functionality of this function are already tested in the tests/integration/test_command_line_client::test_command_get_recursive_and_query which means that the only test if for path=None """ # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) syn.store(File(f, parent=folder_entity)) # Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) syn.store(File(f, parent=project_entity)) # Test recursive get output = synapseutils.syncFromSynapse(syn, project_entity) assert_equals(len(output), len(uploaded_paths)) for f in output: assert_in(f.path, uploaded_paths)
def login(self): """Log in to Synapse and acquire the project entity.""" log.info( "Initiating log in to Synapse and acquiring the project entity.") self.syn.login(email=self.user.SYN_USERNAME, apiKey=self.user.API_KEY) project_name = self.push_config.PROJECT_NAME log.info("""Acquiring Synapse project instance for "{name}".""".format( name=project_name)) try: self.project = self.syn.get(synapse.Project(name=project_name)) except TypeError: self.project = self.syn.store(synapse.Project(name=project_name)) self._build_remote_entity_dag()
def test_create_project(): """Test creating project""" name = str(uuid.uuid1()) proj = synapseclient.Project(name, id="syn1234") with patch.object(SYN, "store", return_value=proj) as patch_store: new_proj = createchallenge.create_project(SYN, proj.name) assert new_proj == proj patch_store.assert_called_once()
def test_command_copy(): """Tests the 'synapse cp' function""" # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity)) schedule_for_cleanup(folder_entity.id) # Create and upload a file in Folder repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' annots = {'test': ['hello_world']} # Create, upload, and set annotations on a file in Folder filename = utils.make_bogus_data_file() schedule_for_cleanup(filename) file_entity = syn.store(synapseclient.File(filename, parent=folder_entity)) externalURL_entity = syn.store(synapseclient.File(repo_url, name='rand', parent=folder_entity, synapseStore=False)) syn.setAnnotations(file_entity, annots) syn.setAnnotations(externalURL_entity, annots) schedule_for_cleanup(file_entity.id) schedule_for_cleanup(externalURL_entity.id) # Test cp function output = run('synapse', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id) output_URL = run('synapse', '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id) copied_id = parse(r'Copied syn\d+ to (syn\d+)', output) copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL) # Verify that our copied files are identical copied_ent = syn.get(copied_id) copied_URL_ent = syn.get(copied_URL_id, downloadFile=False) schedule_for_cleanup(copied_id) schedule_for_cleanup(copied_URL_id) copied_ent_annot = syn.getAnnotations(copied_id) copied_url_annot = syn.getAnnotations(copied_URL_id) copied_prov = syn.getProvenance(copied_id)['used'][0]['reference']['targetId'] copied_url_prov = syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId'] # Make sure copied files are the same assert_equals(copied_prov, file_entity.id) assert_equals(copied_ent_annot, annots) assert_equals(copied_ent.properties.dataFileHandleId, file_entity.properties.dataFileHandleId) # Make sure copied URLs are the same assert_equals(copied_url_prov, externalURL_entity.id) assert_equals(copied_url_annot, annots) assert_equals(copied_URL_ent.externalURL, repo_url) assert_equals(copied_URL_ent.name, 'rand') assert_equals(copied_URL_ent.properties.dataFileHandleId, externalURL_entity.properties.dataFileHandleId) # Verify that errors are being thrown when a # file is copied to a folder/project that has a file with the same filename assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
def test_command_get_recursive_and_query(): """Tests the 'synapse get -r' and 'synapse get -q' functions""" # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) file_entity = synapseclient.File(f, parent=folder_entity) file_entity.location = 'folder' file_entity = syn.store(file_entity) #Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) file_entity = synapseclient.File(f, parent=project_entity) file_entity.location = 'project' file_entity = syn.store(file_entity) ### Test recursive get output = run('synapse', '--skip-checks', 'get', '-r', project_entity.id) #Verify that we downloaded files: new_paths = [ os.path.join('.', folder_entity.name, os.path.basename(f)) for f in uploaded_paths[:-1] ] new_paths.append(os.path.join('.', os.path.basename(uploaded_paths[-1]))) schedule_for_cleanup(folder_entity.name) for downloaded, uploaded in zip(new_paths, uploaded_paths): print uploaded, downloaded assert os.path.exists(downloaded) assert filecmp.cmp(downloaded, uploaded) schedule_for_cleanup(new_paths[0]) ### Test query get output = run( 'synapse', '--skip-checks', 'get', '-q', "select id from file where parentId=='%s' and location=='folder'" % folder_entity.id) #Verify that we downloaded files: new_paths = [ os.path.join('.', os.path.basename(f)) for f in uploaded_paths[:-1] ] for downloaded, uploaded in zip(new_paths, uploaded_paths[:-1]): print uploaded, downloaded assert os.path.exists(downloaded) assert filecmp.cmp(downloaded, uploaded) schedule_for_cleanup(downloaded)
def bootstrap_infra(syn, args): """Create GENIE-like infrastructure""" # Basic setup of the project if args.project_name: # Create the project project = synapseclient.Project(args.project_name) project = syn.store(project) else: project = syn.get(args.project_id) bootstrap.main(syn, project=project, format_registry=args.format_registry_packages, centers=args.centers)
def test_command_copy(): """Tests the 'synapse cp' function""" # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload a file in Folder dummy = utils.make_bogus_data_file() schedule_for_cleanup(dummy) dummy_entity = syn.store(synapseclient.File(dummy, parent=folder_entity)) repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' annots = {'test': 'hello_world'} # Create, upload, and set annotations on a file in Folder filename = utils.make_bogus_data_file() schedule_for_cleanup(filename) file_entity = syn.store(synapseclient.File(filename, parent=folder_entity), used=dummy_entity.id, executed=repo_url) syn.setAnnotations(file_entity, annots) ### Test cp function output = run('synapse', '--skip-checks', 'cp', '--id', file_entity.id, '--parentid', project_entity.id) copied_id = parse(r'Copied syn\d+ to (syn\d+)', output) #Verify that our copied files are identical copied_ent = syn.get(copied_id) schedule_for_cleanup(copied_id) copied_ent_annot = syn.getAnnotations(copied_ent) copied_annot = dict((key, copied_ent_annot[key].pop()) for key in copied_ent_annot if key not in ('uri', 'id', 'creationDate', 'etag')) copied_prov = syn.getProvenance( copied_ent)['used'][0]['reference']['targetId'] assert copied_prov == file_entity.id assert copied_annot == annots #Verify that errors are being thrown when folders/projects are attempted to be copied, #or file is copied to a foler/project that has a file with the same filename assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', '--id', folder_entity.id, '--parentid', project_entity.id) assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', '--id', project_entity.id, '--parentid', project_entity.id) assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', '--id', file_entity.id, '--parentid', project_entity.id)
def test_entity_type_display_name(): assert SynapseProxy.entity_type_display_name( 'org.sagebionetworks.repo.model.Project' ) == SynapseProxy.PROJECT_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( syn.Project()) == SynapseProxy.PROJECT_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name({ 'concreteType': 'org.sagebionetworks.repo.model.Project' }) == SynapseProxy.PROJECT_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( 'org.sagebionetworks.repo.model.Folder' ) == SynapseProxy.FOLDER_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( syn.Folder(parentId='syn0')) == SynapseProxy.FOLDER_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name({ 'concreteType': 'org.sagebionetworks.repo.model.Folder' }) == SynapseProxy.FOLDER_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( 'org.sagebionetworks.repo.model.FileEntity' ) == SynapseProxy.FILE_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( syn.File(parentId='syn0')) == SynapseProxy.FILE_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name({ 'concreteType': 'org.sagebionetworks.repo.model.FileEntity' }) == SynapseProxy.FILE_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( 'org.sagebionetworks.repo.model.Link' ) == SynapseProxy.LINK_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( syn.Link(parentId='syn0', targetId='syn0')) == SynapseProxy.LINK_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name({ 'concreteType': 'org.sagebionetworks.repo.model.Link' }) == SynapseProxy.LINK_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name( 'org.sagebionetworks.repo.model.table.TableEntity' ) == SynapseProxy.TABLE_TYPE_DISPLAY_NAME assert SynapseProxy.entity_type_display_name({ 'concreteType': 'org.sagebionetworks.repo.model.table.TableEntity' }) == SynapseProxy.TABLE_TYPE_DISPLAY_NAME
def test_livesite_main(): """Tests main when live site is not None""" team_map = { 'team_part_id': '1234', 'team_admin_id': '2345', 'team_prereg_id': '3456', 'team_org_id': '4567' } teamid = str(uuid.uuid1()) project = str(uuid.uuid1()) chalid = str(uuid.uuid1()) etag = str(uuid.uuid1()) challenge_obj = Challenge(id=chalid, projectId=project, etag=etag, participantTeamId=teamid) proj = synapseclient.Project(project, id="syn1234") wiki = synapseclient.Wiki(title='', owner=proj, markdown='') challenge_name = str(uuid.uuid1()) with patch.object(createchallenge, "_create_teams", return_value=team_map),\ patch.object(createchallenge, "create_project", return_value=proj) as patch_create_proj,\ patch.object(SYN, "get", return_value=proj),\ patch.object(permissions, "set_entity_permissions") as patch_set_perms,\ patch.object(createchallenge, "create_challenge_widget", return_value=challenge_obj),\ patch.object(createchallenge, "create_evaluation_queue"),\ patch.object(createchallenge, "check_existing_and_delete_wiki"),\ patch.object(synapseutils, "copyWiki", return_value=[{'id': 'foo'}]),\ patch.object(SYN, "getWiki", return_value=wiki),\ patch.object(createchallenge, "_update_wikipage_string", return_value=wiki),\ patch.object(SYN, "store"): components = createchallenge.main(SYN, challenge_name, live_site="syn123") assert patch_set_perms.call_count == 2 assert patch_create_proj.call_count == 1 assert components == { "live_projectid": proj.id, "staging_projectid": proj.id, "admin_teamid": '2345', "organizer_teamid": '4567', "participant_teamid": '1234', "preregistrantrant_teamid": '3456' }
def test__extract_concrete_type(): assert SynapseProxy._extract_concrete_type( syn.Project()) == 'org.sagebionetworks.repo.model.Project' assert SynapseProxy._extract_concrete_type({ 'concreteType': 'org.sagebionetworks.repo.model.Project' }) == 'org.sagebionetworks.repo.model.Project' assert SynapseProxy._extract_concrete_type({ 'type': 'org.sagebionetworks.repo.model.Project' }) == 'org.sagebionetworks.repo.model.Project' with pytest.raises(ValueError) as err: SynapseProxy._extract_concrete_type({}) assert 'Cannot extract type from' in str(err)
def create_project(syn, project_name): """Creates Synapse Project Args: syn: Synpase object project_name: Name of project Returns: Project Entity """ project = synapseclient.Project(project_name) # returns the handle to the project if the user has sufficient priviledge project = syn.store(project) logger.info('Created/Fetched Project {} ({})'.format( project.name, project.id)) return project
def test__create_synapse_resources_recursive(self): """Test recursive calls are made""" project_ent = synapseclient.Project(id="syn5555") folder_ent = synapseclient.Folder(id="syn33333", parentId="syn5555") call_1 = mock.call(name="Genes", parentId="syn5555") call_2 = mock.call(name="testing", parentId="syn33333") with patch.object(self.create_cls, "get_or_create_project", return_value=project_ent) as patch_create_proj,\ patch.object(self.create_cls, "get_or_create_folder", return_value=folder_ent) as patch_create_folder: client._create_synapse_resources(config_list=self.config, creation_cls=self.create_cls) patch_create_proj.assert_called_once_with( name="Test Configuration" ) patch_create_folder.assert_has_calls([call_1, call_2])
def create_project(syn, project_name): ''' Convenience function to create Synapse Project Args: syn: Synpase object project_name: Name of project Returns: Project Entity ''' project = synapseclient.Project(project_name) # returns the handle to the project if the user has sufficient priviledge project = syn.store(project) logger.info('Created/Fetched Project %s (%s)' % (project.name, project.id)) return (project)
def test_syncFromSynapse_Links(): """This function tests recursive download of links as defined in syncFromSynapse most of the functionality of this function are already tested in the tests/integration/test_command_line_client::test_command_get_recursive_and_query which means that the only test if for path=None """ # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create a Folder hiearchy in folder_entity inner_folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=folder_entity)) second_folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) file_entity = syn.store(File(f, parent=project_entity)) # Create links to inner folder syn.store(Link(file_entity.id, parent=folder_entity)) #Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) file_entity = syn.store(File(f, parent=second_folder_entity)) # Create link to inner folder syn.store(Link(file_entity.id, parent=inner_folder_entity)) ### Test recursive get output = synapseutils.syncFromSynapse(syn, folder_entity, followLink=True) assert len(output) == len(uploaded_paths) for f in output: assert f.path in uploaded_paths
def setUp(self): print("Creating private Project...") test_project = sc.Project("Test" + uuid.uuid4().hex) self.project_id = syn.store(test_project).id print("Creating Folder...") folder = sc.Folder("folder", parent=self.project_id) self.folder_id = syn.store(folder).id print("Creating File within Folder...") with tempfile.NamedTemporaryFile() as temp: temp.write("123testingfolder") temp.flush() temp_file = sc.File(temp.name, parent=self.folder_id) self.folder_fileId = syn.store(temp_file).id print("Creating File within Project...") with tempfile.NamedTemporaryFile() as temp: temp.write("123testingproject") temp.flush() temp_file = sc.File(temp.name, parent=self.project_id) self.project_fileId = syn.store(temp_file).id
def test__create_synapse_resources_project(self): """Test project gets created""" project_name = 'Test Configuration' project_config = [{ 'name': project_name, 'type': 'Project' }] expected_config = [{ 'name': project_name, 'type': 'Project', 'id': 'syn12222' }] project_ent = synapseclient.Project(id="syn12222") with patch.object(self.create_cls, "get_or_create_project", return_value=project_ent) as patch_create: client._create_synapse_resources(config_list=project_config, creation_cls=self.create_cls) patch_create.assert_called_once_with(name=project_name) assert project_config == expected_config
def test_syncFromSynapse(): """This function tests recursive download as defined in syncFromSynapse most of the functionality of this function are already tested in the tests/integration/test_command_line_client::test_command_get_recursive_and_query which means that the only test if for path=None """ # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) syn.store(File(f, parent=folder_entity)) # Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) syn.store(File(f, parent=project_entity)) # syncFromSynapse() uses chunkedQuery() which will return results that are eventually consistent # but not always right after the entity is created. start_time = time.time() while len(list(syn.getChildren(project_entity))) != 2: assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC) time.sleep(2) # Test recursive get output = synapseutils.syncFromSynapse(syn, project_entity) assert_equals(len(output), len(uploaded_paths)) for f in output: assert_in(f.path, uploaded_paths)
def create_team_wikis(syn, synid, templateid, tracker_table_synid): """ Function that creates wiki pages from a template by looking at teams that are registered for a challenge. The teams that have a wiki made for them Are stored into a trackerTable that has columns wikiSynId, and teamId Args: synId: Synapse id of challenge project templateId: Synapse id of the template trackerTableSynId: Synapse id of Table that tracks if wiki pages have been made per team """ challenge_ent = syn.get(synid) challenge_obj = utils.get_challenge(challenge_ent) registered_teams = syn._GET_paginated("/challenge/{}/challengeTeam".format( challenge_obj['id'])) for i in registered_teams: submitted_teams = syn.tableQuery( "SELECT * FROM {} where teamId = '{}'".format( tracker_table_synid, i['teamId'])) if len(submitted_teams.asDataFrame()) == 0: team = syn.getTeam(i['teamId']) # The project name is the challenge project name and team name project = syn.store( synapseclient.Project("{} {}".format(challenge_ent.name, team.name))) # Give admin access to the team syn.setPermissions(project, i['teamId'], accessType=[ 'DELETE', 'CHANGE_SETTINGS', 'MODERATE', 'CREATE', 'READ', 'DOWNLOAD', 'UPDATE', 'CHANGE_PERMISSIONS' ]) wiki_copy = synapseutils.copy(syn, templateid, project.id) # syn.sendMessage(i[]) # Store copied synId to tracking table tracking_table = synapseclient.Table( tracker_table_synid, [[wiki_copy[templateid], i['teamId']]]) syn.store(tracking_table)
def create_project(self, name): """Creates a new project in Synapse. Args: name: The name of the project to create. Returns: SynapseRemoteEntity Raises: Exception: Raised if a project with the same name already exists. """ # Check if the project already exists. syn_project_id = SynapseAdapter.client().findEntityId(name=name) if syn_project_id: raise Exception( 'Synapse project already exists with name: {0}'.format(name)) syn_project = SynapseAdapter.client().store( synapseclient.Project(name=name)) return SynapseRemoteEntity(syn_project)
def _create_project(self): name = '_TEMP_{0}_VIEW_PROJECT_'.format(str(uuid.uuid4())) self.view_project = SynapseProxy.client().store(syn.Project(name=name))
import pytest from mock import patch, create_autospec import synapseclient from challengeutils import permissions SYN = create_autospec(synapseclient.Synapse) SET_PERMS = {"set"} @pytest.mark.parametrize( "entity,principalid,permission_level,mapped", [ # tuple with (input, expectedOutput) (synapseclient.Project(), None, "view", permissions.ENTITY_PERMS_MAPPINGS['view']), (synapseclient.Folder(parentId="syn123"), None, "download", permissions.ENTITY_PERMS_MAPPINGS['download']), (synapseclient.Entity(), None, "edit", permissions.ENTITY_PERMS_MAPPINGS['edit']), (synapseclient.Schema(parentId="syn123"), None, "edit_and_delete", permissions.ENTITY_PERMS_MAPPINGS['edit_and_delete']), (synapseclient.File(parentId="syn123"), None, "admin", permissions.ENTITY_PERMS_MAPPINGS['admin']), (synapseclient.Entity(), None, "remove", permissions.ENTITY_PERMS_MAPPINGS['remove']), (synapseclient.Evaluation(contentSource="syn123"), None, "view", permissions.EVALUATION_PERMS_MAPPINGS['view']), (synapseclient.Evaluation(contentSource="syn123"), None, "submit", permissions.EVALUATION_PERMS_MAPPINGS['submit']),
def test_migrate_project(request, syn, schedule_for_cleanup, storage_location_id): test_name = request.node.name project_name = "{}-{}".format(test_name, uuid.uuid4()) project = synapseclient.Project(name=project_name) project_entity = syn.store(project) file_0_path = _create_temp_file() schedule_for_cleanup(file_0_path) file_0_name = "{}-{}".format(test_name, 1) file_0 = synapseclient.File(name=file_0_name, path=file_0_path, parent=project_entity) file_0_entity = syn.store(file_0) default_storage_location_id = file_0_entity._file_handle[ 'storageLocationId'] folder_1_name = "{}-{}-{}".format(test_name, 1, uuid.uuid4()) folder_1 = synapseclient.Folder(parent=project_entity, name=folder_1_name) folder_1_entity = syn.store(folder_1) file_1_path = _create_temp_file() schedule_for_cleanup(file_1_path) file_1_name = "{}-{}".format(test_name, 1) file_1 = synapseclient.File(name=file_1_name, path=file_1_path, parent=folder_1_entity) file_1_entity = syn.store(file_1) file_2_path = _create_temp_file() schedule_for_cleanup(file_2_path) file_2_name = "{}-{}".format(test_name, 2) file_2 = synapseclient.File(name=file_2_name, path=file_2_path, parent=folder_1_entity) file_2_entity = syn.store(file_2) # file 3 shares the same file handle id as file 1 file_3_path = file_1_path file_3_name = "{}-{}".format(test_name, 3) file_3 = synapseclient.File(name=file_3_name, path=file_3_path, parent=folder_1_entity) file_3.dataFileHandleId = file_1_entity.dataFileHandleId file_3_entity = syn.store(file_3) table_1_cols = [ synapseclient.Column(name='file_col_1', columnType='FILEHANDLEID'), synapseclient.Column(name='num', columnType='INTEGER'), synapseclient.Column(name='file_col_2', columnType='FILEHANDLEID'), ] table_1 = syn.store( synapseclient.Schema(name=test_name, columns=table_1_cols, parent=folder_1_entity)) table_1_file_col_1_1 = _create_temp_file() table_1_file_handle_1 = syn.uploadFileHandle(table_1_file_col_1_1, table_1) table_1_file_col_1_2 = _create_temp_file() table_1_file_handle_2 = syn.uploadFileHandle(table_1_file_col_1_2, table_1) table_1_file_col_2_1 = _create_temp_file() table_1_file_handle_3 = syn.uploadFileHandle(table_1_file_col_2_1, table_1) table_1_file_col_2_2 = _create_temp_file() table_1_file_handle_4 = syn.uploadFileHandle(table_1_file_col_2_2, table_1) data = [ [table_1_file_handle_1['id'], 1, table_1_file_handle_2['id']], [table_1_file_handle_3['id'], 2, table_1_file_handle_4['id']], ] table_1_entity = syn.store( synapseclient.RowSet(schema=table_1, rows=[synapseclient.Row(r) for r in data])) db_path = tempfile.NamedTemporaryFile(delete=False).name schedule_for_cleanup(db_path) index_result = synapseutils.index_files_for_migration( syn, project_entity, storage_location_id, db_path, file_version_strategy='new', include_table_files=True, ) counts_by_status = index_result.get_counts_by_status() assert counts_by_status['INDEXED'] == 8 assert counts_by_status['ERRORED'] == 0 migration_result = synapseutils.migrate_indexed_files(syn, db_path, force=True) file_0_entity_updated = syn.get(utils.id_of(file_0_entity), downloadFile=False) file_1_entity_updated = syn.get(utils.id_of(file_1_entity), downloadFile=False) file_2_entity_updated = syn.get(utils.id_of(file_2_entity), downloadFile=False) file_3_entity_updated = syn.get(utils.id_of(file_3_entity), downloadFile=False) file_handles = [ f['_file_handle'] for f in ( file_0_entity_updated, file_1_entity_updated, file_2_entity_updated, file_3_entity_updated, ) ] table_1_id = utils.id_of(table_1_entity) results = syn.tableQuery("select file_col_1, file_col_2 from {}".format( utils.id_of(table_1_entity))) table_file_handles = [] for row in results: for file_handle_id in row[2:]: file_handle = syn._getFileHandleDownload( file_handle_id, table_1_id, objectType='TableEntity')['fileHandle'] table_file_handles.append(file_handle) file_handles.extend(table_file_handles) _assert_storage_location(file_handles, storage_location_id) assert storage_location_id != default_storage_location_id with sqlite3.connect(db_path) as conn: cursor = conn.cursor() query_result = cursor.execute( "select status, count(*) from migrations where type in (?, ?) group by status", (_MigrationType.FILE.value, _MigrationType.TABLE_ATTACHED_FILE.value)).fetchall() counts = {r[0]: r[1] for r in query_result} # should only be one status and they should all be migrated # should be 3 migrated files entities + 4 migrated table attached files assert len(counts) == 1 assert counts[_MigrationStatus.MIGRATED.value] == 8 csv_file = tempfile.NamedTemporaryFile(delete=False) schedule_for_cleanup(csv_file.name) migration_result.as_csv(csv_file.name) with open(csv_file.name, 'r') as csv_file_in: csv_contents = csv_file_in.read() table_1_id = table_1_entity['tableId'] # assert the content of the csv. we don't assert any particular order of the lines # but the presence of the expected lines and the correct # of lines csv_lines = csv_contents.split('\n') assert "id,type,version,row_id,col_name,from_storage_location_id,from_file_handle_id,to_file_handle_id,status,exception" in csv_lines # noqa assert f"{file_0_entity.id},file,,,,{default_storage_location_id},{file_0_entity.dataFileHandleId},{file_0_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa assert f"{file_1_entity.id},file,,,,{default_storage_location_id},{file_1_entity.dataFileHandleId},{file_1_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa assert f"{file_2_entity.id},file,,,,{default_storage_location_id},{file_2_entity.dataFileHandleId},{file_2_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa assert f"{file_3_entity.id},file,,,,{default_storage_location_id},{file_3_entity.dataFileHandleId},{file_3_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa assert f"{table_1_id},table,1,1,file_col_1,{default_storage_location_id},{table_1_file_handle_1['id']},{table_file_handles[0]['id']},MIGRATED," in csv_lines # noqa assert f"{table_1_id},table,1,1,file_col_2,{default_storage_location_id},{table_1_file_handle_2['id']},{table_file_handles[1]['id']},MIGRATED," in csv_lines # noqa assert f"{table_1_id},table,1,2,file_col_1,{default_storage_location_id},{table_1_file_handle_3['id']},{table_file_handles[2]['id']},MIGRATED," in csv_lines # noqa assert f"{table_1_id},table,1,2,file_col_2,{default_storage_location_id},{table_1_file_handle_4['id']},{table_file_handles[3]['id']},MIGRATED," in csv_lines # noqa assert "" in csv_lines # expect trailing newline in a csv
def main(syn): # Basic setup of the project project_name = "Testing Synapse Genie" # Determine the short and long names of the centers. center_abbreviations = ['AAA', 'BBB', 'CCC'] center_names = center_abbreviations # Create the project project = synapseclient.Project(project_name) project = syn.store(project) # Create a folder for log files generated by the GENIE processes # of validation and updating the database tables logs_folder = synapseclient.Folder(name='Logs', parent=project) logs_folder = syn.store(logs_folder) # Folder for individual center folders root_center_folder = synapseclient.Folder(name='Centers', parent=project) root_center_folder = syn.store(root_center_folder) # The folders for each center where they will upload files for validation # and submission. There is one folder per center. # This currently deviates from the original GENIE setup of having an # 'Input' and 'Staging' folder for each center. center_folders = [ synapseclient.Folder(name=name, parent=root_center_folder) for name in center_abbreviations ] center_folders = [syn.store(folder) for folder in center_folders] # Make some fake data that only contains basic text to check # for validation. n_files = 5 # number of files per center to create for folder in center_folders: for idx in range(n_files): tmp = tempfile.NamedTemporaryFile(prefix=f'TEST-{folder.name}', suffix='.txt') with open(tmp.name, mode='w') as fh: fh.write(random.choice(['ERROR', 'VALID', 'NOPE'])) synfile = syn.store(synapseclient.File(tmp.name, parent=folder)) # Set up the table that holds the validation status of all submitted files. status_schema = create_status_table(syn, project) # Set up the table that maps the center abbreviation to the folder where # their data is uploaded. This is used by the GENIE framework to find the # files to validate for a center. center_map_table_defs = [ { 'name': 'name', 'columnType': 'STRING', 'maximumSize': 250 }, { 'name': 'center', 'columnType': 'STRING', 'maximumSize': 50 }, { 'name': 'inputSynId', 'columnType': 'ENTITYID' }, # {'name': 'stagingSynId', # 'columnType': 'ENTITYID'}, { 'name': 'release', 'defaultValue': 'false', 'columnType': 'BOOLEAN' } # {'id': '68438', # 'name': 'mutationInCisFilter', # 'defaultValue': 'true', # 'columnType': 'BOOLEAN', # 'concreteType': 'org.sagebionetworks.repo.model.table.ColumnModel'} ] center_map_cols = [ synapseclient.Column(**col) for col in center_map_table_defs ] center_schema = synapseclient.Schema(name='Center Table', columns=center_map_cols, parent=project) center_schema = syn.store(center_schema) # Add the center folders created above to this table. center_folder_ids = [folder.id for folder in center_folders] center_df = pandas.DataFrame( dict(name=center_names, center=center_abbreviations, inputSynId=center_folder_ids)) tbl = synapseclient.Table(schema=center_schema, values=center_df) tbl = syn.store(tbl) # Create a table that stores the error logs for each submitted file. error_col_defs = [ { 'name': 'id', 'columnType': 'ENTITYID' }, { 'name': 'center', 'columnType': 'STRING', 'maximumSize': 50, 'facetType': 'enumeration' }, { 'name': 'errors', 'columnType': 'LARGETEXT' }, { 'name': 'name', 'columnType': 'STRING', 'maximumSize': 500 }, # {'name': 'versionNumber', # 'columnType': 'STRING', # 'maximumSize': 50}, { 'name': 'fileType', 'columnType': 'STRING', 'maximumSize': 50 } ] error_map_cols = [synapseclient.Column(**col) for col in error_col_defs] error_schema = synapseclient.Schema(name='Error Table', columns=error_map_cols, parent=project) error_schema = syn.store(error_schema) # Create a table that maps the various database tables to a short name. # This table is used in many GENIE functions to find the correct table to update # or get the state of something from. db_map_col_defs = [{ 'name': 'Database', 'columnType': 'STRING', 'maximumSize': 50 }, { 'name': 'Id', 'columnType': 'ENTITYID' }] db_map_cols = [synapseclient.Column(**col) for col in db_map_col_defs] db_map_schema = synapseclient.Schema(name='DB Mapping Table', columns=db_map_cols, parent=project) db_map_schema = syn.store(db_map_schema) # Add dbMapping annotation project.annotations.dbMapping = db_map_schema.tableId project = syn.store(project) # Add the tables we already created to the mapping table. dbmap_df = pandas.DataFrame( dict(Database=[ 'centerMapping', 'validationStatus', 'errorTracker', 'dbMapping', 'logs' ], Id=[ center_schema.id, status_schema.id, error_schema.id, db_map_schema.id, logs_folder.id ])) db_map_tbl = synapseclient.Table(schema=db_map_schema, values=dbmap_df) db_map_tbl = syn.store(db_map_tbl) # Make a top level folder for output. Some processing for # file types copy a file from one place to another. output_folder = synapseclient.Folder(name='Output', parent=project) output_folder = syn.store(output_folder) output_folder_map = [] # default_table_col_defs = status_table_col_defs = [ # {'name': 'PRIMARY_KEY', # 'columnType': 'STRING'} # ] # default_table_cols = [synapseclient.Column(**col) # for col in default_table_col_defs] default_primary_key = 'PRIMARY_KEY' # For each file type format in the format registry, create an output folder and a table. # Some GENIE file types copy a file to a new place, and some update a table. Having both # means that both of these operations will be available at the beginning. # The mapping between the file type and the folder or table have a consistent naming. # The key ('Database' value) is {file_type}_folder or {file_type}_table. # Determine which file formats are going to be used. format_registry = config.collect_format_types(['example_registry']) for file_type, obj in format_registry.items(): file_type_folder = synapseclient.Folder(name=file_type, parent=output_folder) file_type_folder = syn.store(file_type_folder) output_folder_map.append( dict(Database=f"{file_type}_folder", Id=file_type_folder.id)) file_type_schema = synapseclient.Schema(name=file_type, parent=project) file_type_schema.annotations.primaryKey = default_primary_key file_type_schema = syn.store(file_type_schema) output_folder_map.append( dict(Database=f"{file_type}_table", Id=file_type_schema.id)) # Add the folders and tables created to the mapping table. db_map_tbl = synapseclient.Table( schema=db_map_schema, values=pandas.DataFrame(output_folder_map)) db_map_tbl = syn.store(db_map_tbl)
database_mapping = pd.DataFrame(dict(Database=['bed'], Id=['syn8457748'])) symbols = pd.DataFrame( dict(Hugo_Symbol=['AAK1', 'AAED1', 'AAAS', 'AAED1'], ID=['AAK1', 'AAED', 'AAAS', 'AAD'])) # This is the gene positions that all bed dataframe will be processed against table_query_results_map = { ("SELECT * FROM syn10967259", ): createMockTable(database_mapping), ("select Hugo_Symbol, ID from syn8457748 where CENTER = 'SAGE'", ): createMockTable(symbols), } syn = mock.create_autospec(synapseclient.Synapse) syn.tableQuery.side_effect = table_query_results ENTITY = synapseclient.Project("testing", annotations={'dbMapping': ["syn10967259"]}) cna_class = cna(syn, "SAGE") def test_processing(): order = ["Hugo_Symbol", "Entrez_gene_id", "Id1-1", "Id2-1"] expected_cnadf = pd.DataFrame({ "Hugo_Symbol": ['AAED1', 'AAK1', 'AAAS'], "GENIE-SAGE-Id1-1": [-0.5, 2.0, 0.5], "GENIE-SAGE-Id2-1": [1.0, 1.5, -1.5] }) cnadf = pd.DataFrame({ "Hugo_Symbol": ['AAED', 'AAK1', 'AAAS'],