def setup(self):
self.project_entity = synapseclient.Project(name=str(uuid.uuid4()),
id="syn1234")
self.second_project = synapseclient.Project(name=str(uuid.uuid4()),
id="syn2345")
self.file_ent = synapseclient.File(name='File',
parent=self.project_entity.id)
self.file_ent.id = "syn3456"
def test_get_or_create_project__call():
"""Makes sure correct parameters are called"""
project_name = str(uuid.uuid1())
project = synapseclient.Project(name=project_name)
returned = synapseclient.Project(name=project_name,
id=str(uuid.uuid1()))
with patch.object(CREATE_CLS,
"_find_by_obj_or_create",
return_value=returned) as patch_find_or_create:
new_project = CREATE_CLS.get_or_create_project(name=project_name)
assert new_project == returned
patch_find_or_create.assert_called_once_with(project)
def archive_writeup(syn, evaluation, stat="VALIDATED", reArchive=False):
"""
Archive the submissions for the given evaluation queue and
store them in the destination synapse folder.
:param evaluation: a synapse evaluation queue or its ID
:param query: a query that will return the desired submissions.
At least the ID must be returned. Defaults to:
'select * from evaluation_[EVAL_ID] where status=="SCORED"'
"""
if type(evaluation) != synapseclient.Evaluation:
evaluation = syn.getEvaluation(evaluation)
print("\n\nArchiving", evaluation.id, evaluation.name)
print("-" * 60)
for sub, status in syn.getSubmissionBundles(evaluation, status=stat):
# retrieve file into cache and copy it to destination
checkIfArchived = filter(lambda x: x.get("key") == "archived",
status.annotations['stringAnnos'])
if len(list(checkIfArchived)) == 0 or reArchive:
projectEntity = synapseclient.Project(
'Archived {} {} {} {}'.format(
sub.name.replace("&", "+").replace("'", ""),
int(round(time.time() * 1000)), sub.id, sub.entityId))
entity = syn.store(projectEntity)
adminPriv = [
'DELETE', 'DOWNLOAD', 'CREATE', 'READ', 'CHANGE_PERMISSIONS',
'UPDATE', 'MODERATE', 'CHANGE_SETTINGS'
]
syn.setPermissions(entity, "3324230", adminPriv)
synapseutils.copy(syn, sub.entityId, entity.id)
archived = {"archived": entity.id}
status = utils.update_single_submission_status(status, archived)
syn.store(status)
def storeSubmissions(syn, evaluationQueue, synProject=None, filetype="csv"):
"""Store submissions on Synapse and submit to the evaluation queue.
Arguments
---------
syn : synapseclient.Synapse
evaluationQueue : int
Evaluation ID.
synProject : str
Synapse ID of project to store files to.
filetype : str
filename extension to append to filenames (default 'csv').
Returns
-------
None
"""
if not synProject:
logger.info("Creating new Synapse Project")
user_profile = syn.getUserProfile()
project_name = "{}_{}_testSubmissions".format(
user_profile['userName'], evaluationQueue)
synProject = sc.Project(project_name)
synProject = syn.store(synProject).id
logger.info("Synapse project created at {}".format(synProject))
for submission in glob.glob("{}/*.{}".format(TEST_SUBMISSION_PATH, filetype)):
logger.info("Storing {} to {} and submitting to {}".format(
submission, synProject, evaluationQueue))
synFile = sc.File(submission, parent=synProject)
synFile = syn.store(synFile)
syn.submit(evaluationQueue, synFile, name=os.path.basename(submission))
def test__create_synapse_resources_acl(self):
"""Test ACL gets created"""
project_name = 'Test Configuration'
project_config = [{
'name': project_name,
'type': 'Project',
'acl': ['fake']
}]
expected_config = [{
'name': project_name,
'type': 'Project',
'id': 'syn12222',
'acl': ['fake']
}]
project_ent = synapseclient.Project(id="syn12222")
with patch.object(self.create_cls, "get_or_create_project",
return_value=project_ent) as patch_create,\
patch.object(create, "_set_acl") as patch_set:
client._create_synapse_resources(config_list=project_config,
creation_cls=self.create_cls)
patch_create.assert_called_once_with(name=project_name)
assert project_config == expected_config
patch_set.assert_called_once_with(
syn=self.create_cls.syn, entity=project_ent,
acl_config=['fake'])
async def _create(self):
name = '_TEMP_{0}_TEMP_'.format(str(uuid.uuid4()))
logging.info('Creating file view project: {0}'.format(name))
self.view_project = await SynapseProxy.storeAsync(
syn.Project(name=name))
logging.info('Creating file view: {0}'.format(name))
cols = [
syn.Column(name=self.COL_ID, columnType='ENTITYID'),
syn.Column(name=self.COL_DATAFILEHANDLEID,
columnType='FILEHANDLEID'),
syn.Column(name=self.COL_NAME,
columnType='STRING',
maximumSize=256)
]
schema = syn.EntityViewSchema(
name=name,
columns=cols,
properties=None,
parent=self.view_project,
scopes=[self.scope],
includeEntityTypes=[syn.EntityViewType.FILE],
addDefaultViewColumns=False,
addAnnotationColumns=False)
self.view = await SynapseProxy.storeAsync(schema)
def test_syncFromSynapse():
"""This function tests recursive download as defined in syncFromSynapse
most of the functionality of this function are already tested in the
tests/integration/test_command_line_client::test_command_get_recursive_and_query
which means that the only test if for path=None
"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
syn.store(File(f, parent=folder_entity))
# Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
syn.store(File(f, parent=project_entity))
# Test recursive get
output = synapseutils.syncFromSynapse(syn, project_entity)
assert_equals(len(output), len(uploaded_paths))
for f in output:
assert_in(f.path, uploaded_paths)
def login(self):
"""Log in to Synapse and acquire the project entity."""
log.info(
"Initiating log in to Synapse and acquiring the project entity.")
self.syn.login(email=self.user.SYN_USERNAME, apiKey=self.user.API_KEY)
project_name = self.push_config.PROJECT_NAME
log.info("""Acquiring Synapse project instance for "{name}".""".format(
name=project_name))
try:
self.project = self.syn.get(synapse.Project(name=project_name))
except TypeError:
self.project = self.syn.store(synapse.Project(name=project_name))
self._build_remote_entity_dag()
def test_create_project():
"""Test creating project"""
name = str(uuid.uuid1())
proj = synapseclient.Project(name, id="syn1234")
with patch.object(SYN, "store", return_value=proj) as patch_store:
new_proj = createchallenge.create_project(SYN, proj.name)
assert new_proj == proj
patch_store.assert_called_once()
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()),
parent=project_entity))
schedule_for_cleanup(folder_entity.id)
# Create and upload a file in Folder
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': ['hello_world']}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity))
externalURL_entity = syn.store(synapseclient.File(repo_url, name='rand', parent=folder_entity, synapseStore=False))
syn.setAnnotations(file_entity, annots)
syn.setAnnotations(externalURL_entity, annots)
schedule_for_cleanup(file_entity.id)
schedule_for_cleanup(externalURL_entity.id)
# Test cp function
output = run('synapse', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
output_URL = run('synapse', '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL)
# Verify that our copied files are identical
copied_ent = syn.get(copied_id)
copied_URL_ent = syn.get(copied_URL_id, downloadFile=False)
schedule_for_cleanup(copied_id)
schedule_for_cleanup(copied_URL_id)
copied_ent_annot = syn.getAnnotations(copied_id)
copied_url_annot = syn.getAnnotations(copied_URL_id)
copied_prov = syn.getProvenance(copied_id)['used'][0]['reference']['targetId']
copied_url_prov = syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId']
# Make sure copied files are the same
assert_equals(copied_prov, file_entity.id)
assert_equals(copied_ent_annot, annots)
assert_equals(copied_ent.properties.dataFileHandleId, file_entity.properties.dataFileHandleId)
# Make sure copied URLs are the same
assert_equals(copied_url_prov, externalURL_entity.id)
assert_equals(copied_url_annot, annots)
assert_equals(copied_URL_ent.externalURL, repo_url)
assert_equals(copied_URL_ent.name, 'rand')
assert_equals(copied_URL_ent.properties.dataFileHandleId, externalURL_entity.properties.dataFileHandleId)
# Verify that errors are being thrown when a
# file is copied to a folder/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id,
'--destinationId', project_entity.id)
def test_command_get_recursive_and_query():
"""Tests the 'synapse get -r' and 'synapse get -q' functions"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=folder_entity)
file_entity.location = 'folder'
file_entity = syn.store(file_entity)
#Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=project_entity)
file_entity.location = 'project'
file_entity = syn.store(file_entity)
### Test recursive get
output = run('synapse', '--skip-checks', 'get', '-r', project_entity.id)
#Verify that we downloaded files:
new_paths = [
os.path.join('.', folder_entity.name, os.path.basename(f))
for f in uploaded_paths[:-1]
]
new_paths.append(os.path.join('.', os.path.basename(uploaded_paths[-1])))
schedule_for_cleanup(folder_entity.name)
for downloaded, uploaded in zip(new_paths, uploaded_paths):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(new_paths[0])
### Test query get
output = run(
'synapse', '--skip-checks', 'get', '-q',
"select id from file where parentId=='%s' and location=='folder'" %
folder_entity.id)
#Verify that we downloaded files:
new_paths = [
os.path.join('.', os.path.basename(f)) for f in uploaded_paths[:-1]
]
for downloaded, uploaded in zip(new_paths, uploaded_paths[:-1]):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(downloaded)
def bootstrap_infra(syn, args):
"""Create GENIE-like infrastructure"""
# Basic setup of the project
if args.project_name:
# Create the project
project = synapseclient.Project(args.project_name)
project = syn.store(project)
else:
project = syn.get(args.project_id)
bootstrap.main(syn,
project=project,
format_registry=args.format_registry_packages,
centers=args.centers)
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload a file in Folder
dummy = utils.make_bogus_data_file()
schedule_for_cleanup(dummy)
dummy_entity = syn.store(synapseclient.File(dummy, parent=folder_entity))
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': 'hello_world'}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity),
used=dummy_entity.id,
executed=repo_url)
syn.setAnnotations(file_entity, annots)
### Test cp function
output = run('synapse', '--skip-checks', 'cp', '--id', file_entity.id,
'--parentid', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
#Verify that our copied files are identical
copied_ent = syn.get(copied_id)
schedule_for_cleanup(copied_id)
copied_ent_annot = syn.getAnnotations(copied_ent)
copied_annot = dict((key, copied_ent_annot[key].pop())
for key in copied_ent_annot
if key not in ('uri', 'id', 'creationDate', 'etag'))
copied_prov = syn.getProvenance(
copied_ent)['used'][0]['reference']['targetId']
assert copied_prov == file_entity.id
assert copied_annot == annots
#Verify that errors are being thrown when folders/projects are attempted to be copied,
#or file is copied to a foler/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', folder_entity.id, '--parentid', project_entity.id)
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', project_entity.id, '--parentid', project_entity.id)
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', file_entity.id, '--parentid', project_entity.id)
def test_entity_type_display_name():
assert SynapseProxy.entity_type_display_name(
'org.sagebionetworks.repo.model.Project'
) == SynapseProxy.PROJECT_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
syn.Project()) == SynapseProxy.PROJECT_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name({
'concreteType':
'org.sagebionetworks.repo.model.Project'
}) == SynapseProxy.PROJECT_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
'org.sagebionetworks.repo.model.Folder'
) == SynapseProxy.FOLDER_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
syn.Folder(parentId='syn0')) == SynapseProxy.FOLDER_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name({
'concreteType':
'org.sagebionetworks.repo.model.Folder'
}) == SynapseProxy.FOLDER_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
'org.sagebionetworks.repo.model.FileEntity'
) == SynapseProxy.FILE_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
syn.File(parentId='syn0')) == SynapseProxy.FILE_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name({
'concreteType':
'org.sagebionetworks.repo.model.FileEntity'
}) == SynapseProxy.FILE_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
'org.sagebionetworks.repo.model.Link'
) == SynapseProxy.LINK_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
syn.Link(parentId='syn0',
targetId='syn0')) == SynapseProxy.LINK_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name({
'concreteType':
'org.sagebionetworks.repo.model.Link'
}) == SynapseProxy.LINK_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name(
'org.sagebionetworks.repo.model.table.TableEntity'
) == SynapseProxy.TABLE_TYPE_DISPLAY_NAME
assert SynapseProxy.entity_type_display_name({
'concreteType':
'org.sagebionetworks.repo.model.table.TableEntity'
}) == SynapseProxy.TABLE_TYPE_DISPLAY_NAME
def test_livesite_main():
"""Tests main when live site is not None"""
team_map = {
'team_part_id': '1234',
'team_admin_id': '2345',
'team_prereg_id': '3456',
'team_org_id': '4567'
}
teamid = str(uuid.uuid1())
project = str(uuid.uuid1())
chalid = str(uuid.uuid1())
etag = str(uuid.uuid1())
challenge_obj = Challenge(id=chalid,
projectId=project,
etag=etag,
participantTeamId=teamid)
proj = synapseclient.Project(project, id="syn1234")
wiki = synapseclient.Wiki(title='', owner=proj, markdown='')
challenge_name = str(uuid.uuid1())
with patch.object(createchallenge, "_create_teams",
return_value=team_map),\
patch.object(createchallenge, "create_project",
return_value=proj) as patch_create_proj,\
patch.object(SYN, "get", return_value=proj),\
patch.object(permissions,
"set_entity_permissions") as patch_set_perms,\
patch.object(createchallenge, "create_challenge_widget",
return_value=challenge_obj),\
patch.object(createchallenge, "create_evaluation_queue"),\
patch.object(createchallenge, "check_existing_and_delete_wiki"),\
patch.object(synapseutils, "copyWiki",
return_value=[{'id': 'foo'}]),\
patch.object(SYN, "getWiki", return_value=wiki),\
patch.object(createchallenge, "_update_wikipage_string",
return_value=wiki),\
patch.object(SYN, "store"):
components = createchallenge.main(SYN,
challenge_name,
live_site="syn123")
assert patch_set_perms.call_count == 2
assert patch_create_proj.call_count == 1
assert components == {
"live_projectid": proj.id,
"staging_projectid": proj.id,
"admin_teamid": '2345',
"organizer_teamid": '4567',
"participant_teamid": '1234',
"preregistrantrant_teamid": '3456'
}
def test__extract_concrete_type():
assert SynapseProxy._extract_concrete_type(
syn.Project()) == 'org.sagebionetworks.repo.model.Project'
assert SynapseProxy._extract_concrete_type({
'concreteType':
'org.sagebionetworks.repo.model.Project'
}) == 'org.sagebionetworks.repo.model.Project'
assert SynapseProxy._extract_concrete_type({
'type':
'org.sagebionetworks.repo.model.Project'
}) == 'org.sagebionetworks.repo.model.Project'
with pytest.raises(ValueError) as err:
SynapseProxy._extract_concrete_type({})
assert 'Cannot extract type from' in str(err)
def create_project(syn, project_name):
"""Creates Synapse Project
Args:
syn: Synpase object
project_name: Name of project
Returns:
Project Entity
"""
project = synapseclient.Project(project_name)
# returns the handle to the project if the user has sufficient priviledge
project = syn.store(project)
logger.info('Created/Fetched Project {} ({})'.format(
project.name, project.id))
return project
def test__create_synapse_resources_recursive(self):
"""Test recursive calls are made"""
project_ent = synapseclient.Project(id="syn5555")
folder_ent = synapseclient.Folder(id="syn33333", parentId="syn5555")
call_1 = mock.call(name="Genes", parentId="syn5555")
call_2 = mock.call(name="testing", parentId="syn33333")
with patch.object(self.create_cls, "get_or_create_project",
return_value=project_ent) as patch_create_proj,\
patch.object(self.create_cls, "get_or_create_folder",
return_value=folder_ent) as patch_create_folder:
client._create_synapse_resources(config_list=self.config,
creation_cls=self.create_cls)
patch_create_proj.assert_called_once_with(
name="Test Configuration"
)
patch_create_folder.assert_has_calls([call_1, call_2])
def create_project(syn, project_name):
'''
Convenience function to create Synapse Project
Args:
syn: Synpase object
project_name: Name of project
Returns:
Project Entity
'''
project = synapseclient.Project(project_name)
# returns the handle to the project if the user has sufficient priviledge
project = syn.store(project)
logger.info('Created/Fetched Project %s (%s)' % (project.name, project.id))
return (project)
def test_syncFromSynapse_Links():
"""This function tests recursive download of links as defined in syncFromSynapse
most of the functionality of this function are already tested in the
tests/integration/test_command_line_client::test_command_get_recursive_and_query
which means that the only test if for path=None
"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create a Folder hiearchy in folder_entity
inner_folder_entity = syn.store(
Folder(name=str(uuid.uuid4()), parent=folder_entity))
second_folder_entity = syn.store(
Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = syn.store(File(f, parent=project_entity))
# Create links to inner folder
syn.store(Link(file_entity.id, parent=folder_entity))
#Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = syn.store(File(f, parent=second_folder_entity))
# Create link to inner folder
syn.store(Link(file_entity.id, parent=inner_folder_entity))
### Test recursive get
output = synapseutils.syncFromSynapse(syn, folder_entity, followLink=True)
assert len(output) == len(uploaded_paths)
for f in output:
assert f.path in uploaded_paths
def setUp(self):
print("Creating private Project...")
test_project = sc.Project("Test" + uuid.uuid4().hex)
self.project_id = syn.store(test_project).id
print("Creating Folder...")
folder = sc.Folder("folder", parent=self.project_id)
self.folder_id = syn.store(folder).id
print("Creating File within Folder...")
with tempfile.NamedTemporaryFile() as temp:
temp.write("123testingfolder")
temp.flush()
temp_file = sc.File(temp.name, parent=self.folder_id)
self.folder_fileId = syn.store(temp_file).id
print("Creating File within Project...")
with tempfile.NamedTemporaryFile() as temp:
temp.write("123testingproject")
temp.flush()
temp_file = sc.File(temp.name, parent=self.project_id)
self.project_fileId = syn.store(temp_file).id
def test__create_synapse_resources_project(self):
"""Test project gets created"""
project_name = 'Test Configuration'
project_config = [{
'name': project_name,
'type': 'Project'
}]
expected_config = [{
'name': project_name,
'type': 'Project',
'id': 'syn12222'
}]
project_ent = synapseclient.Project(id="syn12222")
with patch.object(self.create_cls, "get_or_create_project",
return_value=project_ent) as patch_create:
client._create_synapse_resources(config_list=project_config,
creation_cls=self.create_cls)
patch_create.assert_called_once_with(name=project_name)
assert project_config == expected_config
def test_syncFromSynapse():
"""This function tests recursive download as defined in syncFromSynapse
most of the functionality of this function are already tested in the
tests/integration/test_command_line_client::test_command_get_recursive_and_query
which means that the only test if for path=None
"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
syn.store(File(f, parent=folder_entity))
# Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
syn.store(File(f, parent=project_entity))
# syncFromSynapse() uses chunkedQuery() which will return results that are eventually consistent
# but not always right after the entity is created.
start_time = time.time()
while len(list(syn.getChildren(project_entity))) != 2:
assert_less(time.time() - start_time, QUERY_TIMEOUT_SEC)
time.sleep(2)
# Test recursive get
output = synapseutils.syncFromSynapse(syn, project_entity)
assert_equals(len(output), len(uploaded_paths))
for f in output:
assert_in(f.path, uploaded_paths)
def create_team_wikis(syn, synid, templateid, tracker_table_synid):
"""
Function that creates wiki pages from a template by looking at teams that
are registered for a challenge. The teams that have a wiki made for them
Are stored into a trackerTable that has columns wikiSynId, and teamId
Args:
synId: Synapse id of challenge project
templateId: Synapse id of the template
trackerTableSynId: Synapse id of Table that tracks if wiki pages
have been made per team
"""
challenge_ent = syn.get(synid)
challenge_obj = utils.get_challenge(challenge_ent)
registered_teams = syn._GET_paginated("/challenge/{}/challengeTeam".format(
challenge_obj['id']))
for i in registered_teams:
submitted_teams = syn.tableQuery(
"SELECT * FROM {} where teamId = '{}'".format(
tracker_table_synid, i['teamId']))
if len(submitted_teams.asDataFrame()) == 0:
team = syn.getTeam(i['teamId'])
# The project name is the challenge project name and team name
project = syn.store(
synapseclient.Project("{} {}".format(challenge_ent.name,
team.name)))
# Give admin access to the team
syn.setPermissions(project,
i['teamId'],
accessType=[
'DELETE', 'CHANGE_SETTINGS', 'MODERATE',
'CREATE', 'READ', 'DOWNLOAD', 'UPDATE',
'CHANGE_PERMISSIONS'
])
wiki_copy = synapseutils.copy(syn, templateid, project.id)
# syn.sendMessage(i[])
# Store copied synId to tracking table
tracking_table = synapseclient.Table(
tracker_table_synid, [[wiki_copy[templateid], i['teamId']]])
syn.store(tracking_table)
def create_project(self, name):
"""Creates a new project in Synapse.
Args:
name: The name of the project to create.
Returns:
SynapseRemoteEntity
Raises:
Exception: Raised if a project with the same name already exists.
"""
# Check if the project already exists.
syn_project_id = SynapseAdapter.client().findEntityId(name=name)
if syn_project_id:
raise Exception(
'Synapse project already exists with name: {0}'.format(name))
syn_project = SynapseAdapter.client().store(
synapseclient.Project(name=name))
return SynapseRemoteEntity(syn_project)
def _create_project(self):
name = '_TEMP_{0}_VIEW_PROJECT_'.format(str(uuid.uuid4()))
self.view_project = SynapseProxy.client().store(syn.Project(name=name))
import pytest
from mock import patch, create_autospec
import synapseclient
from challengeutils import permissions
SYN = create_autospec(synapseclient.Synapse)
SET_PERMS = {"set"}
@pytest.mark.parametrize(
"entity,principalid,permission_level,mapped",
[
# tuple with (input, expectedOutput)
(synapseclient.Project(), None, "view",
permissions.ENTITY_PERMS_MAPPINGS['view']),
(synapseclient.Folder(parentId="syn123"), None, "download",
permissions.ENTITY_PERMS_MAPPINGS['download']),
(synapseclient.Entity(), None, "edit",
permissions.ENTITY_PERMS_MAPPINGS['edit']),
(synapseclient.Schema(parentId="syn123"), None, "edit_and_delete",
permissions.ENTITY_PERMS_MAPPINGS['edit_and_delete']),
(synapseclient.File(parentId="syn123"), None, "admin",
permissions.ENTITY_PERMS_MAPPINGS['admin']),
(synapseclient.Entity(), None, "remove",
permissions.ENTITY_PERMS_MAPPINGS['remove']),
(synapseclient.Evaluation(contentSource="syn123"), None, "view",
permissions.EVALUATION_PERMS_MAPPINGS['view']),
(synapseclient.Evaluation(contentSource="syn123"), None, "submit",
permissions.EVALUATION_PERMS_MAPPINGS['submit']),
def test_migrate_project(request, syn, schedule_for_cleanup,
storage_location_id):
test_name = request.node.name
project_name = "{}-{}".format(test_name, uuid.uuid4())
project = synapseclient.Project(name=project_name)
project_entity = syn.store(project)
file_0_path = _create_temp_file()
schedule_for_cleanup(file_0_path)
file_0_name = "{}-{}".format(test_name, 1)
file_0 = synapseclient.File(name=file_0_name,
path=file_0_path,
parent=project_entity)
file_0_entity = syn.store(file_0)
default_storage_location_id = file_0_entity._file_handle[
'storageLocationId']
folder_1_name = "{}-{}-{}".format(test_name, 1, uuid.uuid4())
folder_1 = synapseclient.Folder(parent=project_entity, name=folder_1_name)
folder_1_entity = syn.store(folder_1)
file_1_path = _create_temp_file()
schedule_for_cleanup(file_1_path)
file_1_name = "{}-{}".format(test_name, 1)
file_1 = synapseclient.File(name=file_1_name,
path=file_1_path,
parent=folder_1_entity)
file_1_entity = syn.store(file_1)
file_2_path = _create_temp_file()
schedule_for_cleanup(file_2_path)
file_2_name = "{}-{}".format(test_name, 2)
file_2 = synapseclient.File(name=file_2_name,
path=file_2_path,
parent=folder_1_entity)
file_2_entity = syn.store(file_2)
# file 3 shares the same file handle id as file 1
file_3_path = file_1_path
file_3_name = "{}-{}".format(test_name, 3)
file_3 = synapseclient.File(name=file_3_name,
path=file_3_path,
parent=folder_1_entity)
file_3.dataFileHandleId = file_1_entity.dataFileHandleId
file_3_entity = syn.store(file_3)
table_1_cols = [
synapseclient.Column(name='file_col_1', columnType='FILEHANDLEID'),
synapseclient.Column(name='num', columnType='INTEGER'),
synapseclient.Column(name='file_col_2', columnType='FILEHANDLEID'),
]
table_1 = syn.store(
synapseclient.Schema(name=test_name,
columns=table_1_cols,
parent=folder_1_entity))
table_1_file_col_1_1 = _create_temp_file()
table_1_file_handle_1 = syn.uploadFileHandle(table_1_file_col_1_1, table_1)
table_1_file_col_1_2 = _create_temp_file()
table_1_file_handle_2 = syn.uploadFileHandle(table_1_file_col_1_2, table_1)
table_1_file_col_2_1 = _create_temp_file()
table_1_file_handle_3 = syn.uploadFileHandle(table_1_file_col_2_1, table_1)
table_1_file_col_2_2 = _create_temp_file()
table_1_file_handle_4 = syn.uploadFileHandle(table_1_file_col_2_2, table_1)
data = [
[table_1_file_handle_1['id'], 1, table_1_file_handle_2['id']],
[table_1_file_handle_3['id'], 2, table_1_file_handle_4['id']],
]
table_1_entity = syn.store(
synapseclient.RowSet(schema=table_1,
rows=[synapseclient.Row(r) for r in data]))
db_path = tempfile.NamedTemporaryFile(delete=False).name
schedule_for_cleanup(db_path)
index_result = synapseutils.index_files_for_migration(
syn,
project_entity,
storage_location_id,
db_path,
file_version_strategy='new',
include_table_files=True,
)
counts_by_status = index_result.get_counts_by_status()
assert counts_by_status['INDEXED'] == 8
assert counts_by_status['ERRORED'] == 0
migration_result = synapseutils.migrate_indexed_files(syn,
db_path,
force=True)
file_0_entity_updated = syn.get(utils.id_of(file_0_entity),
downloadFile=False)
file_1_entity_updated = syn.get(utils.id_of(file_1_entity),
downloadFile=False)
file_2_entity_updated = syn.get(utils.id_of(file_2_entity),
downloadFile=False)
file_3_entity_updated = syn.get(utils.id_of(file_3_entity),
downloadFile=False)
file_handles = [
f['_file_handle'] for f in (
file_0_entity_updated,
file_1_entity_updated,
file_2_entity_updated,
file_3_entity_updated,
)
]
table_1_id = utils.id_of(table_1_entity)
results = syn.tableQuery("select file_col_1, file_col_2 from {}".format(
utils.id_of(table_1_entity)))
table_file_handles = []
for row in results:
for file_handle_id in row[2:]:
file_handle = syn._getFileHandleDownload(
file_handle_id, table_1_id,
objectType='TableEntity')['fileHandle']
table_file_handles.append(file_handle)
file_handles.extend(table_file_handles)
_assert_storage_location(file_handles, storage_location_id)
assert storage_location_id != default_storage_location_id
with sqlite3.connect(db_path) as conn:
cursor = conn.cursor()
query_result = cursor.execute(
"select status, count(*) from migrations where type in (?, ?) group by status",
(_MigrationType.FILE.value,
_MigrationType.TABLE_ATTACHED_FILE.value)).fetchall()
counts = {r[0]: r[1] for r in query_result}
# should only be one status and they should all be migrated
# should be 3 migrated files entities + 4 migrated table attached files
assert len(counts) == 1
assert counts[_MigrationStatus.MIGRATED.value] == 8
csv_file = tempfile.NamedTemporaryFile(delete=False)
schedule_for_cleanup(csv_file.name)
migration_result.as_csv(csv_file.name)
with open(csv_file.name, 'r') as csv_file_in:
csv_contents = csv_file_in.read()
table_1_id = table_1_entity['tableId']
# assert the content of the csv. we don't assert any particular order of the lines
# but the presence of the expected lines and the correct # of lines
csv_lines = csv_contents.split('\n')
assert "id,type,version,row_id,col_name,from_storage_location_id,from_file_handle_id,to_file_handle_id,status,exception" in csv_lines # noqa
assert f"{file_0_entity.id},file,,,,{default_storage_location_id},{file_0_entity.dataFileHandleId},{file_0_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa
assert f"{file_1_entity.id},file,,,,{default_storage_location_id},{file_1_entity.dataFileHandleId},{file_1_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa
assert f"{file_2_entity.id},file,,,,{default_storage_location_id},{file_2_entity.dataFileHandleId},{file_2_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa
assert f"{file_3_entity.id},file,,,,{default_storage_location_id},{file_3_entity.dataFileHandleId},{file_3_entity_updated.dataFileHandleId},MIGRATED," in csv_lines # noqa
assert f"{table_1_id},table,1,1,file_col_1,{default_storage_location_id},{table_1_file_handle_1['id']},{table_file_handles[0]['id']},MIGRATED," in csv_lines # noqa
assert f"{table_1_id},table,1,1,file_col_2,{default_storage_location_id},{table_1_file_handle_2['id']},{table_file_handles[1]['id']},MIGRATED," in csv_lines # noqa
assert f"{table_1_id},table,1,2,file_col_1,{default_storage_location_id},{table_1_file_handle_3['id']},{table_file_handles[2]['id']},MIGRATED," in csv_lines # noqa
assert f"{table_1_id},table,1,2,file_col_2,{default_storage_location_id},{table_1_file_handle_4['id']},{table_file_handles[3]['id']},MIGRATED," in csv_lines # noqa
assert "" in csv_lines # expect trailing newline in a csv
def main(syn):
# Basic setup of the project
project_name = "Testing Synapse Genie"
# Determine the short and long names of the centers.
center_abbreviations = ['AAA', 'BBB', 'CCC']
center_names = center_abbreviations
# Create the project
project = synapseclient.Project(project_name)
project = syn.store(project)
# Create a folder for log files generated by the GENIE processes
# of validation and updating the database tables
logs_folder = synapseclient.Folder(name='Logs', parent=project)
logs_folder = syn.store(logs_folder)
# Folder for individual center folders
root_center_folder = synapseclient.Folder(name='Centers', parent=project)
root_center_folder = syn.store(root_center_folder)
# The folders for each center where they will upload files for validation
# and submission. There is one folder per center.
# This currently deviates from the original GENIE setup of having an
# 'Input' and 'Staging' folder for each center.
center_folders = [
synapseclient.Folder(name=name, parent=root_center_folder)
for name in center_abbreviations
]
center_folders = [syn.store(folder) for folder in center_folders]
# Make some fake data that only contains basic text to check
# for validation.
n_files = 5 # number of files per center to create
for folder in center_folders:
for idx in range(n_files):
tmp = tempfile.NamedTemporaryFile(prefix=f'TEST-{folder.name}',
suffix='.txt')
with open(tmp.name, mode='w') as fh:
fh.write(random.choice(['ERROR', 'VALID', 'NOPE']))
synfile = syn.store(synapseclient.File(tmp.name, parent=folder))
# Set up the table that holds the validation status of all submitted files.
status_schema = create_status_table(syn, project)
# Set up the table that maps the center abbreviation to the folder where
# their data is uploaded. This is used by the GENIE framework to find the
# files to validate for a center.
center_map_table_defs = [
{
'name': 'name',
'columnType': 'STRING',
'maximumSize': 250
},
{
'name': 'center',
'columnType': 'STRING',
'maximumSize': 50
},
{
'name': 'inputSynId',
'columnType': 'ENTITYID'
},
# {'name': 'stagingSynId',
# 'columnType': 'ENTITYID'},
{
'name': 'release',
'defaultValue': 'false',
'columnType': 'BOOLEAN'
}
# {'id': '68438',
# 'name': 'mutationInCisFilter',
# 'defaultValue': 'true',
# 'columnType': 'BOOLEAN',
# 'concreteType': 'org.sagebionetworks.repo.model.table.ColumnModel'}
]
center_map_cols = [
synapseclient.Column(**col) for col in center_map_table_defs
]
center_schema = synapseclient.Schema(name='Center Table',
columns=center_map_cols,
parent=project)
center_schema = syn.store(center_schema)
# Add the center folders created above to this table.
center_folder_ids = [folder.id for folder in center_folders]
center_df = pandas.DataFrame(
dict(name=center_names,
center=center_abbreviations,
inputSynId=center_folder_ids))
tbl = synapseclient.Table(schema=center_schema, values=center_df)
tbl = syn.store(tbl)
# Create a table that stores the error logs for each submitted file.
error_col_defs = [
{
'name': 'id',
'columnType': 'ENTITYID'
},
{
'name': 'center',
'columnType': 'STRING',
'maximumSize': 50,
'facetType': 'enumeration'
},
{
'name': 'errors',
'columnType': 'LARGETEXT'
},
{
'name': 'name',
'columnType': 'STRING',
'maximumSize': 500
},
# {'name': 'versionNumber',
# 'columnType': 'STRING',
# 'maximumSize': 50},
{
'name': 'fileType',
'columnType': 'STRING',
'maximumSize': 50
}
]
error_map_cols = [synapseclient.Column(**col) for col in error_col_defs]
error_schema = synapseclient.Schema(name='Error Table',
columns=error_map_cols,
parent=project)
error_schema = syn.store(error_schema)
# Create a table that maps the various database tables to a short name.
# This table is used in many GENIE functions to find the correct table to update
# or get the state of something from.
db_map_col_defs = [{
'name': 'Database',
'columnType': 'STRING',
'maximumSize': 50
}, {
'name': 'Id',
'columnType': 'ENTITYID'
}]
db_map_cols = [synapseclient.Column(**col) for col in db_map_col_defs]
db_map_schema = synapseclient.Schema(name='DB Mapping Table',
columns=db_map_cols,
parent=project)
db_map_schema = syn.store(db_map_schema)
# Add dbMapping annotation
project.annotations.dbMapping = db_map_schema.tableId
project = syn.store(project)
# Add the tables we already created to the mapping table.
dbmap_df = pandas.DataFrame(
dict(Database=[
'centerMapping', 'validationStatus', 'errorTracker', 'dbMapping',
'logs'
],
Id=[
center_schema.id, status_schema.id, error_schema.id,
db_map_schema.id, logs_folder.id
]))
db_map_tbl = synapseclient.Table(schema=db_map_schema, values=dbmap_df)
db_map_tbl = syn.store(db_map_tbl)
# Make a top level folder for output. Some processing for
# file types copy a file from one place to another.
output_folder = synapseclient.Folder(name='Output', parent=project)
output_folder = syn.store(output_folder)
output_folder_map = []
# default_table_col_defs = status_table_col_defs = [
# {'name': 'PRIMARY_KEY',
# 'columnType': 'STRING'}
# ]
# default_table_cols = [synapseclient.Column(**col)
# for col in default_table_col_defs]
default_primary_key = 'PRIMARY_KEY'
# For each file type format in the format registry, create an output folder and a table.
# Some GENIE file types copy a file to a new place, and some update a table. Having both
# means that both of these operations will be available at the beginning.
# The mapping between the file type and the folder or table have a consistent naming.
# The key ('Database' value) is {file_type}_folder or {file_type}_table.
# Determine which file formats are going to be used.
format_registry = config.collect_format_types(['example_registry'])
for file_type, obj in format_registry.items():
file_type_folder = synapseclient.Folder(name=file_type,
parent=output_folder)
file_type_folder = syn.store(file_type_folder)
output_folder_map.append(
dict(Database=f"{file_type}_folder", Id=file_type_folder.id))
file_type_schema = synapseclient.Schema(name=file_type, parent=project)
file_type_schema.annotations.primaryKey = default_primary_key
file_type_schema = syn.store(file_type_schema)
output_folder_map.append(
dict(Database=f"{file_type}_table", Id=file_type_schema.id))
# Add the folders and tables created to the mapping table.
db_map_tbl = synapseclient.Table(
schema=db_map_schema, values=pandas.DataFrame(output_folder_map))
db_map_tbl = syn.store(db_map_tbl)
database_mapping = pd.DataFrame(dict(Database=['bed'], Id=['syn8457748']))
symbols = pd.DataFrame(
dict(Hugo_Symbol=['AAK1', 'AAED1', 'AAAS', 'AAED1'],
ID=['AAK1', 'AAED', 'AAAS', 'AAD']))
# This is the gene positions that all bed dataframe will be processed against
table_query_results_map = {
("SELECT * FROM syn10967259", ):
createMockTable(database_mapping),
("select Hugo_Symbol, ID from syn8457748 where CENTER = 'SAGE'", ):
createMockTable(symbols),
}
syn = mock.create_autospec(synapseclient.Synapse)
syn.tableQuery.side_effect = table_query_results
ENTITY = synapseclient.Project("testing",
annotations={'dbMapping': ["syn10967259"]})
cna_class = cna(syn, "SAGE")
def test_processing():
order = ["Hugo_Symbol", "Entrez_gene_id", "Id1-1", "Id2-1"]
expected_cnadf = pd.DataFrame({
"Hugo_Symbol": ['AAED1', 'AAK1', 'AAAS'],
"GENIE-SAGE-Id1-1": [-0.5, 2.0, 0.5],
"GENIE-SAGE-Id2-1": [1.0, 1.5, -1.5]
})
cnadf = pd.DataFrame({
"Hugo_Symbol": ['AAED', 'AAK1', 'AAAS'],
