def test_valid__upload_to_synapse():
"""Test upload of file to synapse under right conditions"""
ent = synapseclient.File(id="syn123", parentId="syn222")
with patch.object(syn, "store", return_value=ent) as patch_synstore:
validate._upload_to_synapse(syn, ['foo'], True, parentid="syn123")
patch_synstore.assert_called_once_with(
synapseclient.File('foo', parent="syn123"))
def copy(args,syn):
"""Copies most recent version of a file specifed by args.id to args.parentId"""
ent = syn.get(args.id, downloadFile=False)
profile = syn.getUserProfile().ownerId
#CHECK: Must be a file entity
if ent.entityType!='org.sagebionetworks.repo.model.FileEntity':
raise ValueError('"synapse cp" can only copy files!')
#Grab file handle createdBy annotation to see the user that created fileHandle
createdBy = syn.restGET('/entity/%s/filehandles'%args.id)['list'][0]['createdBy']
#CHECK: If file is in the same parent directory (throw an error)
search = syn.query('select name from file where parentId =="%s"'%args.parentid)['results']
for i in search:
if i['file.name'] == ent.name:
raise ValueError('Filename exists in directory you would like to copy to, either rename or check if file has already been copied!')
#CHECK: If the user created the file, copy the file by using fileHandleId else hard copy
if profile == createdBy:
new_ent = synapseclient.File(name=ent.name, parentId=args.parentid)
new_ent.properties.dataFileHandleId = ent.properties.dataFileHandleId
new_ent = syn._createEntity(new_ent)
else:
ent = syn.get(args.id)
new_ent = synapseclient.File(ent.path, parent=args.parentid)
new_ent = syn.store(new_ent)
syn.setAnnotations(new_ent, ent.annotations)
act = Activity("Copied file", used=args.id)
syn.setProvenance(new_ent['id'], act)
print('Copied %s to %s' %(ent.id, new_ent['id']))
def test_data_pull_non_data_folder(syn_test_helper, mk_tempfile,
mk_uniq_string, mk_kiproject):
syn_project = syn_test_helper.create_project()
syn_folder1 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder1', parent=syn_project))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder1))
syn_folder2 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder2', parent=syn_folder1))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder2))
syn_folder3 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder3', parent=syn_folder2))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder3))
syn_folder4 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder4', parent=syn_folder3))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder4))
syn_folder5 = syn_test_helper.client().store(
synapseclient.Folder(name='Folder5', parent=syn_folder4))
syn_test_helper.client().store(
synapseclient.File(path=mk_tempfile(), parent=syn_folder5))
kiproject = mk_kiproject()
kiproject.data_add(DataUri('syn', syn_folder1.id).uri,
data_type=kiproject.data_types[0])
kiproject.data_pull()
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()),
parent=project_entity))
schedule_for_cleanup(folder_entity.id)
# Create and upload a file in Folder
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': ['hello_world']}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity))
externalURL_entity = syn.store(synapseclient.File(repo_url, name='rand', parent=folder_entity, synapseStore=False))
syn.setAnnotations(file_entity, annots)
syn.setAnnotations(externalURL_entity, annots)
schedule_for_cleanup(file_entity.id)
schedule_for_cleanup(externalURL_entity.id)
# Test cp function
output = run('synapse', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
output_URL = run('synapse', '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL)
# Verify that our copied files are identical
copied_ent = syn.get(copied_id)
copied_URL_ent = syn.get(copied_URL_id, downloadFile=False)
schedule_for_cleanup(copied_id)
schedule_for_cleanup(copied_URL_id)
copied_ent_annot = syn.getAnnotations(copied_id)
copied_url_annot = syn.getAnnotations(copied_URL_id)
copied_prov = syn.getProvenance(copied_id)['used'][0]['reference']['targetId']
copied_url_prov = syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId']
# Make sure copied files are the same
assert_equals(copied_prov, file_entity.id)
assert_equals(copied_ent_annot, annots)
assert_equals(copied_ent.properties.dataFileHandleId, file_entity.properties.dataFileHandleId)
# Make sure copied URLs are the same
assert_equals(copied_url_prov, externalURL_entity.id)
assert_equals(copied_url_annot, annots)
assert_equals(copied_URL_ent.externalURL, repo_url)
assert_equals(copied_URL_ent.name, 'rand')
assert_equals(copied_URL_ent.properties.dataFileHandleId, externalURL_entity.properties.dataFileHandleId)
# Verify that errors are being thrown when a
# file is copied to a folder/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id,
'--destinationId', project_entity.id)
def test_command_get_recursive_and_query():
"""Tests the 'synapse get -r' and 'synapse get -q' functions"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=folder_entity)
file_entity.location = 'folder'
file_entity = syn.store(file_entity)
#Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=project_entity)
file_entity.location = 'project'
file_entity = syn.store(file_entity)
### Test recursive get
output = run('synapse', '--skip-checks', 'get', '-r', project_entity.id)
#Verify that we downloaded files:
new_paths = [
os.path.join('.', folder_entity.name, os.path.basename(f))
for f in uploaded_paths[:-1]
]
new_paths.append(os.path.join('.', os.path.basename(uploaded_paths[-1])))
schedule_for_cleanup(folder_entity.name)
for downloaded, uploaded in zip(new_paths, uploaded_paths):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(new_paths[0])
### Test query get
output = run(
'synapse', '--skip-checks', 'get', '-q',
"select id from file where parentId=='%s' and location=='folder'" %
folder_entity.id)
#Verify that we downloaded files:
new_paths = [
os.path.join('.', os.path.basename(f)) for f in uploaded_paths[:-1]
]
for downloaded, uploaded in zip(new_paths, uploaded_paths[:-1]):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(downloaded)
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload a file in Folder
dummy = utils.make_bogus_data_file()
schedule_for_cleanup(dummy)
dummy_entity = syn.store(synapseclient.File(dummy, parent=folder_entity))
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': 'hello_world'}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity),
used=dummy_entity.id,
executed=repo_url)
syn.setAnnotations(file_entity, annots)
### Test cp function
output = run('synapse', '--skip-checks', 'cp', '--id', file_entity.id,
'--parentid', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
#Verify that our copied files are identical
copied_ent = syn.get(copied_id)
schedule_for_cleanup(copied_id)
copied_ent_annot = syn.getAnnotations(copied_ent)
copied_annot = dict((key, copied_ent_annot[key].pop())
for key in copied_ent_annot
if key not in ('uri', 'id', 'creationDate', 'etag'))
copied_prov = syn.getProvenance(
copied_ent)['used'][0]['reference']['targetId']
assert copied_prov == file_entity.id
assert copied_annot == annots
#Verify that errors are being thrown when folders/projects are attempted to be copied,
#or file is copied to a foler/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', folder_entity.id, '--parentid', project_entity.id)
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', project_entity.id, '--parentid', project_entity.id)
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', file_entity.id, '--parentid', project_entity.id)
def storeFile(
syn,
fileName,
parentId,
center,
fileFormat,
dataSubType,
platform=None,
cBioFileFormat=None,
used=None,
):
"""
# Storing Files along with annotations
"""
logger.info("STORING FILES")
fileEnt = synapseclient.File(fileName, parent=parentId)
fileEnt.center = center
fileEnt.species = "Human"
fileEnt.consortium = "GENIE"
fileEnt.dataType = "genomicVariants"
fileEnt.fundingAgency = "AACR"
fileEnt.assay = "targetGeneSeq"
fileEnt.fileFormat = fileFormat
fileEnt.dataSubType = dataSubType
fileEnt.fileStage = "staging"
fileEnt.platform = platform
if platform is not None:
fileEnt.platform = platform
if cBioFileFormat is not None:
fileEnt.cBioFileFormat = cBioFileFormat
ent = syn.store(fileEnt, used=used)
return ent
def create_filehandle(event, filename, bucket, key, project_id):
parent = get_parent_folder(project_id, key)
eTag = event['Records'][0]['s3']['object']['eTag']
file_id = syn.findEntityId(filename, parent)
if file_id != None:
targetMD5 = syn.get(file_id, downloadFile=False)['md5'];
# create filehandle if it does not exist in Synapse or if existing file was modified (check md5):
if file_id == None or eTag != targetMD5:
size = event['Records'][0]['s3']['object']['size']
contentType = mimetypes.guess_type(filename, strict=False)[0]
storage_id = syn.restGET("/projectSettings/"+project_id+"/type/upload")['locations'][0]
fileHandle = {'concreteType': 'org.sagebionetworks.repo.model.file.S3FileHandle',
'fileName' : filename,
'contentSize' : size,
'contentType' : contentType,
'contentMd5' : eTag,
'bucketName' : bucket,
'key' : key,
'storageLocationId': storage_id}
fileHandle = syn.restPOST('/externalFileHandle/s3', json.dumps(fileHandle), endpoint=syn.fileHandleEndpoint)
f = synapseclient.File(parentId=parent, dataFileHandleId=fileHandle['id'], name=filename, synapseStore=False)
f = syn.store(f)
def submit(args, syn):
"""
Method to allow challenge participants to submit to an evaluation queue.
Examples::
synapse submit --evaluation 'ra_challenge_Q1_leaderboard' -f ~/testing/testing.txt --parentId syn2345030 \
--used syn2351967 --executed syn2351968
synapse submit --evaluation 2343117 -f ~/testing/testing.txt --parentId syn2345030 --used syn2351967 \
--executed syn2351968
"""
# check if evaluation is a number, if so it is assumed to be a evaluationId else it is a evaluationName
if args.evaluation is not None:
try:
args.evaluationID = str(int(args.evaluation))
except ValueError:
args.evaluationName = args.evaluation
# checking if user has entered a evaluation ID or evaluation Name
if args.evaluationID is None and args.evaluationName is None:
raise ValueError('Evaluation ID or Evaluation Name is required\n')
elif args.evaluationID is not None and args.evaluationName is not None:
sys.stderr.write(
'[Warning]: Both Evaluation ID & Evaluation Name are specified \n EvaluationID will be used\n'
)
elif args.evaluationID is None: # get evalID from evalName
try:
args.evaluationID = syn.getEvaluationByName(
args.evaluationName)['id']
except Exception:
raise ValueError('Could not find an evaluation named: %s \n' %
args.evaluationName)
# checking if a entity id or file was specified by the user
if args.entity is None and args.file is None:
raise ValueError(
'Either entityID or filename is required for a submission\n')
elif args.entity is not None and args.file is not None:
sys.stderr.write(
'[Warning]: Both entityID and filename are specified \n entityID will be used\n'
)
elif args.entity is None: # upload the the file to synapse and get synapse entity id for the file
if args.parentid is None:
raise ValueError('parentID required with a file upload\n')
if not os.path.exists(args.file):
raise IOError('file path %s not valid \n' % args.file)
# //ideally this should be factored out
synFile = syn.store(
synapseclient.File(path=args.file, parent=args.parentid),
used=syn._convertProvenanceList(args.used, args.limitSearch),
executed=syn._convertProvenanceList(args.executed,
args.limitSearch))
args.entity = synFile.id
submission = syn.submit(args.evaluationID,
args.entity,
name=args.name,
team=args.teamName)
sys.stdout.write('Submitted (id: %s) entity: %s\t%s to Evaluation: %s\n' %
(submission['id'], submission['entityId'],
submission['name'], submission['evaluationId']))
def storeSubmissions(syn, evaluationQueue, synProject=None, filetype="csv"):
"""Store submissions on Synapse and submit to the evaluation queue.
Arguments
---------
syn : synapseclient.Synapse
evaluationQueue : int
Evaluation ID.
synProject : str
Synapse ID of project to store files to.
filetype : str
filename extension to append to filenames (default 'csv').
Returns
-------
None
"""
if not synProject:
logger.info("Creating new Synapse Project")
user_profile = syn.getUserProfile()
project_name = "{}_{}_testSubmissions".format(
user_profile['userName'], evaluationQueue)
synProject = sc.Project(project_name)
synProject = syn.store(synProject).id
logger.info("Synapse project created at {}".format(synProject))
for submission in glob.glob("{}/*.{}".format(TEST_SUBMISSION_PATH, filetype)):
logger.info("Storing {} to {} and submitting to {}".format(
submission, synProject, evaluationQueue))
synFile = sc.File(submission, parent=synProject)
synFile = syn.store(synFile)
syn.submit(evaluationQueue, synFile, name=os.path.basename(submission))
def test_get_or_create_file__call():
"""Makes sure correct parameters are called"""
file_path = str(uuid.uuid1())
parentid = str(uuid.uuid1())
file_ent = synapseclient.File(path=file_path,
parentId=parentid)
returned = synapseclient.File(path=file_path,
id=str(uuid.uuid1()),
parentId=parentid)
with patch.object(CREATE_CLS,
"_find_by_obj_or_create",
return_value=returned) as patch_find_or_create:
new_file = CREATE_CLS.get_or_create_file(path=file_path,
parentId=parentid)
assert new_file == returned
patch_find_or_create.assert_called_once_with(file_ent)
def storeProcessedMaf(
self, filePath, mafSynId, centerMafSynId, isNarrow=False):
'''
Stores the processed maf
There is a isNarrow option, but note that the number of rows
of the maf file DOES NOT change in this function
Args:
filePath: Path to maf file
mafSynId: database synid
centerMafSynid: center flat file folder synid
isNarrow: Is the file a narrow maf. Defaul to False.
'''
logger.info('STORING %s' % filePath)
database = self.syn.get(mafSynId)
if isNarrow:
try:
update_table = synapseclient.Table(
database.id, filePath, separator="\t")
self.syn.store(update_table)
except SynapseTimeoutError:
# This error occurs because of waiting for table to index.
# Don't worry about this.
pass
else:
self.syn.store(
synapseclient.File(filePath, parentId=centerMafSynId))
return(filePath)
def upload_predictions(submission_filepath, folder_id, direct=False):
'''
Upload prediciton file to synapse
Args:
syn: Synapse object
submission_filepath: File path of submission
folder_id: Synapse id of Team submission folder
Returns:
Synapse File Entity
'''
if direct:
syn_service = get_service_account()
file_ent = synapseclient.File(submission_filepath, parentId=folder_id)
file_ent = syn.store(file_ent)
entity = file_ent
else:
with open(submission_filepath, 'rb') as data_file:
prediction_data = data_file.read()
encoded_prediction_data = base64.b64encode(prediction_data)
url = 'https://gja3h20usl.execute-api.us-east-1.amazonaws.com/v1/predictions'
data = {
'submission_folder': folder_id,
'data': encoded_prediction_data.decode('utf-8')
}
res = requests.post(url, json=data)
entity = json.loads(res.content)
return entity
def write(syn: Synapse, center_mapping_df: pd.DataFrame,
error_tracker_synid: str):
"""Write center errors to a file
Args:
syn: Synapse connection
center_mapping_df: Center mapping dataframe
error_tracker_synid: Error tracking synapse id
"""
center_errors = get_center_invalid_errors(syn, error_tracker_synid)
for center in center_mapping_df["center"]:
logger.info(center)
staging_synid = center_mapping_df["stagingSynId"][
center_mapping_df["center"] == center][0]
with open(center + "_errors.txt", "w") as errorfile:
if center not in center_errors:
errorfile.write("No errors!")
else:
errorfile.write(center_errors[center])
ent = synapseclient.File(center + "_errors.txt",
parentId=staging_synid)
syn.store(ent)
os.remove(center + "_errors.txt")
def test_store__changing_from_Synapse_to_externalURL_by_changing_path():
#create a temp file
temp_path = utils.make_bogus_data_file()
schedule_for_cleanup(temp_path)
ext = syn.store(
synapseclient.File(temp_path, parent=project, synapseStore=True))
ext = syn.get(ext)
assert_equal("org.sagebionetworks.repo.model.file.S3FileHandle",
ext._file_handle.concreteType)
ext.synapseStore = False
ext = syn.store(ext)
#do a get to make sure filehandle has been updated correctly
ext = syn.get(ext.id, downloadFile=True)
assert_equal("org.sagebionetworks.repo.model.file.ExternalFileHandle",
ext._file_handle.concreteType)
assert_equal(utils.as_url(temp_path), ext.externalURL)
assert_equal(False, ext.synapseStore)
#swap back to synapse storage
ext.synapseStore = True
ext = syn.store(ext)
#do a get to make sure filehandle has been updated correctly
ext = syn.get(ext.id, downloadFile=True)
assert_equal("org.sagebionetworks.repo.model.file.S3FileHandle",
ext._file_handle.concreteType)
assert_equal(None, ext.externalURL)
assert_equal(True, ext.synapseStore)
def store_file(syn,
filepath,
parentid,
name=None,
annotations={},
used=None,
executed=None):
"""Storing Files along with annotations
Args:
filepath: Path to file
parentid: Project or Folder Synapse id
name: Name of entity. Defaults to filename
annotations: Synapse annotations to add
used: List of used entitys or links.
executed: List of scripts executed
Returns:
File Entity
"""
logger.info("STORING FILES")
file_ent = synapseclient.File(filepath, parent=parentid, name=name)
file_ent.annotations.update(annotations)
file_ent = syn.store(file_ent, used=used, executed=executed)
return file_ent
def main(wf_name, to_upload, synapse_parent_id):
git_base = _git_base_url()
syn = synapseclient.Synapse()
syn.login(os.environ["SYNAPSE_LOGIN"],
apiKey=os.environ["SYNAPSE_API_KEY"])
s_base_folder, remotes = _accumulate_remotes(synapse_parent_id, syn)
for dirpath, _, filenames in os.walk(to_upload):
remote_dirpath = os.path.join(s_base_folder.name, dirpath)
if filenames:
s_folder, remotes = _remote_folder(remote_dirpath, remotes, syn)
for filename in filenames:
remote_filename = os.path.join(remote_dirpath, filename)
if remote_filename not in remotes:
filename = os.path.join(dirpath, filename)
if os.path.getsize(filename) > 0:
git_url = git_base + filename.replace(
os.path.dirname(to_upload), "")
print("Linking to %s" % git_url)
f = synapseclient.File(git_url,
parent=s_folder,
synapseStore=False)
f.workflow = wf_name
f.workflowOption = "all"
s_filename = syn.store(f)
remotes[remote_filename] = s_filename.id
def test_store__changing_externalURL_by_changing_path():
url = 'https://www.synapse.org/Portal/clear.cache.gif'
ext = syn.store(
synapseclient.File(url,
name="test",
parent=project,
synapseStore=False))
#perform a syn.get so the filename changes
ext = syn.get(ext)
#create a temp file
temp_path = utils.make_bogus_data_file()
schedule_for_cleanup(temp_path)
ext.synapseStore = False
ext.path = temp_path
ext = syn.store(ext)
#do a get to make sure filehandle has been updated correctly
ext = syn.get(ext.id, downloadFile=True)
assert_not_equal(ext.externalURL, url)
assert_equal(utils.normalize_path(temp_path),
utils.file_url_to_path(ext.externalURL))
assert_equal(temp_path, ext.path)
assert_equal(False, ext.synapseStore)
def process_steps(self, filePath, databaseToSynIdMappingDf):
logger.debug("Performing process_steps for {}".format(self._fileType))
folder_id = databaseToSynIdMappingDf.Id[
databaseToSynIdMappingDf['Database'] == self._fileType][0]
table_id = databaseToSynIdMappingDf.Id[
databaseToSynIdMappingDf['Database'] ==
f"{self._fileType}_table"][0]
logger.debug(f"Storing file at {folder_id}")
f = self.syn.store(synapseclient.File(filePath,
parent=folder_id,
annotations=dict(
center=self.center,
fileType=self._fileType)),
forceVersion=False)
# Add information about assay to the table
data = self._get_dataframe(filePath)
data['entity_id'] = f.id
process_functions.updateData(syn=self.syn,
databaseSynId=table_id,
newData=data,
filterBy=self.center,
filterByColumn="center",
col=self._required_columns,
toDelete=True)
return (filePath)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-u", "--username", help="Synapse username")
parser.add_argument("-p", "--password", help="Synapse password")
parser.add_argument("-s",
"--synapse_entity",
default="syn7188267",
help="Synapse ID for the folder of preprocessed data")
parser.add_argument("-w",
"--work_dir",
default="../brca/pipeline-data/data",
help="Parent directory for the working directory")
parser.add_argument(
'-a',
"--artifacts_dir",
help='Artifacts directory with pipeline artifact files.')
parser.add_argument("-k",
"--keep_tmp_dir",
default=False,
help="Keep the temporary directory (do not delete)")
args = parser.parse_args()
#
# Set up a temp workspace and log in to Synapse
temp_dir = args.work_dir + "/enigma/"
os.makedirs(temp_dir)
syn = synapseclient.Synapse()
syn.login(args.username, args.password)
#
# Download all the ENIGMA_last_updated*hg38.tsv files, the preprocessed files
query = 'SELECT id, name FROM entity WHERE parentId=="%s"' \
% (args.synapse_entity)
results = syn.query(query)
for item in results['results']:
if re.search("ENIGMA_last_updated(.)*hg38.tsv", item['entity.name']):
print "fetching", item['entity.name']
syn.get(item['entity.id'], downloadLocation=temp_dir)
#
# Merge the preprocessed files, postprocess the merged file, and upload
# the resulting file to Synapse
merge_cmd = "python enigma-merge_hg38.py -i %s -o %s" % (temp_dir,
temp_dir)
subprocess.check_call(merge_cmd, shell=True)
postprocess_cmd = "enigma_postprocess.py -i %s -o %s -a %s" \
% (temp_dir + "/ENIGMA_combined_hg38.tsv",
temp_dir + "/ENIGMA_combined_postprocessed_hg38.tsv",
args.artifacts_dir)
subprocess.check_call(postprocess_cmd, shell=True)
output_entity = synapseclient.File(
temp_dir + "/ENIGMA_combined_postprocessed_hg38.tsv",
parent=args.synapse_entity)
output_entity = syn.store(output_entity)
#
# Cleanup...
if not args.keep_tmp_dir:
shutil.rmtree(temp_dir)
def setup(self):
self.project_entity = synapseclient.Project(name=str(uuid.uuid4()),
id="syn1234")
self.second_project = synapseclient.Project(name=str(uuid.uuid4()),
id="syn2345")
self.file_ent = synapseclient.File(name='File',
parent=self.project_entity.id)
self.file_ent.id = "syn3456"
def file_(syn, parent):
"""Store a randomly generated file to Synapse"""
f = tempfile.NamedTemporaryFile(suffix=".csv")
with open(f.name, 'wb') as fout:
fout.write(os.urandom(2))
file_ = synapseclient.File(path=f.name, parent=parent)
file_ = syn.store(file_)
return file_
def push_to_synapse(syn, all_participants):
result = all_participants[["healthCode", "userType", "atHomePD"]]
fname = "mpower2_healthcode_categorizations.csv"
result.to_csv(fname, index = False)
f = sc.File(fname, parent = OUTPUT_PARENT,
used=[AT_HOME_PD_USER_LIST, HEALTH_DATA_SUMMARY_TABLE],
executed="https://github.com/Sage-Bionetworks/at-home-pd/"
"blob/master/tag_users/tag_users.py")
syn.store(f)
def test_none__getoncotreelink():
"""Test oncotree link is gotten"""
arg = argparser()
url = "https://www.synapse.org"
link = synapseclient.File("foo", parentId="foo", externalURL=url)
with patch.object(syn, "get", return_value=link) as patch_synget:
oncolink = validate._get_oncotreelink(syn, arg.asDataFrame())
patch_synget.assert_called_once_with(ONCOTREE_ENT)
assert oncolink == url
def setUp(self):
print("Creating private Project...")
test_project = sc.Project("Test" + uuid.uuid4().hex)
self.project_id = syn.store(test_project).id
print("Creating Folder...")
folder = sc.Folder("folder", parent=self.project_id)
self.folder_id = syn.store(folder).id
print("Creating File within Folder...")
with tempfile.NamedTemporaryFile() as temp:
temp.write("123testingfolder")
temp.flush()
temp_file = sc.File(temp.name, parent=self.folder_id)
self.folder_fileId = syn.store(temp_file).id
print("Creating File within Project...")
with tempfile.NamedTemporaryFile() as temp:
temp.write("123testingproject")
temp.flush()
temp_file = sc.File(temp.name, parent=self.project_id)
self.project_fileId = syn.store(temp_file).id
def test_notcontainer__check_parentid_permission_container():
"""Throws error if input if synid of file"""
parentid = "syn123"
file_ent = synapseclient.File("foo", parentId=parentid)
with patch.object(syn, "get", return_value=file_ent),\
pytest.raises(ValueError,
match="Provided Synapse id must be your input folder "
"Synapse id or a Synapse Id of a folder inside "
"your input directory"):
validate._check_parentid_permission_container(syn, parentid)
def _migrate_file_version_test_helper(request, syn, project,
schedule_for_cleanup):
entity_name = request.node.name
temp_path_1 = _create_temp_file()
schedule_for_cleanup(temp_path_1)
file = synapseclient.File(name=entity_name,
path=temp_path_1,
parent=project)
v1 = syn.store(file)
# create another revision
temp_path_2 = _create_temp_file()
schedule_for_cleanup(temp_path_2)
file = synapseclient.File(name=entity_name,
path=temp_path_2,
parent=project)
v2 = syn.store(file)
return v1, v2
def store_full_maf(syn: Synapse, filepath: str, parentid: str):
"""Stores full maf file
Args:
syn: Synapse connection
filepath: Path to file
parentid: Synapse container id
"""
syn.store(synapseclient.File(filepath, parentId=parentid))
def test_download_file_entity__correct_local_state():
mock_cache_path = synapseclient.utils.normalize_path("/i/will/show/you/the/path/yi.txt")
file_entity = synapseclient.File(parentId="syn123")
file_entity.dataFileHandleId = 123
with patch.object(syn.cache, 'get',return_value=mock_cache_path) as mocked_cache_get:
syn._download_file_entity(downloadLocation=None, entity=file_entity, ifcollision="overwrite.local", submission=None)
assert_equals(mock_cache_path, file_entity.path)
assert_equals(os.path.dirname(mock_cache_path), file_entity.cacheDir)
assert_equals(1, len(file_entity.files))
assert_equals(os.path.basename(mock_cache_path), file_entity.files[0])
def store_log_file(syn, log_filename, parentid, store_log=False):
"""Store log file"""
statinfo = os.stat(log_filename)
if statinfo.st_size > 0:
ent = synapseclient.File(log_filename, parent=parentid)
if store_log:
try:
syn.store(ent)
except synapseclient.exceptions.SynapseHTTPError as err:
print(err)
