def test_valid__upload_to_synapse(): """Test upload of file to synapse under right conditions""" ent = synapseclient.File(id="syn123", parentId="syn222") with patch.object(syn, "store", return_value=ent) as patch_synstore: validate._upload_to_synapse(syn, ['foo'], True, parentid="syn123") patch_synstore.assert_called_once_with( synapseclient.File('foo', parent="syn123"))
def copy(args,syn): """Copies most recent version of a file specifed by args.id to args.parentId""" ent = syn.get(args.id, downloadFile=False) profile = syn.getUserProfile().ownerId #CHECK: Must be a file entity if ent.entityType!='org.sagebionetworks.repo.model.FileEntity': raise ValueError('"synapse cp" can only copy files!') #Grab file handle createdBy annotation to see the user that created fileHandle createdBy = syn.restGET('/entity/%s/filehandles'%args.id)['list'][0]['createdBy'] #CHECK: If file is in the same parent directory (throw an error) search = syn.query('select name from file where parentId =="%s"'%args.parentid)['results'] for i in search: if i['file.name'] == ent.name: raise ValueError('Filename exists in directory you would like to copy to, either rename or check if file has already been copied!') #CHECK: If the user created the file, copy the file by using fileHandleId else hard copy if profile == createdBy: new_ent = synapseclient.File(name=ent.name, parentId=args.parentid) new_ent.properties.dataFileHandleId = ent.properties.dataFileHandleId new_ent = syn._createEntity(new_ent) else: ent = syn.get(args.id) new_ent = synapseclient.File(ent.path, parent=args.parentid) new_ent = syn.store(new_ent) syn.setAnnotations(new_ent, ent.annotations) act = Activity("Copied file", used=args.id) syn.setProvenance(new_ent['id'], act) print('Copied %s to %s' %(ent.id, new_ent['id']))
def test_data_pull_non_data_folder(syn_test_helper, mk_tempfile, mk_uniq_string, mk_kiproject): syn_project = syn_test_helper.create_project() syn_folder1 = syn_test_helper.client().store( synapseclient.Folder(name='Folder1', parent=syn_project)) syn_test_helper.client().store( synapseclient.File(path=mk_tempfile(), parent=syn_folder1)) syn_folder2 = syn_test_helper.client().store( synapseclient.Folder(name='Folder2', parent=syn_folder1)) syn_test_helper.client().store( synapseclient.File(path=mk_tempfile(), parent=syn_folder2)) syn_folder3 = syn_test_helper.client().store( synapseclient.Folder(name='Folder3', parent=syn_folder2)) syn_test_helper.client().store( synapseclient.File(path=mk_tempfile(), parent=syn_folder3)) syn_folder4 = syn_test_helper.client().store( synapseclient.Folder(name='Folder4', parent=syn_folder3)) syn_test_helper.client().store( synapseclient.File(path=mk_tempfile(), parent=syn_folder4)) syn_folder5 = syn_test_helper.client().store( synapseclient.Folder(name='Folder5', parent=syn_folder4)) syn_test_helper.client().store( synapseclient.File(path=mk_tempfile(), parent=syn_folder5)) kiproject = mk_kiproject() kiproject.data_add(DataUri('syn', syn_folder1.id).uri, data_type=kiproject.data_types[0]) kiproject.data_pull()
def test_command_copy(): """Tests the 'synapse cp' function""" # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity)) schedule_for_cleanup(folder_entity.id) # Create and upload a file in Folder repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' annots = {'test': ['hello_world']} # Create, upload, and set annotations on a file in Folder filename = utils.make_bogus_data_file() schedule_for_cleanup(filename) file_entity = syn.store(synapseclient.File(filename, parent=folder_entity)) externalURL_entity = syn.store(synapseclient.File(repo_url, name='rand', parent=folder_entity, synapseStore=False)) syn.setAnnotations(file_entity, annots) syn.setAnnotations(externalURL_entity, annots) schedule_for_cleanup(file_entity.id) schedule_for_cleanup(externalURL_entity.id) # Test cp function output = run('synapse', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id) output_URL = run('synapse', '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id) copied_id = parse(r'Copied syn\d+ to (syn\d+)', output) copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL) # Verify that our copied files are identical copied_ent = syn.get(copied_id) copied_URL_ent = syn.get(copied_URL_id, downloadFile=False) schedule_for_cleanup(copied_id) schedule_for_cleanup(copied_URL_id) copied_ent_annot = syn.getAnnotations(copied_id) copied_url_annot = syn.getAnnotations(copied_URL_id) copied_prov = syn.getProvenance(copied_id)['used'][0]['reference']['targetId'] copied_url_prov = syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId'] # Make sure copied files are the same assert_equals(copied_prov, file_entity.id) assert_equals(copied_ent_annot, annots) assert_equals(copied_ent.properties.dataFileHandleId, file_entity.properties.dataFileHandleId) # Make sure copied URLs are the same assert_equals(copied_url_prov, externalURL_entity.id) assert_equals(copied_url_annot, annots) assert_equals(copied_URL_ent.externalURL, repo_url) assert_equals(copied_URL_ent.name, 'rand') assert_equals(copied_URL_ent.properties.dataFileHandleId, externalURL_entity.properties.dataFileHandleId) # Verify that errors are being thrown when a # file is copied to a folder/project that has a file with the same filename assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
def test_command_get_recursive_and_query(): """Tests the 'synapse get -r' and 'synapse get -q' functions""" # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) file_entity = synapseclient.File(f, parent=folder_entity) file_entity.location = 'folder' file_entity = syn.store(file_entity) #Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) file_entity = synapseclient.File(f, parent=project_entity) file_entity.location = 'project' file_entity = syn.store(file_entity) ### Test recursive get output = run('synapse', '--skip-checks', 'get', '-r', project_entity.id) #Verify that we downloaded files: new_paths = [ os.path.join('.', folder_entity.name, os.path.basename(f)) for f in uploaded_paths[:-1] ] new_paths.append(os.path.join('.', os.path.basename(uploaded_paths[-1]))) schedule_for_cleanup(folder_entity.name) for downloaded, uploaded in zip(new_paths, uploaded_paths): print uploaded, downloaded assert os.path.exists(downloaded) assert filecmp.cmp(downloaded, uploaded) schedule_for_cleanup(new_paths[0]) ### Test query get output = run( 'synapse', '--skip-checks', 'get', '-q', "select id from file where parentId=='%s' and location=='folder'" % folder_entity.id) #Verify that we downloaded files: new_paths = [ os.path.join('.', os.path.basename(f)) for f in uploaded_paths[:-1] ] for downloaded, uploaded in zip(new_paths, uploaded_paths[:-1]): print uploaded, downloaded assert os.path.exists(downloaded) assert filecmp.cmp(downloaded, uploaded) schedule_for_cleanup(downloaded)
def test_command_copy(): """Tests the 'synapse cp' function""" # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload a file in Folder dummy = utils.make_bogus_data_file() schedule_for_cleanup(dummy) dummy_entity = syn.store(synapseclient.File(dummy, parent=folder_entity)) repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' annots = {'test': 'hello_world'} # Create, upload, and set annotations on a file in Folder filename = utils.make_bogus_data_file() schedule_for_cleanup(filename) file_entity = syn.store(synapseclient.File(filename, parent=folder_entity), used=dummy_entity.id, executed=repo_url) syn.setAnnotations(file_entity, annots) ### Test cp function output = run('synapse', '--skip-checks', 'cp', '--id', file_entity.id, '--parentid', project_entity.id) copied_id = parse(r'Copied syn\d+ to (syn\d+)', output) #Verify that our copied files are identical copied_ent = syn.get(copied_id) schedule_for_cleanup(copied_id) copied_ent_annot = syn.getAnnotations(copied_ent) copied_annot = dict((key, copied_ent_annot[key].pop()) for key in copied_ent_annot if key not in ('uri', 'id', 'creationDate', 'etag')) copied_prov = syn.getProvenance( copied_ent)['used'][0]['reference']['targetId'] assert copied_prov == file_entity.id assert copied_annot == annots #Verify that errors are being thrown when folders/projects are attempted to be copied, #or file is copied to a foler/project that has a file with the same filename assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', '--id', folder_entity.id, '--parentid', project_entity.id) assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', '--id', project_entity.id, '--parentid', project_entity.id) assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', '--id', file_entity.id, '--parentid', project_entity.id)
def storeFile( syn, fileName, parentId, center, fileFormat, dataSubType, platform=None, cBioFileFormat=None, used=None, ): """ # Storing Files along with annotations """ logger.info("STORING FILES") fileEnt = synapseclient.File(fileName, parent=parentId) fileEnt.center = center fileEnt.species = "Human" fileEnt.consortium = "GENIE" fileEnt.dataType = "genomicVariants" fileEnt.fundingAgency = "AACR" fileEnt.assay = "targetGeneSeq" fileEnt.fileFormat = fileFormat fileEnt.dataSubType = dataSubType fileEnt.fileStage = "staging" fileEnt.platform = platform if platform is not None: fileEnt.platform = platform if cBioFileFormat is not None: fileEnt.cBioFileFormat = cBioFileFormat ent = syn.store(fileEnt, used=used) return ent
def create_filehandle(event, filename, bucket, key, project_id): parent = get_parent_folder(project_id, key) eTag = event['Records'][0]['s3']['object']['eTag'] file_id = syn.findEntityId(filename, parent) if file_id != None: targetMD5 = syn.get(file_id, downloadFile=False)['md5']; # create filehandle if it does not exist in Synapse or if existing file was modified (check md5): if file_id == None or eTag != targetMD5: size = event['Records'][0]['s3']['object']['size'] contentType = mimetypes.guess_type(filename, strict=False)[0] storage_id = syn.restGET("/projectSettings/"+project_id+"/type/upload")['locations'][0] fileHandle = {'concreteType': 'org.sagebionetworks.repo.model.file.S3FileHandle', 'fileName' : filename, 'contentSize' : size, 'contentType' : contentType, 'contentMd5' : eTag, 'bucketName' : bucket, 'key' : key, 'storageLocationId': storage_id} fileHandle = syn.restPOST('/externalFileHandle/s3', json.dumps(fileHandle), endpoint=syn.fileHandleEndpoint) f = synapseclient.File(parentId=parent, dataFileHandleId=fileHandle['id'], name=filename, synapseStore=False) f = syn.store(f)
def submit(args, syn): """ Method to allow challenge participants to submit to an evaluation queue. Examples:: synapse submit --evaluation 'ra_challenge_Q1_leaderboard' -f ~/testing/testing.txt --parentId syn2345030 \ --used syn2351967 --executed syn2351968 synapse submit --evaluation 2343117 -f ~/testing/testing.txt --parentId syn2345030 --used syn2351967 \ --executed syn2351968 """ # check if evaluation is a number, if so it is assumed to be a evaluationId else it is a evaluationName if args.evaluation is not None: try: args.evaluationID = str(int(args.evaluation)) except ValueError: args.evaluationName = args.evaluation # checking if user has entered a evaluation ID or evaluation Name if args.evaluationID is None and args.evaluationName is None: raise ValueError('Evaluation ID or Evaluation Name is required\n') elif args.evaluationID is not None and args.evaluationName is not None: sys.stderr.write( '[Warning]: Both Evaluation ID & Evaluation Name are specified \n EvaluationID will be used\n' ) elif args.evaluationID is None: # get evalID from evalName try: args.evaluationID = syn.getEvaluationByName( args.evaluationName)['id'] except Exception: raise ValueError('Could not find an evaluation named: %s \n' % args.evaluationName) # checking if a entity id or file was specified by the user if args.entity is None and args.file is None: raise ValueError( 'Either entityID or filename is required for a submission\n') elif args.entity is not None and args.file is not None: sys.stderr.write( '[Warning]: Both entityID and filename are specified \n entityID will be used\n' ) elif args.entity is None: # upload the the file to synapse and get synapse entity id for the file if args.parentid is None: raise ValueError('parentID required with a file upload\n') if not os.path.exists(args.file): raise IOError('file path %s not valid \n' % args.file) # //ideally this should be factored out synFile = syn.store( synapseclient.File(path=args.file, parent=args.parentid), used=syn._convertProvenanceList(args.used, args.limitSearch), executed=syn._convertProvenanceList(args.executed, args.limitSearch)) args.entity = synFile.id submission = syn.submit(args.evaluationID, args.entity, name=args.name, team=args.teamName) sys.stdout.write('Submitted (id: %s) entity: %s\t%s to Evaluation: %s\n' % (submission['id'], submission['entityId'], submission['name'], submission['evaluationId']))
def storeSubmissions(syn, evaluationQueue, synProject=None, filetype="csv"): """Store submissions on Synapse and submit to the evaluation queue. Arguments --------- syn : synapseclient.Synapse evaluationQueue : int Evaluation ID. synProject : str Synapse ID of project to store files to. filetype : str filename extension to append to filenames (default 'csv'). Returns ------- None """ if not synProject: logger.info("Creating new Synapse Project") user_profile = syn.getUserProfile() project_name = "{}_{}_testSubmissions".format( user_profile['userName'], evaluationQueue) synProject = sc.Project(project_name) synProject = syn.store(synProject).id logger.info("Synapse project created at {}".format(synProject)) for submission in glob.glob("{}/*.{}".format(TEST_SUBMISSION_PATH, filetype)): logger.info("Storing {} to {} and submitting to {}".format( submission, synProject, evaluationQueue)) synFile = sc.File(submission, parent=synProject) synFile = syn.store(synFile) syn.submit(evaluationQueue, synFile, name=os.path.basename(submission))
def test_get_or_create_file__call(): """Makes sure correct parameters are called""" file_path = str(uuid.uuid1()) parentid = str(uuid.uuid1()) file_ent = synapseclient.File(path=file_path, parentId=parentid) returned = synapseclient.File(path=file_path, id=str(uuid.uuid1()), parentId=parentid) with patch.object(CREATE_CLS, "_find_by_obj_or_create", return_value=returned) as patch_find_or_create: new_file = CREATE_CLS.get_or_create_file(path=file_path, parentId=parentid) assert new_file == returned patch_find_or_create.assert_called_once_with(file_ent)
def storeProcessedMaf( self, filePath, mafSynId, centerMafSynId, isNarrow=False): ''' Stores the processed maf There is a isNarrow option, but note that the number of rows of the maf file DOES NOT change in this function Args: filePath: Path to maf file mafSynId: database synid centerMafSynid: center flat file folder synid isNarrow: Is the file a narrow maf. Defaul to False. ''' logger.info('STORING %s' % filePath) database = self.syn.get(mafSynId) if isNarrow: try: update_table = synapseclient.Table( database.id, filePath, separator="\t") self.syn.store(update_table) except SynapseTimeoutError: # This error occurs because of waiting for table to index. # Don't worry about this. pass else: self.syn.store( synapseclient.File(filePath, parentId=centerMafSynId)) return(filePath)
def upload_predictions(submission_filepath, folder_id, direct=False): ''' Upload prediciton file to synapse Args: syn: Synapse object submission_filepath: File path of submission folder_id: Synapse id of Team submission folder Returns: Synapse File Entity ''' if direct: syn_service = get_service_account() file_ent = synapseclient.File(submission_filepath, parentId=folder_id) file_ent = syn.store(file_ent) entity = file_ent else: with open(submission_filepath, 'rb') as data_file: prediction_data = data_file.read() encoded_prediction_data = base64.b64encode(prediction_data) url = 'https://gja3h20usl.execute-api.us-east-1.amazonaws.com/v1/predictions' data = { 'submission_folder': folder_id, 'data': encoded_prediction_data.decode('utf-8') } res = requests.post(url, json=data) entity = json.loads(res.content) return entity
def write(syn: Synapse, center_mapping_df: pd.DataFrame, error_tracker_synid: str): """Write center errors to a file Args: syn: Synapse connection center_mapping_df: Center mapping dataframe error_tracker_synid: Error tracking synapse id """ center_errors = get_center_invalid_errors(syn, error_tracker_synid) for center in center_mapping_df["center"]: logger.info(center) staging_synid = center_mapping_df["stagingSynId"][ center_mapping_df["center"] == center][0] with open(center + "_errors.txt", "w") as errorfile: if center not in center_errors: errorfile.write("No errors!") else: errorfile.write(center_errors[center]) ent = synapseclient.File(center + "_errors.txt", parentId=staging_synid) syn.store(ent) os.remove(center + "_errors.txt")
def test_store__changing_from_Synapse_to_externalURL_by_changing_path(): #create a temp file temp_path = utils.make_bogus_data_file() schedule_for_cleanup(temp_path) ext = syn.store( synapseclient.File(temp_path, parent=project, synapseStore=True)) ext = syn.get(ext) assert_equal("org.sagebionetworks.repo.model.file.S3FileHandle", ext._file_handle.concreteType) ext.synapseStore = False ext = syn.store(ext) #do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert_equal("org.sagebionetworks.repo.model.file.ExternalFileHandle", ext._file_handle.concreteType) assert_equal(utils.as_url(temp_path), ext.externalURL) assert_equal(False, ext.synapseStore) #swap back to synapse storage ext.synapseStore = True ext = syn.store(ext) #do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert_equal("org.sagebionetworks.repo.model.file.S3FileHandle", ext._file_handle.concreteType) assert_equal(None, ext.externalURL) assert_equal(True, ext.synapseStore)
def store_file(syn, filepath, parentid, name=None, annotations={}, used=None, executed=None): """Storing Files along with annotations Args: filepath: Path to file parentid: Project or Folder Synapse id name: Name of entity. Defaults to filename annotations: Synapse annotations to add used: List of used entitys or links. executed: List of scripts executed Returns: File Entity """ logger.info("STORING FILES") file_ent = synapseclient.File(filepath, parent=parentid, name=name) file_ent.annotations.update(annotations) file_ent = syn.store(file_ent, used=used, executed=executed) return file_ent
def main(wf_name, to_upload, synapse_parent_id): git_base = _git_base_url() syn = synapseclient.Synapse() syn.login(os.environ["SYNAPSE_LOGIN"], apiKey=os.environ["SYNAPSE_API_KEY"]) s_base_folder, remotes = _accumulate_remotes(synapse_parent_id, syn) for dirpath, _, filenames in os.walk(to_upload): remote_dirpath = os.path.join(s_base_folder.name, dirpath) if filenames: s_folder, remotes = _remote_folder(remote_dirpath, remotes, syn) for filename in filenames: remote_filename = os.path.join(remote_dirpath, filename) if remote_filename not in remotes: filename = os.path.join(dirpath, filename) if os.path.getsize(filename) > 0: git_url = git_base + filename.replace( os.path.dirname(to_upload), "") print("Linking to %s" % git_url) f = synapseclient.File(git_url, parent=s_folder, synapseStore=False) f.workflow = wf_name f.workflowOption = "all" s_filename = syn.store(f) remotes[remote_filename] = s_filename.id
def test_store__changing_externalURL_by_changing_path(): url = 'https://www.synapse.org/Portal/clear.cache.gif' ext = syn.store( synapseclient.File(url, name="test", parent=project, synapseStore=False)) #perform a syn.get so the filename changes ext = syn.get(ext) #create a temp file temp_path = utils.make_bogus_data_file() schedule_for_cleanup(temp_path) ext.synapseStore = False ext.path = temp_path ext = syn.store(ext) #do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert_not_equal(ext.externalURL, url) assert_equal(utils.normalize_path(temp_path), utils.file_url_to_path(ext.externalURL)) assert_equal(temp_path, ext.path) assert_equal(False, ext.synapseStore)
def process_steps(self, filePath, databaseToSynIdMappingDf): logger.debug("Performing process_steps for {}".format(self._fileType)) folder_id = databaseToSynIdMappingDf.Id[ databaseToSynIdMappingDf['Database'] == self._fileType][0] table_id = databaseToSynIdMappingDf.Id[ databaseToSynIdMappingDf['Database'] == f"{self._fileType}_table"][0] logger.debug(f"Storing file at {folder_id}") f = self.syn.store(synapseclient.File(filePath, parent=folder_id, annotations=dict( center=self.center, fileType=self._fileType)), forceVersion=False) # Add information about assay to the table data = self._get_dataframe(filePath) data['entity_id'] = f.id process_functions.updateData(syn=self.syn, databaseSynId=table_id, newData=data, filterBy=self.center, filterByColumn="center", col=self._required_columns, toDelete=True) return (filePath)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-u", "--username", help="Synapse username") parser.add_argument("-p", "--password", help="Synapse password") parser.add_argument("-s", "--synapse_entity", default="syn7188267", help="Synapse ID for the folder of preprocessed data") parser.add_argument("-w", "--work_dir", default="../brca/pipeline-data/data", help="Parent directory for the working directory") parser.add_argument( '-a', "--artifacts_dir", help='Artifacts directory with pipeline artifact files.') parser.add_argument("-k", "--keep_tmp_dir", default=False, help="Keep the temporary directory (do not delete)") args = parser.parse_args() # # Set up a temp workspace and log in to Synapse temp_dir = args.work_dir + "/enigma/" os.makedirs(temp_dir) syn = synapseclient.Synapse() syn.login(args.username, args.password) # # Download all the ENIGMA_last_updated*hg38.tsv files, the preprocessed files query = 'SELECT id, name FROM entity WHERE parentId=="%s"' \ % (args.synapse_entity) results = syn.query(query) for item in results['results']: if re.search("ENIGMA_last_updated(.)*hg38.tsv", item['entity.name']): print "fetching", item['entity.name'] syn.get(item['entity.id'], downloadLocation=temp_dir) # # Merge the preprocessed files, postprocess the merged file, and upload # the resulting file to Synapse merge_cmd = "python enigma-merge_hg38.py -i %s -o %s" % (temp_dir, temp_dir) subprocess.check_call(merge_cmd, shell=True) postprocess_cmd = "enigma_postprocess.py -i %s -o %s -a %s" \ % (temp_dir + "/ENIGMA_combined_hg38.tsv", temp_dir + "/ENIGMA_combined_postprocessed_hg38.tsv", args.artifacts_dir) subprocess.check_call(postprocess_cmd, shell=True) output_entity = synapseclient.File( temp_dir + "/ENIGMA_combined_postprocessed_hg38.tsv", parent=args.synapse_entity) output_entity = syn.store(output_entity) # # Cleanup... if not args.keep_tmp_dir: shutil.rmtree(temp_dir)
def setup(self): self.project_entity = synapseclient.Project(name=str(uuid.uuid4()), id="syn1234") self.second_project = synapseclient.Project(name=str(uuid.uuid4()), id="syn2345") self.file_ent = synapseclient.File(name='File', parent=self.project_entity.id) self.file_ent.id = "syn3456"
def file_(syn, parent): """Store a randomly generated file to Synapse""" f = tempfile.NamedTemporaryFile(suffix=".csv") with open(f.name, 'wb') as fout: fout.write(os.urandom(2)) file_ = synapseclient.File(path=f.name, parent=parent) file_ = syn.store(file_) return file_
def push_to_synapse(syn, all_participants): result = all_participants[["healthCode", "userType", "atHomePD"]] fname = "mpower2_healthcode_categorizations.csv" result.to_csv(fname, index = False) f = sc.File(fname, parent = OUTPUT_PARENT, used=[AT_HOME_PD_USER_LIST, HEALTH_DATA_SUMMARY_TABLE], executed="https://github.com/Sage-Bionetworks/at-home-pd/" "blob/master/tag_users/tag_users.py") syn.store(f)
def test_none__getoncotreelink(): """Test oncotree link is gotten""" arg = argparser() url = "https://www.synapse.org" link = synapseclient.File("foo", parentId="foo", externalURL=url) with patch.object(syn, "get", return_value=link) as patch_synget: oncolink = validate._get_oncotreelink(syn, arg.asDataFrame()) patch_synget.assert_called_once_with(ONCOTREE_ENT) assert oncolink == url
def setUp(self): print("Creating private Project...") test_project = sc.Project("Test" + uuid.uuid4().hex) self.project_id = syn.store(test_project).id print("Creating Folder...") folder = sc.Folder("folder", parent=self.project_id) self.folder_id = syn.store(folder).id print("Creating File within Folder...") with tempfile.NamedTemporaryFile() as temp: temp.write("123testingfolder") temp.flush() temp_file = sc.File(temp.name, parent=self.folder_id) self.folder_fileId = syn.store(temp_file).id print("Creating File within Project...") with tempfile.NamedTemporaryFile() as temp: temp.write("123testingproject") temp.flush() temp_file = sc.File(temp.name, parent=self.project_id) self.project_fileId = syn.store(temp_file).id
def test_notcontainer__check_parentid_permission_container(): """Throws error if input if synid of file""" parentid = "syn123" file_ent = synapseclient.File("foo", parentId=parentid) with patch.object(syn, "get", return_value=file_ent),\ pytest.raises(ValueError, match="Provided Synapse id must be your input folder " "Synapse id or a Synapse Id of a folder inside " "your input directory"): validate._check_parentid_permission_container(syn, parentid)
def _migrate_file_version_test_helper(request, syn, project, schedule_for_cleanup): entity_name = request.node.name temp_path_1 = _create_temp_file() schedule_for_cleanup(temp_path_1) file = synapseclient.File(name=entity_name, path=temp_path_1, parent=project) v1 = syn.store(file) # create another revision temp_path_2 = _create_temp_file() schedule_for_cleanup(temp_path_2) file = synapseclient.File(name=entity_name, path=temp_path_2, parent=project) v2 = syn.store(file) return v1, v2
def store_full_maf(syn: Synapse, filepath: str, parentid: str): """Stores full maf file Args: syn: Synapse connection filepath: Path to file parentid: Synapse container id """ syn.store(synapseclient.File(filepath, parentId=parentid))
def test_download_file_entity__correct_local_state(): mock_cache_path = synapseclient.utils.normalize_path("/i/will/show/you/the/path/yi.txt") file_entity = synapseclient.File(parentId="syn123") file_entity.dataFileHandleId = 123 with patch.object(syn.cache, 'get',return_value=mock_cache_path) as mocked_cache_get: syn._download_file_entity(downloadLocation=None, entity=file_entity, ifcollision="overwrite.local", submission=None) assert_equals(mock_cache_path, file_entity.path) assert_equals(os.path.dirname(mock_cache_path), file_entity.cacheDir) assert_equals(1, len(file_entity.files)) assert_equals(os.path.basename(mock_cache_path), file_entity.files[0])
def store_log_file(syn, log_filename, parentid, store_log=False): """Store log file""" statinfo = os.stat(log_filename) if statinfo.st_size > 0: ent = synapseclient.File(log_filename, parent=parentid) if store_log: try: syn.store(ent) except synapseclient.exceptions.SynapseHTTPError as err: print(err)