def test_get_local_file(): """Tests synapse.get() with local a local file """ new_path = utils.make_bogus_data_file() schedule_for_cleanup(new_path) folder = Folder('TestFindFileFolder', parent=project, description='A place to put my junk') folder = syn.createEntity(folder) #Get an nonexistent file in Synapse assert_raises(SynapseError, syn.get, new_path) #Get a file really stored in Synapse ent_folder = syn.store(File(new_path, parent=folder)) ent2 = syn.get(new_path) assert ent_folder.id==ent2.id and ent_folder.versionNumber==ent2.versionNumber #Get a file stored in Multiple locations #should display warning ent = syn.store(File(new_path, parent=project)) ent = syn.get(new_path) #Get a file stored in multiple locations with limit set ent = syn.get(new_path, limitSearch=folder.id) assert ent.id == ent_folder.id and ent.versionNumber==ent_folder.versionNumber #Get a file that exists but such that limitSearch removes them and raises error assert_raises(SynapseError, syn.get, new_path, limitSearch='syn1')
def test_get_and_store_by_name_and_parent_id(): project = create_project() path = utils.make_bogus_data_file() schedule_for_cleanup(path) f = File(path, name='Foobarbat', parent=project) f2 = syn.store(f) f = syn.get(f) assert f.id == f2.id assert f.name == f2.name assert f.parentId == f2.parentId ## new file path = utils.make_bogus_data_file() schedule_for_cleanup(path) ## should create a new version of the previous File entity f3 = File(path, name='Foobarbat', parent=project, description='banana', junk=1234) f3 = syn.store(f3) ## should be an update of the existing entity with the same name and parent assert f3.id == f.id assert f3.description == 'banana' assert f3.junk == [1234] assert filecmp.cmp(path, f3.path)
def test_getChildren(syn, schedule_for_cleanup): # setup a hierarchy for folders # PROJECT # | \ # File Folder # | # File project_name = str(uuid.uuid1()) test_project = syn.store(Project(name=project_name)) folder = syn.store(Folder(name="firstFolder", parent=test_project)) syn.store( File(path="~/doesntMatter.txt", name="file inside folders", parent=folder, synapseStore=False)) project_file = syn.store( File(path="~/doesntMatterAgain.txt", name="file inside project", parent=test_project, synapseStore=False)) schedule_for_cleanup(test_project) expected_id_set = {project_file.id, folder.id} children_id_set = {x['id'] for x in syn.getChildren(test_project.id)} assert expected_id_set == children_id_set
def test_store_activity(): # Create a File and an Activity path = utils.make_bogus_binary_file() schedule_for_cleanup(path) entity = File(path, name='Hinkle horn honking holes', parent=project) honking = Activity(name='Hinkle horn honking', description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.') honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/') honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/') # This doesn't set the ID of the Activity entity = syn.store(entity, activity=honking) # But this does honking = syn.getProvenance(entity.id) # Verify the Activity assert honking['name'] == 'Hinkle horn honking' assert len(honking['used']) == 2 assert honking['used'][0]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL' assert honking['used'][0]['wasExecuted'] == False assert honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482') assert honking['used'][1]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL' assert honking['used'][1]['wasExecuted'] == False # Store another Entity with the same Activity entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg', name='Nettlebed Cave', parent=project, synapseStore=False) entity = syn.store(entity, activity=honking) # The Activities should match honking2 = syn.getProvenance(entity) assert honking['id'] == honking2['id']
def test_syncFromSynapse(): """This function tests recursive download as defined in syncFromSynapse most of the functionality of this function are already tested in the tests/integration/test_command_line_client::test_command_get_recursive_and_query which means that the only test if for path=None """ # Create a Project project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) syn.store(File(f, parent=folder_entity)) # Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) schedule_for_cleanup(f) syn.store(File(f, parent=project_entity)) # Test recursive get output = synapseutils.syncFromSynapse(syn, project_entity) assert_equals(len(output), len(uploaded_paths)) for f in output: assert_in(f.path, uploaded_paths)
def test_synapseStore_flag(): # Store a path to a local file path = utils.make_bogus_data_file() schedule_for_cleanup(path) bogus = File(path, name='Totally bogus data', parent=project, synapseStore=False) bogus = syn.store(bogus) # Verify the thing can be downloaded as a URL bogus = syn.get(bogus, downloadFile=False) assert bogus.name == 'Totally bogus data' assert bogus.path == path, "Path: %s\nExpected: %s" % (bogus.path, path) assert bogus.synapseStore == False # Make sure the test runs on Windows and other OS's if path[0].isalpha() and path[1]==':': # A Windows file URL looks like this: file:///c:/foo/bar/bat.txt expected_url = 'file:///' + path.replace("\\","/") else: expected_url = 'file://' + path assert bogus.externalURL == expected_url, 'URL: %s\nExpected %s' % (bogus.externalURL, expected_url) # A file path that doesn't exist should still work bogus = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False) bogus = syn.store(bogus) assert_raises(IOError, syn.get, bogus) assert bogus.synapseStore == False # Try a URL bogus = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False) bogus = syn.store(bogus) bogus = syn.get(bogus) assert bogus.synapseStore == False
def test_upload__error(self, syn): """Verify that if an item upload fails the error is raised in the main thread and any running Futures are cancelled""" item_1 = _SyncUploadItem(File(path='/tmp/foo', parentId='syn123'), [], [], {}) item_2 = _SyncUploadItem(File(path='/tmp/bar', parentId='syn123'), [], [], {}) items = [item_1, item_2] def syn_store_side_effect(entity, *args, **kwargs): if entity.path == entity.path: raise ValueError() return Mock() uploader = _SyncUploader(syn, get_executor()) original_abort = uploader._abort def abort_side_effect(futures): return original_abort(futures) with patch.object(syn, 'store') as mock_syn_store, \ patch.object(uploader, '_abort') as mock_abort: mock_syn_store.side_effect = syn_store_side_effect mock_abort.side_effect = abort_side_effect with pytest.raises(ValueError): uploader.upload(items) # it would be aborted with Futures mock_abort.assert_called_once_with([ANY]) isinstance(mock_abort.call_args_list[0][0], Future)
def test_extract_file_entity_metadata__ensure_correct_row_metadata(syn): # Test for SYNPY-692, where 'contentType' was incorrectly set on all rows except for the very first row. # create 2 file entities with different metadata entity1 = File(parent='syn123', id='syn456', contentType='text/json', path='path1', name='entity1', synapseStore=True) entity2 = File(parent='syn789', id='syn890', contentType='text/html', path='path2', name='entity2', synapseStore=False) files = [entity1, entity2] # we don't care about provenance metadata in this case with patch.object(synapseutils.sync, "_get_file_entity_provenance_dict", return_value={}): # method under test keys, data = synapseutils.sync._extract_file_entity_metadata( syn, files) # compare source entity metadata gainst the extracted metadata for file_entity, file_row_data in zip(files, data): for key in keys: if key == 'parent': # workaroundd for parent/parentId inconsistency. (SYNPY-697) assert file_entity.get('parentId') == file_row_data.get(key) else: assert file_entity.get(key) == file_row_data.get(key)
def test_store_with_create_or_update_flag(): project = create_project() filepath = utils.make_bogus_binary_file() bogus1 = File(filepath, name='Bogus Test File', parent=project) bogus1 = syn.store(bogus1, createOrUpdate=True) # Create a different file with the same name and parent new_filepath = utils.make_bogus_binary_file() bogus1.path = new_filepath # Expected behavior is that a new version of the first File will be created bogus2 = syn.store(bogus1, createOrUpdate=True) assert bogus2.id == bogus1.id assert bogus2.versionNumber == 2 assert not filecmp.cmp(bogus2.path, filepath) bogus2a = syn.get(bogus2.id) assert bogus2a.id == bogus1.id assert bogus2a.versionNumber == 2 assert filecmp.cmp(bogus2.path, bogus2a.path) # Create yet another file with the same name and parent newer_filepath = utils.make_bogus_binary_file() bogus3 = File(newer_filepath, name='Bogus Test File', parent=project) # Expected behavior is raising an exception with a 409 error assert_raises(requests.exceptions.HTTPError, syn.store, bogus3, createOrUpdate=False)
def _view_setup(cls): # set up a file view folder = syn.store( Folder(name="PartialRowTestFolder" + str(uuid.uuid4()), parent=project)) syn.store( File("~/path/doesnt/matter", name="f1", parent=folder, synapseStore=False)) syn.store( File("~/path/doesnt/matter/again", name="f2", parent=folder, synapseStore=False)) cols = [ Column(name='foo', columnType='STRING', maximumSize=1000), Column(name='bar', columnType='STRING') ] return syn.store( EntityViewSchema(name='PartialRowTestViews' + str(uuid.uuid4()), columns=cols, addDefaultViewColumns=False, parent=project, scopes=[folder]))
def test_syncFromSynapse__manifest_is_root( mock__get_file_entity_provenance_dict, mock_generateManifest, syn): """ Verify manifest argument equal to "root" that pass in to syncFromSynapse, it will create root_manifest file only. """ project = Project(name="the project", parent="whatever", id="syn123") file1 = File(name="a file", parent=project, id="syn456") folder = Folder(name="a folder", parent=project, id="syn789") file2 = File(name="a file2", parent=folder, id="syn789123") # Structure of nested project # project # |---> file1 # |---> folder # |---> file2 entities = { file1.id: file1, folder.id: folder, file2.id: file2, } def syn_get_side_effect(entity, *args, **kwargs): return entities[id_of(entity)] mock__get_file_entity_provenance_dict.return_value = {} with patch.object(syn, "getChildren", side_effect=[[folder, file1], [file2]]),\ patch.object(syn, "get", side_effect=syn_get_side_effect) as patch_syn_get: synapseutils.syncFromSynapse(syn, project, path="./", downloadFile=False, manifest="root") assert patch_syn_get.call_args_list == [ call( file1['id'], downloadLocation="./", ifcollision='overwrite.local', followLink=False, downloadFile=False, ), call( file2['id'], downloadLocation="./a folder", ifcollision='overwrite.local', followLink=False, downloadFile=False, ) ] assert mock_generateManifest.call_count == 1 call_files = mock_generateManifest.call_args_list[0][0][1] assert len(call_files) == 2 assert call_files[0].id == "syn456" assert call_files[1].id == "syn789123"
def test_walk(): walked = [] firstfile = utils.make_bogus_data_file() schedule_for_cleanup(firstfile) project_entity = syn.store(Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) schedule_for_cleanup(folder_entity.id) second_folder = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) schedule_for_cleanup(second_folder.id) file_entity = syn.store(File(firstfile, parent=project_entity)) schedule_for_cleanup(file_entity.id) walked.append(((project_entity.name, project_entity.id), [ (folder_entity.name, folder_entity.id), (second_folder.name, second_folder.id) ], [(file_entity.name, file_entity.id)])) nested_folder = syn.store( Folder(name=str(uuid.uuid4()), parent=folder_entity)) schedule_for_cleanup(nested_folder.id) secondfile = utils.make_bogus_data_file() schedule_for_cleanup(secondfile) second_file = syn.store(File(secondfile, parent=nested_folder)) schedule_for_cleanup(second_file.id) thirdfile = utils.make_bogus_data_file() schedule_for_cleanup(thirdfile) third_file = syn.store(File(thirdfile, parent=second_folder)) schedule_for_cleanup(third_file.id) walked.append(((os.path.join(project_entity.name, folder_entity.name), folder_entity.id), [(nested_folder.name, nested_folder.id)], [])) walked.append( ((os.path.join(os.path.join(project_entity.name, folder_entity.name), nested_folder.name), nested_folder.id), [], [(second_file.name, second_file.id)])) walked.append(((os.path.join(project_entity.name, second_folder.name), second_folder.id), [], [(third_file.name, third_file.id)])) temp = synapseutils.walk(syn, project_entity.id) temp = list(temp) #Must sort the tuples returned, because order matters for the assert #Folders are returned in a different ordering depending on the name for i in walked: for x in i: if type(x) == list: x = x.sort() for i in temp: for x in i: if type(x) == list: x = x.sort() assert i in walked print("CHECK: synapseutils.walk on a file should return empty generator") temp = synapseutils.walk(syn, second_file.id) assert list(temp) == []
def test_synapseStore_flag(): """Test storing entities while setting the synapseStore flag to False""" project = create_project() ## store a path to a local file (synapseStore=False) path = utils.make_bogus_data_file() schedule_for_cleanup(path) f1 = File(path, name='Totally bogus data', parent=project, synapseStore=False) f1 = syn.store(f1) f1a = syn.get(f1.id, downloadFile=False) assert f1a.name == 'Totally bogus data' assert f1a.path == path, 'path=' + str(f1a.path) + '; expected=' + path assert f1a.synapseStore == False ## make sure the test runs on Windows and other OS's if path[0].isalpha() and path[1] == ':': ## a windows file URL looks like this: file:///c:/foo/bar/bat.txt expected_url = 'file:///' + path else: expected_url = 'file://' + path assert f1a.externalURL == expected_url, 'unexpected externalURL: ' + f1a.externalURL ## a file path that doesn't exist should still work f2 = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False) f2 = syn.store(f2) try: syn.get(f2) assert False except Exception as err: assert err.message.startswith("Could not download non-existent file") assert f1a.synapseStore == False ## Try a URL f3 = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False) f3 = syn.store(f3) f3a = syn.get(f3) assert f1a.synapseStore == False
def test_round_trip(syn, project, schedule_for_cleanup): fhid = None filepath = utils.make_bogus_binary_file(MIN_PART_SIZE + 777771) try: fhid = multipart_upload_file(syn, filepath) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert filecmp.cmp(filepath, junk.path) finally: try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def test_multipart_upload_big_string(syn, project, schedule_for_cleanup): cities = [ "Seattle", "Portland", "Vancouver", "Victoria", "San Francisco", "Los Angeles", "New York", "Oaxaca", "Cancún", "Curaçao", "जोधपुर", "অসম", "ལྷ་ས།", "ཐིམ་ཕུ་", "دبي", "አዲስ አበባ", "São Paulo", "Buenos Aires", "Cartagena", "Amsterdam", "Venice", "Rome", "Dubrovnik", "Sarajevo", "Madrid", "Barcelona", "Paris", "Αθήνα", "Ρόδος", "København", "Zürich", "金沢市", "서울", "แม่ฮ่องสอน", "Москва" ] text = "Places I wanna go:\n" while len(text.encode('utf-8')) < MIN_PART_SIZE: text += ", ".join(random.choice(cities) for i in range(5000)) + "\n" fhid = multipart_upload_string(syn, text) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], "FileEntity", tmp_path) with open(junk.path, encoding='utf-8') as f: retrieved_text = f.read() assert retrieved_text == text
def test_ftp_download(): """Test downloading an Entity that points to a file on an FTP server. """ # Another test with an external reference. This is because we only need to test FTP download; not upload. Also so we don't have to maintain an FTP server just for this purpose. # Make an entity that points to an FTP server file. entity = File(parent=project['id'], name='1KB.zip') fileHandle = {} fileHandle['externalURL'] = 'ftp://speedtest.tele2.net/1KB.zip' fileHandle["fileName"] = entity.name fileHandle["contentType"] = "application/zip" fileHandle["contentMd5"] = '0f343b0931126a20f133d67c2b018a3b' fileHandle["contentSize"] = 1024 fileHandle[ "concreteType"] = "org.sagebionetworks.repo.model.file.ExternalFileHandle" fileHandle = syn.restPOST('/externalFileHandle', json.dumps(fileHandle), syn.fileHandleEndpoint) entity.dataFileHandleId = fileHandle['id'] entity = syn.store(entity) # Download the entity and check that MD5 matches expected FTPfile = syn.get(entity.id, downloadLocation=os.getcwd(), downloadFile=True) assert FTPfile.md5 == utils.md5_for_file(FTPfile.path).hexdigest() schedule_for_cleanup(entity) os.remove(FTPfile.path)
def test_syncFromSynapse__given_file_id(test_state): file_path = utils.make_bogus_data_file() test_state.schedule_for_cleanup(file_path) file = test_state.syn.store(File(file_path, name=str(uuid.uuid4()), parent=test_state.project, synapseStore=False)) all_files = synapseutils.syncFromSynapse(test_state.syn, file.id) assert 1 == len(all_files) assert file == all_files[0]
def test_uploadFileEntity(syn, project, schedule_for_cleanup): # Create a FileEntity # Dictionaries default to FileEntity as a type fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) entity = File(name='fooUploadFileEntity', path=fname, parentId=project['id'], description='A test file entity') entity = syn.store(entity) # Download and verify entity = syn.get(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path']) # Check if we upload the wrong type of file handle fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0] assert fh[ 'concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle' # Create a different temporary file fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) # Update existing FileEntity entity.path = fname entity = syn.store(entity) # Download and verify that it is the same file entity = syn.get(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path'])
def test_create_Link_to_entity_with_the_same_parent(): parent = "syn123" file = File("new file", parent=parent, id="syn456") file_bundle = { 'accessControlList': '/repo/v1/entity/syn456/acl', 'entityType': 'org.sagebionetworks.repo.model.FileEntity', 'annotations': '/repo/v1/entity/syn456/annotations', 'uri': '/repo/v1/entity/syn456', 'createdOn': '2018-08-27T20:48:43.562Z', 'parentId': 'syn123', 'versionNumber': 1, 'dataFileHandleId': '3594', 'modifiedOn': '2018-08-27T20:48:44.938Z', 'versionLabel': '1', 'createdBy': '1', 'versions': '/repo/v1/entity/syn456/version', 'name': 'new file', 'concreteType': 'org.sagebionetworks.repo.model.FileEntity', 'etag': '62fd1a76-ed9c-425a-b4a8-1c4e6aad7fc6', 'modifiedBy': '1', 'id': 'syn456', 'versionUrl': '/repo/v1/entity/syn456/version/1' } link = Link(targetId=file, parent=parent) syn = Synapse(skip_checks=True) with patch.object(syn, "_getEntity", return_value=file_bundle): pytest.raises(ValueError, syn.store, link)
def addExternalDataToSynapse(exBucket, exFolder, toLoadSynID, annotYAML, syn): '''Adds data in external S3 bucket to Synapse.''' s3 = boto.connect_s3() H3bucket = s3.get_bucket(exBucket) rnaseqDir = H3bucket.list(prefix=exFolder) loadedEntities = getLoadedEntities(parent=toLoadSynID, syn=syn) BAMannotations = parseProjectAnnotations(yamlPath=annotYAML) ## TODO Add bucket and key to annotations addedCount = 0 for key in rnaseqDir: if os.path.basename(key.name) in loadedEntities: print 'Skipping %s because it is already loaded to Syanpse.' % os.path.basename( key.name) elif key.name.endswith('bam'): s3path = '/'.join(['http://s3.amazonaws.com', exBucket, key.name]) BAMEntity = File(path=s3path, name=os.path.basename(key.name), description='BAM format aligned reads', parent=toLoadSynID, synapseStore=False, annotations=BAMannotations) BAMEntity = syn.store(BAMEntity) addedCount += 1 return (addedCount)
def storeFile(syn, fileName, parentId, center, fileFormat, dataSubType, platform=None, cBioFileFormat=None, used=None): logger.info("STORING FILES") fileEnt = File(fileName, parent=parentId) fileEnt.center = center fileEnt.species = "Human" fileEnt.consortium = 'GENIE' fileEnt.dataType = "genomicVariants" fileEnt.fundingAgency = "AACR" fileEnt.assay = 'targetGeneSeq' fileEnt.fileFormat = fileFormat fileEnt.dataSubType = dataSubType fileEnt.fileStage = "staging" fileEnt.platform = platform if platform is not None: fileEnt.platform = platform if cBioFileFormat is not None: fileEnt.cBioFileFormat = cBioFileFormat ent = syn.store(fileEnt, used=used) return (ent)
def test_synStore_sftpIntegration(): """Creates a File Entity on an sftp server and add the external url. """ filepath = utils.make_bogus_binary_file(1 * MB - 777771) try: file = syn.store(File(filepath, parent=project)) file2 = syn.get(file) assert file.externalURL == file2.externalURL and urlparse( file2.externalURL).scheme == 'sftp' tmpdir = tempfile.mkdtemp() schedule_for_cleanup(tmpdir) ## test filename override file2.fileNameOverride = "whats_new_in_baltimore.data" file2 = syn.store(file2) ## TODO We haven't defined how filename override interacts with ## TODO previously cached files so, side-step that for now by ## TODO making sure the file is not in the cache! syn.cache.remove(file2.dataFileHandleId, delete=True) file3 = syn.get(file, downloadLocation=tmpdir) assert os.path.basename(file3.path) == file2.fileNameOverride ## test that we got an MD5 à la SYNPY-185 assert_is_not_none(file3.md5) fh = syn._getFileHandle(file3.dataFileHandleId) assert_is_not_none(fh['contentMd5']) assert_equals(file3.md5, fh['contentMd5']) finally: try: os.remove(filepath) except Exception: print(traceback.format_exc())
def _manifest_upload(syn, df): items = [] for i, row in df.iterrows(): file = File( path=row['path'], parent=row['parent'], **{key: row[key] for key in FILE_CONSTRUCTOR_FIELDS if key in row}, ) annotations = dict( row.drop(FILE_CONSTRUCTOR_FIELDS + STORE_FUNCTION_FIELDS + REQUIRED_FIELDS + PROVENANCE_FIELDS, errors='ignore')) # if a item in the manifest upload is an empty string we do not want to upload that # as an empty string annotation file.annotations = {k: v for k, v in annotations.items() if v != ''} item = _SyncUploadItem( file, row['used'] if 'used' in row else [], row['executed'] if 'executed' in row else [], {key: row[key] for key in STORE_FUNCTION_FIELDS if key in row}, ) items.append(item) with _sync_executor(syn) as executor: uploader = _SyncUploader(syn, executor) uploader.upload(items) return True
def test_download_check_md5(syn, project, schedule_for_cleanup): tempfile_path = utils.make_bogus_data_file() schedule_for_cleanup(tempfile_path) entity_bad_md5 = syn.store(File(path=tempfile_path, parent=project['id'], synapseStore=False)) pytest.raises(SynapseMd5MismatchError, syn._download_from_URL, entity_bad_md5['externalURL'], tempfile.gettempdir(), entity_bad_md5['dataFileHandleId'], expected_md5="2345a")
def test_syncFromSynapse__downloadFile_is_false(syn): """ Verify when passing the argument downloadFile is equal to False, syncFromSynapse won't download the file to clients' local end. """ project = Project(name="the project", parent="whatever", id="syn123") file = File(name="a file", parent=project, id="syn456") folder = Folder(name="a folder", parent=project, id="syn789") entities = { file.id: file, folder.id: folder, } def syn_get_side_effect(entity, *args, **kwargs): return entities[id_of(entity)] with patch.object(syn, "getChildren", side_effect=[[folder, file], []]),\ patch.object(syn, "get", side_effect=syn_get_side_effect) as patch_syn_get: synapseutils.syncFromSynapse(syn, project, downloadFile=False) patch_syn_get.assert_called_once_with( file['id'], downloadLocation=None, ifcollision='overwrite.local', followLink=False, downloadFile=False, )
def test_store__changing_from_Synapse_to_externalURL_by_changing_path(syn, project, schedule_for_cleanup): # create a temp file temp_path = utils.make_bogus_data_file() schedule_for_cleanup(temp_path) ext = syn.store(File(temp_path, parent=project, synapseStore=True)) ext = syn.get(ext) assert "org.sagebionetworks.repo.model.file.S3FileHandle" == ext._file_handle.concreteType ext.synapseStore = False ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert "org.sagebionetworks.repo.model.file.ExternalFileHandle" == ext._file_handle.concreteType assert utils.as_url(temp_path) == ext.externalURL assert not ext.synapseStore # swap back to synapse storage ext.synapseStore = True ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert "org.sagebionetworks.repo.model.file.S3FileHandle" == ext._file_handle.concreteType assert ext.externalURL is None assert ext.synapseStore
def test_dispose(syn_client, syn_test_helper, new_temp_file): project = syn_client.store(Project(name=syn_test_helper.uniq_name())) folder = syn_client.store( Folder(name=syn_test_helper.uniq_name(prefix='Folder '), parent=project)) file = syn_client.store( File(name=syn_test_helper.uniq_name(prefix='File '), path=new_temp_file, parent=folder)) syn_objects = [project, folder, file] for syn_obj in syn_objects: syn_test_helper.dispose_of(syn_obj) assert syn_obj in syn_test_helper._trash syn_test_helper.dispose() assert len(syn_test_helper._trash) == 0 for syn_obj in syn_objects: with pytest.raises(synapseclient.exceptions.SynapseHTTPError) as ex: syn_client.get(syn_obj, downloadFile=False) err_str = str(ex.value) assert "Not Found" in err_str or "cannot be found" in err_str or "is in trash can" in err_str or "does not exist" in err_str try: os.remove(new_temp_file) except: pass
def upload_file(file_path: str, login: str, parent: str, description: str = None) -> None: """Uploads file to Synapse. Password must be stored in environment variable SYNAPSE_PASS""" description = '' if None else description f = File(file_path, description=description, parent=parent) syn = _syn_login(login) syn.store(f)
def test_store_file_handle_update_metadata(): original_file_path = utils.make_bogus_data_file() schedule_for_cleanup(original_file_path) #upload the project entity = syn.store(File(original_file_path, parent=project)) old_file_handle = entity._file_handle #create file handle to replace the old one replacement_file_path = utils.make_bogus_data_file() schedule_for_cleanup(replacement_file_path) new_file_handle = syn.uploadFileHandle(replacement_file_path, parent=project) entity.dataFileHandleId = new_file_handle['id'] new_entity = syn.store(entity) #make sure _file_handle info was changed (_file_handle values are all changed at once so just verifying id change is sufficient) assert_equal(new_file_handle['id'], new_entity._file_handle['id']) assert_not_equal(old_file_handle['id'], new_entity._file_handle['id']) #check that local_state was updated assert_equal(replacement_file_path, new_entity.path) assert_equal(os.path.dirname(replacement_file_path), new_entity.cacheDir) assert_equal([os.path.basename(replacement_file_path)], new_entity.files)
def test_resume_partial_download(syn, project, schedule_for_cleanup): original_file = utils.make_bogus_data_file(40000) entity = File(original_file, parent=project['id']) entity = syn.store(entity) # stash the original file for comparison later shutil.move(original_file, original_file+'.original') original_file += '.original' schedule_for_cleanup(original_file) temp_dir = tempfile.gettempdir() url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id) path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId, expected_md5=entity.md5) # simulate an imcomplete download by putting the # complete file back into its temporary location tmp_path = utils.temp_download_filename(temp_dir, entity.dataFileHandleId) shutil.move(path, tmp_path) # ...and truncating it to some fraction of its original size with open(tmp_path, 'r+') as f: f.truncate(3*os.path.getsize(original_file)//7) # this should complete the partial download path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId, expected_md5=entity.md5) assert filecmp.cmp(original_file, path), "File comparison failed"