def test_uploadFileEntity(syn, project, schedule_for_cleanup): # Create a FileEntity # Dictionaries default to FileEntity as a type fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) entity = File(name='fooUploadFileEntity', path=fname, parentId=project['id'], description='A test file entity') entity = syn.store(entity) # Download and verify entity = syn.get(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path']) # Check if we upload the wrong type of file handle fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0] assert fh[ 'concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle' # Create a different temporary file fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) # Update existing FileEntity entity.path = fname entity = syn.store(entity) # Download and verify that it is the same file entity = syn.get(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path'])
def test_store_file_handle_update_metadata(syn, project, schedule_for_cleanup): original_file_path = utils.make_bogus_data_file() schedule_for_cleanup(original_file_path) # upload the project entity = syn.store(File(original_file_path, parent=project)) old_file_handle = entity._file_handle # create file handle to replace the old one replacement_file_path = utils.make_bogus_data_file() schedule_for_cleanup(replacement_file_path) new_file_handle = syn.uploadFileHandle(replacement_file_path, parent=project) entity.dataFileHandleId = new_file_handle['id'] new_entity = syn.store(entity) # make sure _file_handle info was changed # (_file_handle values are all changed at once so just verifying id change is sufficient) assert new_file_handle['id'] == new_entity._file_handle['id'] assert old_file_handle['id'] != new_entity._file_handle['id'] # check that local_state was updated assert replacement_file_path == new_entity.path assert os.path.dirname(replacement_file_path) == new_entity.cacheDir assert [os.path.basename(replacement_file_path)] == new_entity.files
def test_walk(syn, schedule_for_cleanup): walked = [] firstfile = utils.make_bogus_data_file() schedule_for_cleanup(firstfile) project_entity = syn.store(Project(name=str(uuid.uuid4()))) schedule_for_cleanup(project_entity.id) folder_entity = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) schedule_for_cleanup(folder_entity.id) second_folder = syn.store( Folder(name=str(uuid.uuid4()), parent=project_entity)) schedule_for_cleanup(second_folder.id) file_entity = syn.store(File(firstfile, parent=project_entity)) schedule_for_cleanup(file_entity.id) walked.append(((project_entity.name, project_entity.id), [ (folder_entity.name, folder_entity.id), (second_folder.name, second_folder.id) ], [(file_entity.name, file_entity.id)])) nested_folder = syn.store( Folder(name=str(uuid.uuid4()), parent=folder_entity)) schedule_for_cleanup(nested_folder.id) secondfile = utils.make_bogus_data_file() schedule_for_cleanup(secondfile) second_file = syn.store(File(secondfile, parent=nested_folder)) schedule_for_cleanup(second_file.id) thirdfile = utils.make_bogus_data_file() schedule_for_cleanup(thirdfile) third_file = syn.store(File(thirdfile, parent=second_folder)) schedule_for_cleanup(third_file.id) walked.append(((os.path.join(project_entity.name, folder_entity.name), folder_entity.id), [(nested_folder.name, nested_folder.id)], [])) walked.append( ((os.path.join(os.path.join(project_entity.name, folder_entity.name), nested_folder.name), nested_folder.id), [], [(second_file.name, second_file.id)])) walked.append(((os.path.join(project_entity.name, second_folder.name), second_folder.id), [], [(third_file.name, third_file.id)])) temp = synapseutils.walk(syn, project_entity.id) temp = list(temp) # Must sort the tuples returned, because order matters for the assert # Folders are returned in a different ordering depending on the name for i in walked: for x in i: if type(x) == list: x.sort() for i in temp: for x in i: if type(x) == list: x.sort() assert i in walked temp = synapseutils.walk(syn, second_file.id) assert list(temp) == []
def init(self, syn, schedule_for_cleanup): self.syn = syn self.schedule_for_cleanup = schedule_for_cleanup # Create a Project self.project_entity = syn.store(Project(name=str(uuid.uuid4()))) filename = utils.make_bogus_data_file() attachname = utils.make_bogus_data_file() file_entity = self.syn.store(File(filename, parent=self.project_entity)) self.schedule_for_cleanup(self.project_entity.id) self.schedule_for_cleanup(filename) self.schedule_for_cleanup(file_entity.id) # Create mock wiki md = """ This is a test wiki ======================= Blabber jabber blah blah boo. syn123 syn456 """ wiki = Wiki(owner=self.project_entity, title='A Test Wiki', markdown=md, attachments=[attachname]) wiki = self.syn.store(wiki) # Create a Wiki sub-page subwiki = Wiki(owner=self.project_entity, title='A sub-wiki', markdown='%s' % file_entity.id, parentWikiId=wiki.id) self.subwiki = self.syn.store(subwiki) second_md = """ Testing internal links ====================== [test](#!Synapse:%s/wiki/%s) %s) """ % (self.project_entity.id, self.subwiki.id, file_entity.id) sub_subwiki = Wiki(owner=self.project_entity, title='A sub-sub-wiki', markdown=second_md, parentWikiId=self.subwiki.id, attachments=[attachname]) self.sub_subwiki = self.syn.store(sub_subwiki) # Set up the second project self.second_project = self.syn.store(Project(name=str(uuid.uuid4()))) self.schedule_for_cleanup(self.second_project.id) self.fileMapping = {'syn123': 'syn12345', 'syn456': 'syn45678'} self.first_headers = self.syn.getWikiHeaders(self.project_entity)
def test_uploadFile_given_dictionary(syn, project, schedule_for_cleanup): # Make a Folder Entity the old fashioned way folder = { 'concreteType': Folder._synapse_entity_type, 'parentId': project['id'], 'name': 'fooDictionary', 'foo': 334455 } entity = syn.store(folder) # Download and verify that it is the same file entity = syn.get(entity) assert entity.parentId == project.id assert entity.foo[0] == 334455 # Update via a dictionary path = utils.make_bogus_data_file() schedule_for_cleanup(path) rareCase = {} rareCase.update(entity.annotations) rareCase.update(entity.properties) rareCase.update(entity.local_state()) rareCase[ 'description'] = 'Updating with a plain dictionary should be rare.' # Verify it works entity = syn.store(rareCase) assert entity.description == rareCase['description'] assert entity.name == 'fooDictionary' syn.get(entity['id'])
def test_synapseStore_flag(syn, project, schedule_for_cleanup): # Store a path to a local file path = utils.make_bogus_data_file() schedule_for_cleanup(path) bogus = File(path, name='Totally bogus data', parent=project, synapseStore=False) bogus = syn.store(bogus) # Verify the thing can be downloaded as a URL bogus = syn.get(bogus, downloadFile=False) assert bogus.name == 'Totally bogus data' assert bogus.path == path, "Path: %s\nExpected: %s" % (bogus.path, path) assert not bogus.synapseStore # Make sure the test runs on Windows and other OS's if path[0].isalpha() and path[1] == ':': # A Windows file URL looks like this: file:///c:/foo/bar/bat.txt expected_url = 'file:///' + path.replace("\\", "/") else: expected_url = 'file://' + path assert bogus.externalURL == expected_url, 'URL: %s\nExpected %s' % (bogus.externalURL, expected_url) # A file path that doesn't exist should still work bogus = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False) bogus = syn.store(bogus) pytest.raises(IOError, syn.get, bogus) assert not bogus.synapseStore # Try a URL bogus = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False) bogus = syn.store(bogus) bogus = syn.get(bogus) assert not bogus.synapseStore
def test_syncFromSynapse__given_file_id(test_state): file_path = utils.make_bogus_data_file() test_state.schedule_for_cleanup(file_path) file = test_state.syn.store(File(file_path, name=str(uuid.uuid4()), parent=test_state.project, synapseStore=False)) all_files = synapseutils.syncFromSynapse(test_state.syn, file.id) assert 1 == len(all_files) assert file == all_files[0]
def test_store__changing_from_Synapse_to_externalURL_by_changing_path(syn, project, schedule_for_cleanup): # create a temp file temp_path = utils.make_bogus_data_file() schedule_for_cleanup(temp_path) ext = syn.store(File(temp_path, parent=project, synapseStore=True)) ext = syn.get(ext) assert "org.sagebionetworks.repo.model.file.S3FileHandle" == ext._file_handle.concreteType ext.synapseStore = False ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert "org.sagebionetworks.repo.model.file.ExternalFileHandle" == ext._file_handle.concreteType assert utils.as_url(temp_path) == ext.externalURL assert not ext.synapseStore # swap back to synapse storage ext.synapseStore = True ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert "org.sagebionetworks.repo.model.file.S3FileHandle" == ext._file_handle.concreteType assert ext.externalURL is None assert ext.synapseStore
def test_get_local_file(syn, project, schedule_for_cleanup): new_path = utils.make_bogus_data_file() schedule_for_cleanup(new_path) folder = Folder('TestFindFileFolder', parent=project, description='A place to put my junk') folder = syn.store(folder) # Get an nonexistent file in Synapse pytest.raises(SynapseError, syn.get, new_path) # Get a file really stored in Synapse ent_folder = syn.store(File(new_path, parent=folder)) ent2 = syn.get(new_path) assert ent_folder.id == ent2.id assert ent_folder.versionNumber == ent2.versionNumber # Get a file stored in Multiple locations #should display warning syn.store(File(new_path, parent=project)) syn.get(new_path) # Get a file stored in multiple locations with limit set ent = syn.get(new_path, limitSearch=folder.id) assert ent.id == ent_folder.id assert ent.versionNumber == ent_folder.versionNumber # Get a file that exists but such that limitSearch removes them and raises error pytest.raises(SynapseError, syn.get, new_path, limitSearch='syn1')
def test_configPath(test_state): """Test using a user-specified configPath for Synapse configuration file.""" tmp_config_file = tempfile.NamedTemporaryFile(suffix='.synapseConfig', delete=False) shutil.copyfile(client.CONFIG_FILE, tmp_config_file.name) # Create a File filename = utils.make_bogus_data_file() test_state.schedule_for_cleanup(filename) output = run(test_state, 'synapse' '--skip-checks', '--configPath', tmp_config_file.name, 'add', '-name', 'BogusFileEntityTwo', '-description', 'Bogus data to test file upload', '-parentid', test_state.project.id, filename) file_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output) # Verify that we stored the file in Synapse f1 = test_state.syn.get(file_entity_id) fh = test_state.syn._get_file_handle_as_creator(f1.dataFileHandleId) assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
def test_resume_partial_download(syn, project, schedule_for_cleanup): original_file = utils.make_bogus_data_file(40000) entity = File(original_file, parent=project['id']) entity = syn.store(entity) # stash the original file for comparison later shutil.move(original_file, original_file+'.original') original_file += '.original' schedule_for_cleanup(original_file) temp_dir = tempfile.gettempdir() url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id) path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId, expected_md5=entity.md5) # simulate an imcomplete download by putting the # complete file back into its temporary location tmp_path = utils.temp_download_filename(temp_dir, entity.dataFileHandleId) shutil.move(path, tmp_path) # ...and truncating it to some fraction of its original size with open(tmp_path, 'r+') as f: f.truncate(3*os.path.getsize(original_file)//7) # this should complete the partial download path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId, expected_md5=entity.md5) assert filecmp.cmp(original_file, path), "File comparison failed"
def test_download_check_md5(syn, project, schedule_for_cleanup): tempfile_path = utils.make_bogus_data_file() schedule_for_cleanup(tempfile_path) entity_bad_md5 = syn.store(File(path=tempfile_path, parent=project['id'], synapseStore=False)) pytest.raises(SynapseMd5MismatchError, syn._download_from_URL, entity_bad_md5['externalURL'], tempfile.gettempdir(), entity_bad_md5['dataFileHandleId'], expected_md5="2345a")
def test_download_local_file_URL_path(syn, project, schedule_for_cleanup): path = utils.make_bogus_data_file() schedule_for_cleanup(path) filehandle = create_external_file_handle(syn, path, mimetype=None, file_size=None) localFileEntity = syn.store(File(dataFileHandleId=filehandle['id'], parent=project)) e = syn.get(localFileEntity.id) assert path == utils.normalize_path(e.path)
def _create_temp_file_with_cleanup(schedule_for_cleanup, specific_file_text=None): if specific_file_text: with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as file: file.write(specific_file_text) filename = file.name else: filename = utils.make_bogus_data_file() schedule_for_cleanup(filename) return filename
def test_syncFromSynapse_Links(test_state): """This function tests recursive download of links as defined in syncFromSynapse most of the functionality of this function are already tested in the tests/integration/test_command_line_client::test_command_get_recursive_and_query which means that the only test if for path=None """ # Create a Project project_entity = test_state.syn.store(Project(name=str(uuid.uuid4()))) test_state.schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create a Folder hierarchy in folder_entity inner_folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=folder_entity)) second_folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) test_state.schedule_for_cleanup(f) file_entity = test_state.syn.store(File(f, parent=project_entity)) # Create links to inner folder test_state.syn.store(Link(file_entity.id, parent=folder_entity)) # Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) test_state.schedule_for_cleanup(f) file_entity = test_state.syn.store(File(f, parent=second_folder_entity)) # Create link to inner folder test_state.syn.store(Link(file_entity.id, parent=inner_folder_entity)) # Test recursive get output = synapseutils.syncFromSynapse(test_state.syn, folder_entity, followLink=True) assert len(output) == len(uploaded_paths) for f in output: assert f.path in uploaded_paths
def __init__(self): self.syn = syn self.project = syn.store(Project(name=str(uuid.uuid4()))) self.folder = syn.store(Folder(name=str(uuid.uuid4()), parent=self.project)) self.schedule_for_cleanup = schedule_for_cleanup # Create testfiles for upload self.f1 = utils.make_bogus_data_file(n=10) self.f2 = utils.make_bogus_data_file(n=10) self.f3 = 'https://www.synapse.org' self.header = 'path parent used executed activityName synapseStore foo\n' self.row1 = '%s %s %s "%s;https://www.example.com" provName bar\n' % ( self.f1, self.project.id, self.f2, self.f3 ) self.row2 = '%s %s "syn12" "syn123;https://www.example.com" provName2 bar\n' % ( self.f2, self.folder.id ) self.row3 = '%s %s "syn12" prov2 False baz\n' % (self.f3, self.folder.id) self.row4 = '%s %s %s act 2\n' % (self.f3, self.project.id, self.f1) # Circular reference self.row5 = '%s syn12 \n' % (self.f3) # Wrong parent
def test_syncFromSynapse(test_state): """This function tests recursive download as defined in syncFromSynapse most of the functionality of this function are already tested in the tests/integration/test_command_line_client::test_command_get_recursive_and_query which means that the only test if for path=None """ # Create a Project project_entity = test_state.syn.store(Project(name=str(uuid.uuid4()))) test_state.schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload two files in Folder uploaded_paths = [] for i in range(2): f = utils.make_bogus_data_file() uploaded_paths.append(f) test_state.schedule_for_cleanup(f) test_state.syn.store(File(f, parent=folder_entity)) # Add a file in the project level as well f = utils.make_bogus_data_file() uploaded_paths.append(f) test_state.schedule_for_cleanup(f) test_state.syn.store(File(f, parent=project_entity)) # syncFromSynapse() uses chunkedQuery() which will return results that are eventually consistent # but not always right after the entity is created. start_time = time.time() while len(list(test_state.syn.getChildren(project_entity))) != 2: assert time.time() - start_time < QUERY_TIMEOUT_SEC time.sleep(2) # Test recursive get output = synapseutils.syncFromSynapse(test_state.syn, project_entity) assert len(output) == len(uploaded_paths) for f in output: assert f.path in uploaded_paths
def test_http_download__range_request_error(syn, project): # SYNPY-525 file_path = utils.make_bogus_data_file() file_entity = syn.store(File(file_path, parent=project)) syn.cache.purge(time.time()) # download once and rename to temp file to simulate range exceed file_entity = syn.get(file_entity) shutil.move(file_entity.path, utils.temp_download_filename(file_entity.path, file_entity.dataFileHandleId)) file_entity = syn.get(file_entity) assert file_path != file_entity.path assert filecmp.cmp(file_path, file_entity.path)
def test_download_multithreaded(syn, project, schedule_for_cleanup): # Create a FileEntity # Dictionaries default to FileEntity as a type fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) entity = File(name='testMultiThreadDownload' + str(uuid.uuid4()), path=fname, parentId=project['id']) entity = syn.store(entity) # Download and verify syn.multi_threaded = True entity = syn.get(entity) assert entity['files'][0] == os.path.basename(fname) assert filecmp.cmp(fname, entity['path']) syn.multi_threaded = False
def test_provenance(syn, project, schedule_for_cleanup): # Create a File Entity fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) data_entity = syn.store(File(fname, parent=project['id'])) # Create a File Entity of Code fd, path = tempfile.mkstemp(suffix=".py") with os.fdopen(fd, 'w') as f: f.write( utils.normalize_lines(""" ## Chris's fabulous random data generator ############################################################ import random random.seed(12345) data = [random.gauss(mu=0.0, sigma=1.0) for i in range(100)] """)) schedule_for_cleanup(path) code_entity = syn.store(File(path, parent=project['id'])) # Create a new Activity asserting that the Code Entity was 'used' activity = Activity(name='random.gauss', description='Generate some random numbers') activity.used(code_entity, wasExecuted=True) activity.used( { 'name': 'Superhack', 'url': 'https://github.com/joe_coder/Superhack' }, wasExecuted=True) activity = syn.setProvenance(data_entity, activity) # Retrieve and verify the saved Provenance record retrieved_activity = syn.getProvenance(data_entity) assert retrieved_activity == activity # Test Activity update new_description = 'Generate random numbers like a gangsta' retrieved_activity['description'] = new_description updated_activity = syn.updateActivity(retrieved_activity) assert updated_activity['name'] == retrieved_activity['name'] assert updated_activity['description'] == new_description # Test delete syn.deleteProvenance(data_entity) pytest.raises(SynapseHTTPError, syn.getProvenance, data_entity['id'])
def test_entity_version(syn, project, schedule_for_cleanup): # Make an Entity and make sure the version is one entity = File(parent=project['id']) entity['path'] = utils.make_bogus_data_file() schedule_for_cleanup(entity['path']) entity = syn.store(entity) syn.set_annotations(Annotations(entity, entity.etag, {'fizzbuzz': 111222})) entity = syn.get(entity) assert entity.versionNumber == 1 # Update the Entity and make sure the version is incremented entity.foo = 998877 entity['name'] = 'foobarbat' entity['description'] = 'This is a test entity...' entity = syn.store(entity, forceVersion=True, versionLabel="Prada remix") assert entity.versionNumber == 2 # Get the older data and verify the random stuff is still there annotations = syn.get_annotations(entity, version=1) assert annotations['fizzbuzz'][0] == 111222 returnEntity = syn.get(entity, version=1) assert returnEntity.versionNumber == 1 assert returnEntity['fizzbuzz'][0] == 111222 assert 'foo' not in returnEntity # Try the newer Entity returnEntity = syn.get(entity) assert returnEntity.versionNumber == 2 assert returnEntity['foo'][0] == 998877 assert returnEntity['name'] == 'foobarbat' assert returnEntity['description'] == 'This is a test entity...' assert returnEntity['versionLabel'] == 'Prada remix' # Try the older Entity again returnEntity = syn.get(entity, version=1) assert returnEntity.versionNumber == 1 assert returnEntity['fizzbuzz'][0] == 111222 assert 'foo' not in returnEntity # Delete version 2 syn.delete(entity, version=2) returnEntity = syn.get(entity) assert returnEntity.versionNumber == 1
def test_md5_query(syn, project, schedule_for_cleanup): # Add the same Entity several times path = utils.make_bogus_data_file() schedule_for_cleanup(path) repeated = File(path, parent=project['id'], description='Same data over and over again') # Retrieve the data via MD5 num = 5 stored = [] for i in range(num): repeated.name = 'Repeated data %d.dat' % i stored.append(syn.store(repeated).id) # Although we expect num results, it is possible for the MD5 to be non-unique results = syn.md5Query(utils.md5_for_file(path).hexdigest()) assert str(sorted(stored)) == str(sorted([res['id'] for res in results])) assert len(results) == num
def thread_keep_storing_one_File(syn, project, schedule_for_cleanup): """Makes one file and stores it over and over again.""" # Make a local file to continuously store path = utils.make_bogus_data_file() schedule_for_cleanup(path) myPrecious = File(path, parent=project, description='This bogus file is MINE', mwa="hahahah") while syn.test_keepRunning: stored = store_catch_412_HTTPError(syn, myPrecious) if stored is not None: myPrecious = stored else: try: myPrecious = syn.get(myPrecious) except ValueError as ve: raise ValueError(str(myPrecious)) from ve sleep_for_a_bit()
def thread_get_and_update_file_from_Project(syn, project, schedule_for_cleanup): """Fetches one item from the Project and updates it with a new file.""" while syn.test_keepRunning: id = get_all_ids_from_Project(syn, project) if len(id) <= 0: continue id = id[random.randrange(len(id))] entity = syn.get(id) # Replace the file and re-store path = utils.make_bogus_data_file() schedule_for_cleanup(path) entity.path = path entity = store_catch_412_HTTPError(syn, entity) if entity is not None: assert os.stat(entity.path) == os.stat(path) sleep_for_a_bit()
def test_syncFromSynapse__children_contain_non_file(test_state): proj = test_state.syn.store(Project(name="test_syncFromSynapse_children_non_file" + str(uuid.uuid4()))) test_state.schedule_for_cleanup(proj) temp_file = utils.make_bogus_data_file() test_state.schedule_for_cleanup(temp_file) file_entity = test_state.syn.store( File( temp_file, name="temp_file_test_syncFromSynapse_children_non_file" + str(uuid.uuid4()), parent=proj ) ) test_state.syn.store(Schema(name="table_test_syncFromSynapse", parent=proj)) temp_folder = tempfile.mkdtemp() test_state.schedule_for_cleanup(temp_folder) files_list = synapseutils.syncFromSynapse(test_state.syn, proj, temp_folder) assert 1 == len(files_list) assert file_entity == files_list[0]
def test_store__changing_externalURL_by_changing_path(syn, project, schedule_for_cleanup): url = 'https://www.synapse.org/Portal/clear.cache.gif' ext = syn.store(File(url, name="test", parent=project, synapseStore=False)) # perform a syn.get so the filename changes ext = syn.get(ext) # create a temp file temp_path = utils.make_bogus_data_file() schedule_for_cleanup(temp_path) ext.synapseStore = False ext.path = temp_path ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert ext.externalURL != url assert utils.normalize_path(temp_path) == utils.file_url_to_path(ext.externalURL) assert temp_path == utils.normalize_path(ext.path) assert not ext.synapseStore
def test_create_and_update_file_view(syn, project, schedule_for_cleanup): # Create a folder folder = Folder(str(uuid.uuid4()), parent=project, description='creating a file-view') folder = syn.store(folder) # Create dummy file with annotations in our folder path = utils.make_bogus_data_file() file_annotations = dict(fileFormat='jpg', dataType='image', artist='Banksy', medium='print', title='Girl With Ballon') schedule_for_cleanup(path) a_file = File(path, parent=folder, annotations=file_annotations) a_file = syn.store(a_file) schedule_for_cleanup(a_file) # Add new columns for the annotations on this file and get their IDs my_added_cols = [ syn.store(Column(name=k, columnType="STRING")) for k in file_annotations.keys() ] my_added_cols_ids = [c['id'] for c in my_added_cols] view_default_ids = [ c['id'] for c in syn._get_default_view_columns( "entityview", EntityViewType.FILE.value) ] col_ids = my_added_cols_ids + view_default_ids scopeIds = [folder['id'].lstrip('syn')] # Create an empty entity-view with defined scope as folder entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=True, addAnnotationColumns=False, type='file', columns=my_added_cols, parent=project) entity_view = syn.store(entity_view) schedule_for_cleanup(entity_view) assert set(scopeIds) == set(entity_view.scopeIds) assert set(col_ids) == set(entity_view.columnIds) assert EntityViewType.FILE.value == entity_view.viewTypeMask # get the current view-schema view = syn.tableQuery("select * from %s" % entity_view.id) schedule_for_cleanup(view.filepath) view_dict = list( csv.DictReader(io.open(view.filepath, encoding="utf-8", newline=''))) # check that all of the annotations were retrieved from the view assert set(file_annotations.keys()).issubset(set(view_dict[0].keys())) updated_a_file = syn.get(a_file.id, downloadFile=False) # Check that the values are the same as what was set # Both in the view and on the entity itself for k, v in file_annotations.items(): assert view_dict[0][k] == v assert updated_a_file.annotations[k][0] == v # Make a change to the view and store view_dict[0]['fileFormat'] = 'PNG' with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as temp: schedule_for_cleanup(temp.name) temp_filename = temp.name with io.open(temp_filename, mode='w', encoding="utf-8", newline='') as temp_file: dw = csv.DictWriter(temp_file, fieldnames=view_dict[0].keys(), quoting=csv.QUOTE_NONNUMERIC, lineterminator=str(os.linesep)) dw.writeheader() dw.writerows(view_dict) temp_file.flush() syn.store(Table(entity_view.id, temp_filename)) new_view_dict = list( csv.DictReader(io.open(temp_filename, encoding="utf-8", newline=''))) assert new_view_dict[0]['fileFormat'] == 'PNG' # query for the change start_time = time.time() new_view_results = syn.tableQuery("select * from %s" % entity_view.id) schedule_for_cleanup(new_view_results.filepath) new_view_dict = list( csv.DictReader( io.open(new_view_results.filepath, encoding="utf-8", newline=''))) # query until change is seen. while new_view_dict[0]['fileFormat'] != 'PNG': # check timeout assert time.time() - start_time < QUERY_TIMEOUT_SEC # query again new_view_results = syn.tableQuery("select * from %s" % entity_view.id) new_view_dict = list( csv.DictReader( io.open(new_view_results.filepath, encoding="utf-8", newline=''))) # paranoid check assert new_view_dict[0]['fileFormat'] == 'PNG'
def test_command_line_using_paths(test_state): # Create a Project project_entity = test_state.syn.store(Project(name=str(uuid.uuid4()))) test_state.schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) # Create and upload a file in Folder filename = utils.make_bogus_data_file() test_state.schedule_for_cleanup(filename) file_entity = test_state.syn.store(File(filename, parent=folder_entity)) # Verify that we can use show with a filename output = run(test_state, 'synapse' '--skip-checks', 'show', filename) id = parse(r'File: %s\s+\((syn\d+)\)\s+' % os.path.split(filename)[1], output) assert file_entity.id == id # Verify that limitSearch works by making sure we get the file entity # that's inside the folder file_entity2 = test_state.syn.store(File(filename, parent=project_entity)) output = run(test_state, 'synapse' '--skip-checks', 'get', '--limitSearch', folder_entity.id, filename) id = parse(r'Associated file: .* with synapse ID (syn\d+)', output) name = parse(r'Associated file: (.*) with synapse ID syn\d+', output) assert file_entity.id == id assert utils.equal_paths(name, filename) # Verify that set-provenance works with filepath repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' output = run(test_state, 'synapse' '--skip-checks', 'set-provenance', '-id', file_entity2.id, '-name', 'TestActivity', '-description', 'A very excellent provenance', '-used', filename, '-executed', repo_url, '-limitSearch', folder_entity.id) parse(r'Set provenance record (\d+) on entity syn\d+', output) output = run(test_state, 'synapse' '--skip-checks', 'get-provenance', '-id', file_entity2.id) activity = json.loads(output) assert activity['name'] == 'TestActivity' assert activity['description'] == 'A very excellent provenance' # Verify that store works with provenance specified with filepath repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' filename2 = utils.make_bogus_data_file() test_state.schedule_for_cleanup(filename2) output = run(test_state, 'synapse' '--skip-checks', 'add', filename2, '-parentid', project_entity.id, '-used', filename, '-executed', '%s %s' % (repo_url, filename)) entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output) output = run(test_state, 'synapse' '--skip-checks', 'get-provenance', '-id', entity_id) activity = json.loads(output) a = [a for a in activity['used'] if not a['wasExecuted']] assert a[0]['reference']['targetId'] in [file_entity.id, file_entity2.id] # Test associate command # I have two files in Synapse filename and filename2 path = tempfile.mkdtemp() test_state.schedule_for_cleanup(path) shutil.copy(filename, path) shutil.copy(filename2, path) run(test_state, 'synapse' '--skip-checks', 'associate', path, '-r') run(test_state, 'synapse' '--skip-checks', 'show', filename)
def test_command_copy(test_state): """Tests the 'synapse cp' function""" # Create a Project project_entity = test_state.syn.store(Project(name=str(uuid.uuid4()))) test_state.schedule_for_cleanup(project_entity.id) # Create a Folder in Project folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity)) test_state.schedule_for_cleanup(folder_entity.id) # Create and upload a file in Folder repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' annots = {'test': ['hello_world']} # Create, upload, and set annotations on a file in Folder filename = utils.make_bogus_data_file() test_state.schedule_for_cleanup(filename) file_entity = test_state.syn.store(File(filename, parent=folder_entity)) externalURL_entity = test_state.syn.store(File(repo_url, name='rand', parent=folder_entity, synapseStore=False)) test_state.syn.set_annotations(Annotations(file_entity, file_entity.etag, annots)) test_state.syn.set_annotations(Annotations(externalURL_entity, externalURL_entity.etag, annots)) test_state.schedule_for_cleanup(file_entity.id) test_state.schedule_for_cleanup(externalURL_entity.id) # Test cp function output = run(test_state, 'synapse' '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id) output_URL = run(test_state, 'synapse' '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id) copied_id = parse(r'Copied syn\d+ to (syn\d+)', output) copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL) # Verify that our copied files are identical copied_ent = test_state.syn.get(copied_id) copied_URL_ent = test_state.syn.get(copied_URL_id, downloadFile=False) test_state.schedule_for_cleanup(copied_id) test_state.schedule_for_cleanup(copied_URL_id) copied_ent_annot = test_state.syn.get_annotations(copied_id) copied_url_annot = test_state.syn.get_annotations(copied_URL_id) copied_prov = test_state.syn.getProvenance(copied_id)['used'][0]['reference']['targetId'] copied_url_prov = test_state.syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId'] # Make sure copied files are the same assert copied_prov == file_entity.id assert copied_ent_annot == annots assert copied_ent.properties.dataFileHandleId == file_entity.properties.dataFileHandleId # Make sure copied URLs are the same assert copied_url_prov == externalURL_entity.id assert copied_url_annot == annots assert copied_URL_ent.externalURL == repo_url assert copied_URL_ent.name == 'rand' assert copied_URL_ent.properties.dataFileHandleId == externalURL_entity.properties.dataFileHandleId # Verify that errors are being thrown when a # file is copied to a folder/project that has a file with the same filename pytest.raises(ValueError, run, test_state, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
def test_command_line_client(test_state): print("TESTING CMD LINE CLIENT") # Create a Project output = run(test_state, 'synapse', '--skip-checks', 'create', '-name', str(uuid.uuid4()), '-description', 'test of command line client', 'Project') project_id = parse(r'Created entity:\s+(syn\d+)\s+', output) test_state.schedule_for_cleanup(project_id) # Create a File filename = utils.make_bogus_data_file() test_state.schedule_for_cleanup(filename) output = run(test_state, 'synapse', '--skip-checks', 'add', '-name', 'BogusFileEntity', '-description', 'Bogus data to test file upload', '-parentid', project_id, filename) file_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output) # Verify that we stored the file in Synapse f1 = test_state.syn.get(file_entity_id) fh = test_state.syn._get_file_handle_as_creator(f1.dataFileHandleId) assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle' # Get File from the command line output = run(test_state, 'synapse', '--skip-checks', 'get', file_entity_id) downloaded_filename = parse(r'Downloaded file:\s+(.*)', output) test_state.schedule_for_cleanup(downloaded_filename) assert os.path.exists(downloaded_filename) assert filecmp.cmp(filename, downloaded_filename) # Update the File filename = utils.make_bogus_data_file() test_state.schedule_for_cleanup(filename) output = run(test_state, 'synapse', '--skip-checks', 'store', '--id', file_entity_id, filename) # Get the File again output = run(test_state, 'synapse', '--skip-checks', 'get', file_entity_id) downloaded_filename = parse(r'Downloaded file:\s+(.*)', output) test_state.schedule_for_cleanup(downloaded_filename) assert os.path.exists(downloaded_filename) assert filecmp.cmp(filename, downloaded_filename) # Store the same file and don't force a new version # Get the existing file to determine it's current version current_file = test_state.syn.get(file_entity_id, downloadFile=False) current_version = current_file.versionNumber # Store it without forcing version output = run(test_state, 'synapse', '--skip-checks', 'store', '--noForceVersion', '--id', file_entity_id, filename) # Get the File again and check that the version did not change new_file = test_state.syn.get(file_entity_id, downloadFile=False) new_version = new_file.versionNumber assert current_version == new_version # Move the file to new folder folder = test_state.syn.store(Folder(parentId=project_id)) output = run(test_state, 'synapse', 'mv', '--id', file_entity_id, '--parentid', folder.id) movedFile = test_state.syn.get(file_entity_id, downloadFile=False) assert movedFile.parentId == folder.id # Test Provenance repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient' output = run(test_state, 'synapse', '--skip-checks', 'set-provenance', '-id', file_entity_id, '-name', 'TestActivity', '-description', 'A very excellent provenance', '-used', file_entity_id, '-executed', repo_url) output = run(test_state, 'synapse', '--skip-checks', 'get-provenance', '--id', file_entity_id) activity = json.loads(output) assert activity['name'] == 'TestActivity' assert activity['description'] == 'A very excellent provenance' used = utils._find_used(activity, lambda used: 'reference' in used) assert used['reference']['targetId'] == file_entity_id used = utils._find_used(activity, lambda used: 'url' in used) assert used['url'] == repo_url assert used['wasExecuted'] # Note: Tests shouldn't have external dependencies # but this is a pretty picture of Singapore singapore_url = 'http://upload.wikimedia.org/wikipedia/commons/' \ 'thumb/3/3e/1_singapore_city_skyline_dusk_panorama_2011.jpg' \ '/1280px-1_singapore_city_skyline_dusk_panorama_2011.jpg' # Test external file handle output = run(test_state, 'synapse', '--skip-checks', 'add', '-name', 'Singapore', '-description', 'A nice picture of Singapore', '-parentid', project_id, singapore_url) exteral_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output) # Verify that we created an external file handle f2 = test_state.syn.get(exteral_entity_id) fh = test_state.syn._get_file_handle_as_creator(f2.dataFileHandleId) assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.ExternalFileHandle' output = run(test_state, 'synapse', '--skip-checks', 'get', exteral_entity_id) downloaded_filename = parse(r'Downloaded file:\s+(.*)', output) test_state.schedule_for_cleanup(downloaded_filename) assert os.path.exists(downloaded_filename) # Delete the Project run(test_state, 'synapse' '--skip-checks', 'delete', project_id)