def test_uploadFileEntity(syn, project, schedule_for_cleanup):
# Create a FileEntity
# Dictionaries default to FileEntity as a type
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
entity = File(name='fooUploadFileEntity',
path=fname,
parentId=project['id'],
description='A test file entity')
entity = syn.store(entity)
# Download and verify
entity = syn.get(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
# Check if we upload the wrong type of file handle
fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0]
assert fh[
'concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
# Create a different temporary file
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
# Update existing FileEntity
entity.path = fname
entity = syn.store(entity)
# Download and verify that it is the same file
entity = syn.get(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
def test_store_file_handle_update_metadata(syn, project, schedule_for_cleanup):
original_file_path = utils.make_bogus_data_file()
schedule_for_cleanup(original_file_path)
# upload the project
entity = syn.store(File(original_file_path, parent=project))
old_file_handle = entity._file_handle
# create file handle to replace the old one
replacement_file_path = utils.make_bogus_data_file()
schedule_for_cleanup(replacement_file_path)
new_file_handle = syn.uploadFileHandle(replacement_file_path, parent=project)
entity.dataFileHandleId = new_file_handle['id']
new_entity = syn.store(entity)
# make sure _file_handle info was changed
# (_file_handle values are all changed at once so just verifying id change is sufficient)
assert new_file_handle['id'] == new_entity._file_handle['id']
assert old_file_handle['id'] != new_entity._file_handle['id']
# check that local_state was updated
assert replacement_file_path == new_entity.path
assert os.path.dirname(replacement_file_path) == new_entity.cacheDir
assert [os.path.basename(replacement_file_path)] == new_entity.files
def test_walk(syn, schedule_for_cleanup):
walked = []
firstfile = utils.make_bogus_data_file()
schedule_for_cleanup(firstfile)
project_entity = syn.store(Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
folder_entity = syn.store(
Folder(name=str(uuid.uuid4()), parent=project_entity))
schedule_for_cleanup(folder_entity.id)
second_folder = syn.store(
Folder(name=str(uuid.uuid4()), parent=project_entity))
schedule_for_cleanup(second_folder.id)
file_entity = syn.store(File(firstfile, parent=project_entity))
schedule_for_cleanup(file_entity.id)
walked.append(((project_entity.name, project_entity.id), [
(folder_entity.name, folder_entity.id),
(second_folder.name, second_folder.id)
], [(file_entity.name, file_entity.id)]))
nested_folder = syn.store(
Folder(name=str(uuid.uuid4()), parent=folder_entity))
schedule_for_cleanup(nested_folder.id)
secondfile = utils.make_bogus_data_file()
schedule_for_cleanup(secondfile)
second_file = syn.store(File(secondfile, parent=nested_folder))
schedule_for_cleanup(second_file.id)
thirdfile = utils.make_bogus_data_file()
schedule_for_cleanup(thirdfile)
third_file = syn.store(File(thirdfile, parent=second_folder))
schedule_for_cleanup(third_file.id)
walked.append(((os.path.join(project_entity.name,
folder_entity.name), folder_entity.id),
[(nested_folder.name, nested_folder.id)], []))
walked.append(
((os.path.join(os.path.join(project_entity.name, folder_entity.name),
nested_folder.name), nested_folder.id), [],
[(second_file.name, second_file.id)]))
walked.append(((os.path.join(project_entity.name, second_folder.name),
second_folder.id), [], [(third_file.name, third_file.id)]))
temp = synapseutils.walk(syn, project_entity.id)
temp = list(temp)
# Must sort the tuples returned, because order matters for the assert
# Folders are returned in a different ordering depending on the name
for i in walked:
for x in i:
if type(x) == list:
x.sort()
for i in temp:
for x in i:
if type(x) == list:
x.sort()
assert i in walked
temp = synapseutils.walk(syn, second_file.id)
assert list(temp) == []
def init(self, syn, schedule_for_cleanup):
self.syn = syn
self.schedule_for_cleanup = schedule_for_cleanup
# Create a Project
self.project_entity = syn.store(Project(name=str(uuid.uuid4())))
filename = utils.make_bogus_data_file()
attachname = utils.make_bogus_data_file()
file_entity = self.syn.store(File(filename, parent=self.project_entity))
self.schedule_for_cleanup(self.project_entity.id)
self.schedule_for_cleanup(filename)
self.schedule_for_cleanup(file_entity.id)
# Create mock wiki
md = """
This is a test wiki
=======================
Blabber jabber blah blah boo.
syn123
syn456
"""
wiki = Wiki(owner=self.project_entity, title='A Test Wiki', markdown=md,
attachments=[attachname])
wiki = self.syn.store(wiki)
# Create a Wiki sub-page
subwiki = Wiki(owner=self.project_entity, title='A sub-wiki',
markdown='%s' % file_entity.id, parentWikiId=wiki.id)
self.subwiki = self.syn.store(subwiki)
second_md = """
Testing internal links
======================
[test](#!Synapse:%s/wiki/%s)
%s)
""" % (self.project_entity.id, self.subwiki.id, file_entity.id)
sub_subwiki = Wiki(owner=self.project_entity, title='A sub-sub-wiki', markdown=second_md,
parentWikiId=self.subwiki.id, attachments=[attachname])
self.sub_subwiki = self.syn.store(sub_subwiki)
# Set up the second project
self.second_project = self.syn.store(Project(name=str(uuid.uuid4())))
self.schedule_for_cleanup(self.second_project.id)
self.fileMapping = {'syn123': 'syn12345', 'syn456': 'syn45678'}
self.first_headers = self.syn.getWikiHeaders(self.project_entity)
def test_uploadFile_given_dictionary(syn, project, schedule_for_cleanup):
# Make a Folder Entity the old fashioned way
folder = {
'concreteType': Folder._synapse_entity_type,
'parentId': project['id'],
'name': 'fooDictionary',
'foo': 334455
}
entity = syn.store(folder)
# Download and verify that it is the same file
entity = syn.get(entity)
assert entity.parentId == project.id
assert entity.foo[0] == 334455
# Update via a dictionary
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
rareCase = {}
rareCase.update(entity.annotations)
rareCase.update(entity.properties)
rareCase.update(entity.local_state())
rareCase[
'description'] = 'Updating with a plain dictionary should be rare.'
# Verify it works
entity = syn.store(rareCase)
assert entity.description == rareCase['description']
assert entity.name == 'fooDictionary'
syn.get(entity['id'])
def test_synapseStore_flag(syn, project, schedule_for_cleanup):
# Store a path to a local file
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
bogus = File(path, name='Totally bogus data', parent=project, synapseStore=False)
bogus = syn.store(bogus)
# Verify the thing can be downloaded as a URL
bogus = syn.get(bogus, downloadFile=False)
assert bogus.name == 'Totally bogus data'
assert bogus.path == path, "Path: %s\nExpected: %s" % (bogus.path, path)
assert not bogus.synapseStore
# Make sure the test runs on Windows and other OS's
if path[0].isalpha() and path[1] == ':':
# A Windows file URL looks like this: file:///c:/foo/bar/bat.txt
expected_url = 'file:///' + path.replace("\\", "/")
else:
expected_url = 'file://' + path
assert bogus.externalURL == expected_url, 'URL: %s\nExpected %s' % (bogus.externalURL, expected_url)
# A file path that doesn't exist should still work
bogus = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False)
bogus = syn.store(bogus)
pytest.raises(IOError, syn.get, bogus)
assert not bogus.synapseStore
# Try a URL
bogus = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False)
bogus = syn.store(bogus)
bogus = syn.get(bogus)
assert not bogus.synapseStore
def test_syncFromSynapse__given_file_id(test_state):
file_path = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(file_path)
file = test_state.syn.store(File(file_path, name=str(uuid.uuid4()), parent=test_state.project, synapseStore=False))
all_files = synapseutils.syncFromSynapse(test_state.syn, file.id)
assert 1 == len(all_files)
assert file == all_files[0]
def test_store__changing_from_Synapse_to_externalURL_by_changing_path(syn, project, schedule_for_cleanup):
# create a temp file
temp_path = utils.make_bogus_data_file()
schedule_for_cleanup(temp_path)
ext = syn.store(File(temp_path, parent=project, synapseStore=True))
ext = syn.get(ext)
assert "org.sagebionetworks.repo.model.file.S3FileHandle" == ext._file_handle.concreteType
ext.synapseStore = False
ext = syn.store(ext)
# do a get to make sure filehandle has been updated correctly
ext = syn.get(ext.id, downloadFile=True)
assert "org.sagebionetworks.repo.model.file.ExternalFileHandle" == ext._file_handle.concreteType
assert utils.as_url(temp_path) == ext.externalURL
assert not ext.synapseStore
# swap back to synapse storage
ext.synapseStore = True
ext = syn.store(ext)
# do a get to make sure filehandle has been updated correctly
ext = syn.get(ext.id, downloadFile=True)
assert "org.sagebionetworks.repo.model.file.S3FileHandle" == ext._file_handle.concreteType
assert ext.externalURL is None
assert ext.synapseStore
def test_get_local_file(syn, project, schedule_for_cleanup):
new_path = utils.make_bogus_data_file()
schedule_for_cleanup(new_path)
folder = Folder('TestFindFileFolder', parent=project, description='A place to put my junk')
folder = syn.store(folder)
# Get an nonexistent file in Synapse
pytest.raises(SynapseError, syn.get, new_path)
# Get a file really stored in Synapse
ent_folder = syn.store(File(new_path, parent=folder))
ent2 = syn.get(new_path)
assert ent_folder.id == ent2.id
assert ent_folder.versionNumber == ent2.versionNumber
# Get a file stored in Multiple locations #should display warning
syn.store(File(new_path, parent=project))
syn.get(new_path)
# Get a file stored in multiple locations with limit set
ent = syn.get(new_path, limitSearch=folder.id)
assert ent.id == ent_folder.id
assert ent.versionNumber == ent_folder.versionNumber
# Get a file that exists but such that limitSearch removes them and raises error
pytest.raises(SynapseError, syn.get, new_path, limitSearch='syn1')
def test_configPath(test_state):
"""Test using a user-specified configPath for Synapse configuration file."""
tmp_config_file = tempfile.NamedTemporaryFile(suffix='.synapseConfig', delete=False)
shutil.copyfile(client.CONFIG_FILE, tmp_config_file.name)
# Create a File
filename = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(filename)
output = run(test_state,
'synapse'
'--skip-checks',
'--configPath',
tmp_config_file.name,
'add',
'-name',
'BogusFileEntityTwo',
'-description',
'Bogus data to test file upload',
'-parentid',
test_state.project.id,
filename)
file_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output)
# Verify that we stored the file in Synapse
f1 = test_state.syn.get(file_entity_id)
fh = test_state.syn._get_file_handle_as_creator(f1.dataFileHandleId)
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
def test_resume_partial_download(syn, project, schedule_for_cleanup):
original_file = utils.make_bogus_data_file(40000)
entity = File(original_file, parent=project['id'])
entity = syn.store(entity)
# stash the original file for comparison later
shutil.move(original_file, original_file+'.original')
original_file += '.original'
schedule_for_cleanup(original_file)
temp_dir = tempfile.gettempdir()
url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id)
path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId,
expected_md5=entity.md5)
# simulate an imcomplete download by putting the
# complete file back into its temporary location
tmp_path = utils.temp_download_filename(temp_dir, entity.dataFileHandleId)
shutil.move(path, tmp_path)
# ...and truncating it to some fraction of its original size
with open(tmp_path, 'r+') as f:
f.truncate(3*os.path.getsize(original_file)//7)
# this should complete the partial download
path = syn._download_from_URL(url, destination=temp_dir, fileHandleId=entity.dataFileHandleId,
expected_md5=entity.md5)
assert filecmp.cmp(original_file, path), "File comparison failed"
def test_download_check_md5(syn, project, schedule_for_cleanup):
tempfile_path = utils.make_bogus_data_file()
schedule_for_cleanup(tempfile_path)
entity_bad_md5 = syn.store(File(path=tempfile_path, parent=project['id'], synapseStore=False))
pytest.raises(SynapseMd5MismatchError, syn._download_from_URL, entity_bad_md5['externalURL'], tempfile.gettempdir(),
entity_bad_md5['dataFileHandleId'], expected_md5="2345a")
def test_download_local_file_URL_path(syn, project, schedule_for_cleanup):
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
filehandle = create_external_file_handle(syn, path, mimetype=None, file_size=None)
localFileEntity = syn.store(File(dataFileHandleId=filehandle['id'], parent=project))
e = syn.get(localFileEntity.id)
assert path == utils.normalize_path(e.path)
def _create_temp_file_with_cleanup(schedule_for_cleanup, specific_file_text=None):
if specific_file_text:
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as file:
file.write(specific_file_text)
filename = file.name
else:
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
return filename
def test_syncFromSynapse_Links(test_state):
"""This function tests recursive download of links as defined in syncFromSynapse
most of the functionality of this function are already tested in the
tests/integration/test_command_line_client::test_command_get_recursive_and_query
which means that the only test if for path=None
"""
# Create a Project
project_entity = test_state.syn.store(Project(name=str(uuid.uuid4())))
test_state.schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create a Folder hierarchy in folder_entity
inner_folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=folder_entity))
second_folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
test_state.schedule_for_cleanup(f)
file_entity = test_state.syn.store(File(f, parent=project_entity))
# Create links to inner folder
test_state.syn.store(Link(file_entity.id, parent=folder_entity))
# Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
test_state.schedule_for_cleanup(f)
file_entity = test_state.syn.store(File(f, parent=second_folder_entity))
# Create link to inner folder
test_state.syn.store(Link(file_entity.id, parent=inner_folder_entity))
# Test recursive get
output = synapseutils.syncFromSynapse(test_state.syn, folder_entity, followLink=True)
assert len(output) == len(uploaded_paths)
for f in output:
assert f.path in uploaded_paths
def __init__(self):
self.syn = syn
self.project = syn.store(Project(name=str(uuid.uuid4())))
self.folder = syn.store(Folder(name=str(uuid.uuid4()), parent=self.project))
self.schedule_for_cleanup = schedule_for_cleanup
# Create testfiles for upload
self.f1 = utils.make_bogus_data_file(n=10)
self.f2 = utils.make_bogus_data_file(n=10)
self.f3 = 'https://www.synapse.org'
self.header = 'path parent used executed activityName synapseStore foo\n'
self.row1 = '%s %s %s "%s;https://www.example.com" provName bar\n' % (
self.f1, self.project.id, self.f2, self.f3
)
self.row2 = '%s %s "syn12" "syn123;https://www.example.com" provName2 bar\n' % (
self.f2, self.folder.id
)
self.row3 = '%s %s "syn12" prov2 False baz\n' % (self.f3, self.folder.id)
self.row4 = '%s %s %s act 2\n' % (self.f3, self.project.id, self.f1) # Circular reference
self.row5 = '%s syn12 \n' % (self.f3) # Wrong parent
def test_syncFromSynapse(test_state):
"""This function tests recursive download as defined in syncFromSynapse
most of the functionality of this function are already tested in the
tests/integration/test_command_line_client::test_command_get_recursive_and_query
which means that the only test if for path=None
"""
# Create a Project
project_entity = test_state.syn.store(Project(name=str(uuid.uuid4())))
test_state.schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
test_state.schedule_for_cleanup(f)
test_state.syn.store(File(f, parent=folder_entity))
# Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
test_state.schedule_for_cleanup(f)
test_state.syn.store(File(f, parent=project_entity))
# syncFromSynapse() uses chunkedQuery() which will return results that are eventually consistent
# but not always right after the entity is created.
start_time = time.time()
while len(list(test_state.syn.getChildren(project_entity))) != 2:
assert time.time() - start_time < QUERY_TIMEOUT_SEC
time.sleep(2)
# Test recursive get
output = synapseutils.syncFromSynapse(test_state.syn, project_entity)
assert len(output) == len(uploaded_paths)
for f in output:
assert f.path in uploaded_paths
def test_http_download__range_request_error(syn, project):
# SYNPY-525
file_path = utils.make_bogus_data_file()
file_entity = syn.store(File(file_path, parent=project))
syn.cache.purge(time.time())
# download once and rename to temp file to simulate range exceed
file_entity = syn.get(file_entity)
shutil.move(file_entity.path, utils.temp_download_filename(file_entity.path, file_entity.dataFileHandleId))
file_entity = syn.get(file_entity)
assert file_path != file_entity.path
assert filecmp.cmp(file_path, file_entity.path)
def test_download_multithreaded(syn, project, schedule_for_cleanup):
# Create a FileEntity
# Dictionaries default to FileEntity as a type
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
entity = File(name='testMultiThreadDownload' + str(uuid.uuid4()),
path=fname,
parentId=project['id'])
entity = syn.store(entity)
# Download and verify
syn.multi_threaded = True
entity = syn.get(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
syn.multi_threaded = False
def test_provenance(syn, project, schedule_for_cleanup):
# Create a File Entity
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
data_entity = syn.store(File(fname, parent=project['id']))
# Create a File Entity of Code
fd, path = tempfile.mkstemp(suffix=".py")
with os.fdopen(fd, 'w') as f:
f.write(
utils.normalize_lines("""
## Chris's fabulous random data generator
############################################################
import random
random.seed(12345)
data = [random.gauss(mu=0.0, sigma=1.0) for i in range(100)]
"""))
schedule_for_cleanup(path)
code_entity = syn.store(File(path, parent=project['id']))
# Create a new Activity asserting that the Code Entity was 'used'
activity = Activity(name='random.gauss',
description='Generate some random numbers')
activity.used(code_entity, wasExecuted=True)
activity.used(
{
'name': 'Superhack',
'url': 'https://github.com/joe_coder/Superhack'
},
wasExecuted=True)
activity = syn.setProvenance(data_entity, activity)
# Retrieve and verify the saved Provenance record
retrieved_activity = syn.getProvenance(data_entity)
assert retrieved_activity == activity
# Test Activity update
new_description = 'Generate random numbers like a gangsta'
retrieved_activity['description'] = new_description
updated_activity = syn.updateActivity(retrieved_activity)
assert updated_activity['name'] == retrieved_activity['name']
assert updated_activity['description'] == new_description
# Test delete
syn.deleteProvenance(data_entity)
pytest.raises(SynapseHTTPError, syn.getProvenance, data_entity['id'])
def test_entity_version(syn, project, schedule_for_cleanup):
# Make an Entity and make sure the version is one
entity = File(parent=project['id'])
entity['path'] = utils.make_bogus_data_file()
schedule_for_cleanup(entity['path'])
entity = syn.store(entity)
syn.set_annotations(Annotations(entity, entity.etag, {'fizzbuzz': 111222}))
entity = syn.get(entity)
assert entity.versionNumber == 1
# Update the Entity and make sure the version is incremented
entity.foo = 998877
entity['name'] = 'foobarbat'
entity['description'] = 'This is a test entity...'
entity = syn.store(entity, forceVersion=True, versionLabel="Prada remix")
assert entity.versionNumber == 2
# Get the older data and verify the random stuff is still there
annotations = syn.get_annotations(entity, version=1)
assert annotations['fizzbuzz'][0] == 111222
returnEntity = syn.get(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Try the newer Entity
returnEntity = syn.get(entity)
assert returnEntity.versionNumber == 2
assert returnEntity['foo'][0] == 998877
assert returnEntity['name'] == 'foobarbat'
assert returnEntity['description'] == 'This is a test entity...'
assert returnEntity['versionLabel'] == 'Prada remix'
# Try the older Entity again
returnEntity = syn.get(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Delete version 2
syn.delete(entity, version=2)
returnEntity = syn.get(entity)
assert returnEntity.versionNumber == 1
def test_md5_query(syn, project, schedule_for_cleanup):
# Add the same Entity several times
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path,
parent=project['id'],
description='Same data over and over again')
# Retrieve the data via MD5
num = 5
stored = []
for i in range(num):
repeated.name = 'Repeated data %d.dat' % i
stored.append(syn.store(repeated).id)
# Although we expect num results, it is possible for the MD5 to be non-unique
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
assert str(sorted(stored)) == str(sorted([res['id'] for res in results]))
assert len(results) == num
def thread_keep_storing_one_File(syn, project, schedule_for_cleanup):
"""Makes one file and stores it over and over again."""
# Make a local file to continuously store
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
myPrecious = File(path, parent=project, description='This bogus file is MINE', mwa="hahahah")
while syn.test_keepRunning:
stored = store_catch_412_HTTPError(syn, myPrecious)
if stored is not None:
myPrecious = stored
else:
try:
myPrecious = syn.get(myPrecious)
except ValueError as ve:
raise ValueError(str(myPrecious)) from ve
sleep_for_a_bit()
def thread_get_and_update_file_from_Project(syn, project, schedule_for_cleanup):
"""Fetches one item from the Project and updates it with a new file."""
while syn.test_keepRunning:
id = get_all_ids_from_Project(syn, project)
if len(id) <= 0:
continue
id = id[random.randrange(len(id))]
entity = syn.get(id)
# Replace the file and re-store
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity.path = path
entity = store_catch_412_HTTPError(syn, entity)
if entity is not None:
assert os.stat(entity.path) == os.stat(path)
sleep_for_a_bit()
def test_syncFromSynapse__children_contain_non_file(test_state):
proj = test_state.syn.store(Project(name="test_syncFromSynapse_children_non_file" + str(uuid.uuid4())))
test_state.schedule_for_cleanup(proj)
temp_file = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(temp_file)
file_entity = test_state.syn.store(
File(
temp_file,
name="temp_file_test_syncFromSynapse_children_non_file" + str(uuid.uuid4()),
parent=proj
)
)
test_state.syn.store(Schema(name="table_test_syncFromSynapse", parent=proj))
temp_folder = tempfile.mkdtemp()
test_state.schedule_for_cleanup(temp_folder)
files_list = synapseutils.syncFromSynapse(test_state.syn, proj, temp_folder)
assert 1 == len(files_list)
assert file_entity == files_list[0]
def test_store__changing_externalURL_by_changing_path(syn, project, schedule_for_cleanup):
url = 'https://www.synapse.org/Portal/clear.cache.gif'
ext = syn.store(File(url, name="test", parent=project, synapseStore=False))
# perform a syn.get so the filename changes
ext = syn.get(ext)
# create a temp file
temp_path = utils.make_bogus_data_file()
schedule_for_cleanup(temp_path)
ext.synapseStore = False
ext.path = temp_path
ext = syn.store(ext)
# do a get to make sure filehandle has been updated correctly
ext = syn.get(ext.id, downloadFile=True)
assert ext.externalURL != url
assert utils.normalize_path(temp_path) == utils.file_url_to_path(ext.externalURL)
assert temp_path == utils.normalize_path(ext.path)
assert not ext.synapseStore
def test_create_and_update_file_view(syn, project, schedule_for_cleanup):
# Create a folder
folder = Folder(str(uuid.uuid4()),
parent=project,
description='creating a file-view')
folder = syn.store(folder)
# Create dummy file with annotations in our folder
path = utils.make_bogus_data_file()
file_annotations = dict(fileFormat='jpg',
dataType='image',
artist='Banksy',
medium='print',
title='Girl With Ballon')
schedule_for_cleanup(path)
a_file = File(path, parent=folder, annotations=file_annotations)
a_file = syn.store(a_file)
schedule_for_cleanup(a_file)
# Add new columns for the annotations on this file and get their IDs
my_added_cols = [
syn.store(Column(name=k, columnType="STRING"))
for k in file_annotations.keys()
]
my_added_cols_ids = [c['id'] for c in my_added_cols]
view_default_ids = [
c['id'] for c in syn._get_default_view_columns(
"entityview", EntityViewType.FILE.value)
]
col_ids = my_added_cols_ids + view_default_ids
scopeIds = [folder['id'].lstrip('syn')]
# Create an empty entity-view with defined scope as folder
entity_view = EntityViewSchema(name=str(uuid.uuid4()),
scopeIds=scopeIds,
addDefaultViewColumns=True,
addAnnotationColumns=False,
type='file',
columns=my_added_cols,
parent=project)
entity_view = syn.store(entity_view)
schedule_for_cleanup(entity_view)
assert set(scopeIds) == set(entity_view.scopeIds)
assert set(col_ids) == set(entity_view.columnIds)
assert EntityViewType.FILE.value == entity_view.viewTypeMask
# get the current view-schema
view = syn.tableQuery("select * from %s" % entity_view.id)
schedule_for_cleanup(view.filepath)
view_dict = list(
csv.DictReader(io.open(view.filepath, encoding="utf-8", newline='')))
# check that all of the annotations were retrieved from the view
assert set(file_annotations.keys()).issubset(set(view_dict[0].keys()))
updated_a_file = syn.get(a_file.id, downloadFile=False)
# Check that the values are the same as what was set
# Both in the view and on the entity itself
for k, v in file_annotations.items():
assert view_dict[0][k] == v
assert updated_a_file.annotations[k][0] == v
# Make a change to the view and store
view_dict[0]['fileFormat'] = 'PNG'
with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as temp:
schedule_for_cleanup(temp.name)
temp_filename = temp.name
with io.open(temp_filename, mode='w', encoding="utf-8",
newline='') as temp_file:
dw = csv.DictWriter(temp_file,
fieldnames=view_dict[0].keys(),
quoting=csv.QUOTE_NONNUMERIC,
lineterminator=str(os.linesep))
dw.writeheader()
dw.writerows(view_dict)
temp_file.flush()
syn.store(Table(entity_view.id, temp_filename))
new_view_dict = list(
csv.DictReader(io.open(temp_filename, encoding="utf-8", newline='')))
assert new_view_dict[0]['fileFormat'] == 'PNG'
# query for the change
start_time = time.time()
new_view_results = syn.tableQuery("select * from %s" % entity_view.id)
schedule_for_cleanup(new_view_results.filepath)
new_view_dict = list(
csv.DictReader(
io.open(new_view_results.filepath, encoding="utf-8", newline='')))
# query until change is seen.
while new_view_dict[0]['fileFormat'] != 'PNG':
# check timeout
assert time.time() - start_time < QUERY_TIMEOUT_SEC
# query again
new_view_results = syn.tableQuery("select * from %s" % entity_view.id)
new_view_dict = list(
csv.DictReader(
io.open(new_view_results.filepath,
encoding="utf-8",
newline='')))
# paranoid check
assert new_view_dict[0]['fileFormat'] == 'PNG'
def test_command_line_using_paths(test_state):
# Create a Project
project_entity = test_state.syn.store(Project(name=str(uuid.uuid4())))
test_state.schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload a file in Folder
filename = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(filename)
file_entity = test_state.syn.store(File(filename, parent=folder_entity))
# Verify that we can use show with a filename
output = run(test_state,
'synapse'
'--skip-checks', 'show', filename)
id = parse(r'File: %s\s+\((syn\d+)\)\s+' % os.path.split(filename)[1], output)
assert file_entity.id == id
# Verify that limitSearch works by making sure we get the file entity
# that's inside the folder
file_entity2 = test_state.syn.store(File(filename, parent=project_entity))
output = run(test_state,
'synapse'
'--skip-checks', 'get',
'--limitSearch', folder_entity.id,
filename)
id = parse(r'Associated file: .* with synapse ID (syn\d+)', output)
name = parse(r'Associated file: (.*) with synapse ID syn\d+', output)
assert file_entity.id == id
assert utils.equal_paths(name, filename)
# Verify that set-provenance works with filepath
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
output = run(test_state,
'synapse'
'--skip-checks', 'set-provenance',
'-id', file_entity2.id,
'-name', 'TestActivity',
'-description', 'A very excellent provenance',
'-used', filename,
'-executed', repo_url,
'-limitSearch', folder_entity.id)
parse(r'Set provenance record (\d+) on entity syn\d+', output)
output = run(test_state,
'synapse'
'--skip-checks', 'get-provenance',
'-id', file_entity2.id)
activity = json.loads(output)
assert activity['name'] == 'TestActivity'
assert activity['description'] == 'A very excellent provenance'
# Verify that store works with provenance specified with filepath
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
filename2 = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(filename2)
output = run(test_state,
'synapse'
'--skip-checks', 'add', filename2,
'-parentid', project_entity.id,
'-used', filename,
'-executed', '%s %s' % (repo_url, filename))
entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output)
output = run(test_state,
'synapse'
'--skip-checks', 'get-provenance',
'-id', entity_id)
activity = json.loads(output)
a = [a for a in activity['used'] if not a['wasExecuted']]
assert a[0]['reference']['targetId'] in [file_entity.id, file_entity2.id]
# Test associate command
# I have two files in Synapse filename and filename2
path = tempfile.mkdtemp()
test_state.schedule_for_cleanup(path)
shutil.copy(filename, path)
shutil.copy(filename2, path)
run(test_state,
'synapse'
'--skip-checks', 'associate', path, '-r')
run(test_state,
'synapse'
'--skip-checks', 'show', filename)
def test_command_copy(test_state):
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = test_state.syn.store(Project(name=str(uuid.uuid4())))
test_state.schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = test_state.syn.store(Folder(name=str(uuid.uuid4()),
parent=project_entity))
test_state.schedule_for_cleanup(folder_entity.id)
# Create and upload a file in Folder
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': ['hello_world']}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(filename)
file_entity = test_state.syn.store(File(filename, parent=folder_entity))
externalURL_entity = test_state.syn.store(File(repo_url, name='rand', parent=folder_entity, synapseStore=False))
test_state.syn.set_annotations(Annotations(file_entity, file_entity.etag, annots))
test_state.syn.set_annotations(Annotations(externalURL_entity, externalURL_entity.etag, annots))
test_state.schedule_for_cleanup(file_entity.id)
test_state.schedule_for_cleanup(externalURL_entity.id)
# Test cp function
output = run(test_state,
'synapse'
'--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
output_URL = run(test_state,
'synapse'
'--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL)
# Verify that our copied files are identical
copied_ent = test_state.syn.get(copied_id)
copied_URL_ent = test_state.syn.get(copied_URL_id, downloadFile=False)
test_state.schedule_for_cleanup(copied_id)
test_state.schedule_for_cleanup(copied_URL_id)
copied_ent_annot = test_state.syn.get_annotations(copied_id)
copied_url_annot = test_state.syn.get_annotations(copied_URL_id)
copied_prov = test_state.syn.getProvenance(copied_id)['used'][0]['reference']['targetId']
copied_url_prov = test_state.syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId']
# Make sure copied files are the same
assert copied_prov == file_entity.id
assert copied_ent_annot == annots
assert copied_ent.properties.dataFileHandleId == file_entity.properties.dataFileHandleId
# Make sure copied URLs are the same
assert copied_url_prov == externalURL_entity.id
assert copied_url_annot == annots
assert copied_URL_ent.externalURL == repo_url
assert copied_URL_ent.name == 'rand'
assert copied_URL_ent.properties.dataFileHandleId == externalURL_entity.properties.dataFileHandleId
# Verify that errors are being thrown when a
# file is copied to a folder/project that has a file with the same filename
pytest.raises(ValueError, run, test_state, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id,
'--destinationId', project_entity.id)
def test_command_line_client(test_state):
print("TESTING CMD LINE CLIENT")
# Create a Project
output = run(test_state,
'synapse',
'--skip-checks',
'create',
'-name',
str(uuid.uuid4()),
'-description',
'test of command line client',
'Project')
project_id = parse(r'Created entity:\s+(syn\d+)\s+', output)
test_state.schedule_for_cleanup(project_id)
# Create a File
filename = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(filename)
output = run(test_state,
'synapse',
'--skip-checks',
'add',
'-name',
'BogusFileEntity',
'-description',
'Bogus data to test file upload',
'-parentid',
project_id,
filename)
file_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output)
# Verify that we stored the file in Synapse
f1 = test_state.syn.get(file_entity_id)
fh = test_state.syn._get_file_handle_as_creator(f1.dataFileHandleId)
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
# Get File from the command line
output = run(test_state,
'synapse',
'--skip-checks',
'get',
file_entity_id)
downloaded_filename = parse(r'Downloaded file:\s+(.*)', output)
test_state.schedule_for_cleanup(downloaded_filename)
assert os.path.exists(downloaded_filename)
assert filecmp.cmp(filename, downloaded_filename)
# Update the File
filename = utils.make_bogus_data_file()
test_state.schedule_for_cleanup(filename)
output = run(test_state,
'synapse',
'--skip-checks',
'store',
'--id',
file_entity_id,
filename)
# Get the File again
output = run(test_state,
'synapse',
'--skip-checks',
'get',
file_entity_id)
downloaded_filename = parse(r'Downloaded file:\s+(.*)', output)
test_state.schedule_for_cleanup(downloaded_filename)
assert os.path.exists(downloaded_filename)
assert filecmp.cmp(filename, downloaded_filename)
# Store the same file and don't force a new version
# Get the existing file to determine it's current version
current_file = test_state.syn.get(file_entity_id, downloadFile=False)
current_version = current_file.versionNumber
# Store it without forcing version
output = run(test_state,
'synapse',
'--skip-checks',
'store',
'--noForceVersion',
'--id',
file_entity_id,
filename)
# Get the File again and check that the version did not change
new_file = test_state.syn.get(file_entity_id, downloadFile=False)
new_version = new_file.versionNumber
assert current_version == new_version
# Move the file to new folder
folder = test_state.syn.store(Folder(parentId=project_id))
output = run(test_state,
'synapse',
'mv',
'--id',
file_entity_id,
'--parentid',
folder.id)
movedFile = test_state.syn.get(file_entity_id, downloadFile=False)
assert movedFile.parentId == folder.id
# Test Provenance
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
output = run(test_state,
'synapse',
'--skip-checks',
'set-provenance',
'-id',
file_entity_id,
'-name',
'TestActivity',
'-description',
'A very excellent provenance',
'-used',
file_entity_id,
'-executed',
repo_url)
output = run(test_state,
'synapse',
'--skip-checks',
'get-provenance',
'--id',
file_entity_id)
activity = json.loads(output)
assert activity['name'] == 'TestActivity'
assert activity['description'] == 'A very excellent provenance'
used = utils._find_used(activity, lambda used: 'reference' in used)
assert used['reference']['targetId'] == file_entity_id
used = utils._find_used(activity, lambda used: 'url' in used)
assert used['url'] == repo_url
assert used['wasExecuted']
# Note: Tests shouldn't have external dependencies
# but this is a pretty picture of Singapore
singapore_url = 'http://upload.wikimedia.org/wikipedia/commons/' \
'thumb/3/3e/1_singapore_city_skyline_dusk_panorama_2011.jpg' \
'/1280px-1_singapore_city_skyline_dusk_panorama_2011.jpg'
# Test external file handle
output = run(test_state,
'synapse',
'--skip-checks',
'add',
'-name',
'Singapore',
'-description',
'A nice picture of Singapore',
'-parentid',
project_id,
singapore_url)
exteral_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output)
# Verify that we created an external file handle
f2 = test_state.syn.get(exteral_entity_id)
fh = test_state.syn._get_file_handle_as_creator(f2.dataFileHandleId)
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.ExternalFileHandle'
output = run(test_state,
'synapse',
'--skip-checks',
'get',
exteral_entity_id)
downloaded_filename = parse(r'Downloaded file:\s+(.*)', output)
test_state.schedule_for_cleanup(downloaded_filename)
assert os.path.exists(downloaded_filename)
# Delete the Project
run(test_state,
'synapse'
'--skip-checks', 'delete', project_id)
