def test_uploadFileEntity():
# Create a FileEntity
# Dictionaries default to FileEntity as a type
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
entity = {'name' : 'fooUploadFileEntity', \
'description' : 'A test file entity', \
'parentId' : project['id']}
entity = syn.uploadFile(entity, fname)
# Download and verify
entity = syn.downloadEntity(entity)
print entity['files']
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
# Check if we upload the wrong type of file handle
fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0]
assert fh[
'concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
# Create a different temporary file
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
# Update existing FileEntity
entity = syn.uploadFile(entity, fname)
# Download and verify that it is the same file
entity = syn.downloadEntity(entity)
print entity['files']
assert_equals(entity['files'][0], os.path.basename(fname))
assert filecmp.cmp(fname, entity['path'])
def test_get_and_store_by_name_and_parent_id():
project = create_project()
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
f = File(path, name='Foobarbat', parent=project)
f2 = syn.store(f)
f = syn.get(f)
assert f.id == f2.id
assert f.name == f2.name
assert f.parentId == f2.parentId
## new file
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
## should create a new version of the previous File entity
f3 = File(path,
name='Foobarbat',
parent=project,
description='banana',
junk=1234)
f3 = syn.store(f3)
## should be an update of the existing entity with the same name and parent
assert f3.id == f.id
assert f3.description == 'banana'
assert f3.junk == [1234]
assert filecmp.cmp(path, f3.path)
def test_syncFromSynapse():
"""This function tests recursive download as defined in syncFromSynapse
most of the functionality of this function are already tested in the
tests/integration/test_command_line_client::test_command_get_recursive_and_query
which means that the only test if for path=None
"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = syn.store(File(f, parent=folder_entity))
#Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = syn.store(File(f, parent=project_entity))
### Test recursive get
output = synapseutils.syncFromSynapse(syn, project_entity)
assert len(output) == len(uploaded_paths)
for f in output:
print(f.path)
assert f.path in uploaded_paths
def test_uploadFileEntity():
# Create a FileEntity
# Dictionaries default to FileEntity as a type
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
entity = {'name' : 'fooUploadFileEntity', \
'description' : 'A test file entity', \
'parentId' : project['id']}
entity = syn.uploadFile(entity, fname)
# Download and verify
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
# Check if we upload the wrong type of file handle
fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0]
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
# Create a different temporary file
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
# Update existing FileEntity
entity = syn.uploadFile(entity, fname)
# Download and verify that it is the same file
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
def test_get_and_store_by_name_and_parent_id():
project = create_project()
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
f = File(path, name='Foobarbat', parent=project)
f2 = syn.store(f)
f = syn.get(f)
assert f.id == f2.id
assert f.name == f2.name
assert f.parentId == f2.parentId
## new file
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
## should create a new version of the previous File entity
f3 = File(path, name='Foobarbat', parent=project, description='banana', junk=1234)
f3 = syn.store(f3)
## should be an update of the existing entity with the same name and parent
assert f3.id == f.id
assert f3.description == 'banana'
assert f3.junk == [1234]
assert filecmp.cmp(path, f3.path)
def test_Entity():
# Test CRUD on Entity objects, Project, Folder, File with createEntity/getEntity/updateEntity
project_name = str(uuid.uuid4())
project = Project(project_name, description='Bogus testing project')
project = syn.createEntity(project)
schedule_for_cleanup(project)
folder = Folder('Test Folder', parent=project, description='A place to put my junk', foo=1000)
folder = syn.createEntity(folder)
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
a_file = File(path, parent=folder, description='Random data for testing', foo='An arbitrary value', bar=[33,44,55], bday=Datetime(2013,3,15))
a_file = syn._createFileEntity(a_file)
## local object state should be preserved
assert a_file.path == path
## check the project entity
project = syn.getEntity(project)
assert project.name == project_name
## check the folder entity
folder = syn.getEntity(folder.id)
assert folder.name == 'Test Folder'
assert folder.parentId == project.id
assert folder.foo[0] == 1000
## check the file entity
a_file = syn.getEntity(a_file)
assert a_file['foo'][0] == 'An arbitrary value'
assert a_file['bar'] == [33,44,55]
assert a_file['bday'][0] == Datetime(2013,3,15)
## make sure file comes back intact
a_file = syn.downloadEntity(a_file)
assert filecmp.cmp(path, a_file.path)
#TODO We're forgotten the local file path
a_file.path = path
## update the file entity
a_file['foo'] = 'Another arbitrary chunk of text data'
a_file['new_key'] = 'A newly created value'
a_file = syn.updateEntity(a_file)
assert a_file['foo'][0] == 'Another arbitrary chunk of text data'
assert a_file['bar'] == [33,44,55]
assert a_file['bday'][0] == Datetime(2013,3,15)
assert a_file.new_key[0] == 'A newly created value'
assert a_file.path == path
## upload a new file
new_path = utils.make_bogus_data_file()
schedule_for_cleanup(new_path)
a_file = syn.uploadFile(a_file, new_path)
## make sure file comes back intact
a_file = syn.downloadEntity(a_file)
assert filecmp.cmp(new_path, a_file.path)
def test_uploadFileEntity():
projectEntity = create_project()
# Defaults to FileEntity
fname = utils.make_bogus_data_file()
entity = {'name' : 'foo', \
'description' : 'A test file entity', \
'parentId' : projectEntity['id']}
# Create new FileEntity
entity = syn.uploadFile(entity, fname)
# Download and verify
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
# Check if we upload the wrong type of file handle
fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0]
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
os.remove(fname)
# Create a different temporary file
fname = utils.make_bogus_data_file()
# Update existing FileEntity
entity = syn.uploadFile(entity, fname)
# Download and verify that it is the same filename
entity = syn.downloadEntity(entity)
print entity['files'][0]
print os.path.basename(fname)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
os.remove(fname)
def test_wikiAttachment():
# Upload a file to be attached to a Wiki
filename = utils.make_bogus_data_file()
attachname = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
schedule_for_cleanup(attachname)
fileHandle = syn._uploadFileToFileHandleService(filename)
# Create and store a Wiki
# The constructor should accept both file handles and file paths
md = """
This is a test wiki
=======================
Blabber jabber blah blah boo.
"""
wiki = Wiki(owner=project, title='A Test Wiki', markdown=md,
fileHandles=[fileHandle['id']],
attachments=[attachname])
wiki = syn.store(wiki)
# Create a Wiki sub-page
subwiki = Wiki(owner=project, title='A sub-wiki',
markdown='nothing', parentWikiId=wiki.id)
subwiki = syn.store(subwiki)
# Retrieve the root Wiki from Synapse
wiki2 = syn.getWiki(project)
assert wiki == wiki2
# Retrieve the sub Wiki from Synapse
wiki2 = syn.getWiki(project, subpageId=subwiki.id)
assert subwiki == wiki2
# Try making an update
wiki['title'] = 'A New Title'
wiki['markdown'] = wiki['markdown'] + "\nNew stuff here!!!\n"
wiki = syn.store(wiki)
assert wiki['title'] == 'A New Title'
assert wiki['markdown'].endswith("\nNew stuff here!!!\n")
# Check the Wiki's metadata
headers = syn.getWikiHeaders(project)
assert headers['totalNumberOfResults'] == 2
assert headers['results'][0]['title'] in (wiki['title'], subwiki['title'])
# # Retrieve the file attachment
# tmpdir = tempfile.mkdtemp()
# file_props = syn._downloadWikiAttachment(project, wiki,
# os.path.basename(filename), dest_dir=tmpdir)
# path = file_props['path']
# assert os.path.exists(path)
# assert filecmp.cmp(original_path, path)
# Clean up
# syn._deleteFileHandle(fileHandle)
syn.delete(wiki)
syn.delete(subwiki)
assert_raises(SynapseHTTPError, syn.getWiki, project)
def test_command_get_recursive_and_query():
"""Tests the 'synapse get -r' and 'synapse get -q' functions"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=folder_entity)
file_entity.location = 'folder'
file_entity = syn.store(file_entity)
#Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=project_entity)
file_entity.location = 'project'
file_entity = syn.store(file_entity)
### Test recursive get
output = run('synapse', '--skip-checks', 'get', '-r', project_entity.id)
#Verify that we downloaded files:
new_paths = [
os.path.join('.', folder_entity.name, os.path.basename(f))
for f in uploaded_paths[:-1]
]
new_paths.append(os.path.join('.', os.path.basename(uploaded_paths[-1])))
schedule_for_cleanup(folder_entity.name)
for downloaded, uploaded in zip(new_paths, uploaded_paths):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(new_paths[0])
### Test query get
output = run(
'synapse', '--skip-checks', 'get', '-q',
"select id from file where parentId=='%s' and location=='folder'" %
folder_entity.id)
#Verify that we downloaded files:
new_paths = [
os.path.join('.', os.path.basename(f)) for f in uploaded_paths[:-1]
]
for downloaded, uploaded in zip(new_paths, uploaded_paths[:-1]):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(downloaded)
def test_command_line_store_and_submit():
"""Test command line client store command."""
## Create a project
output = run('synapse store --name "%s" --description "test of store command" --type Project' % str(str(uuid.uuid4())))
project_id = parse(r'Created entity:\s+(syn\d+)\s+', output)
schedule_for_cleanup(project_id)
## Create and upload a file
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
output = run('synapse store --name "BogusFileEntity" --description "Bogus data to test file upload" --parentid %s --file %s' % (project_id, filename))
file_entity_id = parse(r'Created entity:\s+(syn\d+)\s+', output)
## Verify that we stored the file in Synapse
f1 = syn.get(file_entity_id)
fh = syn._getFileHandle(f1.dataFileHandleId)
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
## Create an Evaluation to submit to
eval = synapseclient.evaluation.Evaluation(name=str(str(uuid.uuid4())), contentSource=project_id)
eval = syn.store(eval)
syn.addEvaluationParticipant(eval)
## Submit a bogus file
output = run('synapse submit --evaluation %s --name Some random name --entity %s' %(eval.id, file_entity_id))
submission_id = parse(r'Submitted \(id: (\d+)\) entity:\s+', output)
## Update the file
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
output = run('synapse store --id %s --file %s' % (file_entity_id, filename,))
updated_entity_id = parse(r'Updated entity:\s+(syn\d+)', output)
## Tests shouldn't have external dependencies, but here it's required
ducky_url = 'http://upload.wikimedia.org/wikipedia/commons/9/93/Rubber_Duck.jpg'
## Test external file handle
## This time, omit the quotation marks
output = run('synapse store --name Rubber Ducky --description I like rubber duckies --parentid %s --file %s' % (project_id, ducky_url))
exteral_entity_id = parse(r'Created entity:\s+(syn\d+)\s+', output)
## Verify that we created an external file handle
f2 = syn.get(exteral_entity_id)
fh = syn._getFileHandle(f2.dataFileHandleId)
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.ExternalFileHandle'
## Delete project
output = run('synapse delete %s' % project_id)
def test_walk():
walked = []
firstfile = utils.make_bogus_data_file()
schedule_for_cleanup(firstfile)
project_entity = syn.store(Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
folder_entity = syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
schedule_for_cleanup(folder_entity.id)
second_folder = syn.store(Folder(name=str(uuid.uuid4()), parent=project_entity))
schedule_for_cleanup(second_folder.id)
file_entity = syn.store(File(firstfile, parent=project_entity))
schedule_for_cleanup(file_entity.id)
walked.append(((project_entity.name,project_entity.id),
[(folder_entity.name, folder_entity.id),
(second_folder.name, second_folder.id)],
[(file_entity.name, file_entity.id)]))
nested_folder = syn.store(Folder(name=str(uuid.uuid4()), parent=folder_entity))
schedule_for_cleanup(nested_folder.id)
secondfile = utils.make_bogus_data_file()
schedule_for_cleanup(secondfile)
second_file = syn.store(File(secondfile, parent=nested_folder))
schedule_for_cleanup(second_file.id)
thirdfile = utils.make_bogus_data_file()
schedule_for_cleanup(thirdfile)
third_file = syn.store(File(thirdfile, parent=second_folder))
schedule_for_cleanup(third_file.id)
walked.append(((os.path.join(project_entity.name,folder_entity.name),folder_entity.id),
[(nested_folder.name,nested_folder.id)],
[]))
walked.append(((os.path.join(os.path.join(project_entity.name,folder_entity.name),nested_folder.name),nested_folder.id),
[],
[(second_file.name,second_file.id)]))
walked.append(((os.path.join(project_entity.name,second_folder.name),second_folder.id),
[],
[(third_file.name,third_file.id)]))
temp = synu.walk(syn, project_entity.id)
temp = list(temp)
#Must sort the tuples returned, because order matters for the assert
#Folders are returned in a different ordering depending on the name
for i in walked:
for x in i:
if type(x) == list:
x = x.sort()
for i in temp:
assert i in walked
print("CHECK: Cannot synu.walk a file returns empty generator")
temp = synu.walk(syn, second_file.id)
assert list(temp) == []
def test_command_get_recursive_and_query():
"""Tests the 'synapse get -r' and 'synapse get -q' functions"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()),
parent=project_entity))
# Create and upload two files in Folder
uploaded_paths = []
for i in range(2):
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=folder_entity)
file_entity.location = 'folder'
file_entity = syn.store(file_entity)
#Add a file in the project level as well
f = utils.make_bogus_data_file()
uploaded_paths.append(f)
schedule_for_cleanup(f)
file_entity = synapseclient.File(f, parent=project_entity)
file_entity.location = 'project'
file_entity = syn.store(file_entity)
### Test recursive get
output = run('synapse', '--skip-checks',
'get', '-r',
project_entity.id)
#Verify that we downloaded files:
new_paths = [os.path.join('.', folder_entity.name, os.path.basename(f)) for f in uploaded_paths[:-1]]
new_paths.append(os.path.join('.', os.path.basename(uploaded_paths[-1])))
schedule_for_cleanup(folder_entity.name)
for downloaded, uploaded in zip(new_paths, uploaded_paths):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(new_paths[0])
### Test query get
output = run('synapse', '--skip-checks',
'get', '-q', "select id from file where parentId=='%s' and location=='folder'" %
folder_entity.id)
#Verify that we downloaded files:
new_paths = [os.path.join('.', os.path.basename(f)) for f in uploaded_paths[:-1]]
for downloaded, uploaded in zip(new_paths, uploaded_paths[:-1]):
print uploaded, downloaded
assert os.path.exists(downloaded)
assert filecmp.cmp(downloaded, uploaded)
schedule_for_cleanup(downloaded)
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(
synapseclient.Folder(name=str(uuid.uuid4()), parent=project_entity))
# Create and upload a file in Folder
dummy = utils.make_bogus_data_file()
schedule_for_cleanup(dummy)
dummy_entity = syn.store(synapseclient.File(dummy, parent=folder_entity))
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': 'hello_world'}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity),
used=dummy_entity.id,
executed=repo_url)
syn.setAnnotations(file_entity, annots)
### Test cp function
output = run('synapse', '--skip-checks', 'cp', '--id', file_entity.id,
'--parentid', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
#Verify that our copied files are identical
copied_ent = syn.get(copied_id)
schedule_for_cleanup(copied_id)
copied_ent_annot = syn.getAnnotations(copied_ent)
copied_annot = dict((key, copied_ent_annot[key].pop())
for key in copied_ent_annot
if key not in ('uri', 'id', 'creationDate', 'etag'))
copied_prov = syn.getProvenance(
copied_ent)['used'][0]['reference']['targetId']
assert copied_prov == file_entity.id
assert copied_annot == annots
#Verify that errors are being thrown when folders/projects are attempted to be copied,
#or file is copied to a foler/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', folder_entity.id, '--parentid', project_entity.id)
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', project_entity.id, '--parentid', project_entity.id)
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp',
'--id', file_entity.id, '--parentid', project_entity.id)
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()),
parent=project_entity))
# Create and upload a file in Folder
dummy = utils.make_bogus_data_file()
schedule_for_cleanup(dummy)
dummy_entity = syn.store(synapseclient.File(dummy, parent=folder_entity))
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test':'hello_world'}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity),used=dummy_entity.id,executed=repo_url)
syn.setAnnotations(file_entity,annots)
### Test cp function
output = run('synapse', '--skip-checks',
'cp', '--id',file_entity.id,
'--parentid',project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)',output)
#Verify that our copied files are identical
copied_ent = syn.get(copied_id)
schedule_for_cleanup(copied_id)
copied_ent_annot = syn.getAnnotations(copied_ent)
copied_annot = dict((key,copied_ent_annot[key].pop()) for key in copied_ent_annot if key not in ('uri','id','creationDate','etag'))
copied_prov = syn.getProvenance(copied_ent)['used'][0]['reference']['targetId']
assert copied_prov == file_entity.id
assert copied_annot == annots
#Verify that errors are being thrown when folders/projects are attempted to be copied,
#or file is copied to a foler/project that has a file with the same filename
assert_raises(ValueError,run, 'synapse', '--debug', '--skip-checks',
'cp', '--id',folder_entity.id,
'--parentid',project_entity.id)
assert_raises(ValueError,run, 'synapse', '--debug', '--skip-checks',
'cp', '--id',project_entity.id,
'--parentid',project_entity.id)
assert_raises(ValueError,run, 'synapse', '--debug', '--skip-checks',
'cp', '--id',file_entity.id,
'--parentid',project_entity.id)
def test_get_local_file():
"""Tests synapse.get() with local a local file """
new_path = utils.make_bogus_data_file()
schedule_for_cleanup(new_path)
folder = Folder('TestFindFileFolder', parent=project, description='A place to put my junk')
folder = syn.createEntity(folder)
#Get an nonexistent file in Synapse
assert_raises(SynapseError, syn.get, new_path)
#Get a file really stored in Synapse
ent_folder = syn.store(File(new_path, parent=folder))
ent2 = syn.get(new_path)
assert ent_folder.id==ent2.id and ent_folder.versionNumber==ent2.versionNumber
#Get a file stored in Multiple locations #should display warning
ent = syn.store(File(new_path, parent=project))
ent = syn.get(new_path)
#Get a file stored in multiple locations with limit set
ent = syn.get(new_path, limitSearch=folder.id)
assert ent.id == ent_folder.id and ent.versionNumber==ent_folder.versionNumber
#Get a file that exists but such that limitSearch removes them and raises error
assert_raises(SynapseError, syn.get, new_path, limitSearch='syn1')
def test_configPath():
"""Test using a user-specified configPath for Synapse configuration file."""
tmp_config_file = tempfile.NamedTemporaryFile(suffix='.synapseConfig', delete=False)
shutil.copyfile(synapseclient.client.CONFIG_FILE, tmp_config_file.name)
# Create a File
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
output = run('synapse',
'--skip-checks',
'--configPath',
tmp_config_file.name,
'add',
'-name',
'BogusFileEntityTwo',
'-description',
'Bogus data to test file upload',
'-parentid',
project.id,
filename)
file_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output)
# Verify that we stored the file in Synapse
f1 = syn.get(file_entity_id)
fh = syn._getFileHandle(f1.dataFileHandleId)
assert_equals(fh['concreteType'], 'org.sagebionetworks.repo.model.file.S3FileHandle')
def test_slow_unlocker():
"""Manually grabs a lock and makes sure the get/store methods are blocked."""
# Make a file to manually lock
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
contention = File(path, parent=syn.test_parent)
contention = syn.store(contention)
# Lock the Cache Map
cacheDir = cache.determine_cache_directory(contention)
cache.obtain_lock_and_read_cache(cacheDir)
# Start a few calls to get/store that should not complete yet
store_thread = wrap_function_as_child_thread(lambda: store_catch_412_HTTPError(contention))
get_thread = wrap_function_as_child_thread(lambda: syn.get(contention))
thread.start_new_thread(store_thread, ())
thread.start_new_thread(get_thread, ())
time.sleep(min(5, cache.CACHE_LOCK_TIME / 2))
# Make sure the threads did not finish
assert syn.test_threadsRunning > 0
cache.write_cache_then_release_lock(cacheDir)
# Let the threads go
while syn.test_threadsRunning > 0:
time.sleep(1)
collect_errors_and_fail()
def test_configPath():
"""Test using a user-specified configPath for Synapse configuration file.
"""
tmp_config_file = tempfile.NamedTemporaryFile(suffix='.synapseConfig', delete=False)
shutil.copyfile(synapseclient.client.CONFIG_FILE, tmp_config_file.name)
# Create a File
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
output = run('synapse',
'--skip-checks',
'--configPath',
tmp_config_file.name,
'add',
'-name',
'BogusFileEntityTwo',
'-description',
'Bogus data to test file upload',
'-parentid',
project.id,
filename)
file_entity_id = parse(r'Created/Updated entity:\s+(syn\d+)\s+', output)
# Verify that we stored the file in Synapse
f1 = syn.get(file_entity_id)
fh = syn._getFileHandle(f1.dataFileHandleId)
assert fh['concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
def test_uploadFile_given_dictionary():
# Make a Folder Entity the old fashioned way
folder = {'concreteType': Folder._synapse_entity_type,
'parentId' : project['id'],
'name' : 'fooDictionary',
'foo' : 334455}
entity = syn.store(folder)
# Download and verify that it is the same file
entity = syn.get(entity)
assert entity.parentId == project.id
assert entity.foo[0] == 334455
# Update via a dictionary
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
rareCase = {}
rareCase.update(entity.annotations)
rareCase.update(entity.properties)
rareCase.update(entity.local_state())
rareCase['description'] = 'Updating with a plain dictionary should be rare.'
# Verify it works
entity = syn.store(rareCase)
assert entity.description == rareCase['description']
assert entity.name == 'fooDictionary'
entity = syn.get(entity['id'])
def test_uploadFile_given_dictionary():
# Make a Folder Entity the old fashioned way
folder = {
'concreteType': Folder._synapse_entity_type,
'parentId': project['id'],
'name': 'fooDictionary',
'foo': 334455
}
entity = syn.store(folder)
# Download and verify that it is the same file
entity = syn.get(entity)
assert entity.parentId == project.id
assert entity.foo[0] == 334455
# Update via a dictionary
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
rareCase = {}
rareCase.update(entity.annotations)
rareCase.update(entity.properties)
rareCase.update(entity.local_state())
rareCase[
'description'] = 'Updating with a plain dictionary should be rare.'
# Verify it works
entity = syn.store(rareCase)
assert entity.description == rareCase['description']
assert entity.name == 'fooDictionary'
entity = syn.get(entity['id'])
def test_store_activity():
"""Test storing entities with Activities"""
project = create_project()
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
f = File(path, name='Hinkle horn honking holes', parent=project)
honking = Activity(name='Hinkle horn honking', description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.')
honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/')
honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/')
## doesn't set the ID of the activity
f = syn.store(f, activity=honking)
honking = syn.getProvenance(f.id)
## now, we have an activity ID
assert honking['name'] == 'Hinkle horn honking'
assert len(honking['used']) == 2
assert honking['used'][0]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][0]['wasExecuted'] == False
assert honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482')
assert honking['used'][1]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][1]['wasExecuted'] == False
## store another entity with the same activity
f2 = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg', name='Nettlebed Cave', parent=project)
f2 = syn.store(f2, activity=honking)
honking2 = syn.getProvenance(f2)
assert honking['id'] == honking2['id']
def test_resume_partial_download():
original_file = utils.make_bogus_data_file(40000)
original_md5 = utils.md5_for_file(original_file).hexdigest()
entity = File(original_file, parent=project['id'])
entity = syn.store(entity)
## stash the original file for comparison later
shutil.move(original_file, original_file+'.original')
original_file += '.original'
schedule_for_cleanup(original_file)
temp_dir = tempfile.gettempdir()
url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id)
path = syn._download(url, destination=temp_dir, file_handle_id=entity.dataFileHandleId, expected_md5=entity.md5)
## simulate an imcomplete download by putting the
## complete file back into its temporary location
tmp_path = utils.temp_download_filename(temp_dir, entity.dataFileHandleId)
shutil.move(path, tmp_path)
## ...and truncating it to some fraction of its original size
with open(tmp_path, 'r+') as f:
f.truncate(3*os.path.getsize(original_file)//7)
## this should complete the partial download
path = syn._download(url, destination=temp_dir, file_handle_id=entity.dataFileHandleId, expected_md5=entity.md5)
assert filecmp.cmp(original_file, path), "File comparison failed"
def test_synapseStore_flag():
# Store a path to a local file
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
bogus = File(path, name='Totally bogus data', parent=project, synapseStore=False)
bogus = syn.store(bogus)
# Verify the thing can be downloaded as a URL
bogus = syn.get(bogus, downloadFile=False)
assert bogus.name == 'Totally bogus data'
assert bogus.path == path, "Path: %s\nExpected: %s" % (bogus.path, path)
assert bogus.synapseStore == False
# Make sure the test runs on Windows and other OS's
if path[0].isalpha() and path[1]==':':
# A Windows file URL looks like this: file:///c:/foo/bar/bat.txt
expected_url = 'file:///' + path.replace("\\","/")
else:
expected_url = 'file://' + path
assert bogus.externalURL == expected_url, 'URL: %s\nExpected %s' % (bogus.externalURL, expected_url)
# A file path that doesn't exist should still work
bogus = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False)
bogus = syn.store(bogus)
assert_raises(IOError, syn.get, bogus)
assert bogus.synapseStore == False
# Try a URL
bogus = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False)
bogus = syn.store(bogus)
bogus = syn.get(bogus)
assert bogus.synapseStore == False
def test_download_table_files():
cols = [
Column(name='artist', columnType='STRING', maximumSize=50),
Column(name='album', columnType='STRING', maximumSize=50),
Column(name='year', columnType='INTEGER'),
Column(name='catalog', columnType='STRING', maximumSize=50),
Column(name='cover', columnType='FILEHANDLEID')]
schema = syn.store(Schema(name='Jazz Albums', columns=cols, parent=project))
schedule_for_cleanup(schema)
data = [["John Coltrane", "Blue Train", 1957, "BLP 1577", "coltraneBlueTrain.jpg"],
["Sonny Rollins", "Vol. 2", 1957, "BLP 1558", "rollinsBN1558.jpg"],
["Sonny Rollins", "Newk's Time", 1958, "BLP 4001", "rollinsBN4001.jpg"],
["Kenny Burrel", "Kenny Burrel", 1956, "BLP 1543", "burrellWarholBN1543.jpg"]]
## upload files and store file handle ids
original_files = []
for row in data:
path = utils.make_bogus_data_file()
original_files.append(path)
schedule_for_cleanup(path)
file_handle = syn._chunkedUploadFile(path)
row[4] = file_handle['id']
row_reference_set = syn.store(RowSet(columns=cols, schema=schema, rows=[Row(r) for r in data]))
## retrieve the files for each row and verify that they are identical to the originals
results = syn.tableQuery('select artist, album, year, catalog, cover from %s'%schema.id, resultsAs="rowset")
for i, row in enumerate(results):
print "%s_%s" % (row.rowId, row.versionNumber), row.values
file_info = syn.downloadTableFile(results, rowId=row.rowId, versionNumber=row.versionNumber, column='cover', downloadLocation='.')
assert filecmp.cmp(original_files[i], file_info['path'])
schedule_for_cleanup(file_info['path'])
def test_caching_of_locationables_containing_zip_files():
"""Test for SYNR-728, cache.retrieve_local_file_info sets cacheDir and files incorrectly for zip files"""
data = Data(name='qwertyqwer', parent=project['id'])
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
zip_path = os.path.join(os.path.dirname(path), 'Archive.zip')
schedule_for_cleanup(zip_path)
import zipfile
with zipfile.ZipFile(zip_path, 'w') as zf:
zf.write(path, os.path.basename(path))
data['path'] = zip_path
data = syn.store(data)
assert data.path == zip_path
## should cacheDir and files be filled in here?
## remove the files
os.remove(path)
os.remove(zip_path)
## get the file and store it in the cache. This also has the side
## effect of unzipping archive files.
data2 = syn.get(data.id)
## this time it's retreived from the cache. We should still get
## the same cacheDir and files as before
data3 = syn.get(data.id)
assert data2.cacheDir == data3.cacheDir
assert data2.files == data3.files
def test_synapseStore_flag():
# Store a path to a local file
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
bogus = File(path, name='Totally bogus data', parent=project, synapseStore=False)
bogus = syn.store(bogus)
# Verify the thing can be downloaded as a URL
bogus = syn.get(bogus, downloadFile=False)
assert bogus.name == 'Totally bogus data'
assert bogus.path == path, "Path: %s\nExpected: %s" % (bogus.path, path)
assert bogus.synapseStore == False
# Make sure the test runs on Windows and other OS's
if path[0].isalpha() and path[1]==':':
# A Windows file URL looks like this: file:///c:/foo/bar/bat.txt
expected_url = 'file:///' + path
else:
expected_url = 'file://' + path
assert bogus.externalURL == expected_url, 'URL: %s\nExpected %s' % (bogus.externalURL, expected_URL)
# A file path that doesn't exist should still work
bogus = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False)
bogus = syn.store(bogus)
assert_raises(IOError, syn.get, bogus)
assert bogus.synapseStore == False
# Try a URL
bogus = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False)
bogus = syn.store(bogus)
bogus = syn.get(bogus)
assert bogus.synapseStore == False
def test_slow_unlocker():
"""Manually grabs a lock and makes sure the get/store methods are blocked."""
# Make a file to manually lock
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
contention = File(path, parent=syn.test_parent)
contention = syn.store(contention)
# Lock the Cache Map
cacheDir = cache.determine_cache_directory(contention['dataFileHandleId'])
cache.obtain_lock_and_read_cache(cacheDir)
# Start a few calls to get/store that should not complete yet
store_thread = wrap_function_as_child_thread(lambda: store_catch_412_HTTPError(contention))
get_thread = wrap_function_as_child_thread(lambda: syn.get(contention))
thread.start_new_thread(store_thread, ())
thread.start_new_thread(get_thread, ())
time.sleep(min(5, cache.CACHE_LOCK_TIME / 2))
# Make sure the threads did not finish
assert syn.test_threadsRunning > 0
cache.write_cache_then_release_lock(cacheDir)
# Let the threads go
while syn.test_threadsRunning > 0:
time.sleep(1)
collect_errors_and_fail()
def test_get_local_file():
"""Tests synapse.get() with local a local file """
new_path = utils.make_bogus_data_file()
schedule_for_cleanup(new_path)
folder = Folder('TestFindFileFolder', parent=project, description='A place to put my junk')
folder = syn.createEntity(folder)
#Get an nonexistent file in Synapse
assert_raises(SynapseError, syn.get, new_path)
#Get a file really stored in Synapse
ent_folder = syn.store(File(new_path, parent=folder))
ent2 = syn.get(new_path)
assert ent_folder.id==ent2.id and ent_folder.versionNumber==ent2.versionNumber
#Get a file stored in Multiple locations #should display warning
ent = syn.store(File(new_path, parent=project))
ent = syn.get(new_path)
#Get a file stored in multiple locations with limit set
ent = syn.get(new_path, limitSearch=folder.id)
assert ent.id == ent_folder.id and ent.versionNumber==ent_folder.versionNumber
#Get a file that exists but such that limitSearch removes them and raises error
assert_raises(SynapseError, syn.get, new_path, limitSearch='syn1')
def test_download_table_files():
cols = [
Column(name='artist', columnType='STRING', maximumSize=50),
Column(name='album', columnType='STRING', maximumSize=50),
Column(name='year', columnType='INTEGER'),
Column(name='catalog', columnType='STRING', maximumSize=50),
Column(name='cover', columnType='FILEHANDLEID')]
schema = syn.store(Schema(name='Jazz Albums', columns=cols, parent=project))
schedule_for_cleanup(schema)
data = [["John Coltrane", "Blue Train", 1957, "BLP 1577", "coltraneBlueTrain.jpg"],
["Sonny Rollins", "Vol. 2", 1957, "BLP 1558", "rollinsBN1558.jpg"],
["Sonny Rollins", "Newk's Time", 1958, "BLP 4001", "rollinsBN4001.jpg"],
["Kenny Burrel", "Kenny Burrel", 1956, "BLP 1543", "burrellWarholBN1543.jpg"]]
## upload files and store file handle ids
original_files = []
for row in data:
path = utils.make_bogus_data_file()
original_files.append(path)
schedule_for_cleanup(path)
file_handle = syn._uploadToFileHandleService(path)
row[4] = file_handle['id']
row_reference_set = syn.store(RowSet(columns=cols, schema=schema, rows=[Row(r) for r in data]))
## retrieve the files for each row and verify that they are identical to the originals
results = syn.tableQuery('select artist, album, year, catalog, cover from %s'%schema.id, resultsAs="rowset")
for i, row in enumerate(results):
print("%s_%s" % (row.rowId, row.versionNumber), row.values)
file_info = syn.downloadTableFile(results, rowId=row.rowId, versionNumber=row.versionNumber, column='cover')
assert filecmp.cmp(original_files[i], file_info['path'])
schedule_for_cleanup(file_info['path'])
## test that cached copies are returned for already downloaded files
original_downloadFile_method = syn._downloadFile
with patch("synapseclient.Synapse._downloadFile") as _downloadFile_mock:
_downloadFile_mock.side_effect = original_downloadFile_method
results = syn.tableQuery("select artist, album, year, catalog, cover from %s where artist = 'John Coltrane'"%schema.id, resultsAs="rowset")
for i, row in enumerate(results):
print("%s_%s" % (row.rowId, row.versionNumber), row.values)
file_info = syn.downloadTableFile(results, rowId=row.rowId, versionNumber=row.versionNumber, column='cover')
assert filecmp.cmp(original_files[i], file_info['path'])
assert not _downloadFile_mock.called, "Should have used cached copy of file and not called _downloadFile"
## test download table column
results = syn.tableQuery('select * from %s' % schema.id)
## uncache 2 out of 4 files
for i, row in enumerate(results):
if i % 2 == 0:
syn.cache.remove(row[6])
file_map = syn.downloadTableColumns(results, ['cover'])
assert len(file_map) == 4
for row in results:
filecmp.cmp(original_files[i], file_map[row[6]])
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()),
parent=project_entity))
schedule_for_cleanup(folder_entity.id)
# Create and upload a file in Folder
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': ['hello_world']}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity))
externalURL_entity = syn.store(synapseclient.File(repo_url, name='rand', parent=folder_entity, synapseStore=False))
syn.setAnnotations(file_entity, annots)
syn.setAnnotations(externalURL_entity, annots)
schedule_for_cleanup(file_entity.id)
schedule_for_cleanup(externalURL_entity.id)
# Test cp function
output = run('synapse', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
output_URL = run('synapse', '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL)
# Verify that our copied files are identical
copied_ent = syn.get(copied_id)
copied_URL_ent = syn.get(copied_URL_id, downloadFile=False)
schedule_for_cleanup(copied_id)
schedule_for_cleanup(copied_URL_id)
copied_ent_annot = syn.getAnnotations(copied_id)
copied_url_annot = syn.getAnnotations(copied_URL_id)
copied_prov = syn.getProvenance(copied_id)['used'][0]['reference']['targetId']
copied_url_prov = syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId']
# Make sure copied files are the same
assert_equals(copied_prov, file_entity.id)
assert_equals(copied_ent_annot, annots)
assert_equals(copied_ent.properties.dataFileHandleId, file_entity.properties.dataFileHandleId)
# Make sure copied URLs are the same
assert_equals(copied_url_prov, externalURL_entity.id)
assert_equals(copied_url_annot, annots)
assert_equals(copied_URL_ent.externalURL, repo_url)
assert_equals(copied_URL_ent.name, 'rand')
assert_equals(copied_URL_ent.properties.dataFileHandleId, externalURL_entity.properties.dataFileHandleId)
# Verify that errors are being thrown when a
# file is copied to a folder/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id,
'--destinationId', project_entity.id)
def test_command_copy():
"""Tests the 'synapse cp' function"""
# Create a Project
project_entity = syn.store(synapseclient.Project(name=str(uuid.uuid4())))
schedule_for_cleanup(project_entity.id)
# Create a Folder in Project
folder_entity = syn.store(synapseclient.Folder(name=str(uuid.uuid4()),
parent=project_entity))
schedule_for_cleanup(folder_entity.id)
# Create and upload a file in Folder
repo_url = 'https://github.com/Sage-Bionetworks/synapsePythonClient'
annots = {'test': ['hello_world']}
# Create, upload, and set annotations on a file in Folder
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
file_entity = syn.store(synapseclient.File(filename, parent=folder_entity))
externalURL_entity = syn.store(synapseclient.File(repo_url, name='rand', parent=folder_entity, synapseStore=False))
syn.setAnnotations(file_entity, annots)
syn.setAnnotations(externalURL_entity, annots)
schedule_for_cleanup(file_entity.id)
schedule_for_cleanup(externalURL_entity.id)
# Test cp function
output = run('synapse', '--skip-checks', 'cp', file_entity.id, '--destinationId', project_entity.id)
output_URL = run('synapse', '--skip-checks', 'cp', externalURL_entity.id, '--destinationId', project_entity.id)
copied_id = parse(r'Copied syn\d+ to (syn\d+)', output)
copied_URL_id = parse(r'Copied syn\d+ to (syn\d+)', output_URL)
# Verify that our copied files are identical
copied_ent = syn.get(copied_id)
copied_URL_ent = syn.get(copied_URL_id, downloadFile=False)
schedule_for_cleanup(copied_id)
schedule_for_cleanup(copied_URL_id)
copied_ent_annot = syn.getAnnotations(copied_id)
copied_url_annot = syn.getAnnotations(copied_URL_id)
copied_prov = syn.getProvenance(copied_id)['used'][0]['reference']['targetId']
copied_url_prov = syn.getProvenance(copied_URL_id)['used'][0]['reference']['targetId']
# Make sure copied files are the same
assert_equals(copied_prov, file_entity.id)
assert_equals(copied_ent_annot, annots)
assert_equals(copied_ent.properties.dataFileHandleId, file_entity.properties.dataFileHandleId)
# Make sure copied URLs are the same
assert_equals(copied_url_prov, externalURL_entity.id)
assert_equals(copied_url_annot, annots)
assert_equals(copied_URL_ent.externalURL, repo_url)
assert_equals(copied_URL_ent.name, 'rand')
assert_equals(copied_URL_ent.properties.dataFileHandleId, externalURL_entity.properties.dataFileHandleId)
# Verify that errors are being thrown when a
# file is copied to a folder/project that has a file with the same filename
assert_raises(ValueError, run, 'synapse', '--debug', '--skip-checks', 'cp', file_entity.id,
'--destinationId', project_entity.id)
def test_get_entity_owned_by_another_user():
if 'username' not in other_user or 'password' not in other_user:
sys.stderr.write(
'\nWarning: no test-authentication configured. skipping test_get_entity_owned_by_another.\n'
)
return
try:
syn_other = synapseclient.Synapse(skip_checks=True)
syn_other.login(other_user['username'], other_user['password'])
project = Project(name=str(uuid.uuid4()))
project = syn_other.store(project)
filepath = utils.make_bogus_data_file()
a_file = File(filepath,
parent=project,
description='asdf qwer',
foo=1234)
a_file = syn_other.store(a_file)
current_user_id = int(syn.getUserProfile()['ownerId'])
# Update the acl to give the current user read permissions
syn_other.setPermissions(a_file,
current_user_id,
accessType=['READ', 'DOWNLOAD'],
modify_benefactor=True)
# Test whether the benefactor's ACL was modified
assert_equals(set(syn_other.getPermissions(project, current_user_id)),
set(['READ', 'DOWNLOAD']))
# Add a new permission to a user with existing permissions
# make this change on the entity itself, not its benefactor
syn_other.setPermissions(a_file,
current_user_id,
accessType=['READ', 'UPDATE', 'DOWNLOAD'],
modify_benefactor=False,
warn_if_inherits=False)
permissions = syn_other.getPermissions(a_file, current_user_id)
assert 'READ' in permissions
assert 'UPDATE' in permissions
assert len(permissions) == 3
syn_other.setPermissions(a_file,
current_user_id,
accessType=['READ', 'DOWNLOAD'])
assert_equals(set(syn_other.getPermissions(a_file, current_user_id)),
set(['DOWNLOAD', 'READ']))
other_users_file = syn.get(a_file.id)
a_file = syn_other.get(a_file.id)
assert other_users_file == a_file
finally:
syn_other.logout()
def _create_temp_file_with_cleanup(specific_file_text = None):
if specific_file_text:
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as file:
file.write(specific_file_text)
filename = file.name
else:
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
return filename
def _create_temp_file_with_cleanup(specific_file_text=None):
if specific_file_text:
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as file:
file.write(specific_file_text)
filename = file.name
else:
filename = utils.make_bogus_data_file()
schedule_for_cleanup(filename)
return filename
def test_store_dictionary():
project = {
'entityType': 'org.sagebionetworks.repo.model.Project',
'name': str(uuid.uuid4()),
'description': 'Project from dictionary'
}
project = syn.store(project)
schedule_for_cleanup(project)
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity = {
'entityType': 'org.sagebionetworks.repo.model.Data',
'name': 'foo',
'parentId': project['id'],
'foo': 334455,
'path': path
}
data = syn.store(entity)
data = syn.get(data)
assert data.parentId == project.id
assert data.foo[0] == 334455
assert filecmp.cmp(path, os.path.join(data['cacheDir'], data['files'][0]))
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity = {}
entity.update(data.annotations)
entity.update(data.properties)
entity.update(data.local_state())
entity['path'] = path
entity['description'] = 'Updating with a plain dictionary should be rare.'
data = syn.store(entity)
assert data.description == entity['description']
assert data.name == 'foo'
data = syn.get(data['id'])
assert filecmp.cmp(path, os.path.join(data['cacheDir'], data['files'][0]))
def test_uploadFile_given_dictionary():
# Make a Data Entity the old fashioned way
data = {
'concreteType': Data._synapse_entity_type,
'parentId': project['id'],
'name': 'fooDictionary',
'foo': 334455
}
entity = syn.createEntity(data)
# Create and upload a temporary file
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
syn.uploadFile(entity, fname)
# Download and verify that it is the same file
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname,
os.path.join(entity['cacheDir'], entity['files'][0]))
assert entity.parentId == project.id
assert entity.foo[0] == 334455
# Update via a dictionary
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
rareCase = {}
rareCase.update(entity.annotations)
rareCase.update(entity.properties)
rareCase.update(entity.local_state())
rareCase['path'] = path
rareCase[
'description'] = 'Updating with a plain dictionary should be rare.'
# Verify it works
entity = syn.store(rareCase)
assert entity.description == rareCase['description']
assert entity.name == 'fooDictionary'
entity = syn.get(entity['id'])
assert filecmp.cmp(path,
os.path.join(entity['cacheDir'], entity['files'][0]))
def test_download_check_md5():
tempfile_path = utils.make_bogus_data_file()
schedule_for_cleanup(tempfile_path)
entity = File(parent=project['id'])
entity['path'] = tempfile_path
entity = syn.store(entity)
syn._downloadFileHandle(entity['dataFileHandleId'], entity['id'],
'FileEntity', tempfile.gettempdir())
tempfile_path2 = utils.make_bogus_data_file()
schedule_for_cleanup(tempfile_path2)
entity_bad_md5 = syn.store(
File(path=tempfile_path2, parent=project['id'], synapseStore=False))
assert_raises(SynapseMd5MismatchError,
syn._download_from_URL,
entity_bad_md5['externalURL'],
tempfile.gettempdir(),
entity_bad_md5['dataFileHandleId'],
expected_md5="2345a")
def test_store_dictionary():
project = { 'entityType': 'org.sagebionetworks.repo.model.Project',
'name':str(uuid.uuid4()),
'description':'Project from dictionary'}
project = syn.store(project)
schedule_for_cleanup(project)
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity = { 'entityType': 'org.sagebionetworks.repo.model.Data',
'name':'foo',
'parentId':project['id'],
'foo':334455,
'path':path }
data = syn.store(entity)
data = syn.get(data)
assert data.parentId == project.id
assert data.foo[0] == 334455
assert filecmp.cmp(path, os.path.join(data['cacheDir'], data['files'][0]))
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity = {}
entity.update(data.annotations)
entity.update(data.properties)
entity.update(data.local_state())
entity['path'] = path
entity['description'] = 'Updating with a plain dictionary should be rare.'
data = syn.store(entity)
assert data.description == entity['description']
assert data.name == 'foo'
data = syn.get(data['id'])
assert filecmp.cmp(path, os.path.join(data['cacheDir'], data['files'][0]))
def test_uploadFile_given_dictionary():
project = create_project()
data = DATA_JSON.copy()
data['parentId']= project.id
filename = utils.make_bogus_data_file()
data = syn.uploadFile(data, filename)
entity = syn.downloadEntity(data['id'])
assert entity['files'][0]==os.path.basename(filename)
assert filecmp.cmp(filename, os.path.join(entity['cacheDir'],entity['files'][0]))
os.remove(filename)
def test_download_table_files():
cols = [
Column(name='artist', columnType='STRING', maximumSize=50),
Column(name='album', columnType='STRING', maximumSize=50),
Column(name='year', columnType='INTEGER'),
Column(name='catalog', columnType='STRING', maximumSize=50),
Column(name='cover', columnType='FILEHANDLEID')
]
schema = syn.store(Schema(name='Jazz Albums', columns=cols,
parent=project))
schedule_for_cleanup(schema)
data = [[
"John Coltrane", "Blue Train", 1957, "BLP 1577",
"coltraneBlueTrain.jpg"
], ["Sonny Rollins", "Vol. 2", 1957, "BLP 1558", "rollinsBN1558.jpg"],
[
"Sonny Rollins", "Newk's Time", 1958, "BLP 4001",
"rollinsBN4001.jpg"
],
[
"Kenny Burrel", "Kenny Burrel", 1956, "BLP 1543",
"burrellWarholBN1543.jpg"
]]
## upload files and store file handle ids
original_files = []
for row in data:
path = utils.make_bogus_data_file()
original_files.append(path)
schedule_for_cleanup(path)
file_handle = syn._chunkedUploadFile(path)
row[4] = file_handle['id']
row_reference_set = syn.store(
RowSet(columns=cols, schema=schema, rows=[Row(r) for r in data]))
## retrieve the files for each row and verify that they are identical to the originals
results = syn.tableQuery(
'select artist, album, year, catalog, cover from %s' % schema.id,
resultsAs="rowset")
for i, row in enumerate(results):
print "%s_%s" % (row.rowId, row.versionNumber), row.values
file_info = syn.downloadTableFile(results,
rowId=row.rowId,
versionNumber=row.versionNumber,
column='cover',
downloadLocation='.')
assert filecmp.cmp(original_files[i], file_info['path'])
schedule_for_cleanup(file_info['path'])
def test_uploadFile_given_dictionary():
project = create_project()
data = DATA_JSON.copy()
data['parentId'] = project.id
filename = utils.make_bogus_data_file()
data = syn.uploadFile(data, filename)
entity = syn.downloadEntity(data['id'])
assert entity['files'][0] == os.path.basename(filename)
assert filecmp.cmp(filename,
os.path.join(entity['cacheDir'], entity['files'][0]))
os.remove(filename)
def test_uploadFile():
project = create_project()
## here, entity has been saved to Synapse and entity is an Entity object
entity = create_data_entity(project['id'])
#create a temporary file
fname = utils.make_bogus_data_file()
syn.uploadFile(entity, fname)
#Download and verify that it is the same filename
entity = syn.downloadEntity(entity)
assert entity['files'][0]==os.path.basename(fname)
assert filecmp.cmp(fname, os.path.join(entity['cacheDir'],entity['files'][0]))
os.remove(fname)
def test_uploadFile_given_dictionary():
# Make a Data Entity the old fashioned way
data = {'concreteType': Data._synapse_entity_type,
'parentId' : project['id'],
'name' : 'fooDictionary',
'foo' : 334455}
entity = syn.createEntity(data)
# Create and upload a temporary file
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
syn.uploadFile(entity, fname)
# Download and verify that it is the same file
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, os.path.join(entity['cacheDir'],entity['files'][0]))
assert entity.parentId == project.id
assert entity.foo[0] == 334455
# Update via a dictionary
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
rareCase = {}
rareCase.update(entity.annotations)
rareCase.update(entity.properties)
rareCase.update(entity.local_state())
rareCase['path'] = path
rareCase['description'] = 'Updating with a plain dictionary should be rare.'
# Verify it works
entity = syn.store(rareCase)
assert entity.description == rareCase['description']
assert entity.name == 'fooDictionary'
entity = syn.get(entity['id'])
assert filecmp.cmp(path, os.path.join(entity['cacheDir'], entity['files'][0]))
def test_uploadFileEntity():
projectEntity = create_project()
# Defaults to FileEntity
fname = utils.make_bogus_data_file()
entity = {'name' : 'foo', \
'description' : 'A test file entity', \
'parentId' : projectEntity['id']}
# Create new FileEntity
entity = syn.uploadFile(entity, fname)
# Download and verify
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
# Check if we upload the wrong type of file handle
fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0]
assert fh[
'concreteType'] == 'org.sagebionetworks.repo.model.file.S3FileHandle'
os.remove(fname)
# Create a different temporary file
fname = utils.make_bogus_data_file()
# Update existing FileEntity
entity = syn.uploadFile(entity, fname)
# Download and verify that it is the same filename
entity = syn.downloadEntity(entity)
print entity['files'][0]
print os.path.basename(fname)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname, entity['path'])
os.remove(fname)
def test_synapseStore_flag():
"""Test storing entities while setting the synapseStore flag to False"""
project = create_project()
## store a path to a local file (synapseStore=False)
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
f1 = File(path,
name='Totally bogus data',
parent=project,
synapseStore=False)
f1 = syn.store(f1)
f1a = syn.get(f1.id, downloadFile=False)
assert f1a.name == 'Totally bogus data'
assert f1a.path == path, 'path=' + str(f1a.path) + '; expected=' + path
assert f1a.synapseStore == False
## make sure the test runs on Windows and other OS's
if path[0].isalpha() and path[1] == ':':
## a windows file URL looks like this: file:///c:/foo/bar/bat.txt
expected_url = 'file:///' + path
else:
expected_url = 'file://' + path
assert f1a.externalURL == expected_url, 'unexpected externalURL: ' + f1a.externalURL
## a file path that doesn't exist should still work
f2 = File('/path/to/local/file1.xyz',
parentId=project.id,
synapseStore=False)
f2 = syn.store(f2)
try:
syn.get(f2)
assert False
except Exception as err:
assert err.message.startswith("Could not download non-existent file")
assert f1a.synapseStore == False
## Try a URL
f3 = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient',
parent=project,
synapseStore=False)
f3 = syn.store(f3)
f3a = syn.get(f3)
assert f1a.synapseStore == False
def test_provenance():
# Create a Data Entity
fname = utils.make_bogus_data_file()
schedule_for_cleanup(fname)
data_entity = syn.createEntity(Data(parent=project['id']))
data_entity = syn.uploadFile(data_entity, fname)
# Create a Code Entity
fd, path = tempfile.mkstemp(suffix=".py")
os.write(
fd, """
## Chris's fabulous random data generator
############################################################
import random
random.seed(12345)
data = [random.gauss(mu=0.0, sigma=1.0) for i in range(100)]
""")
os.close(fd)
schedule_for_cleanup(path)
code_entity = syn.createEntity(Code(parent=project['id']))
code_entity = syn.uploadFile(code_entity, path)
# Create a new Activity asserting that the Code Entity was 'used'
activity = Activity(name='random.gauss',
description='Generate some random numbers')
activity.used(code_entity, wasExecuted=True)
activity.used(
{
'name': 'Superhack',
'url': 'https://github.com/joe_coder/Superhack'
},
wasExecuted=True)
activity = syn.setProvenance(data_entity, activity)
# Retrieve and verify the saved Provenance record
retrieved_activity = syn.getProvenance(data_entity)
assert retrieved_activity == activity
# Test Activity update
new_description = 'Generate random numbers like a gangsta'
retrieved_activity['description'] = new_description
updated_activity = syn.updateActivity(retrieved_activity)
assert updated_activity['name'] == retrieved_activity['name']
assert updated_activity['description'] == new_description
# Test delete
syn.deleteProvenance(data_entity)
assert_raises(SynapseHTTPError, syn.getProvenance, data_entity['id'])
def test_path_in_annotations_of_data_entities_bug():
# test for SYNR-610, path, files and cacheDir appearing in annotations
data1 = syn.createEntity(Data(name='Yet more totally bogus data', parent=project))
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity = syn.uploadFile(data1, path)
## entity should have a path, but not files and location at this point
assert 'path' in entity and entity['path'] is not None
data2 = syn.get(data1.id)
## These shouldn't be in annotations: "files" and "path" and "cacheDir"
assert not 'files' in data2.annotations
assert not 'path' in data2.annotations
assert not 'cacheDir' in data2.annotations
def test_uploadFile():
project = create_project()
## here, entity has been saved to Synapse and entity is an Entity object
entity = create_data_entity(project['id'])
#create a temporary file
fname = utils.make_bogus_data_file()
syn.uploadFile(entity, fname)
#Download and verify that it is the same filename
entity = syn.downloadEntity(entity)
assert entity['files'][0] == os.path.basename(fname)
assert filecmp.cmp(fname,
os.path.join(entity['cacheDir'], entity['files'][0]))
os.remove(fname)
def test_deprecated_entity_types():
"""Test Data Entity object"""
project = create_project()
data = Data(parent=project)
data = syn.createEntity(data)
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
data = syn.uploadFile(data, path)
## make sure file comes back intact
data = syn.downloadEntity(data)
assert filecmp.cmp(path, os.path.join(data['cacheDir'], data['files'][0]))
def test_deprecated_entity_types():
"""Test Data Entity object"""
project = create_project()
data = Data(parent=project)
data = syn.createEntity(data)
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
data = syn.uploadFile(data, path)
## make sure file comes back intact
data = syn.downloadEntity(data)
assert filecmp.cmp(path, os.path.join(data['cacheDir'], data['files'][0]))
def test_http_download__range_request_error():
# SYNPY-525
file_path = utils.make_bogus_data_file()
file_entity = syn.store(File(file_path, parent=project))
syn.cache.purge(time.time())
#download once and rename to temp file to simulate range exceed
file_entity = syn.get(file_entity)
shutil.move(
file_entity.path,
utils.temp_download_filename(file_entity.path,
file_entity.dataFileHandleId))
file_entity = syn.get(file_entity)
assert_not_equal(file_path, file_entity.path)
assert filecmp.cmp(file_path, file_entity.path)
def test_md5_query():
# Add the same Entity several times
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path, parent=project['id'], description='Same data over and over again')
# Retrieve the data via MD5
num = 5
stored = []
for i in range(num):
repeated.name = 'Repeated data %d.dat' % i
stored.append(syn.store(repeated).id)
# Although we expect num results, it is possible for the MD5 to be non-unique
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
assert str(sorted(stored)) == str(sorted([res['id'] for res in results]))
assert len(results) == num
def test_path_in_annotations_of_data_entities_bug():
# test for SYNR-610, path, files and cacheDir appearing in annotations
data1 = syn.createEntity(
Data(name='Yet more totally bogus data', parent=project))
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity = syn.uploadFile(data1, path)
## entity should have a path, but not files and location at this point
assert 'path' in entity and entity['path'] is not None
data2 = syn.get(data1.id)
## These shouldn't be in annotations: "files" and "path" and "cacheDir"
assert not 'files' in data2.annotations
assert not 'path' in data2.annotations
assert not 'cacheDir' in data2.annotations
def test_entity_version():
# Make an Entity and make sure the version is one
entity = File(parent=project['id'])
entity['path'] = utils.make_bogus_data_file()
schedule_for_cleanup(entity['path'])
entity = syn.createEntity(entity)
syn.setAnnotations(entity, {'fizzbuzz': 111222})
entity = syn.getEntity(entity)
assert entity.versionNumber == 1
# Update the Entity and make sure the version is incremented
entity.foo = 998877
entity['name'] = 'foobarbat'
entity['description'] = 'This is a test entity...'
entity = syn.updateEntity(entity,
incrementVersion=True,
versionLabel="Prada remix")
assert entity.versionNumber == 2
# Get the older data and verify the random stuff is still there
annotations = syn.getAnnotations(entity, version=1)
assert annotations['fizzbuzz'][0] == 111222
returnEntity = syn.getEntity(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Try the newer Entity
returnEntity = syn.getEntity(entity)
assert returnEntity.versionNumber == 2
assert returnEntity['foo'][0] == 998877
assert returnEntity['name'] == 'foobarbat'
assert returnEntity['description'] == 'This is a test entity...'
assert returnEntity['versionLabel'] == 'Prada remix'
# Try the older Entity again
returnEntity = syn.downloadEntity(entity, version=1)
assert returnEntity.versionNumber == 1
assert returnEntity['fizzbuzz'][0] == 111222
assert 'foo' not in returnEntity
# Delete version 2
syn.delete(entity, version=2)
returnEntity = syn.getEntity(entity)
assert returnEntity.versionNumber == 1
def test_download_check_md5():
entity = File(parent=project['id'])
entity['path'] = utils.make_bogus_data_file()
schedule_for_cleanup(entity['path'])
entity = syn.store(entity)
print('expected MD5:', entity['md5'])
url = '%s/entity/%s/file' % (syn.repoEndpoint, entity.id)
syn._downloadFile(url, destination=tempfile.gettempdir(), expected_md5=entity['md5'])
try:
syn._downloadFile(url, destination=tempfile.gettempdir(), expected_md5='10000000000000000000000000000001')
except SynapseMd5MismatchError as ex1:
print("Expected exception:", ex1)
else:
assert False, "Should have raised SynapseMd5MismatchError"
def thread_keep_storing_one_File():
"""Makes one file and stores it over and over again."""
# Make a local file to continuously store
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
myPrecious = File(path, parent=syn.test_parent, description='This bogus file is MINE', mwa="hahahah")
while syn.test_keepRunning:
stored = store_catch_412_HTTPError(myPrecious)
if stored is not None:
myPrecious = stored
# print "I've stored %s" % myPrecious.id
else:
myPrecious = syn.get(myPrecious)
# print "Grrr... Someone modified my %s" % myPrecious.id
sleep_for_a_bit()
def thread_keep_storing_one_File():
"""Makes one file and stores it over and over again."""
# Make a local file to continuously store
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
myPrecious = File(path, parent=syn.test_parent, description='This bogus file is MINE', mwa="hahahah")
while syn.test_keepRunning:
stored = store_catch_412_HTTPError(myPrecious)
if stored is not None:
myPrecious = stored
# print("I've stored %s" % myPrecious.id)
else:
myPrecious = syn.get(myPrecious)
# print("Grrr... Someone modified my %s" % myPrecious.id)
sleep_for_a_bit()
def test_store_activity():
"""Test storing entities with Activities"""
project = create_project()
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
f = File(path, name='Hinkle horn honking holes', parent=project)
honking = Activity(
name='Hinkle horn honking',
description=
'Nettlebed Cave is a limestone cave located on the South Island of New Zealand.'
)
honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/')
honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/')
## doesn't set the ID of the activity
f = syn.store(f, activity=honking)
honking = syn.getProvenance(f.id)
## now, we have an activity ID
assert honking['name'] == 'Hinkle horn honking'
assert len(honking['used']) == 2
assert honking['used'][0][
'concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][0]['wasExecuted'] == False
assert honking['used'][0]['url'].startswith(
'http://www.flickr.com/photos/bevanbfree/3482')
assert honking['used'][1][
'concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][1]['wasExecuted'] == False
## store another entity with the same activity
f2 = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg',
name='Nettlebed Cave',
parent=project)
f2 = syn.store(f2, activity=honking)
honking2 = syn.getProvenance(f2)
assert honking['id'] == honking2['id']
def test_md5_query():
# Add the same Entity several times
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
repeated = File(path,
parent=project['id'],
description='Same data over and over again')
# Retrieve the data via MD5
num = 5
stored = []
for i in range(num):
repeated.name = 'Repeated data %d.dat' % i
stored.append(syn.store(repeated).id)
# Although we expect num results, it is possible for the MD5 to be non-unique
results = syn.md5Query(utils.md5_for_file(path).hexdigest())
assert str(sorted(stored)) == str(sorted([res['id'] for res in results]))
assert len(results) == num
def thread_get_and_update_file_from_Project():
"""Fetches one item from the Project and updates it with a new file."""
while syn.test_keepRunning:
id = get_all_ids_from_Project()
if len(id) <= 0:
continue
id = id[random.randrange(len(id))]
entity = syn.get(id)
# Replace the file and re-store
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
entity.path = path
entity = store_catch_412_HTTPError(entity)
if entity is not None:
assert os.stat(entity.path) == os.stat(path)
sleep_for_a_bit()