def test_single_thread_upload(syn):
synapseclient.core.config.single_threaded = True
try:
filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + 1)
assert multipart_upload_file(syn, filepath) is not None
finally:
synapseclient.core.config.single_threaded = False
def test_store_activity(syn, project, schedule_for_cleanup):
# Create a File and an Activity
path = utils.make_bogus_binary_file()
schedule_for_cleanup(path)
entity = File(path, name='Hinkle horn honking holes', parent=project)
honking = Activity(name='Hinkle horn honking',
description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.')
honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/')
honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/')
# This doesn't set the ID of the Activity
entity = syn.store(entity, activity=honking)
# But this does
honking = syn.getProvenance(entity.id)
# Verify the Activity
assert honking['name'] == 'Hinkle horn honking'
assert len(honking['used']) == 2
assert honking['used'][0]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert not honking['used'][0]['wasExecuted']
assert honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482')
assert honking['used'][1]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert not honking['used'][1]['wasExecuted']
# Store another Entity with the same Activity
entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg',
name='Nettlebed Cave', parent=project, synapseStore=False)
entity = syn.store(entity, activity=honking)
# The Activities should match
honking2 = syn.getProvenance(entity)
assert honking['id'] == honking2['id']
def test_round_trip(syn, project, schedule_for_cleanup):
fhid = None
filepath = utils.make_bogus_binary_file(MIN_PART_SIZE + 777771)
try:
fhid = multipart_upload_file(syn, filepath)
# Download the file and compare it with the original
junk = File(parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity',
tmp_path)
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_randomly_failing_parts(syn, project, schedule_for_cleanup):
"""Verify that we can recover gracefully with some randomly inserted errors
while uploading parts."""
fail_every = 3 # fail every nth request
fail_cycle = random.randint(
0,
fail_every - 1) # randomly vary which n of the request cycle we fail
fhid = None
filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 +
(MIN_PART_SIZE / 2))
put_count = 0
normal_put = requests.Session.put
def _put_chunk_or_fail_randomly(self, url, *args, **kwargs):
# fail every nth put to aws s3
if 's3.amazonaws.com' not in url:
return normal_put(self, url, *args, **kwargs)
nonlocal put_count
put_count += 1
if (put_count + fail_cycle) % fail_every == 0:
raise IOError("Ooops! Artificial upload failure for testing.")
return normal_put(self, url, *args, **kwargs)
with mock.patch('requests.Session.put',
side_effect=_put_chunk_or_fail_randomly,
autospec=True):
try:
fhid = multipart_upload_file(syn,
filepath,
part_size=MIN_PART_SIZE)
# Download the file and compare it with the original
junk = File(parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk['path'] = syn._downloadFileHandle(fhid, junk['id'],
'FileEntity', tmp_path)
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_store_isRestricted_flag(syn, project, schedule_for_cleanup):
# Store a file with access requirements
path = utils.make_bogus_binary_file()
schedule_for_cleanup(path)
entity = File(path, name='Secret human data', parent=project)
# We don't want to spam ACT with test emails
with patch('synapseclient.client.Synapse._createAccessRequirementIfNone') as intercepted:
entity = syn.store(entity, isRestricted=True)
assert intercepted.called
def test_synGet_sftpIntegration(syn, project):
# Create file by uploading directly to sftp and creating entity from URL
server_prefix = get_sftp_server_prefix()
username, password = syn._getUserCredentials(server_prefix)
server_url = server_prefix + get_user_home_path(username) + '/test_synGet_sftpIntegration/' + str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * utils.MB - 777771)
url = SFTPWrapper.upload_file(filepath, url=server_url, username=username, password=password)
file = syn.store(File(path=url, parent=project, synapseStore=False))
junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
filecmp.cmp(filepath, junk.path)
def test_get_with_downloadLocation_and_ifcollision(syn, project,
schedule_for_cleanup):
# Store a File and delete it locally
filepath = utils.make_bogus_binary_file()
bogus = File(filepath, name='Bogus Test File', parent=project)
bogus = syn.store(bogus)
os.remove(filepath)
# Compare stuff to this one
normalBogus = syn.get(bogus)
# Download to the temp folder, should be the same
otherBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath))
assert otherBogus.id == normalBogus.id
assert filecmp.cmp(otherBogus.path, normalBogus.path)
# Invalidate the downloaded file's timestamps
os.utime(otherBogus.path, (0, 0))
badtimestamps = os.path.getmtime(otherBogus.path)
# Download again, should change the modification time
overwriteBogus = syn.get(bogus,
downloadLocation=os.path.dirname(filepath),
ifcollision="overwrite.local")
overwriteModTime = os.path.getmtime(overwriteBogus.path)
assert badtimestamps != overwriteModTime
# Download again, should not change the modification time
otherBogus = syn.get(bogus,
downloadLocation=os.path.dirname(filepath),
ifcollision="keep.local")
assert overwriteModTime == os.path.getmtime(overwriteBogus.path)
# "keep.local" should have made the path invalid since it is keeping a potentially modified version
assert otherBogus.path is None
assert otherBogus.cacheDir is None
assert 0 == len(otherBogus.files)
# Invalidate the timestamps again
os.utime(overwriteBogus.path, (0, 0))
badtimestamps = os.path.getmtime(overwriteBogus.path)
# Download once more, but rename
renamedBogus = syn.get(bogus,
downloadLocation=os.path.dirname(filepath),
ifcollision="keep.both")
assert overwriteBogus.path != renamedBogus.path
assert filecmp.cmp(overwriteBogus.path, renamedBogus.path)
# Clean up
os.remove(overwriteBogus.path)
os.remove(renamedBogus.path)
def test_utils_sftp_upload_and_download(syn):
"""Tries to upload a file to an sftp file """
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_utils_sftp_upload_and_download/' + str(
uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * utils.MB - 777771)
tempdir = tempfile.mkdtemp()
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)
try:
url = SFTPWrapper.upload_file(filepath,
url=serverURL,
username=username,
password=password)
# Get with a specified localpath
junk = SFTPWrapper.download_file(url,
tempdir,
username=username,
password=password)
filecmp.cmp(filepath, junk)
# Get without specifying path
junk2 = SFTPWrapper.download_file(url,
username=username,
password=password)
filecmp.cmp(filepath, junk2)
# Get with a specified localpath as file
junk3 = SFTPWrapper.download_file(url,
os.path.join(tempdir, 'bar.dat'),
username=username,
password=password)
filecmp.cmp(filepath, junk3)
finally:
try:
if 'junk' in locals():
os.remove(junk)
if 'junk2' in locals():
os.remove(junk2)
if 'junk3' in locals():
os.remove(junk3)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
try:
shutil.rmtree(tempdir)
except Exception:
print(traceback.format_exc())
def test_pool_provider_is_used_in__multipart_upload():
mocked_get_chunk_function = MagicMock(side_effect=[1, 2, 3, 4])
file_size = 1 * MB
filepath = make_bogus_binary_file(n=file_size)
md5 = md5_for_file(filepath).hexdigest()
status = {'partsState': {}, 'uploadId': {}, 'state': 'COMPLETED'}
pool = MagicMock()
with patch.object(syn, "restPOST", return_value=status),\
patch.object(pool_provider, "get_pool", return_value=pool) as mock_provider:
_multipart_upload(syn, filepath, "application/octet-stream",
mocked_get_chunk_function, md5, file_size)
mock_provider.assert_called()
pool.map.assert_called()
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
file_size = 1 * MB
filepath = make_bogus_binary_file(n=file_size)
chunksize = 64 * 1024
nchunks = int(math.ceil(float(file_size) / chunksize))
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
for i in range(1, nchunks + 1):
out.write(get_file_chunk(filepath, i, chunksize))
assert_true(filecmp.cmp(filepath, out.name))
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_synStore_sftpIntegration(syn, project, schedule_for_cleanup):
"""Creates a File Entity on an sftp server and add the external url. """
filepath = utils.make_bogus_binary_file(1 * utils.MB - 777771)
try:
file = syn.store(File(filepath, parent=project))
file2 = syn.get(file)
assert file.externalURL == file2.externalURL
assert urlparse(file2.externalURL).scheme == 'sftp'
tmpdir = tempfile.mkdtemp()
schedule_for_cleanup(tmpdir)
# test that we got an MD5 à la SYNPY-185
assert file2.md5 is not None
fh = syn._get_file_handle_as_creator(file2.dataFileHandleId)
assert fh['contentMd5'] is not None
assert file2.md5 == fh['contentMd5']
finally:
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_download_file_false(syn, project, schedule_for_cleanup):
RENAME_SUFFIX = 'blah'
# Upload a file
filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(filepath)
schedule_for_cleanup(filepath + RENAME_SUFFIX)
file = File(filepath, name='SYNR 619', parent=project)
file = syn.store(file)
# Now hide the file from the cache and download with downloadFile=False
os.rename(filepath, filepath + RENAME_SUFFIX)
file = syn.get(file.id, downloadFile=False)
# Change something and reupload the file's metadata
file.name = "Only change the name, not the file"
reupload = syn.store(file)
assert reupload.path is None, "Path field should be null: %s" % reupload.path
# This should still get the correct file
reupload = syn.get(reupload.id)
assert filecmp.cmp(filepath + RENAME_SUFFIX, reupload.path)
assert reupload.name == file.name
def test_store_with_flags(syn, project, schedule_for_cleanup):
# -- CreateOrUpdate flag for Projects --
# If we store a project with the same name, it should become an update
projUpdate = Project(project.name)
projUpdate.updatedThing = 'Yep, sho\'nuf it\'s updated!'
projUpdate = syn.store(projUpdate, createOrUpdate=True)
assert project.id == projUpdate.id
assert projUpdate.updatedThing == ['Yep, sho\'nuf it\'s updated!']
# Store a File
filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(filepath)
origBogus = File(filepath, name='Bogus Test File', parent=project)
origBogus = syn.store(origBogus, createOrUpdate=True)
assert origBogus.versionNumber == 1
# Modify existing annotations by createOrUpdate
del projUpdate['parentId']
del projUpdate['id']
projUpdate.updatedThing = 'Updated again'
projUpdate.addedThing = 'Something new'
projUpdate = syn.store(projUpdate, createOrUpdate=True)
assert project.id == projUpdate.id
assert projUpdate.updatedThing == ['Updated again']
# -- ForceVersion flag --
# Re-store the same thing and don't up the version
mutaBogus = syn.store(origBogus, forceVersion=False)
assert mutaBogus.versionNumber == 1
# Re-store again, essentially the same condition
mutaBogus = syn.store(mutaBogus, createOrUpdate=True, forceVersion=False)
assert mutaBogus.versionNumber == 1, "expected version 1 but got version %s" % mutaBogus.versionNumber
# And again, but up the version this time
mutaBogus = syn.store(mutaBogus, forceVersion=True)
assert mutaBogus.versionNumber == 2
# Create file with different contents and store it with force version false
# This should be ignored because contents (and md5) are different
different_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(different_filepath)
mutaBogus = File(different_filepath, name='Bogus Test File',
parent=project)
mutaBogus = syn.store(mutaBogus, forceVersion=False)
assert mutaBogus.versionNumber == 3
# -- CreateOrUpdate flag for files --
# Store a different file with the same name and parent
# Expected behavior is that a new version of the first File will be created
new_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(new_filepath)
mutaBogus.path = new_filepath
mutaBogus = syn.store(mutaBogus, createOrUpdate=True)
assert mutaBogus.id == origBogus.id
assert mutaBogus.versionNumber == 4
assert not filecmp.cmp(mutaBogus.path, filepath)
# Make doubly sure the File was uploaded
checkBogus = syn.get(mutaBogus.id)
assert checkBogus.id == origBogus.id
assert checkBogus.versionNumber == 4
assert filecmp.cmp(mutaBogus.path, checkBogus.path)
# Create yet another file with the same name and parent
# Expected behavior is raising an exception with a 409 error
newer_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(newer_filepath)
badBogus = File(newer_filepath, name='Bogus Test File', parent=project)
pytest.raises(SynapseHTTPError, syn.store, badBogus, createOrUpdate=False)
# -- Storing after syn.get(..., downloadFile=False) --
ephemeralBogus = syn.get(mutaBogus, downloadFile=False)
ephemeralBogus.description = 'Snorklewacker'
ephemeralBogus.shoe_size = 11.5
ephemeralBogus = syn.store(ephemeralBogus)
ephemeralBogus = syn.get(ephemeralBogus, downloadFile=False)
assert ephemeralBogus.description == 'Snorklewacker'
assert ephemeralBogus.shoe_size == [11.5]
