def test_large_file_upload(file_to_upload_size=11*utils.KB, filepath=None):
clean_up_file = False
try:
project = syn.store(Project("File Upload Load Test " + datetime.now().strftime("%Y-%m-%d %H%M%S%f")))
if filepath:
## keep a file around so we don't have to regenerate it.
if not os.path.exists(filepath):
filepath = utils.make_bogus_binary_file(file_to_upload_size, filepath=filepath, printprogress=True)
print('Made bogus file: ', filepath)
else:
## generate a temporary file and clean it up when we're done
clean_up_file = True
filepath = utils.make_bogus_binary_file(file_to_upload_size, printprogress=True)
print('Made bogus file: ', filepath)
try:
junk = syn.store(File(filepath, parent=project))
fh = syn._getFileHandle(junk['dataFileHandleId'])
syn.printEntity(fh)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
finally:
try:
if 'filepath' in locals() and clean_up_file:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_store_with_create_or_update_flag():
project = create_project()
filepath = utils.make_bogus_binary_file()
bogus1 = File(filepath, name='Bogus Test File', parent=project)
bogus1 = syn.store(bogus1, createOrUpdate=True)
# Create a different file with the same name and parent
new_filepath = utils.make_bogus_binary_file()
bogus1.path = new_filepath
# Expected behavior is that a new version of the first File will be created
bogus2 = syn.store(bogus1, createOrUpdate=True)
assert bogus2.id == bogus1.id
assert bogus2.versionNumber == 2
assert not filecmp.cmp(bogus2.path, filepath)
bogus2a = syn.get(bogus2.id)
assert bogus2a.id == bogus1.id
assert bogus2a.versionNumber == 2
assert filecmp.cmp(bogus2.path, bogus2a.path)
# Create yet another file with the same name and parent
newer_filepath = utils.make_bogus_binary_file()
bogus3 = File(newer_filepath, name='Bogus Test File', parent=project)
# Expected behavior is raising an exception with a 409 error
assert_raises(requests.exceptions.HTTPError, syn.store, bogus3, createOrUpdate=False)
def test_round_trip():
fh = None
filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
print 'Made bogus file: ', filepath
try:
fh = syn._chunkedUploadFile(filepath, verbose=False)
# print 'FileHandle:'
# syn.printEntity(fh)
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
junk.update(syn._downloadFileEntity(junk, filepath))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print traceback.format_exc()
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
if fh:
# print 'Deleting fileHandle', fh['id']
syn._deleteFileHandle(fh)
def test_store_activity():
# Create a File and an Activity
path = utils.make_bogus_binary_file()
schedule_for_cleanup(path)
entity = File(path, name='Hinkle horn honking holes', parent=project)
honking = Activity(name='Hinkle horn honking',
description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.')
honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/')
honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/')
# This doesn't set the ID of the Activity
entity = syn.store(entity, activity=honking)
# But this does
honking = syn.getProvenance(entity.id)
# Verify the Activity
assert honking['name'] == 'Hinkle horn honking'
assert len(honking['used']) == 2
assert honking['used'][0]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][0]['wasExecuted'] == False
assert honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482')
assert honking['used'][1]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][1]['wasExecuted'] == False
# Store another Entity with the same Activity
entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg',
name='Nettlebed Cave', parent=project, synapseStore=False)
entity = syn.store(entity, activity=honking)
# The Activities should match
honking2 = syn.getProvenance(entity)
assert honking['id'] == honking2['id']
def test_single_thread_upload():
synapseclient.config.single_threaded = True
try:
filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE * 2 + 1)
assert_is_not_none(multipart_upload(syn, filepath))
finally:
synapseclient.config.single_threaded = False
def test_round_trip():
fhid = None
filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE + 777771)
print('Made bogus file: ', filepath)
try:
fhid = multipart_upload(syn, filepath)
print('FileHandle: {fhid}'.format(fhid=fhid))
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk.update(syn._downloadFileEntity(junk, tmp_path))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_synStore_sftpIntegration():
"""Creates a File Entity on an sftp server and add the external url. """
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
try:
file = syn.store(File(filepath, parent=project))
file2 = syn.get(file)
assert file.externalURL == file2.externalURL and urlparse(file2.externalURL).scheme == "sftp"
tmpdir = tempfile.mkdtemp()
schedule_for_cleanup(tmpdir)
## test filename override
file2.fileNameOverride = "whats_new_in_baltimore.data"
file2 = syn.store(file2)
## TODO We haven't defined how filename override interacts with
## TODO previously cached files so, side-step that for now by
## TODO making sure the file is not in the cache!
syn.cache.remove(file2.dataFileHandleId, delete=True)
file3 = syn.get(file, downloadLocation=tmpdir)
assert os.path.basename(file3.path) == file2.fileNameOverride
## test that we got an MD5 à la SYNPY-185
assert_is_not_none(file3.md5)
fh = syn._getFileHandle(file3.dataFileHandleId)
assert_is_not_none(fh["contentMd5"])
assert_equals(file3.md5, fh["contentMd5"])
finally:
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_store_isRestricted_flag():
# Store a file with access requirements
path = utils.make_bogus_binary_file()
schedule_for_cleanup(path)
entity = File(path, name='Secret human data', parent=project)
# We don't want to spam ACT with test emails
with patch('synapseclient.client.Synapse._createAccessRequirementIfNone') as intercepted:
entity = syn.store(entity, isRestricted=True)
assert intercepted.called
def test_store_isRestricted_flag():
# Store a file with access requirements
path = utils.make_bogus_binary_file()
schedule_for_cleanup(path)
entity = File(path, name='Secret human data', parent=project)
# We don't want to spam ACT with test emails
with patch('synapseclient.client.Synapse._createAccessRequirementIfNone'
) as intercepted:
entity = syn.store(entity, isRestricted=True)
assert intercepted.called
def test_synGet_sftpIntegration():
#Create file by uploading directly to sftp and creating entity from URL
serverURL='sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/'+str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)
print '\n\tMade bogus file: ', filepath
url = syn._sftpUploadFile(filepath, url=serverURL)
file = syn.store(File(path=url, parent=project, synapseStore=False))
print '\nDownloading file', os.getcwd(), filepath
junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
filecmp.cmp(filepath, junk.path)
def test_synGet_sftpIntegration():
# Create file by uploading directly to sftp and creating entity from URL
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_synGet_sftpIntegration/' + str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)
url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
file = syn.store(File(path=url, parent=project, synapseStore=False))
junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
filecmp.cmp(filepath, junk.path)
def test_synStore_sftpIntegration():
"""Creates a File Entity on an sftp server and add the external url. """
filepath = utils.make_bogus_binary_file(1*MB - 777771)
try:
file = syn.store(File(filepath, parent=project))
file2 = syn.get(file)
assert file.externalURL==file2.externalURL and urlparse.urlparse(file2.externalURL).scheme=='sftp'
finally:
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
def test_large_file_upload(file_to_upload_size=11 * utils.KB, filepath=None):
clean_up_file = False
try:
project = syn.store(
Project("File Upload Load Test " +
datetime.now().strftime("%Y-%m-%d %H%M%S%f")))
if filepath:
## keep a file around so we don't have to regenerate it.
if not os.path.exists(filepath):
filepath = utils.make_bogus_binary_file(file_to_upload_size,
filepath=filepath,
printprogress=True)
print('Made bogus file: ', filepath)
else:
## generate a temporary file and clean it up when we're done
clean_up_file = True
filepath = utils.make_bogus_binary_file(file_to_upload_size,
printprogress=True)
print('Made bogus file: ', filepath)
try:
junk = syn.store(File(filepath, parent=project))
fh = syn._getFileHandle(junk['dataFileHandleId'])
syn.printEntity(fh)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
finally:
try:
if 'filepath' in locals() and clean_up_file:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_synGet_sftpIntegration():
#Create file by uploading directly to sftp and creating entity from URL
serverURL = 'sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/' + str(
uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
print('\n\tMade bogus file: ', filepath)
url = syn._sftpUploadFile(filepath, url=serverURL)
file = syn.store(File(path=url, parent=project, synapseStore=False))
print('\nDownloading file', os.getcwd(), filepath)
junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
filecmp.cmp(filepath, junk.path)
def test_synGet_sftpIntegration():
#Create file by uploading directly to sftp and creating entity from URL
serverURL='sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/'+str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)
username, password = syn._getUserCredentials(serverURL)
url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
file = syn.store(File(path=url, parent=project, synapseStore=False))
junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
filecmp.cmp(filepath, junk.path)
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
filepath = utils.make_bogus_binary_file(n=1 * MB)
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
for i in range(1, nchunks(filepath, chunksize=64 * 1024) + 1):
out.write(get_chunk(filepath, i, chunksize=64 * 1024))
assert filecmp.cmp(filepath, out.name)
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_synStore_sftpIntegration():
"""Creates a File Entity on an sftp server and add the external url. """
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
try:
file = syn.store(File(filepath, parent=project))
file2 = syn.get(file)
assert file.externalURL == file2.externalURL and urlparse.urlparse(
file2.externalURL).scheme == 'sftp'
finally:
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
filepath = utils.make_bogus_binary_file(n=1*MB)
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
for i in range(1, nchunks(filepath, chunksize=64*1024)+1):
out.write(get_chunk(filepath, i, chunksize=64*1024))
assert filecmp.cmp(filepath, out.name)
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_utils_sftp_upload_and_download():
"""Tries to upload a file to an sftp file """
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_utils_sftp_upload_and_download/' + str(
uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
tempdir = tempfile.mkdtemp()
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)
try:
url = SFTPWrapper.upload_file(filepath,
url=serverURL,
username=username,
password=password)
# Get with a specified localpath
junk = SFTPWrapper.download_file(url,
tempdir,
username=username,
password=password)
filecmp.cmp(filepath, junk)
# Get without specifying path
junk2 = SFTPWrapper.download_file(url,
username=username,
password=password)
filecmp.cmp(filepath, junk2)
# Get with a specified localpath as file
junk3 = SFTPWrapper.download_file(url,
os.path.join(tempdir, 'bar.dat'),
username=username,
password=password)
filecmp.cmp(filepath, junk3)
finally:
try:
if 'junk' in locals():
os.remove(junk)
if 'junk2' in locals():
os.remove(junk2)
if 'junk3' in locals():
os.remove(junk3)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
try:
shutil.rmtree(tempdir)
except Exception:
print(traceback.format_exc())
def test_store_with_force_version_flag():
project = create_project()
filepath = utils.make_bogus_binary_file()
bogus1 = File(filepath, name='Bogus Test File', parent=project)
# Expect to get version 1 back
bogus1 = syn.store(bogus1, forceVersion=False)
assert bogus1.versionNumber == 1
# Re-store the same thing and don't up the version
bogus2 = syn.store(bogus1, forceVersion=False)
assert bogus1.versionNumber == 1
# Create a different file with the same name and parent
new_filepath = utils.make_bogus_binary_file()
bogus2.path = new_filepath
# Expected behavior is that a new version of the first File will be created
bogus2 = syn.store(bogus2, forceVersion=False)
assert bogus2.id == bogus1.id
assert bogus2.versionNumber == 2
assert not filecmp.cmp(bogus2.path, filepath)
def test_randomly_failing_parts():
FAILURE_RATE = 1.0 / 3.0
fhid = None
multipart_upload_module.MIN_PART_SIZE = 5 * MB
multipart_upload_module.MAX_RETRIES = 20
filepath = utils.make_bogus_binary_file(
multipart_upload_module.MIN_PART_SIZE * 2 + 777771)
print('Made bogus file: ', filepath)
normal_put_chunk = None
def _put_chunk_or_fail_randomly(url, chunk, verbose=False):
if random.random() < FAILURE_RATE:
raise IOError("Ooops! Artificial upload failure for testing.")
else:
return normal_put_chunk(url, chunk, verbose)
## Mock _put_chunk to fail randomly
normal_put_chunk = multipart_upload_module._put_chunk
multipart_upload_module._put_chunk = _put_chunk_or_fail_randomly
try:
fhid = multipart_upload(syn, filepath)
print('FileHandle: {fhid}'.format(fhid=fhid))
# Download the file and compare it with the original
junk = File(parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity',
tmp_path)
assert filecmp.cmp(filepath, junk.path)
finally:
## Un-mock _put_chunk
if normal_put_chunk:
multipart_upload_module._put_chunk = normal_put_chunk
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_pool_provider_is_used_in__multipart_upload():
mocked_get_chunk_function = MagicMock(side_effect=[1, 2, 3, 4])
file_size = 1 * MB
filepath = make_bogus_binary_file(n=file_size)
md5 = md5_for_file(filepath).hexdigest()
status = {'partsState': {}, 'uploadId': {}, 'state': 'COMPLETED'}
pool = MagicMock()
with patch.object(syn, "restPOST", return_value=status),\
patch.object(pool_provider, "get_pool", return_value=pool) as mock_provider:
_multipart_upload(syn, filepath, "application/octet-stream",
mocked_get_chunk_function, md5, file_size)
mock_provider.assert_called()
pool.map.assert_called()
def test_store_with_force_version_flag():
project = create_project()
filepath = utils.make_bogus_binary_file()
bogus1 = File(filepath, name='Bogus Test File', parent=project)
# Expect to get version 1 back
bogus1 = syn.store(bogus1, forceVersion=False)
assert bogus1.versionNumber == 1
# Re-store the same thing and don't up the version
bogus2 = syn.store(bogus1, forceVersion=False)
assert bogus1.versionNumber == 1
# Create a different file with the same name and parent
new_filepath = utils.make_bogus_binary_file()
bogus2.path = new_filepath
# Expected behavior is that a new version of the first File will be created
bogus2 = syn.store(bogus2, forceVersion=False)
assert bogus2.id == bogus1.id
assert bogus2.versionNumber == 2
assert not filecmp.cmp(bogus2.path, filepath)
def test_pool_provider_is_used_in__multipart_upload():
mocked_get_chunk_function = MagicMock(side_effect=[1, 2, 3, 4])
file_size = 1*MB
filepath = make_bogus_binary_file(n=file_size)
md5 = md5_for_file(filepath).hexdigest()
status = {'partsState': {},
'uploadId': {},
'state': 'COMPLETED'}
pool = MagicMock()
with patch.object(syn, "restPOST", return_value=status),\
patch.object(pool_provider, "get_pool", return_value=pool) as mock_provider:
_multipart_upload(syn, filepath, "application/octet-stream", mocked_get_chunk_function, md5, file_size)
mock_provider.assert_called()
pool.map.assert_called()
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
filepath = utils.make_bogus_binary_file()
with open(filepath, 'rb') as f, tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
for chunk in utils.chunks(f):
buff = chunk.read(4*KB)
while buff:
out.write(buff)
buff = chunk.read(4*KB)
assert filecmp.cmp(filepath, out.name)
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_synGet_sftpIntegration():
# Create file by uploading directly to sftp and creating entity from URL
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_synGet_sftpIntegration/' + str(
uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)
url = SFTPWrapper.upload_file(filepath,
url=serverURL,
username=username,
password=password)
file = syn.store(File(path=url, parent=project, synapseStore=False))
junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True)
filecmp.cmp(filepath, junk.path)
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
file_size = 1 * MB
filepath = make_bogus_binary_file(n=file_size)
chunksize = 64 * 1024
nchunks = int(math.ceil(float(file_size) / chunksize))
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
for i in range(1, nchunks + 1):
out.write(get_file_chunk(filepath, i, chunksize))
assert_true(filecmp.cmp(filepath, out.name))
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_upload_speed(uploadSize=60 + 777771, threadCount=5):
import time
fh = None
filepath = utils.make_bogus_binary_file(uploadSize*MB)
try:
t0 = time.time()
fh = syn._uploadToFileHandleService(filepath, threadCount=threadCount)
dt = time.time()-t0
finally:
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
if fh:
syn._deleteFileHandle(fh)
return dt
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
file_size = 1*MB
filepath = make_bogus_binary_file(n=file_size)
chunksize = 64*1024
nchunks = int(math.ceil(float(file_size) / chunksize))
with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out:
for i in range(1, nchunks+1):
out.write(get_file_chunk(filepath, i, chunksize))
assert_true(filecmp.cmp(filepath, out.name))
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_randomly_failing_parts():
FAILURE_RATE = 1.0/3.0
fhid = None
multipart_upload_module.MIN_PART_SIZE = 5*MB
multipart_upload_module.MAX_RETRIES = 20
filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE*2 + 777771)
print('Made bogus file: ', filepath)
normal_put_chunk = None
def _put_chunk_or_fail_randomly(url, chunk, verbose=False):
if random.random() < FAILURE_RATE:
raise IOError("Ooops! Artificial upload failure for testing.")
else:
return normal_put_chunk(url, chunk, verbose)
## Mock _put_chunk to fail randomly
normal_put_chunk = multipart_upload_module._put_chunk
multipart_upload_module._put_chunk = _put_chunk_or_fail_randomly
try:
fhid = multipart_upload(syn, filepath)
print('FileHandle: {fhid}'.format(fhid=fhid))
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk.update(syn._downloadFileEntity(junk, tmp_path))
assert filecmp.cmp(filepath, junk.path)
finally:
## Un-mock _put_chunk
if normal_put_chunk:
multipart_upload_module._put_chunk = normal_put_chunk
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_get_with_downloadLocation_and_ifcollision():
project = create_project()
# Make the file to get then delete it
filepath = utils.make_bogus_binary_file()
bogus = File(filepath, name='Bogus Test File', parent=project)
bogus = syn.store(bogus)
os.remove(filepath)
# Compare stuff to this one
normalBogus = syn.get(bogus)
# Download to the temp folder, should be the same
otherBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath))
assert otherBogus.id == normalBogus.id
assert filecmp.cmp(otherBogus.path, normalBogus.path)
# Invalidate the downloaded file's timestamps
os.utime(otherBogus.path, (0, 0))
badtimestamps = os.path.getmtime(otherBogus.path)
# Download again, should change the modification time
overwriteBogus = syn.get(bogus,
downloadLocation=os.path.dirname(filepath),
ifcollision="overwrite.local")
overwriteModTime = os.path.getmtime(overwriteBogus.path)
assert badtimestamps != overwriteModTime
# Download again, should not change the modification time
otherBogus = syn.get(bogus,
downloadLocation=os.path.dirname(filepath),
ifcollision="keep.local")
assert overwriteModTime == os.path.getmtime(otherBogus.path)
# Invalidate the timestamps again
os.utime(otherBogus.path, (0, 0))
badtimestamps = os.path.getmtime(otherBogus.path)
# Download once more, but rename
renamedBogus = syn.get(bogus,
downloadLocation=os.path.dirname(filepath),
ifcollision="keep.both")
assert otherBogus.path != renamedBogus.path
assert filecmp.cmp(otherBogus.path, renamedBogus.path)
os.remove(otherBogus.path)
os.remove(renamedBogus.path)
def manually_check_retry_on_key_does_not_exist():
## This is a manual test -- don't know how to automate this one.
## To run: nosetests -vs tests/integration/test_chunked_upload.py:manually_check_retry_on_key_does_not_exist
## We're testing the retrying of key-does-not-exist errors from S3.
## Expected behavior: Retries several times, getting a error message:
## 'The specified key does not exist.', then fails with a stack trace.
i = 1
filepath = utils.make_bogus_binary_file(6*MB, verbose=True)
try:
token = syn._createChunkedFileUploadToken(filepath, 'application/octet-stream')
chunkRequest, url = syn._createChunkedFileUploadChunkURL(i, token)
## never upload the chunk, so we will get an error 'The specified key does not exist.'
chunkResult = syn._addChunkToFile(chunkRequest, verbose=True)
finally:
os.remove(filepath)
def manually_check_retry_on_key_does_not_exist():
## This is a manual test -- don't know how to automate this one.
## To run: nosetests -vs tests/integration/test_chunked_upload.py:manually_check_retry_on_key_does_not_exist
## We're testing the retrying of key-does-not-exist errors from S3.
## Expected behavior: Retries several times, getting a error message:
## 'The specified key does not exist.', then fails with a stack trace.
i = 1
filepath = utils.make_bogus_binary_file(6*MB, verbose=True)
try:
token = syn._createChunkedFileUploadToken(filepath, 'application/octet-stream')
chunkRequest, url = syn._createChunkedFileUploadChunkURL(i, token)
## never upload the chunk, so we will get an error 'The specified key does not exist.'
chunkResult = syn._addChunkToFile(chunkRequest, verbose=True)
finally:
os.remove(filepath)
def test_upload_speed(uploadSize=60 + 777771, threadCount=5):
import time
fh = None
filepath = utils.make_bogus_binary_file(uploadSize*MB)
print('Made bogus file: ', filepath)
try:
t0 = time.time()
fh = syn._uploadToFileHandleService(filepath, threadCount=threadCount)
dt = time.time()-t0
finally:
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
if fh:
syn._deleteFileHandle(fh)
return dt
def test_chunks():
# Read a file in chunks, write the chunks out, and compare to the original
try:
filepath = utils.make_bogus_binary_file()
with open(filepath,
'rb') as f, tempfile.NamedTemporaryFile(mode='wb',
delete=False) as out:
for chunk in utils.chunks(f):
buff = chunk.read(4 * KB)
while buff:
out.write(buff)
buff = chunk.read(4 * KB)
assert filecmp.cmp(filepath, out.name)
finally:
if 'filepath' in locals() and filepath:
os.remove(filepath)
if 'out' in locals() and out:
os.remove(out.name)
def test_utils_sftp_upload_and_download():
"""Tries to upload a file to an sftp file """
serverURL = "sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/" + str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
tempdir = tempfile.mkdtemp()
try:
print("\n\tMade bogus file: ", filepath)
url = syn._sftpUploadFile(filepath, url=serverURL)
print("\tStored URL:", url)
print("\tDownloading")
# Get with a specified localpath
junk = syn._sftpDownloadFile(url, tempdir)
print("\tComparing:", junk, filepath)
filecmp.cmp(filepath, junk)
# Get without specifying path
print("\tDownloading")
junk2 = syn._sftpDownloadFile(url)
print("\tComparing:", junk2, filepath)
filecmp.cmp(filepath, junk2)
# Get with a specified localpath as file
print("\tDownloading")
junk3 = syn._sftpDownloadFile(url, os.path.join(tempdir, "bar.dat"))
print("\tComparing:", junk3, filepath)
filecmp.cmp(filepath, junk3)
finally:
try:
if "junk" in locals():
os.remove(junk)
if "junk2" in locals():
os.remove(junk2)
if "junk3" in locals():
os.remove(junk3)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
try:
shutil.rmtree(tempdir)
except Exception:
print(traceback.format_exc())
def test_utils_sftp_upload_and_download():
"""Tries to upload a file to an sftp file """
serverURL = 'sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/' + str(
uuid.uuid1())
filepath = utils.make_bogus_binary_file(1 * MB - 777771)
tempdir = tempfile.mkdtemp()
try:
print('\n\tMade bogus file: ', filepath)
url = syn._sftpUploadFile(filepath, url=serverURL)
print('\tStored URL:', url)
print('\tDownloading', )
#Get with a specified localpath
junk = syn._sftpDownloadFile(url, tempdir)
print('\tComparing:', junk, filepath)
filecmp.cmp(filepath, junk)
#Get without specifying path
print('\tDownloading', )
junk2 = syn._sftpDownloadFile(url)
print('\tComparing:', junk2, filepath)
filecmp.cmp(filepath, junk2)
#Get with a specified localpath as file
print('\tDownloading', )
junk3 = syn._sftpDownloadFile(url, os.path.join(tempdir, 'bar.dat'))
print('\tComparing:', junk3, filepath)
filecmp.cmp(filepath, junk3)
finally:
try:
if 'junk' in locals(): os.remove(junk)
if 'junk2' in locals(): os.remove(junk2)
if 'junk3' in locals(): os.remove(junk3)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
try:
shutil.rmtree(tempdir)
except Exception:
print(traceback.format_exc())
def test_get_with_downloadLocation_and_ifcollision():
project = create_project()
# Make the file to get then delete it
filepath = utils.make_bogus_binary_file()
bogus = File(filepath, name='Bogus Test File', parent=project)
bogus = syn.store(bogus)
os.remove(filepath)
# Compare stuff to this one
normalBogus = syn.get(bogus)
# Download to the temp folder, should be the same
otherBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath))
assert otherBogus.id == normalBogus.id
assert filecmp.cmp(otherBogus.path, normalBogus.path)
# Invalidate the downloaded file's timestamps
os.utime(otherBogus.path, (0, 0))
badtimestamps = os.path.getmtime(otherBogus.path)
# Download again, should change the modification time
overwriteBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath), ifcollision="overwrite.local")
overwriteModTime = os.path.getmtime(overwriteBogus.path)
assert badtimestamps != overwriteModTime
# Download again, should not change the modification time
otherBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath), ifcollision="keep.local")
assert overwriteModTime == os.path.getmtime(otherBogus.path)
# Invalidate the timestamps again
os.utime(otherBogus.path, (0, 0))
badtimestamps = os.path.getmtime(otherBogus.path)
# Download once more, but rename
renamedBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath), ifcollision="keep.both")
assert otherBogus.path != renamedBogus.path
assert filecmp.cmp(otherBogus.path, renamedBogus.path)
os.remove(otherBogus.path)
os.remove(renamedBogus.path)
def test_store_activity():
# Create a File and an Activity
path = utils.make_bogus_binary_file()
schedule_for_cleanup(path)
entity = File(path, name='Hinkle horn honking holes', parent=project)
honking = Activity(
name='Hinkle horn honking',
description=
'Nettlebed Cave is a limestone cave located on the South Island of New Zealand.'
)
honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/')
honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/')
# This doesn't set the ID of the Activity
entity = syn.store(entity, activity=honking)
# But this does
honking = syn.getProvenance(entity.id)
# Verify the Activity
assert honking['name'] == 'Hinkle horn honking'
assert len(honking['used']) == 2
assert honking['used'][0][
'concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][0]['wasExecuted'] == False
assert honking['used'][0]['url'].startswith(
'http://www.flickr.com/photos/bevanbfree/3482')
assert honking['used'][1][
'concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL'
assert honking['used'][1]['wasExecuted'] == False
# Store another Entity with the same Activity
entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg',
name='Nettlebed Cave',
parent=project,
synapseStore=False)
entity = syn.store(entity, activity=honking)
# The Activities should match
honking2 = syn.getProvenance(entity)
assert honking['id'] == honking2['id']
def test_synStore_sftpIntegration():
"""Creates a File Entity on an sftp server and add the external url. """
filepath = utils.make_bogus_binary_file(1*MB - 777771)
try:
file = syn.store(File(filepath, parent=project))
file2 = syn.get(file)
assert file.externalURL==file2.externalURL and urlparse(file2.externalURL).scheme=='sftp'
tmpdir = tempfile.mkdtemp()
schedule_for_cleanup(tmpdir)
## test that we got an MD5 à la SYNPY-185
assert_is_not_none(file2.md5)
fh = syn._getFileHandle(file2.dataFileHandleId)
assert_is_not_none(fh['contentMd5'])
assert_equals(file2.md5, fh['contentMd5'])
finally:
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_synStore_sftpIntegration():
"""Creates a File Entity on an sftp server and add the external url. """
filepath = utils.make_bogus_binary_file(1*MB - 777771)
try:
file = syn.store(File(filepath, parent=project))
file2 = syn.get(file)
assert_equals(file.externalURL, file2.externalURL)
assert_equals(urlparse(file2.externalURL).scheme, 'sftp')
tmpdir = tempfile.mkdtemp()
schedule_for_cleanup(tmpdir)
# test that we got an MD5 à la SYNPY-185
assert_is_not_none(file2.md5)
fh = syn._getFileHandle(file2.dataFileHandleId)
assert_is_not_none(fh['contentMd5'])
assert_equals(file2.md5, fh['contentMd5'])
finally:
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_utils_sftp_upload_and_download():
"""Tries to upload a file to an sftp file """
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_utils_sftp_upload_and_download/' + str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)
tempdir = tempfile.mkdtemp()
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)
try:
url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
# Get with a specified localpath
junk = SFTPWrapper.download_file(url, tempdir, username=username, password=password)
filecmp.cmp(filepath, junk)
# Get without specifying path
junk2 = SFTPWrapper.download_file(url, username=username, password=password)
filecmp.cmp(filepath, junk2)
# Get with a specified localpath as file
junk3 = SFTPWrapper.download_file(url, os.path.join(tempdir, 'bar.dat'), username=username, password=password)
filecmp.cmp(filepath, junk3)
finally:
try:
if 'junk' in locals():
os.remove(junk)
if 'junk2' in locals():
os.remove(junk2)
if 'junk3' in locals():
os.remove(junk3)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
try:
shutil.rmtree(tempdir)
except Exception:
print(traceback.format_exc())
def test_upload_chunk__expired_url():
upload_parts = [{
'uploadPresignedUrl': 'https://www.fake.url/fake/news',
'partNumber': 420
}, {
'uploadPresignedUrl': 'https://www.google.com',
'partNumber': 421
}, {
'uploadPresignedUrl': 'https://rito.pls/',
'partNumber': 422
}, {
'uploadPresignedUrl': 'https://never.lucky.gg',
'partNumber': 423
}]
with patch.object(multipart_upload, "_put_chunk",
side_effect=SynapseHTTPError("useless message",response=MagicMock(status_code=403))) as mocked_put_chunk, \
patch.object(warnings, "warn") as mocked_warn, \
patch.object(multipart_upload, '_start_multipart_upload',
return_value=DictObject({'partsState': '0', 'uploadId': '1', 'state': 'COMPLETED', 'resultFileHandleId': '1'})), \
patch.object(multipart_upload, "_get_presigned_urls", return_value=upload_parts):
file_size = 1 * MB
filepath = make_bogus_binary_file(n=file_size)
try:
multipart_upload.multipart_upload(syn, filepath)
finally:
if os.path.isfile(filepath):
os.remove(filepath)
mocked_warn.assert_called_with(
'The pre-signed upload URL has expired. Restarting upload...\n')
# 4 URLs, 7 retries.
assert mocked_warn.call_count == 28
# assert _put_chunk was called at least once
assert_greater_equal(len(mocked_put_chunk.call_args_list), 1)
def test_download_file_false():
RENAME_SUFFIX = 'blah'
# Upload a file
filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(filepath)
schedule_for_cleanup(filepath + RENAME_SUFFIX)
file = File(filepath, name='SYNR 619', parent=project)
file = syn.store(file)
# Now hide the file from the cache and download with downloadFile=False
os.rename(filepath, filepath + RENAME_SUFFIX)
file = syn.get(file.id, downloadFile=False)
# Change something and reupload the file's metadata
file.name = "Only change the name, not the file"
reupload = syn.store(file)
assert reupload.path is None, "Path field should be null: %s" % reupload.path
# This should still get the correct file
reupload = syn.get(reupload.id)
assert filecmp.cmp(filepath + RENAME_SUFFIX, reupload.path)
assert reupload.name == file.name
def test_download_file_false():
RENAME_SUFFIX = 'blah'
# Upload a file
filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(filepath)
schedule_for_cleanup(filepath + RENAME_SUFFIX)
file = File(filepath, name='SYNR 619', parent=project)
file = syn.store(file)
# Now hide the file from the cache and download with downloadFile=False
os.rename(filepath, filepath + RENAME_SUFFIX)
file = syn.get(file.id, downloadFile=False)
# Change something and reupload the file's metadata
file.name = "Only change the name, not the file"
reupload = syn.store(file)
assert reupload.path is None, "Path field should be null: %s" % reupload.path
# This should still get the correct file
reupload = syn.get(reupload.id)
assert filecmp.cmp(filepath + RENAME_SUFFIX, reupload.path)
assert reupload.name == file.name
def test_round_trip():
fh = None
filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
print 'Made bogus file: ', filepath
try:
fh = syn._chunkedUploadFile(filepath, verbose=False)
print '=' * 60
print 'FileHandle:'
syn.printEntity(fh)
print 'creating project and file'
project = create_project()
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
print 'downloading file'
junk.update(syn._downloadFileEntity(junk, filepath))
print 'comparing files'
assert filecmp.cmp(filepath, junk.path)
print 'ok!'
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception as ex:
print ex
try:
os.remove(filepath)
except Exception as ex:
print ex
if fh:
print 'Deleting fileHandle', fh['id']
syn._deleteFileHandle(fh)
def test_utils_sftp_upload_and_download():
"""Tries to upload a file to an sftp file """
serverURL='sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/'+str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)
tempdir = tempfile.mkdtemp()
username, password = syn._getUserCredentials(serverURL)
try:
url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
#Get with a specified localpath
junk = SFTPWrapper.download_file(url, tempdir, username=username, password=password)
filecmp.cmp(filepath, junk)
#Get without specifying path
junk2 = SFTPWrapper.download_file(url, username=username, password=password)
filecmp.cmp(filepath, junk2)
#Get with a specified localpath as file
junk3 = SFTPWrapper.download_file(url, os.path.join(tempdir, 'bar.dat'), username=username, password=password)
filecmp.cmp(filepath, junk3)
finally:
try:
if 'junk' in locals(): os.remove(junk)
if 'junk2' in locals(): os.remove(junk2)
if 'junk3' in locals(): os.remove(junk3)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
try:
shutil.rmtree(tempdir)
except Exception:
print(traceback.format_exc())
def test_store_with_flags():
# -- CreateOrUpdate flag for Projects --
# If we store a project with the same name, it should become an update
projUpdate = Project(project.name)
projUpdate.updatedThing = 'Yep, sho\'nuf it\'s updated!'
projUpdate = syn.store(projUpdate, createOrUpdate=True)
assert project.id == projUpdate.id
assert projUpdate.updatedThing == ['Yep, sho\'nuf it\'s updated!']
# Store a File
filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(filepath)
origBogus = File(filepath, name='Bogus Test File', parent=project)
origBogus = syn.store(origBogus, createOrUpdate=True)
assert origBogus.versionNumber == 1
# Modify existing annotations by createOrUpdate
del projUpdate['parentId']
del projUpdate['id']
projUpdate.updatedThing = 'Updated again'
projUpdate.addedThing = 'Something new'
projUpdate = syn.store(projUpdate, createOrUpdate=True)
assert project.id == projUpdate.id
assert projUpdate.updatedThing == ['Updated again']
# -- ForceVersion flag --
# Re-store the same thing and don't up the version
mutaBogus = syn.store(origBogus, forceVersion=False)
assert mutaBogus.versionNumber == 1
# Re-store again, essentially the same condition
mutaBogus = syn.store(mutaBogus, createOrUpdate=True, forceVersion=False)
assert mutaBogus.versionNumber == 1
# And again, but up the version this time
mutaBogus = syn.store(mutaBogus, forceVersion=True)
assert mutaBogus.versionNumber == 2
# -- CreateOrUpdate flag for files --
# Store a different file with the same name and parent
# Expected behavior is that a new version of the first File will be created
new_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(new_filepath)
mutaBogus.path = new_filepath
mutaBogus = syn.store(mutaBogus, createOrUpdate=True)
assert mutaBogus.id == origBogus.id
assert mutaBogus.versionNumber == 3
assert not filecmp.cmp(mutaBogus.path, filepath)
# Make doubly sure the File was uploaded
checkBogus = syn.get(mutaBogus.id)
assert checkBogus.id == origBogus.id
assert checkBogus.versionNumber == 3
assert filecmp.cmp(mutaBogus.path, checkBogus.path)
# Create yet another file with the same name and parent
# Expected behavior is raising an exception with a 409 error
newer_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(newer_filepath)
badBogus = File(newer_filepath, name='Bogus Test File', parent=project)
assert_raises(SynapseHTTPError, syn.store, badBogus, createOrUpdate=False)
# -- Storing after syn.get(..., downloadFile=False) --
ephemeralBogus = syn.get(mutaBogus, downloadFile=False)
ephemeralBogus.description = 'Snorklewacker'
ephemeralBogus.shoe_size = 11.5
ephemeralBogus = syn.store(ephemeralBogus)
ephemeralBogus = syn.get(ephemeralBogus, downloadFile=False)
assert ephemeralBogus.description == 'Snorklewacker'
assert ephemeralBogus.shoe_size == [11.5]
def test_store_with_flags():
# -- CreateOrUpdate flag for Projects --
# If we store a project with the same name, it should become an update
projUpdate = Project(project.name)
projUpdate.updatedThing = 'Yep, sho\'nuf it\'s updated!'
projUpdate = syn.store(projUpdate, createOrUpdate=True)
assert project.id == projUpdate.id
assert projUpdate.updatedThing == ['Yep, sho\'nuf it\'s updated!']
# Store a File
filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(filepath)
origBogus = File(filepath, name='Bogus Test File', parent=project)
origBogus = syn.store(origBogus, createOrUpdate=True)
assert origBogus.versionNumber == 1
# -- ForceVersion flag --
# Re-store the same thing and don't up the version
mutaBogus = syn.store(origBogus, forceVersion=False)
assert mutaBogus.versionNumber == 1
# Re-store again, essentially the same condition
mutaBogus = syn.store(mutaBogus, createOrUpdate=True, forceVersion=False)
assert mutaBogus.versionNumber == 1
# And again, but up the version this time
mutaBogus = syn.store(mutaBogus, forceVersion=True)
assert mutaBogus.versionNumber == 2
# -- CreateOrUpdate flag for files --
# Store a different file with the same name and parent
# Expected behavior is that a new version of the first File will be created
new_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(new_filepath)
mutaBogus.path = new_filepath
mutaBogus = syn.store(mutaBogus, createOrUpdate=True)
assert mutaBogus.id == origBogus.id
assert mutaBogus.versionNumber == 3
assert not filecmp.cmp(mutaBogus.path, filepath)
# Make doubly sure the File was uploaded
checkBogus = syn.get(mutaBogus.id)
assert checkBogus.id == origBogus.id
assert checkBogus.versionNumber == 3
assert filecmp.cmp(mutaBogus.path, checkBogus.path)
# Create yet another file with the same name and parent
# Expected behavior is raising an exception with a 409 error
newer_filepath = utils.make_bogus_binary_file()
schedule_for_cleanup(newer_filepath)
badBogus = File(newer_filepath, name='Bogus Test File', parent=project)
assert_raises(SynapseHTTPError, syn.store, badBogus, createOrUpdate=False)
# -- Storing after syn.get(..., downloadFile=False) --
ephemeralBogus = syn.get(mutaBogus, downloadFile=False)
ephemeralBogus.description = 'Snorklewacker'
ephemeralBogus.shoe_size = 11.5
ephemeralBogus = syn.store(ephemeralBogus)
ephemeralBogus = syn.get(ephemeralBogus, downloadFile=False)
assert ephemeralBogus.description == 'Snorklewacker'
assert ephemeralBogus.shoe_size == [11.5]