def test_multipart_upload_big_string():
cities = ["Seattle", "Portland", "Vancouver", "Victoria",
"San Francisco", "Los Angeles", "New York",
"Oaxaca", "Cancún", "Curaçao", "जोधपुर",
"অসম", "ལྷ་ས།", "ཐིམ་ཕུ་", "دبي", "አዲስ አበባ",
"São Paulo", "Buenos Aires", "Cartagena",
"Amsterdam", "Venice", "Rome", "Dubrovnik",
"Sarajevo", "Madrid", "Barcelona", "Paris",
"Αθήνα", "Ρόδος", "København", "Zürich",
"金沢市", "서울", "แม่ฮ่องสอน", "Москва"]
text = "Places I wanna go:\n"
while len(text.encode('utf-8')) < multipart_upload_module.MIN_PART_SIZE:
text += ", ".join( random.choice(cities) for i in range(5000) ) + "\n"
fhid = multipart_upload_string(syn, text)
print('FileHandle: {fhid}'.format(fhid=fhid))
# Download the file and compare it with the original
junk = File("message.txt", parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk.update(syn._downloadFileEntity(junk, tmp_path))
with open(junk.path, encoding='utf-8') as f:
retrieved_text = f.read()
assert retrieved_text == text
def test_round_trip():
fhid = None
filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE + 777771)
print('Made bogus file: ', filepath)
try:
fhid = multipart_upload(syn, filepath)
print('FileHandle: {fhid}'.format(fhid=fhid))
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk.update(syn._downloadFileEntity(junk, tmp_path))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_round_trip():
fh = None
filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
print 'Made bogus file: ', filepath
try:
fh = syn._chunkedUploadFile(filepath, verbose=False)
# print 'FileHandle:'
# syn.printEntity(fh)
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
junk.update(syn._downloadFileEntity(junk, filepath))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print traceback.format_exc()
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
if fh:
# print 'Deleting fileHandle', fh['id']
syn._deleteFileHandle(fh)
def test_upload_string():
## This tests the utility that uploads a _string_ rather than
## a file on disk, to S3.
fh = None
content = "My dog has fleas.\n"
f = tempfile.NamedTemporaryFile(suffix=".txt", delete=False)
f.write(content)
f.close()
filepath=f.name
print 'Made bogus file: ', filepath
try:
fh = syn._uploadStringToFile(content)
# print 'FileHandle:'
# syn.printEntity(fh)
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
junk.update(syn._downloadFileEntity(junk, filepath))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print traceback.format_exc()
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
def test_upload_string():
## This tests the utility that uploads a _string_ rather than
## a file on disk, to S3.
fh = None
content = "My dog has fleas.\n"
f = tempfile.NamedTemporaryFile(suffix=".txt", delete=False)
f.write(content)
f.close()
filepath = f.name
print 'Made bogus file: ', filepath
try:
fh = syn._uploadStringToFile(content)
# print 'FileHandle:'
# syn.printEntity(fh)
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
junk.update(syn._downloadFileEntity(junk, filepath))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print traceback.format_exc()
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
def test_round_trip():
fh = None
filepath = utils.make_bogus_binary_file(6 * MB + 777771)
print 'Made bogus file: ', filepath
try:
fh = syn._chunkedUploadFile(filepath)
# print 'FileHandle:'
# syn.printEntity(fh)
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
junk.update(syn._downloadFileEntity(junk, filepath))
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print traceback.format_exc()
try:
os.remove(filepath)
except Exception:
print traceback.format_exc()
def test_randomly_failing_parts():
FAILURE_RATE = 1.0/3.0
fhid = None
multipart_upload_module.MIN_PART_SIZE = 5*MB
multipart_upload_module.MAX_RETRIES = 20
filepath = utils.make_bogus_binary_file(multipart_upload_module.MIN_PART_SIZE*2 + 777771)
print('Made bogus file: ', filepath)
normal_put_chunk = None
def _put_chunk_or_fail_randomly(url, chunk, verbose=False):
if random.random() < FAILURE_RATE:
raise IOError("Ooops! Artificial upload failure for testing.")
else:
return normal_put_chunk(url, chunk, verbose)
## Mock _put_chunk to fail randomly
normal_put_chunk = multipart_upload_module._put_chunk
multipart_upload_module._put_chunk = _put_chunk_or_fail_randomly
try:
fhid = multipart_upload(syn, filepath)
print('FileHandle: {fhid}'.format(fhid=fhid))
# Download the file and compare it with the original
junk = File(filepath, parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk.update(syn._downloadFileEntity(junk, tmp_path))
assert filecmp.cmp(filepath, junk.path)
finally:
## Un-mock _put_chunk
if normal_put_chunk:
multipart_upload_module._put_chunk = normal_put_chunk
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def test_round_trip():
fh = None
filepath = utils.make_bogus_binary_file(6*MB + 777771, verbose=True)
print 'Made bogus file: ', filepath
try:
fh = syn._chunkedUploadFile(filepath, verbose=False)
print '=' * 60
print 'FileHandle:'
syn.printEntity(fh)
print 'creating project and file'
project = create_project()
junk = File(filepath, parent=project, dataFileHandleId=fh['id'])
junk.properties.update(syn._createEntity(junk.properties))
print 'downloading file'
junk.update(syn._downloadFileEntity(junk, filepath))
print 'comparing files'
assert filecmp.cmp(filepath, junk.path)
print 'ok!'
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception as ex:
print ex
try:
os.remove(filepath)
except Exception as ex:
print ex
if fh:
print 'Deleting fileHandle', fh['id']
syn._deleteFileHandle(fh)
