def test_getWithEntityBundle(*mocks):
mocks = [item for item in mocks]
is_loco_mock = mocks.pop()
cache_location_guess_mock = mocks.pop()
download_file_mock = mocks.pop()
# -- Change downloadLocation but do not download more than once --
is_loco_mock.return_value = False
bundle = {"entity" : {"name": "anonymous",
"dataFileHandleId": "-1337",
"concreteType": "org.sagebionetworks.repo.model.FileEntity",
"parentId": "syn12345"},
"fileHandles": [{u'concreteType': u'org.sagebionetworks.repo.model.file.S3FileHandle',
u'fileName': u'anonymous',
u'contentMd5': u'1698d26000d60816caab15169efcd23a',
u'id': u'-1337'}],
"annotations": {}}
# Make the cache point to some temporary location
cacheDir = synapseclient.cache.determine_cache_directory(bundle['entity'])
# Pretend that the file is downloaded by the first call to syn._downloadFileEntity
# The temp file should be added to the cache by the first syn._getWithEntityBundle() call
f, cachedFile = tempfile.mkstemp()
os.close(f)
defaultLocation = os.path.join(cacheDir, bundle['entity']['name'])
cache_location_guess_mock.return_value = (cacheDir, defaultLocation, cachedFile)
# Make sure the Entity is updated with the cached file path
def _downloadFileEntity(entity, path, submission):
# We're disabling the download, but the given path should be within the cache
assert path == defaultLocation
return {"path": cachedFile}
download_file_mock.side_effect = _downloadFileEntity
# Make sure the cache does not already exist
cacheMap = os.path.join(cacheDir, '.cacheMap')
if os.path.exists(cacheMap):
os.remove(cacheMap)
syn._getWithEntityBundle(entityBundle=bundle, entity=None, downloadLocation=cacheDir, ifcollision="overwrite.local")
syn._getWithEntityBundle(entityBundle=bundle, entity=None, ifcollision="overwrite.local")
e = syn._getWithEntityBundle(entityBundle=bundle, entity=None, downloadLocation=cacheDir, ifcollision="overwrite.local")
assert download_file_mock.call_count == 1
assert e.name == bundle["entity"]["name"]
assert e.parentId == bundle["entity"]["parentId"]
assert e.cacheDir == cacheDir
assert bundle['entity']['name'] in e.files
assert e.path == os.path.join(cacheDir, bundle["entity"]["name"])
## test preservation of local state
url = 'http://foo.com/secretstuff.txt'
e = File(name='anonymous', parentId="syn12345", synapseStore=False, externalURL=url)
e.local_state({'zap':'pow'})
e = syn._getWithEntityBundle(entityBundle=bundle, entity=e)
assert e.local_state()['zap'] == 'pow'
assert e.synapseStore == False
assert e.externalURL == url
def test_getWithEntityBundle(download_file_mock):
## Note: one thing that remains unexplained is why the previous version of
## this test worked if you had a .cacheMap file of the form:
## {"/Users/chris/.synapseCache/663/-1337/anonymous": "2014-09-15T22:54:57.000Z",
## "/var/folders/ym/p7cr7rrx4z7fw36sxv04pqh00000gq/T/tmpJ4nz8U": "2014-09-15T23:27:25.000Z"}
## ...but failed if you didn't.
## TODO: Uncomment failing asserts after SYNR-790 and SYNR-697 are fixed
bundle = {
'entity': {
'id': 'syn10101',
'name': 'anonymous',
'dataFileHandleId': '-1337',
'concreteType': 'org.sagebionetworks.repo.model.FileEntity',
'parentId': 'syn12345'},
'fileHandles': [{
'concreteType': 'org.sagebionetworks.repo.model.file.S3FileHandle',
'fileName': 'anonymous',
'contentType': 'application/flapdoodle',
'contentMd5': '1698d26000d60816caab15169efcd23a',
'id': '-1337'}],
'annotations': {}}
fileHandle = bundle['fileHandles'][0]['id']
cacheDir = syn.cache.get_cache_dir(fileHandle)
print "cacheDir=", cacheDir
# Make sure the .cacheMap file does not already exist
cacheMap = os.path.join(cacheDir, '.cacheMap')
if os.path.exists(cacheMap):
print "removing cacheMap file: ", cacheMap
os.remove(cacheMap)
def _downloadFileEntity(entity, path, submission):
print "mock downloading file to:", path
## touch file at path
with open(path, 'a'):
os.utime(path, None)
dest_dir, filename = os.path.split(path)
return {"path": path,
"files": [filename],
"cacheDir": dest_dir}
download_file_mock.side_effect = _downloadFileEntity
# 1. ----------------------------------------------------------------------
# download file to an alternate location
temp_dir1 = tempfile.mkdtemp()
print "temp_dir1=", temp_dir1
e = syn._getWithEntityBundle(entityBundle=bundle,
downloadLocation=temp_dir1,
ifcollision="overwrite.local")
print e
assert e.name == bundle["entity"]["name"]
assert e.parentId == bundle["entity"]["parentId"]
assert e.cacheDir == temp_dir1
assert bundle["fileHandles"][0]["fileName"] in e.files
assert e.path == os.path.join(temp_dir1, bundle["fileHandles"][0]["fileName"])
# 2. ----------------------------------------------------------------------
# get without specifying downloadLocation
e = syn._getWithEntityBundle(entityBundle=bundle, ifcollision="overwrite.local")
print e
assert e.name == bundle["entity"]["name"]
assert e.parentId == bundle["entity"]["parentId"]
assert bundle["fileHandles"][0]["fileName"] in e.files
# 3. ----------------------------------------------------------------------
# download to another location
temp_dir2 = tempfile.mkdtemp()
assert temp_dir2 != temp_dir1
e = syn._getWithEntityBundle(entityBundle=bundle,
downloadLocation=temp_dir2,
ifcollision="overwrite.local")
print "temp_dir2=", temp_dir2
print e
assert_in(bundle["fileHandles"][0]["fileName"], e.files)
assert e.path is not None
assert_equal( os.path.dirname(e.path), temp_dir2 )
# 4. ----------------------------------------------------------------------
## test preservation of local state
url = 'http://foo.com/secretstuff.txt'
e = File(name='anonymous', parentId="syn12345", synapseStore=False, externalURL=url)
e.local_state({'zap':'pow'})
e = syn._getWithEntityBundle(entityBundle=bundle, entity=e)
assert e.local_state()['zap'] == 'pow'
assert e.synapseStore == False
assert e.externalURL == url
def test_getWithEntityBundle(download_file_mock,
get_file_URL_and_metadata_mock):
## Note: one thing that remains unexplained is why the previous version of
## this test worked if you had a .cacheMap file of the form:
## {"/Users/chris/.synapseCache/663/-1337/anonymous": "2014-09-15T22:54:57.000Z",
## "/var/folders/ym/p7cr7rrx4z7fw36sxv04pqh00000gq/T/tmpJ4nz8U": "2014-09-15T23:27:25.000Z"}
## ...but failed if you didn't.
bundle = {
'entity': {
'id': 'syn10101',
'name': 'anonymous',
'dataFileHandleId': '-1337',
'concreteType': 'org.sagebionetworks.repo.model.FileEntity',
'parentId': 'syn12345'
},
'fileHandles': [{
'concreteType': 'org.sagebionetworks.repo.model.file.S3FileHandle',
'fileName': 'anonymous',
'contentType': 'application/flapdoodle',
'contentMd5': '1698d26000d60816caab15169efcd23a',
'id': '-1337'
}],
'annotations': {}
}
fileHandle = bundle['fileHandles'][0]['id']
cacheDir = syn.cache.get_cache_dir(fileHandle)
print("cacheDir=", cacheDir)
# Make sure the .cacheMap file does not already exist
cacheMap = os.path.join(cacheDir, '.cacheMap')
if os.path.exists(cacheMap):
print("removing cacheMap file: ", cacheMap)
os.remove(cacheMap)
def _downloadFileHandle(fileHandleId,
objectId,
objectType,
path,
retries=5):
print("mock downloading file to:", path)
## touch file at path
with open(path, 'a'):
os.utime(path, None)
dest_dir, filename = os.path.split(path)
syn.cache.add(fileHandle, path)
return path
def _getFileHandleDownload(fileHandleId,
objectId,
objectType='FileHandle'):
print("getting metadata for:", fileHandleId)
return {
'fileHandle': bundle['fileHandles'][0],
'fileHandleId': fileHandleId,
'preSignedURL': 'http://example.com'
}
download_file_mock.side_effect = _downloadFileHandle
get_file_URL_and_metadata_mock.side_effect = _getFileHandleDownload
# 1. ----------------------------------------------------------------------
# download file to an alternate location
temp_dir1 = tempfile.mkdtemp()
print("temp_dir1=", temp_dir1)
e = syn._getWithEntityBundle(entityBundle=bundle,
downloadLocation=temp_dir1,
ifcollision="overwrite.local")
print(e)
assert_equal(e.name, bundle["entity"]["name"])
assert_equal(e.parentId, bundle["entity"]["parentId"])
assert_equal(
utils.normalize_path(os.path.abspath(os.path.dirname(e.path))),
utils.normalize_path(temp_dir1))
assert_equal(bundle["fileHandles"][0]["fileName"],
os.path.basename(e.path))
assert_equal(
utils.normalize_path(os.path.abspath(e.path)),
utils.normalize_path(
os.path.join(temp_dir1, bundle["fileHandles"][0]["fileName"])))
# 2. ----------------------------------------------------------------------
# get without specifying downloadLocation
e = syn._getWithEntityBundle(entityBundle=bundle,
ifcollision="overwrite.local")
print(e)
assert_equal(e.name, bundle["entity"]["name"])
assert_equal(e.parentId, bundle["entity"]["parentId"])
assert bundle["fileHandles"][0]["fileName"] in e.files
# 3. ----------------------------------------------------------------------
# download to another location
temp_dir2 = tempfile.mkdtemp()
assert temp_dir2 != temp_dir1
e = syn._getWithEntityBundle(entityBundle=bundle,
downloadLocation=temp_dir2,
ifcollision="overwrite.local")
print("temp_dir2=", temp_dir2)
print(e)
assert_in(bundle["fileHandles"][0]["fileName"], e.files)
assert e.path is not None
assert utils.equal_paths(os.path.dirname(e.path), temp_dir2)
# 4. ----------------------------------------------------------------------
## test preservation of local state
url = 'http://foo.com/secretstuff.txt'
#need to create a bundle with externalURL
externalURLBundle = dict(bundle)
externalURLBundle['fileHandles'][0]['externalURL'] = url
e = File(name='anonymous',
parentId="syn12345",
synapseStore=False,
externalURL=url)
e.local_state({'zap': 'pow'})
e = syn._getWithEntityBundle(entityBundle=externalURLBundle, entity=e)
assert_equal(e.local_state()['zap'], 'pow')
assert_equal(e.synapseStore, False)
assert_equal(e.externalURL, url)
def test_getWithEntityBundle(self, download_file_mock, get_file_URL_and_metadata_mock):
# Note: one thing that remains unexplained is why the previous version of
# this test worked if you had a .cacheMap file of the form:
# {"/Users/chris/.synapseCache/663/-1337/anonymous": "2014-09-15T22:54:57.000Z",
# "/var/folders/ym/p7cr7rrx4z7fw36sxv04pqh00000gq/T/tmpJ4nz8U": "2014-09-15T23:27:25.000Z"}
# ...but failed if you didn't.
bundle = {
'entity': {
'id': 'syn10101',
'name': 'anonymous',
'dataFileHandleId': '-1337',
'concreteType': 'org.sagebionetworks.repo.model.FileEntity',
'parentId': 'syn12345'},
'fileHandles': [{
'concreteType': 'org.sagebionetworks.repo.model.file.S3FileHandle',
'fileName': 'anonymous',
'contentType': 'application/flapdoodle',
'contentMd5': '1698d26000d60816caab15169efcd23a',
'id': '-1337'}],
'annotations': {}}
fileHandle = bundle['fileHandles'][0]['id']
cacheDir = syn.cache.get_cache_dir(fileHandle)
# Make sure the .cacheMap file does not already exist
cacheMap = os.path.join(cacheDir, '.cacheMap')
if os.path.exists(cacheMap):
os.remove(cacheMap)
def _downloadFileHandle(fileHandleId, objectId, objectType, path, retries=5):
# touch file at path
with open(path, 'a'):
os.utime(path, None)
os.path.split(path)
syn.cache.add(fileHandle, path)
return path
def _getFileHandleDownload(fileHandleId, objectId, objectType='FileHandle'):
return {'fileHandle': bundle['fileHandles'][0], 'fileHandleId': fileHandleId,
'preSignedURL': 'http://example.com'}
download_file_mock.side_effect = _downloadFileHandle
get_file_URL_and_metadata_mock.side_effect = _getFileHandleDownload
# 1. ----------------------------------------------------------------------
# download file to an alternate location
temp_dir1 = tempfile.mkdtemp()
e = syn._getWithEntityBundle(entityBundle=bundle,
downloadLocation=temp_dir1,
ifcollision="overwrite.local")
assert_equal(e.name, bundle["entity"]["name"])
assert_equal(e.parentId, bundle["entity"]["parentId"])
assert_equal(utils.normalize_path(os.path.abspath(os.path.dirname(e.path))), utils.normalize_path(temp_dir1))
assert_equal(bundle["fileHandles"][0]["fileName"], os.path.basename(e.path))
assert_equal(utils.normalize_path(os.path.abspath(e.path)),
utils.normalize_path(os.path.join(temp_dir1, bundle["fileHandles"][0]["fileName"])))
# 2. ----------------------------------------------------------------------
# get without specifying downloadLocation
e = syn._getWithEntityBundle(entityBundle=bundle, ifcollision="overwrite.local")
assert_equal(e.name, bundle["entity"]["name"])
assert_equal(e.parentId, bundle["entity"]["parentId"])
assert_in(bundle["fileHandles"][0]["fileName"], e.files)
# 3. ----------------------------------------------------------------------
# download to another location
temp_dir2 = tempfile.mkdtemp()
assert_not_equals(temp_dir2, temp_dir1)
e = syn._getWithEntityBundle(entityBundle=bundle,
downloadLocation=temp_dir2,
ifcollision="overwrite.local")
assert_in(bundle["fileHandles"][0]["fileName"], e.files)
assert_is_not_none(e.path)
assert_true(utils.equal_paths(os.path.dirname(e.path), temp_dir2))
# 4. ----------------------------------------------------------------------
# test preservation of local state
url = 'http://foo.com/secretstuff.txt'
# need to create a bundle with externalURL
externalURLBundle = dict(bundle)
externalURLBundle['fileHandles'][0]['externalURL'] = url
e = File(name='anonymous', parentId="syn12345", synapseStore=False, externalURL=url)
e.local_state({'zap': 'pow'})
e = syn._getWithEntityBundle(entityBundle=externalURLBundle, entity=e)
assert_equal(e.local_state()['zap'], 'pow')
assert_equal(e.synapseStore, False)
assert_equal(e.externalURL, url)
def test_getWithEntityBundle(*mocks):
mocks = [item for item in mocks]
is_loco_mock = mocks.pop()
cache_location_guess_mock = mocks.pop()
download_file_mock = mocks.pop()
# -- Change downloadLocation but do not download more than once --
is_loco_mock.return_value = False
bundle = {
"entity": {
"name": "anonymous",
"dataFileHandleId": "-1337",
"concreteType": "org.sagebionetworks.repo.model.FileEntity",
"parentId": "syn12345"
},
"fileHandles": [{
u'concreteType':
u'org.sagebionetworks.repo.model.file.S3FileHandle',
u'fileName': u'anonymous',
u'contentMd5': u'1698d26000d60816caab15169efcd23a',
u'id': u'-1337'
}],
"annotations": {}
}
# Make the cache point to some temporary location
cacheDir = synapseclient.cache.determine_cache_directory(bundle['entity'])
# Pretend that the file is downloaded by the first call to syn._downloadFileEntity
# The temp file should be added to the cache by the first syn._getWithEntityBundle() call
f, cachedFile = tempfile.mkstemp()
os.close(f)
defaultLocation = os.path.join(cacheDir, bundle['entity']['name'])
cache_location_guess_mock.return_value = (cacheDir, defaultLocation,
cachedFile)
# Make sure the Entity is updated with the cached file path
def _downloadFileEntity(entity, path, submission):
# We're disabling the download, but the given path should be within the cache
assert path == defaultLocation
return {"path": cachedFile}
download_file_mock.side_effect = _downloadFileEntity
# Make sure the cache does not already exist
cacheMap = os.path.join(cacheDir, '.cacheMap')
if os.path.exists(cacheMap):
os.remove(cacheMap)
syn._getWithEntityBundle(entityBundle=bundle,
entity=None,
downloadLocation=cacheDir,
ifcollision="overwrite.local")
syn._getWithEntityBundle(entityBundle=bundle,
entity=None,
ifcollision="overwrite.local")
e = syn._getWithEntityBundle(entityBundle=bundle,
entity=None,
downloadLocation=cacheDir,
ifcollision="overwrite.local")
assert download_file_mock.call_count == 1
assert e.name == bundle["entity"]["name"]
assert e.parentId == bundle["entity"]["parentId"]
assert e.cacheDir == cacheDir
assert bundle['entity']['name'] in e.files
assert e.path == os.path.join(cacheDir, bundle["entity"]["name"])
## test preservation of local state
url = 'http://foo.com/secretstuff.txt'
e = File(name='anonymous',
parentId="syn12345",
synapseStore=False,
externalURL=url)
e.local_state({'zap': 'pow'})
e = syn._getWithEntityBundle(entityBundle=bundle, entity=e)
assert e.local_state()['zap'] == 'pow'
assert e.synapseStore == False
assert e.externalURL == url
