def test_check_file_ready_pull(self):
"""test that checks to see if a file is ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready, True)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a bad response
ready = cart_utils.check_file_size_needed("", test_file,
test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, "error")
def test_check_file_not_ready_pull(self):
"""test that checks to see if a file is not ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(0L, 24L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "tape",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
resp_bad = """{
"bytes_per_level": "(0L, 33L, 33L, 0L, 0L)",
"ctime": "1444938177",
"file": "/myemsl-dev/bundle/file.2",
"filesize": "33",
"message": "File was found",
"mtime": "1444938133"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready, False)
#now check for an error by sending a bad response
ready = cart_utils.check_file_ready_pull('', test_file, test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, 'error')
#now check for an error with storage media
ready = cart_utils.check_file_ready_pull(resp_bad, test_file,
test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, 'error')
def test_check_file_ready_pull(self):
"""test that checks to see if a file is ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready['enough_space'], True)
self.assertNotEqual(test_file.status, 'error')
def status_file_task(file_id):
"""Status a file from the archive. If ready then pull the file"""
Cart.database_connect()
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
#check to see if file is available to pull from archive interface
archive_request = ArchiveRequests()
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to status file with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
ready = cart_utils.check_file_ready_pull(response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0. No coverage on recall since it just calls the method again
if ready < 0:
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
elif not ready: # pragma: no cover
Cart.database_close()
status_file_task.delay(file_id)
return
#ready so try to pull file
pull_file.delay(file_id, ready['filepath'], ready['modtime'], False)
def pull_file(file_id, record_error):
"""Pull a file from the archive """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, "staging", False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
#stage the file on the archive. True on success, False on fail
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to stage with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
Cart.database_close()
return
#check to see if file is available to pull from archive interface
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to status file with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
response = 'False'
size_needed = cart_utils.check_file_size_needed(response, cart_file, mycart)
mod_time = cart_utils.check_file_modified_time(response, cart_file, mycart)
#Return from function if the values couldnt be parsed (-1 return)
if size_needed < 0 or mod_time < 0:
Cart.database_close()
return
ready_to_pull = cart_utils.check_file_ready_pull(
response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0
if ready_to_pull < 0:
Cart.database_close()
return
elif not ready_to_pull:
pull_file.delay(file_id, False)
Cart.database_close()
return
#create the path the file will be downloaded to
abs_cart_file_path = os.path.join(
VOLUME_PATH, str(mycart.id), mycart.cart_uid, cart_file.bundle_path)
path_created = cart_utils.create_download_path(
cart_file, mycart, abs_cart_file_path)
#Check size here and make sure enough space is available.
enough_space = cart_utils.check_space_requirements(
cart_file, mycart, size_needed, True)
if path_created and enough_space:
try:
archive_request.pull_file(cart_file.file_name, abs_cart_file_path)
cart_utils.set_file_status(cart_file, mycart, "staged", False)
Cart.database_close()
except requests.exceptions.RequestException as ex:
#if request fails...try a second time, if that fails write error
if record_error:
error_msg = "Failed to pull with error: " + str(ex)
cart_utils.set_file_status(
cart_file, mycart, "error", error_msg)
Cart.database_close()
else:
pull_file.delay(file_id, True)
Cart.database_close()
os.utime(abs_cart_file_path, (int(float(mod_time)), int(float(mod_time))))