def test_bad_status(self, mock_stage_file, mock_status_file):
"""test the bad status of a archive file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.side_effect = requests.exceptions.RequestException(
mock.Mock(status=500), 'Error')
file_id = test_file.id
status_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'error')
def test_prep_bundle_staging(self):
"""test getting bundle ready with a file in staging state"""
data = json.loads(cart_json_helper())
file_ids = data['fileids']
Cart.database_connect()
mycart = Cart(cart_uid=343, status='staging')
mycart.save()
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
get_files_locally(mycart.id)
for cart_file in File.select().where(File.cart == mycart.id):
cart_file.status = 'staging'
cart_file.save()
cart_utils.prepare_bundle(
mycart.id) #hitting more coverage, set files to staged
for cart_file in File.select().where(File.cart == mycart.id):
cart_file.status = 'staged'
cart_file.save()
cart_utils.prepare_bundle(mycart.id) #call again after file update
status = mycart.status
cartid = mycart.id
while status == 'staging':
mycart = Cart.get(Cart.id == cartid)
status = mycart.status
Cart.database_close()
self.assertEqual(status, 'ready')
def update_cart_files(cls, cart, file_ids):
"""Update the files associated to a cart"""
with Cart.atomic():
for f_id in file_ids:
filepath = cls.fix_absolute_path(f_id["path"])
File.create(
cart=cart, file_name=f_id["id"], bundle_path=filepath)
cart.updated_date = datetime.datetime.now()
cart.save()
def test_status_cart_deleted(self):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
deleted_date='2017-05-03 00:00:00')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
file_id = test_file.id
status_file_task(file_id)
self.assertEqual(True, True)
def test_bad_pull_value(self, mock_pull):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_pull.side_effect = ValueError('Error with hash pulling file')
file_id = test_file.id
pull_file(file_id, '/tmp/1/1.txt', '9999999', False)
cart_after = Cart.get(Cart.id == test_cart.id)
status = cart_after.status
self.assertEqual(status, 'error')
def prepare_bundle(cls, cartid):
"""Checks to see if all the files are staged locally
before calling the bundling action. If not will call
itself to continue the waiting process
"""
Cart.database_connect()
bundle_flag = True
for c_file in File.select().where(File.cart == cartid):
if c_file.status == 'error':
#error pulling file so set cart error and return
try:
mycart = Cart.get(Cart.id == cartid)
mycart.status = 'error'
mycart.error = 'Failed to pull file(s)'
mycart.updated_date = datetime.datetime.now()
mycart.save()
Cart.database_close()
return
except DoesNotExist: # pragma: no cover
#case if record no longer exists
#creating this case in unit testing requires deletion and creation
#occuring nearly simultaneously, as such cant create unit test
Cart.database_close()
return
elif c_file.status != 'staged':
bundle_flag = False
cls.tar_files(cartid, bundle_flag)
Cart.database_close()
def test_check_file_size_needed(self):
"""test that the file size returned from the archive is parsed right"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
file_size = cart_utils.check_file_size_needed(
response, test_file, test_cart)
self.assertEqual(file_size, 24)
self.assertNotEqual(test_file.status, 'error')
#now check for an error by sending a bad response
file_size = cart_utils.check_file_size_needed(
'', test_file, test_cart)
self.assertEqual(file_size, -1)
self.assertEqual(test_file.status, 'error')
def get_files_locally(cartid):
"""Pull the files to the local system from the backend """
#tell each file to be pulled
Cart.database_connect()
for cart_file in File.select().where(File.cart == cartid):
stage_file_task.delay(cart_file.id)
Cart.database_close()
def test_check_file_size_needed(self):
"""test that the file size returned from the archive is parsed right"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
file_size = cart_utils.check_file_size_needed(response, test_file,
test_cart)
self.assertEqual(file_size, 24)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a bad response
file_size = cart_utils.check_file_size_needed("", test_file,
test_cart)
self.assertEqual(file_size, -1)
self.assertEqual(test_file.status, "error")
def stage_file_task(file_id):
"""Stage the file from the archive, then call status """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, 'staging', False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to stage with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
#successful stage so move on to status
Cart.database_close()
status_file_task.delay(file_id)
def prepare_bundle(cartid):
"""Checks to see if all the files are staged locally
before calling the bundling action. If not will call
itself to continue the waiting process
"""
Cart.database_connect()
bundle_flag = True
for c_file in File.select().where(File.cart == cartid):
if c_file.status == "error":
#error pulling file so set cart error and return
try:
mycart = Cart.get(Cart.id == cartid)
mycart.status = "error"
mycart.error = "Failed to pull file(s)"
mycart.updated_date = datetime.datetime.now()
mycart.save()
Cart.database_close()
return
except DoesNotExist:
#case if record no longer exists
Cart.database_close()
return
elif c_file.status != "staged":
bundle_flag = False
if not bundle_flag:
#if not ready to bundle recall this task
prepare_bundle.delay(cartid)
else:
#All files are local...try to tar
tar_files.delay(cartid)
Cart.database_close()
def get_files_locally(cartid):
"""Pull the files to the local system from the backend """
#tell each file to be pulled
Cart.database_connect()
for cart_file in File.select().where(File.cart == cartid):
pull_file.delay(cart_file.id, False)
Cart.database_close()
def update_cart_files(cls, cart, file_ids):
"""Update the files associated to a cart"""
with Cart.atomic():
for f_id in file_ids:
try:
filepath = cls.fix_absolute_path(f_id['path'])
hashtype = f_id['hashtype']
hashval = f_id['hashsum']
File.create(cart=cart,
file_name=f_id['id'],
bundle_path=filepath,
hash_type=hashtype,
hash_value=hashval)
cart.updated_date = datetime.datetime.now()
cart.save()
except (NameError, KeyError) as ex:
return ex #return error so that the cart can be updated
return None
def test_set_file_status(self):
"""test that trys to set a specific files status"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging",
bundle_path="/tmp/1/")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
cart_utils.set_file_status(test_file, test_cart, "error", "fake error")
self.assertEqual(test_file.status, "error")
self.assertEqual(test_file.error, "fake error")
def test_bad_create_download_path(self, mock_create_bundle):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
mock_create_bundle.side_effect = OSError(mock.Mock(), 'Error')
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
self.assertEqual(success, False)
def test_bad_pull(self, mock_stage_file, mock_status_file, mock_pull_file,
mock_utime):
"""test the bad pull of a archive file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.return_value = """{
"bytes_per_level": "(10L, 0L)",
"ctime": "1444629567",
"file": "1.txt",
"file_storage_media": "disk",
"filesize": "10",
"message": "File was found",
"mtime": "1444937154"
}"""
mock_pull_file.side_effect = requests.exceptions.RequestException(
mock.Mock(status=500), 'Error')
mock_utime.return_value = True
file_id = test_file.id
stage_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'error')
def test_bad_ready_to_pull(self, mock_stage_file, mock_status_file,
mock_check_file):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.return_value = """{
"bytes_per_level": "(10L, 0L)",
"ctime": "1444629567",
"file": "1.txt",
"file_storage_media": "disk",
"filesize": "10",
"message": "File was found",
"mtime": "1444937154"
}"""
mock_check_file.return_value = -1
file_id = test_file.id
stage_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'staging')
def test_check_space_bad_path(self, mock_disk_usage):
"""test that the error when a bad path"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
mock_disk_usage.side_effect = psutil.Error(mock.Mock())
rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
False)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, 'error')
def test_create_download_path(self):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
directory_name = os.path.dirname(test_file.bundle_path)
self.assertEqual(success, True)
self.assertEqual(os.path.isdir(directory_name), True)
os.rmdir(directory_name)
self.assertEqual(os.path.isdir(directory_name), False)
def test_set_file_status(self):
"""test that trys to set a specific files status"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
cart_utils.set_file_status(test_file, test_cart, 'error',
'fake error')
self.assertEqual(test_file.status, 'error')
self.assertEqual(test_file.error, 'fake error')
def test_create_download_path(self):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
directory_name = os.path.dirname(test_file.bundle_path)
self.assertEqual(success, True)
self.assertEqual(os.path.isdir(directory_name), True)
os.rmdir(directory_name)
self.assertEqual(os.path.isdir(directory_name), False)
def test_status_details_fail(self):
"""test status details fail"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
#say file is way to big
retval = cart_utils.check_status_details(
test_cart, test_file, 99999999999999999999999999999, 1)
self.assertEqual(retval, -1)
def test_check_space_requirements(self):
"""test that there is enough space on the volume for the file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
rtn = cart_utils.check_space_requirements(test_file, test_cart,
10, False)
self.assertEqual(rtn, True)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a way to large size needed number
rtn = cart_utils.check_space_requirements(test_file, test_cart,
9999999999999999999999, False)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, "error")
def test_cart_deleted_date(self):
"""test getting bundle ready with a file in staging state"""
data = json.loads(cart_json_helper())
file_ids = data['fileids']
Cart.database_connect()
mycart = Cart(cart_uid=444, status='staging')
mycart.save()
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
get_files_locally(mycart.id)
mycart.status = 'deleted'
mycart.deleted_date = datetime.datetime.now()
mycart.save()
status = mycart.status
for cart_file in File.select().where(File.cart == mycart.id):
pull_file(cart_file.id, '/tmp/some/Path', '1111', False)
Cart.database_close()
self.assertEqual(status, 'deleted')
def test_check_space_requirements(self):
"""test that there is enough space on the volume for the file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
False)
self.assertEqual(rtn, True)
self.assertNotEqual(test_file.status, 'error')
#now check for an error by sending a way to large size needed number
rtn = cart_utils.check_space_requirements(test_file, test_cart,
9999999999999999999999,
True)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, 'error')
def pull_file(file_id, filepath, modtime, record_error):
"""Pull a file from the archive """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
try:
archive_request.pull_file(cart_file.file_name, filepath,
cart_file.hash_value, cart_file.hash_type)
cart_utils.set_file_status(cart_file, mycart, 'staged', False)
os.utime(filepath, (int(float(modtime)), int(float(modtime))))
Cart.database_close()
except requests.exceptions.RequestException as ex:
#if request fails...try a second time, if that fails write error
if record_error:
error_msg = 'Failed to pull with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
else:
pull_file.delay(file_id, filepath, modtime, True)
Cart.database_close()
except ValueError as ex:
error_msg = 'Failed to pull with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
cart_utils.prepare_bundle(mycart.id)
def test_prep_bundle_error(self):
"""test getting bundle ready with a file in error state"""
data = json.loads(cart_json_helper())
file_ids = data['fileids']
Cart.database_connect()
mycart = Cart(cart_uid=343, status='staging')
mycart.save()
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
get_files_locally(mycart.id)
for cart_file in File.select().where(File.cart == mycart.id):
cart_file.status = 'error'
cart_file.save()
cart_utils.prepare_bundle(mycart.id)
status = mycart.status
cartid = mycart.id
while status == 'staging':
mycart = Cart.get(Cart.id == cartid)
status = mycart.status
Cart.database_close()
self.assertEqual(status, 'error')
def test_check_file_not_ready_pull(self):
"""test that checks to see if a file is not ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(0L, 24L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "tape",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
resp_bad = """{
"bytes_per_level": "(0L, 33L, 33L, 0L, 0L)",
"ctime": "1444938177",
"file": "/myemsl-dev/bundle/file.2",
"filesize": "33",
"message": "File was found",
"mtime": "1444938133"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready, False)
#now check for an error by sending a bad response
ready = cart_utils.check_file_ready_pull('', test_file, test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, 'error')
#now check for an error with storage media
ready = cart_utils.check_file_ready_pull(resp_bad, test_file,
test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, 'error')
def test_check_file_ready_pull(self):
"""test that checks to see if a file is ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready['enough_space'], True)
self.assertNotEqual(test_file.status, 'error')
def status_file_task(file_id):
"""Status a file from the archive. If ready then pull the file"""
Cart.database_connect()
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
#check to see if file is available to pull from archive interface
archive_request = ArchiveRequests()
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to status file with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
ready = cart_utils.check_file_ready_pull(response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0. No coverage on recall since it just calls the method again
if ready < 0:
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
elif not ready: # pragma: no cover
Cart.database_close()
status_file_task.delay(file_id)
return
#ready so try to pull file
pull_file.delay(file_id, ready['filepath'], ready['modtime'], False)
def pull_file(file_id, record_error):
"""Pull a file from the archive """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, "staging", False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
#stage the file on the archive. True on success, False on fail
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to stage with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
Cart.database_close()
return
#check to see if file is available to pull from archive interface
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to status file with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
response = 'False'
size_needed = cart_utils.check_file_size_needed(response, cart_file, mycart)
mod_time = cart_utils.check_file_modified_time(response, cart_file, mycart)
#Return from function if the values couldnt be parsed (-1 return)
if size_needed < 0 or mod_time < 0:
Cart.database_close()
return
ready_to_pull = cart_utils.check_file_ready_pull(
response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0
if ready_to_pull < 0:
Cart.database_close()
return
elif not ready_to_pull:
pull_file.delay(file_id, False)
Cart.database_close()
return
#create the path the file will be downloaded to
abs_cart_file_path = os.path.join(
VOLUME_PATH, str(mycart.id), mycart.cart_uid, cart_file.bundle_path)
path_created = cart_utils.create_download_path(
cart_file, mycart, abs_cart_file_path)
#Check size here and make sure enough space is available.
enough_space = cart_utils.check_space_requirements(
cart_file, mycart, size_needed, True)
if path_created and enough_space:
try:
archive_request.pull_file(cart_file.file_name, abs_cart_file_path)
cart_utils.set_file_status(cart_file, mycart, "staged", False)
Cart.database_close()
except requests.exceptions.RequestException as ex:
#if request fails...try a second time, if that fails write error
if record_error:
error_msg = "Failed to pull with error: " + str(ex)
cart_utils.set_file_status(
cart_file, mycart, "error", error_msg)
Cart.database_close()
else:
pull_file.delay(file_id, True)
Cart.database_close()
os.utime(abs_cart_file_path, (int(float(mod_time)), int(float(mod_time))))
def test_cart_orm_db_setup(self):
"""call database_setup"""
with test_database(self.sqlite_db, (Cart, File), create_tables=False):
database_setup(2)
self.assertTrue(Cart.table_exists())
self.assertTrue(File.table_exists())
def test_cart_orm_db_setup(self):
"""call database_setup"""
with test_database(self.sqlite_db, (Cart, File), create_tables=False):
database_setup(2)
self.assertTrue(Cart.table_exists())
self.assertTrue(File.table_exists())
