def remove_cart(cls, uid):
"""Call when a DELETE request comes in. Verifies there is a cart
to delete then removes it
"""
deleted_flag = True
iterator = 0 #used to verify at least one cart deleted
Cart.database_connect()
carts = (Cart
.select()
.where(
(Cart.cart_uid == str(uid)) &
(Cart.deleted_date.is_null(True))))
for cart in carts:
iterator += 1
success = cls.delete_cart_bundle(cart)
if not success:
deleted_flag = False
Cart.database_close()
if deleted_flag and iterator > 0:
return "Cart Deleted Successfully"
elif deleted_flag:
return "Cart with uid: " + str(uid) + """
was previously deleted or no longer exists"""
else:
return "Error with deleting Cart"
def test_bad_pull_value(self, mock_pull):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_pull.side_effect = ValueError('Error with hash pulling file')
file_id = test_file.id
pull_file(file_id, '/tmp/1/1.txt', '9999999', False)
cart_after = Cart.get(Cart.id == test_cart.id)
status = cart_after.status
self.assertEqual(status, 'error')
def get_files_locally(cartid):
"""Pull the files to the local system from the backend """
#tell each file to be pulled
Cart.database_connect()
for cart_file in File.select().where(File.cart == cartid):
stage_file_task.delay(cart_file.id)
Cart.database_close()
def stage_file_task(file_id):
"""Stage the file from the archive, then call status """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, 'staging', False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to stage with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
#successful stage so move on to status
Cart.database_close()
status_file_task.delay(file_id)
def get_files_locally(cartid):
"""Pull the files to the local system from the backend """
#tell each file to be pulled
Cart.database_connect()
for cart_file in File.select().where(File.cart == cartid):
pull_file.delay(cart_file.id, False)
Cart.database_close()
def test_prep_bundle_staging(self):
"""test getting bundle ready with a file in staging state"""
data = json.loads(cart_json_helper())
file_ids = data['fileids']
Cart.database_connect()
mycart = Cart(cart_uid=343, status='staging')
mycart.save()
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
get_files_locally(mycart.id)
for cart_file in File.select().where(File.cart == mycart.id):
cart_file.status = 'staging'
cart_file.save()
cart_utils.prepare_bundle(
mycart.id) #hitting more coverage, set files to staged
for cart_file in File.select().where(File.cart == mycart.id):
cart_file.status = 'staged'
cart_file.save()
cart_utils.prepare_bundle(mycart.id) #call again after file update
status = mycart.status
cartid = mycart.id
while status == 'staging':
mycart = Cart.get(Cart.id == cartid)
status = mycart.status
Cart.database_close()
self.assertEqual(status, 'ready')
def prepare_bundle(cartid):
"""Checks to see if all the files are staged locally
before calling the bundling action. If not will call
itself to continue the waiting process
"""
Cart.database_connect()
bundle_flag = True
for c_file in File.select().where(File.cart == cartid):
if c_file.status == "error":
#error pulling file so set cart error and return
try:
mycart = Cart.get(Cart.id == cartid)
mycart.status = "error"
mycart.error = "Failed to pull file(s)"
mycart.updated_date = datetime.datetime.now()
mycart.save()
Cart.database_close()
return
except DoesNotExist:
#case if record no longer exists
Cart.database_close()
return
elif c_file.status != "staged":
bundle_flag = False
if not bundle_flag:
#if not ready to bundle recall this task
prepare_bundle.delay(cartid)
else:
#All files are local...try to tar
tar_files.delay(cartid)
Cart.database_close()
def prepare_bundle(cls, cartid):
"""Checks to see if all the files are staged locally
before calling the bundling action. If not will call
itself to continue the waiting process
"""
Cart.database_connect()
bundle_flag = True
for c_file in File.select().where(File.cart == cartid):
if c_file.status == 'error':
#error pulling file so set cart error and return
try:
mycart = Cart.get(Cart.id == cartid)
mycart.status = 'error'
mycart.error = 'Failed to pull file(s)'
mycart.updated_date = datetime.datetime.now()
mycart.save()
Cart.database_close()
return
except DoesNotExist: # pragma: no cover
#case if record no longer exists
#creating this case in unit testing requires deletion and creation
#occuring nearly simultaneously, as such cant create unit test
Cart.database_close()
return
elif c_file.status != 'staged':
bundle_flag = False
cls.tar_files(cartid, bundle_flag)
Cart.database_close()
def create_cart(file_ids, uid):
"""Create the cart or update previous"""
Cart.database_connect()
mycart = Cart(cart_uid=uid, status='staging')
mycart.save()
stage_files.delay(file_ids, mycart.id)
Cart.database_close()
def test_check_file_size_needed(self):
"""test that the file size returned from the archive is parsed right"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
file_size = cart_utils.check_file_size_needed(
response, test_file, test_cart)
self.assertEqual(file_size, 24)
self.assertNotEqual(test_file.status, 'error')
#now check for an error by sending a bad response
file_size = cart_utils.check_file_size_needed(
'', test_file, test_cart)
self.assertEqual(file_size, -1)
self.assertEqual(test_file.status, 'error')
def stage_files(file_ids, mycart_id):
"""Tell the files to be staged on the backend system """
Cart.database_connect()
#with update or new, need to add in files
mycart = Cart.get(Cart.id == mycart_id)
cart_utils = Cartutils()
file_id_error = cart_utils.update_cart_files(mycart, file_ids)
if not file_id_error:
get_files_locally.delay(mycart.id)
else:
mycart.status = 'error'
mycart.error = 'Error parsing file Ids with error: ' + str(
file_id_error)
mycart.updated_date = datetime.datetime.now()
mycart.save()
Cart.database_close()
def test_check_file_size_needed(self):
"""test that the file size returned from the archive is parsed right"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
file_size = cart_utils.check_file_size_needed(response, test_file,
test_cart)
self.assertEqual(file_size, 24)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a bad response
file_size = cart_utils.check_file_size_needed("", test_file,
test_cart)
self.assertEqual(file_size, -1)
self.assertEqual(test_file.status, "error")
def test_status_cart_deleted(self):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
deleted_date='2017-05-03 00:00:00')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
file_id = test_file.id
status_file_task(file_id)
self.assertEqual(True, True)
def test_bad_status(self, mock_stage_file, mock_status_file):
"""test the bad status of a archive file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.side_effect = requests.exceptions.RequestException(
mock.Mock(status=500), 'Error')
file_id = test_file.id
status_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'error')
def test_cart_int_get(self):
"""Testing the cart interface get method"""
saved_state = {}
def start_response(*args):
"""stub for start_response to do some checking"""
self.assertEqual(args[0], '200 OK')
self.assertEqual(args[1][0][0], 'Content-Type')
self.assertEqual(args[1][0][1], 'application/octet-stream')
def file_wrapper(rfd, blksize):
"""stub for file_wrapper to do some checking"""
self.assertEqual(blksize, 1 << 20)
self.assertEqual(type(rfd), file)
saved_state['fd'] = rfd
return iter(lambda: rfd.read(blksize), '')
env = {
'PATH_INFO': '/123',
'wsgi.file_wrapper': file_wrapper,
'QUERY_STRING': 'filename=my_file.tar'
}
with test_database(self.sqlite_db, (Cart, File)):
sample_cart = Cart()
sample_cart.cart_uid = 123
sample_cart.bundle_path = mkdtemp()
sample_cart.status = 'ready'
sample_cart.save(force_insert=True)
self.sqlite_db.close()
cgen = CartGenerator()
tarfile_read = cgen.get(env, start_response)
for buf in tarfile_read:
self.assertTrue(len(buf) > 0)
saved_state['fd'].close()
def test_read_cart_ioerrror(self):
"""Testing the cart interface get against not ready cart"""
def start_response(*args):
"""stub for start_response to do some checking"""
self.assertEqual(args[0], '404 Not Found')
self.assertEqual(args[1][0][0], 'Content-Type')
self.assertEqual(args[1][0][1], 'application/json')
env = {'PATH_INFO': '/123', 'QUERY_STRING': ''}
with test_database(self.sqlite_db, (Cart, File)):
sample_cart = Cart()
sample_cart.cart_uid = 123
sample_cart.bundle_path = mkdtemp()
sample_cart.status = 'ready'
sample_cart.save(force_insert=True)
self.sqlite_db.close()
cgen = CartGenerator()
orig_fork = os.fork
def bad_fork():
"""The get method trys to fork so replacing it with a method that fails.
"""
raise IOError('failed to fork')
os.fork = bad_fork
data = cgen.get(env, start_response)
self.assertEqual(
loads(data)['message'], 'The cart bundle does not exist')
os.fork = orig_fork
def update_cart_files(cls, cart, file_ids):
"""Update the files associated to a cart"""
with Cart.atomic():
for f_id in file_ids:
filepath = cls.fix_absolute_path(f_id["path"])
File.create(
cart=cart, file_name=f_id["id"], bundle_path=filepath)
cart.updated_date = datetime.datetime.now()
cart.save()
def available_cart(uid):
"""Checks if the asked for cart tar is available
returns the path to tar if yes, false if not. None if no cart"""
Cart.database_connect()
cart_bundle_path = False
try:
mycart = (Cart.select().where((Cart.cart_uid == str(uid)) & (
Cart.deleted_date.is_null(True))).order_by(
Cart.creation_date.desc()).get())
except DoesNotExist:
#case if no record exists yet in database
mycart = None
cart_bundle_path = None
if mycart and mycart.status == 'ready':
cart_bundle_path = mycart.bundle_path
Cart.database_close()
return cart_bundle_path
def cart_status(uid):
"""Get the status of a specified cart"""
Cart.database_connect()
status = None
try:
mycart = (Cart.select().where((Cart.cart_uid == str(uid)) & (
Cart.deleted_date.is_null(True))).order_by(
Cart.creation_date.desc()).get())
except DoesNotExist:
#case if no record exists yet in database
mycart = None
status = ['error', 'No cart with uid ' + uid + ' found']
if mycart:
#send the status and any available error text
status = [mycart.status, mycart.error]
Cart.database_close()
return status
def remove_cart(cls, uid):
"""Call when a DELETE request comes in. Verifies there is a cart
to delete then removes it
"""
deleted_flag = True
iterator = 0 #used to verify at least one cart deleted
Cart.database_connect()
carts = (Cart.select().where((Cart.cart_uid == str(uid))
& (Cart.deleted_date.is_null(True))))
for cart in carts:
iterator += 1
success = cls.delete_cart_bundle(cart)
if not success:
deleted_flag = False
Cart.database_close()
if deleted_flag and iterator > 0:
return 'Cart Deleted Successfully'
elif deleted_flag:
return False #already deleted
return None #unknown error
def test_lru_cart_delete(self):
"""test that trys to delete a cart"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_cart2 = Cart.create(cart_uid='2',
status='staging',
bundle_path='/tmp/2/',
updated_date=1)
cart_utils = Cartutils()
os.makedirs(test_cart2.bundle_path, 0o777)
retval = cart_utils.lru_cart_delete(test_cart)
self.assertEqual(retval, True)
test_c2 = Cart.get(Cart.id == test_cart2.id)
self.assertEqual(test_c2.status, 'deleted')
#also hit error block when nothing to delete
retval = cart_utils.lru_cart_delete(test_cart)
self.assertEqual(retval, False)
def available_cart(uid):
"""Checks if the asked for cart tar is available
returns the path to tar if yes, false if not"""
Cart.database_connect()
cart_bundle_path = False
try:
mycart = (Cart
.select()
.where(
(Cart.cart_uid == str(uid)) &
(Cart.deleted_date.is_null(True)))
.order_by(Cart.creation_date.desc())
.get())
except DoesNotExist:
#case if no record exists yet in database
mycart = None
if mycart and mycart.status == "ready":
cart_bundle_path = mycart.bundle_path
Cart.database_close()
return cart_bundle_path
def test_bad_create_download_path(self, mock_create_bundle):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
mock_create_bundle.side_effect = OSError(mock.Mock(), 'Error')
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
self.assertEqual(success, False)
def test_set_file_status(self):
"""test that trys to set a specific files status"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging",
bundle_path="/tmp/1/")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
cart_utils.set_file_status(test_file, test_cart, "error", "fake error")
self.assertEqual(test_file.status, "error")
self.assertEqual(test_file.error, "fake error")
def test_delete_cart_bundle(self):
"""test that trys to delete a cart bundle"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging",
bundle_path="/tmp/1/")
cart_utils = Cartutils()
os.makedirs(test_cart.bundle_path, 0777)
deleted = cart_utils.delete_cart_bundle(test_cart)
self.assertEqual(test_cart.status, "deleted")
self.assertEqual(deleted, True)
self.assertEqual(os.path.isdir(test_cart.bundle_path), False)
def test_stage_files(self):
"""test getting bundle files ready"""
data = json.loads(cart_json_helper())
file_ids = data['fileids']
Cart.database_connect()
mycart = Cart(cart_uid=747, status='staging')
mycart.save()
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
stage_files(file_ids, mycart.id)
cart_utils.prepare_bundle(mycart.id)
status = mycart.status
cartid = mycart.id
while status == 'staging':
mycart = Cart.get(Cart.id == cartid)
status = mycart.status
Cart.database_close()
self.assertEqual(status, 'ready')
def test_delete_cart_bundle(self):
"""test that trys to delete a cart bundle"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
cart_utils = Cartutils()
os.makedirs(test_cart.bundle_path, 0o777)
deleted = cart_utils.delete_cart_bundle(test_cart)
self.assertEqual(test_cart.status, 'deleted')
self.assertEqual(deleted, True)
self.assertEqual(os.path.isdir(test_cart.bundle_path), False)
def stage_files(file_ids, uid):
"""Tell the files to be staged on the backend system """
Cart.database_connect()
mycart = Cart(cart_uid=uid, status="staging")
mycart.save()
#with update or new, need to add in files
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
get_files_locally.delay(mycart.id)
prepare_bundle.delay(mycart.id)
Cart.database_close()
def cart_status(uid):
"""Get the status of a specified cart"""
Cart.database_connect()
status = None
try:
mycart = (Cart
.select()
.where(
(Cart.cart_uid == str(uid)) &
(Cart.deleted_date.is_null(True)))
.order_by(Cart.creation_date.desc())
.get())
except DoesNotExist:
#case if no record exists yet in database
mycart = None
status = ["error", "No cart with uid " + uid + " found"]
if mycart:
#send the status and any available error text
status = [mycart.status, mycart.error]
Cart.database_close()
return status
def test_delete_cart_bundle_fail(self, mock_rmtree):
"""test that trys to delete a cart bundle but fails"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
cart_utils = Cartutils()
os.makedirs(test_cart.bundle_path, 0o777)
mock_rmtree.side_effect = OSError(mock.Mock(), 'Error')
deleted = cart_utils.delete_cart_bundle(test_cart)
self.assertNotEqual(test_cart.status, 'deleted')
self.assertEqual(deleted, False)
self.assertEqual(os.path.isdir(test_cart.bundle_path), True)
def test_check_space_bad_path(self, mock_disk_usage):
"""test that the error when a bad path"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
mock_disk_usage.side_effect = psutil.Error(mock.Mock())
rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
False)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, 'error')
def status_file_task(file_id):
"""Status a file from the archive. If ready then pull the file"""
Cart.database_connect()
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
#check to see if file is available to pull from archive interface
archive_request = ArchiveRequests()
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to status file with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
ready = cart_utils.check_file_ready_pull(response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0. No coverage on recall since it just calls the method again
if ready < 0:
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
elif not ready: # pragma: no cover
Cart.database_close()
status_file_task.delay(file_id)
return
#ready so try to pull file
pull_file.delay(file_id, ready['filepath'], ready['modtime'], False)
def test_create_download_path(self):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
directory_name = os.path.dirname(test_file.bundle_path)
self.assertEqual(success, True)
self.assertEqual(os.path.isdir(directory_name), True)
os.rmdir(directory_name)
self.assertEqual(os.path.isdir(directory_name), False)
def test_prep_bundle_error(self):
"""test getting bundle ready with a file in error state"""
data = json.loads(cart_json_helper())
file_ids = data['fileids']
Cart.database_connect()
mycart = Cart(cart_uid=343, status='staging')
mycart.save()
cart_utils = Cartutils()
cart_utils.update_cart_files(mycart, file_ids)
get_files_locally(mycart.id)
for cart_file in File.select().where(File.cart == mycart.id):
cart_file.status = 'error'
cart_file.save()
cart_utils.prepare_bundle(mycart.id)
status = mycart.status
cartid = mycart.id
while status == 'staging':
mycart = Cart.get(Cart.id == cartid)
status = mycart.status
Cart.database_close()
self.assertEqual(status, 'error')
def test_set_file_status(self):
"""test that trys to set a specific files status"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
cart_utils.set_file_status(test_file, test_cart, 'error',
'fake error')
self.assertEqual(test_file.status, 'error')
self.assertEqual(test_file.error, 'fake error')
def test_status_details_fail(self):
"""test status details fail"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
#say file is way to big
retval = cart_utils.check_status_details(
test_cart, test_file, 99999999999999999999999999999, 1)
self.assertEqual(retval, -1)
def test_cart_no_hash_passed(self):
"""test error with cart with no hash passed"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
cart_utils = Cartutils()
data = json.loads(
'{"fileids": [{"id":"foo.txt", "path":"1/2/3/foo.txt", "hashtype":"md5"}]}'
)
file_ids = data['fileids']
retval = cart_utils.update_cart_files(test_cart, file_ids)
self.assertNotEqual(retval, None)
def test_create_download_path(self):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
directory_name = os.path.dirname(test_file.bundle_path)
self.assertEqual(success, True)
self.assertEqual(os.path.isdir(directory_name), True)
os.rmdir(directory_name)
self.assertEqual(os.path.isdir(directory_name), False)
def lru_cart_delete(cls, mycart):
"""Delete the least recently used cart that isnt this one.
Only delete one cart per call
"""
try:
lru_time = datetime.datetime.now() - datetime.timedelta(
seconds=int(LRU_BUFFER_TIME))
del_cart = (
Cart.select().where((Cart.id != mycart.id)
& (Cart.deleted_date.is_null(True))
& (Cart.updated_date < lru_time)).order_by(
Cart.creation_date).get())
return cls.delete_cart_bundle(del_cart)
except DoesNotExist:
#case if no cart exists that can be deleted
return False
def test_cart_int_delete(self):
"""Testing the cart interface delete"""
def start_response(*args):
"""stub for start_response to do some checking"""
self.assertEqual(args[0], '200 OK')
self.assertEqual(args[1][0][0], 'Content-Type')
self.assertEqual(args[1][0][1], 'application/json')
env = {'PATH_INFO': '/123', 'QUERY_STRING': ''}
with test_database(self.sqlite_db, (Cart, File)):
sample_cart = Cart()
sample_cart.cart_uid = 123
sample_cart.bundle_path = mkdtemp()
sample_cart.status = 'ready'
sample_cart.save(force_insert=True)
sample_cart.reload()
path_to_files = os.path.join(VOLUME_PATH, str(sample_cart.id))
os.makedirs(path_to_files)
self.sqlite_db.close()
cgen = CartGenerator()
data = cgen.delete_cart(env, start_response)
self.assertEqual(
loads(data)['message'], 'Cart Deleted Successfully')
def test_check_space_requirements(self):
"""test that there is enough space on the volume for the file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
rtn = cart_utils.check_space_requirements(test_file, test_cart,
10, False)
self.assertEqual(rtn, True)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a way to large size needed number
rtn = cart_utils.check_space_requirements(test_file, test_cart,
9999999999999999999999, False)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, "error")
def update_cart_files(cls, cart, file_ids):
"""Update the files associated to a cart"""
with Cart.atomic():
for f_id in file_ids:
try:
filepath = cls.fix_absolute_path(f_id['path'])
hashtype = f_id['hashtype']
hashval = f_id['hashsum']
File.create(cart=cart,
file_name=f_id['id'],
bundle_path=filepath,
hash_type=hashtype,
hash_value=hashval)
cart.updated_date = datetime.datetime.now()
cart.save()
except (NameError, KeyError) as ex:
return ex #return error so that the cart can be updated
return None
def tar_files(cartid):
"""Start to bundle all the files together
Due to streaming the tar we dont need to try and bundle
everything together"""
Cart.database_connect()
try:
mycart = Cart.get(Cart.id == cartid)
bundle_path = os.path.join(
VOLUME_PATH, str(mycart.id), (mycart.cart_uid))
mycart.status = "ready"
mycart.bundle_path = bundle_path
mycart.updated_date = datetime.datetime.now()
mycart.save()
except DoesNotExist:
#case if record no longer exists
Cart.database_close()
return
Cart.database_close()
def pull_file(file_id, record_error):
"""Pull a file from the archive """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, "staging", False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
#stage the file on the archive. True on success, False on fail
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to stage with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
Cart.database_close()
return
#check to see if file is available to pull from archive interface
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to status file with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
response = 'False'
size_needed = cart_utils.check_file_size_needed(response, cart_file, mycart)
mod_time = cart_utils.check_file_modified_time(response, cart_file, mycart)
#Return from function if the values couldnt be parsed (-1 return)
if size_needed < 0 or mod_time < 0:
Cart.database_close()
return
ready_to_pull = cart_utils.check_file_ready_pull(
response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0
if ready_to_pull < 0:
Cart.database_close()
return
elif not ready_to_pull:
pull_file.delay(file_id, False)
Cart.database_close()
return
#create the path the file will be downloaded to
abs_cart_file_path = os.path.join(
VOLUME_PATH, str(mycart.id), mycart.cart_uid, cart_file.bundle_path)
path_created = cart_utils.create_download_path(
cart_file, mycart, abs_cart_file_path)
#Check size here and make sure enough space is available.
enough_space = cart_utils.check_space_requirements(
cart_file, mycart, size_needed, True)
if path_created and enough_space:
try:
archive_request.pull_file(cart_file.file_name, abs_cart_file_path)
cart_utils.set_file_status(cart_file, mycart, "staged", False)
Cart.database_close()
except requests.exceptions.RequestException as ex:
#if request fails...try a second time, if that fails write error
if record_error:
error_msg = "Failed to pull with error: " + str(ex)
cart_utils.set_file_status(
cart_file, mycart, "error", error_msg)
Cart.database_close()
else:
pull_file.delay(file_id, True)
Cart.database_close()
os.utime(abs_cart_file_path, (int(float(mod_time)), int(float(mod_time))))
def test_cart_orm_db_setup(self):
"""call database_setup"""
with test_database(self.sqlite_db, (Cart, File), create_tables=False):
database_setup(2)
self.assertTrue(Cart.table_exists())
self.assertTrue(File.table_exists())
