def update_cart_files(cls, cart, file_ids):
"""Update the files associated to a cart"""
with Cart.atomic():
for f_id in file_ids:
filepath = cls.fix_absolute_path(f_id["path"])
File.create(
cart=cart, file_name=f_id["id"], bundle_path=filepath)
cart.updated_date = datetime.datetime.now()
cart.save()
def test_check_file_size_needed(self):
"""test that the file size returned from the archive is parsed right"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
file_size = cart_utils.check_file_size_needed(response, test_file,
test_cart)
self.assertEqual(file_size, 24)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a bad response
file_size = cart_utils.check_file_size_needed("", test_file,
test_cart)
self.assertEqual(file_size, -1)
self.assertEqual(test_file.status, "error")
def test_bad_pull_value(self, mock_pull):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_pull.side_effect = ValueError('Error with hash pulling file')
file_id = test_file.id
pull_file(file_id, '/tmp/1/1.txt', '9999999', False)
cart_after = Cart.get(Cart.id == test_cart.id)
status = cart_after.status
self.assertEqual(status, 'error')
def test_check_file_size_needed(self):
"""test that the file size returned from the archive is parsed right"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
file_size = cart_utils.check_file_size_needed(
response, test_file, test_cart)
self.assertEqual(file_size, 24)
self.assertNotEqual(test_file.status, 'error')
#now check for an error by sending a bad response
file_size = cart_utils.check_file_size_needed(
'', test_file, test_cart)
self.assertEqual(file_size, -1)
self.assertEqual(test_file.status, 'error')
def test_bad_status(self, mock_stage_file, mock_status_file):
"""test the bad status of a archive file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.side_effect = requests.exceptions.RequestException(
mock.Mock(status=500), 'Error')
file_id = test_file.id
status_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'error')
def test_status_cart_deleted(self):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
deleted_date='2017-05-03 00:00:00')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
file_id = test_file.id
status_file_task(file_id)
self.assertEqual(True, True)
def update_cart_files(cls, cart, file_ids):
"""Update the files associated to a cart"""
with Cart.atomic():
for f_id in file_ids:
try:
filepath = cls.fix_absolute_path(f_id['path'])
hashtype = f_id['hashtype']
hashval = f_id['hashsum']
File.create(cart=cart,
file_name=f_id['id'],
bundle_path=filepath,
hash_type=hashtype,
hash_value=hashval)
cart.updated_date = datetime.datetime.now()
cart.save()
except (NameError, KeyError) as ex:
return ex #return error so that the cart can be updated
return None
def test_set_file_status(self):
"""test that trys to set a specific files status"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging",
bundle_path="/tmp/1/")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
cart_utils.set_file_status(test_file, test_cart, "error", "fake error")
self.assertEqual(test_file.status, "error")
self.assertEqual(test_file.error, "fake error")
def test_bad_create_download_path(self, mock_create_bundle):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
mock_create_bundle.side_effect = OSError(mock.Mock(), 'Error')
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
self.assertEqual(success, False)
def test_check_space_bad_path(self, mock_disk_usage):
"""test that the error when a bad path"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
mock_disk_usage.side_effect = psutil.Error(mock.Mock())
rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
False)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, 'error')
def test_create_download_path(self):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
directory_name = os.path.dirname(test_file.bundle_path)
self.assertEqual(success, True)
self.assertEqual(os.path.isdir(directory_name), True)
os.rmdir(directory_name)
self.assertEqual(os.path.isdir(directory_name), False)
def test_status_details_fail(self):
"""test status details fail"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
#say file is way to big
retval = cart_utils.check_status_details(
test_cart, test_file, 99999999999999999999999999999, 1)
self.assertEqual(retval, -1)
def test_set_file_status(self):
"""test that trys to set a specific files status"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1',
status='staging',
bundle_path='/tmp/1/')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
cart_utils.set_file_status(test_file, test_cart, 'error',
'fake error')
self.assertEqual(test_file.status, 'error')
self.assertEqual(test_file.error, 'fake error')
def test_create_download_path(self):
"""test the creation of the download path for a cart file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
success = cart_utils.create_download_path(test_file, test_cart,
test_file.bundle_path)
directory_name = os.path.dirname(test_file.bundle_path)
self.assertEqual(success, True)
self.assertEqual(os.path.isdir(directory_name), True)
os.rmdir(directory_name)
self.assertEqual(os.path.isdir(directory_name), False)
def test_check_space_requirements(self):
"""test that there is enough space on the volume for the file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status="staging")
test_file = File.create(cart=test_cart, file_name="1.txt",
bundle_path="/tmp/1/1.txt")
cart_utils = Cartutils()
rtn = cart_utils.check_space_requirements(test_file, test_cart,
10, False)
self.assertEqual(rtn, True)
self.assertNotEqual(test_file.status, "error")
#now check for an error by sending a way to large size needed number
rtn = cart_utils.check_space_requirements(test_file, test_cart,
9999999999999999999999, False)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, "error")
def test_check_space_requirements(self):
"""test that there is enough space on the volume for the file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
False)
self.assertEqual(rtn, True)
self.assertNotEqual(test_file.status, 'error')
#now check for an error by sending a way to large size needed number
rtn = cart_utils.check_space_requirements(test_file, test_cart,
9999999999999999999999,
True)
self.assertEqual(rtn, False)
self.assertEqual(test_file.status, 'error')
def test_bad_pull(self, mock_stage_file, mock_status_file, mock_pull_file,
mock_utime):
"""test the bad pull of a archive file"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.return_value = """{
"bytes_per_level": "(10L, 0L)",
"ctime": "1444629567",
"file": "1.txt",
"file_storage_media": "disk",
"filesize": "10",
"message": "File was found",
"mtime": "1444937154"
}"""
mock_pull_file.side_effect = requests.exceptions.RequestException(
mock.Mock(status=500), 'Error')
mock_utime.return_value = True
file_id = test_file.id
stage_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'error')
def test_check_file_not_ready_pull(self):
"""test that checks to see if a file is not ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(0L, 24L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "tape",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
resp_bad = """{
"bytes_per_level": "(0L, 33L, 33L, 0L, 0L)",
"ctime": "1444938177",
"file": "/myemsl-dev/bundle/file.2",
"filesize": "33",
"message": "File was found",
"mtime": "1444938133"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready, False)
#now check for an error by sending a bad response
ready = cart_utils.check_file_ready_pull('', test_file, test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, 'error')
#now check for an error with storage media
ready = cart_utils.check_file_ready_pull(resp_bad, test_file,
test_cart)
self.assertEqual(ready, -1)
self.assertEqual(test_file.status, 'error')
def test_check_file_ready_pull(self):
"""test that checks to see if a file is ready to pull
by checking the archive response"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
response = """{
"bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
"ctime": "1444938166",
"file": "/myemsl-dev/bundle/file.1",
"file_storage_media": "disk",
"filesize": "24",
"message": "File was found",
"mtime": "1444938166"
}"""
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
cart_utils = Cartutils()
ready = cart_utils.check_file_ready_pull(response, test_file,
test_cart)
self.assertEqual(ready['enough_space'], True)
self.assertNotEqual(test_file.status, 'error')
def test_bad_ready_to_pull(self, mock_stage_file, mock_status_file,
mock_check_file):
"""test a error return from a file not ready to pull"""
with test_database(SqliteDatabase(':memory:'), (Cart, File)):
test_cart = Cart.create(cart_uid='1', status='staging')
test_file = File.create(cart=test_cart,
file_name='1.txt',
bundle_path='/tmp/1/1.txt')
def fake_database_connect(cls):
"""dont error with connect"""
return cls
def fake_database_close(cls):
"""dont actually close"""
return cls
cart.cart_orm.CartBase.database_connect = MethodType(
fake_database_connect, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.database_close = MethodType(
fake_database_close, cart.cart_orm.CartBase)
cart.cart_orm.CartBase.throw_error = False
mock_stage_file.return_value = True
mock_status_file.return_value = """{
"bytes_per_level": "(10L, 0L)",
"ctime": "1444629567",
"file": "1.txt",
"file_storage_media": "disk",
"filesize": "10",
"message": "File was found",
"mtime": "1444937154"
}"""
mock_check_file.return_value = -1
file_id = test_file.id
stage_file_task(file_id)
cart_file = File.get(File.id == file_id)
status = cart_file.status
self.assertEqual(status, 'staging')
