def test_archive_status(self):
"""
Test pulling a file
"""
stage_file_obj = {
"bytes_per_level": "0",
"ctime": "Sun, 06 Nov 1994 08:49:37 GMT",
"file_storage_media": "0",
"filesize": "8",
"file": "1",
"mtime": "Sun, 06 Nov 1994 08:49:37 GMT",
"message": "File was found"
}
adding_headers = {
"x-pacifica-messsage": "File was found",
'x-pacifica-ctime': 'Sun, 06 Nov 1994 08:49:37 GMT',
'x-pacifica-bytes-per-level': '0',
'x-pacifica-file-storage-media': '0',
'last-modified': 'Sun, 06 Nov 1994 08:49:37 GMT'
}
httpretty.register_uri(httpretty.HEAD, "%s/1"%(self.endpoint_url),
status=200,
body='blahblah',
adding_headers=adding_headers)
archreq = ArchiveRequests()
status = loads(archreq.status_file("1"))
for key in status.keys():
self.assertEqual(status[key], stage_file_obj[key])
self.assertEqual(httpretty.last_request().method, "HEAD")
def stage_file_task(file_id):
"""Stage the file from the archive, then call status """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, 'staging', False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to stage with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
#successful stage so move on to status
Cart.database_close()
status_file_task.delay(file_id)
def test_archive_status(self):
"""
Test pulling a file
"""
stage_file_obj = {
'bytes_per_level': '0',
'ctime': 'Sun, 06 Nov 1994 08:49:37 GMT',
'file_storage_media': '0',
'filesize': '8',
'file': '1',
'mtime': 'Sun, 06 Nov 1994 08:49:37 GMT',
'message': 'File was found'
}
adding_headers = {
'x-pacifica-messsage': 'File was found',
'x-pacifica-ctime': 'Sun, 06 Nov 1994 08:49:37 GMT',
'x-pacifica-bytes-per-level': '0',
'x-pacifica-file-storage-media': '0',
'last-modified': 'Sun, 06 Nov 1994 08:49:37 GMT'
}
httpretty.register_uri(httpretty.HEAD,
'%s/1' % (self.endpoint_url),
status=200,
body='blahblah',
adding_headers=adding_headers)
archreq = ArchiveRequests()
status = loads(archreq.status_file('1'))
for key in status.keys():
self.assertEqual(status[key], stage_file_obj[key])
self.assertEqual(httpretty.last_request().method, 'HEAD')
def test_archive_stage(self):
"""
Test staging a file
"""
response_body = {'message': 'File was staged', 'file': '1'}
httpretty.register_uri(httpretty.POST,
'%s/1' % (self.endpoint_url),
body=dumps(response_body),
content_type='application/json')
archreq = ArchiveRequests()
archreq.stage_file('1')
self.assertEqual(httpretty.last_request().method, 'POST')
def test_archive_stage_fail(self):
"""
Test staging a file failure
"""
response_body = {'message': 'error', 'file': '1'}
httpretty.register_uri(httpretty.POST,
'%s/1' % (self.endpoint_url),
status=500,
body=dumps(response_body),
content_type='application/json')
archreq = ArchiveRequests()
with self.assertRaises(requests.exceptions.RequestException):
archreq.stage_file('fakeFileName')
def test_archive_stage(self):
"""
Test staging a file
"""
response_body = {
'message': 'File was staged',
'file': "1"
}
httpretty.register_uri(httpretty.POST, "%s/1"%(self.endpoint_url),
body=dumps(response_body),
content_type="application/json")
archreq = ArchiveRequests()
archreq.stage_file("1")
self.assertEqual(httpretty.last_request().method, "POST")
def test_archive_get_bad_hash(self):
"""
Test pulling a file with a bad hash value
"""
response_body = "This is the body of the file in the archive."
httpretty.register_uri(httpretty.GET,
'%s/1' % (self.endpoint_url),
body=response_body,
content_type='application/octet-stream')
temp_dir = mkdtemp()
archreq = ArchiveRequests()
hashval = '5b'
hashtype = 'md5'
with self.assertRaises(ValueError):
archreq.pull_file('1', '%s/1' % (temp_dir), hashval, hashtype)
def test_archive_get(self):
"""
Test pulling a file
"""
response_body = """
This is the body of the file in the archive.
"""
httpretty.register_uri(httpretty.GET, "%s/1"%(self.endpoint_url),
body=response_body,
content_type='application/octet-stream')
temp_dir = mkdtemp()
archreq = ArchiveRequests()
archreq.pull_file("1", "%s/1"%(temp_dir))
testfd = open("%s/1"%(temp_dir), "r")
self.assertEqual(testfd.read(), response_body)
def test_archive_get(self):
"""
Test pulling a file
"""
response_body = "This is the body of the file in the archive."
httpretty.register_uri(httpretty.GET,
'%s/1' % (self.endpoint_url),
body=response_body,
content_type='application/octet-stream')
temp_dir = mkdtemp()
archreq = ArchiveRequests()
hashval = '5bf018b3c598df19b5f4363fc55f2f89'
hashtype = 'md5'
archreq.pull_file('1', '%s/1' % (temp_dir), hashval, hashtype)
testfd = open('%s/1' % (temp_dir), 'r')
self.assertEqual(testfd.read(), response_body)
def pull_file(file_id, filepath, modtime, record_error):
"""Pull a file from the archive """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
try:
archive_request.pull_file(cart_file.file_name, filepath,
cart_file.hash_value, cart_file.hash_type)
cart_utils.set_file_status(cart_file, mycart, 'staged', False)
os.utime(filepath, (int(float(modtime)), int(float(modtime))))
Cart.database_close()
except requests.exceptions.RequestException as ex:
#if request fails...try a second time, if that fails write error
if record_error:
error_msg = 'Failed to pull with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
else:
pull_file.delay(file_id, filepath, modtime, True)
Cart.database_close()
except ValueError as ex:
error_msg = 'Failed to pull with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
cart_utils.prepare_bundle(mycart.id)
def status_file_task(file_id):
"""Status a file from the archive. If ready then pull the file"""
Cart.database_connect()
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
Cart.database_close()
return
#check to see if file is available to pull from archive interface
archive_request = ArchiveRequests()
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = 'Failed to status file with error: ' + str(ex)
cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
ready = cart_utils.check_file_ready_pull(response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0. No coverage on recall since it just calls the method again
if ready < 0:
Cart.database_close()
cart_utils.prepare_bundle(mycart.id)
return
elif not ready: # pragma: no cover
Cart.database_close()
status_file_task.delay(file_id)
return
#ready so try to pull file
pull_file.delay(file_id, ready['filepath'], ready['modtime'], False)
def pull_file(file_id, record_error):
"""Pull a file from the archive """
Cart.database_connect()
try:
cart_file = File.get(File.id == file_id)
mycart = cart_file.cart
cart_utils = Cartutils()
cart_utils.set_file_status(cart_file, mycart, "staging", False)
#make sure cart wasnt deleted before pulling file
if mycart.deleted_date:
return
except DoesNotExist:
Cart.database_close()
return
archive_request = ArchiveRequests()
#stage the file on the archive. True on success, False on fail
try:
archive_request.stage_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to stage with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
Cart.database_close()
return
#check to see if file is available to pull from archive interface
try:
response = archive_request.status_file(cart_file.file_name)
except requests.exceptions.RequestException as ex:
error_msg = "Failed to status file with error: " + str(ex)
cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
response = 'False'
size_needed = cart_utils.check_file_size_needed(response, cart_file, mycart)
mod_time = cart_utils.check_file_modified_time(response, cart_file, mycart)
#Return from function if the values couldnt be parsed (-1 return)
if size_needed < 0 or mod_time < 0:
Cart.database_close()
return
ready_to_pull = cart_utils.check_file_ready_pull(
response, cart_file, mycart)
#Check to see if ready to pull. If not recall this to check again
# error on less then 0
if ready_to_pull < 0:
Cart.database_close()
return
elif not ready_to_pull:
pull_file.delay(file_id, False)
Cart.database_close()
return
#create the path the file will be downloaded to
abs_cart_file_path = os.path.join(
VOLUME_PATH, str(mycart.id), mycart.cart_uid, cart_file.bundle_path)
path_created = cart_utils.create_download_path(
cart_file, mycart, abs_cart_file_path)
#Check size here and make sure enough space is available.
enough_space = cart_utils.check_space_requirements(
cart_file, mycart, size_needed, True)
if path_created and enough_space:
try:
archive_request.pull_file(cart_file.file_name, abs_cart_file_path)
cart_utils.set_file_status(cart_file, mycart, "staged", False)
Cart.database_close()
except requests.exceptions.RequestException as ex:
#if request fails...try a second time, if that fails write error
if record_error:
error_msg = "Failed to pull with error: " + str(ex)
cart_utils.set_file_status(
cart_file, mycart, "error", error_msg)
Cart.database_close()
else:
pull_file.delay(file_id, True)
Cart.database_close()
os.utime(abs_cart_file_path, (int(float(mod_time)), int(float(mod_time))))
