def test_archive_get_bad_hash(self): """ Test pulling a file with a bad hash value """ response_body = "This is the body of the file in the archive." httpretty.register_uri(httpretty.GET, '%s/1' % (self.endpoint_url), body=response_body, content_type='application/octet-stream') temp_dir = mkdtemp() archreq = ArchiveRequests() hashval = '5b' hashtype = 'md5' with self.assertRaises(ValueError): archreq.pull_file('1', '%s/1' % (temp_dir), hashval, hashtype)
def test_archive_get(self): """ Test pulling a file """ response_body = """ This is the body of the file in the archive. """ httpretty.register_uri(httpretty.GET, "%s/1"%(self.endpoint_url), body=response_body, content_type='application/octet-stream') temp_dir = mkdtemp() archreq = ArchiveRequests() archreq.pull_file("1", "%s/1"%(temp_dir)) testfd = open("%s/1"%(temp_dir), "r") self.assertEqual(testfd.read(), response_body)
def test_archive_get(self): """ Test pulling a file """ response_body = "This is the body of the file in the archive." httpretty.register_uri(httpretty.GET, '%s/1' % (self.endpoint_url), body=response_body, content_type='application/octet-stream') temp_dir = mkdtemp() archreq = ArchiveRequests() hashval = '5bf018b3c598df19b5f4363fc55f2f89' hashtype = 'md5' archreq.pull_file('1', '%s/1' % (temp_dir), hashval, hashtype) testfd = open('%s/1' % (temp_dir), 'r') self.assertEqual(testfd.read(), response_body)
def pull_file(file_id, filepath, modtime, record_error): """Pull a file from the archive """ Cart.database_connect() try: cart_file = File.get(File.id == file_id) mycart = cart_file.cart cart_utils = Cartutils() #make sure cart wasnt deleted before pulling file if mycart.deleted_date: Cart.database_close() return except DoesNotExist: Cart.database_close() return archive_request = ArchiveRequests() try: archive_request.pull_file(cart_file.file_name, filepath, cart_file.hash_value, cart_file.hash_type) cart_utils.set_file_status(cart_file, mycart, 'staged', False) os.utime(filepath, (int(float(modtime)), int(float(modtime)))) Cart.database_close() except requests.exceptions.RequestException as ex: #if request fails...try a second time, if that fails write error if record_error: error_msg = 'Failed to pull with error: ' + str(ex) cart_utils.set_file_status(cart_file, mycart, 'error', error_msg) Cart.database_close() cart_utils.prepare_bundle(mycart.id) else: pull_file.delay(file_id, filepath, modtime, True) Cart.database_close() except ValueError as ex: error_msg = 'Failed to pull with error: ' + str(ex) cart_utils.set_file_status(cart_file, mycart, 'error', error_msg) Cart.database_close() cart_utils.prepare_bundle(mycart.id) cart_utils.prepare_bundle(mycart.id)
def pull_file(file_id, record_error): """Pull a file from the archive """ Cart.database_connect() try: cart_file = File.get(File.id == file_id) mycart = cart_file.cart cart_utils = Cartutils() cart_utils.set_file_status(cart_file, mycart, "staging", False) #make sure cart wasnt deleted before pulling file if mycart.deleted_date: return except DoesNotExist: Cart.database_close() return archive_request = ArchiveRequests() #stage the file on the archive. True on success, False on fail try: archive_request.stage_file(cart_file.file_name) except requests.exceptions.RequestException as ex: error_msg = "Failed to stage with error: " + str(ex) cart_utils.set_file_status(cart_file, mycart, "error", error_msg) Cart.database_close() return #check to see if file is available to pull from archive interface try: response = archive_request.status_file(cart_file.file_name) except requests.exceptions.RequestException as ex: error_msg = "Failed to status file with error: " + str(ex) cart_utils.set_file_status(cart_file, mycart, "error", error_msg) response = 'False' size_needed = cart_utils.check_file_size_needed(response, cart_file, mycart) mod_time = cart_utils.check_file_modified_time(response, cart_file, mycart) #Return from function if the values couldnt be parsed (-1 return) if size_needed < 0 or mod_time < 0: Cart.database_close() return ready_to_pull = cart_utils.check_file_ready_pull( response, cart_file, mycart) #Check to see if ready to pull. If not recall this to check again # error on less then 0 if ready_to_pull < 0: Cart.database_close() return elif not ready_to_pull: pull_file.delay(file_id, False) Cart.database_close() return #create the path the file will be downloaded to abs_cart_file_path = os.path.join( VOLUME_PATH, str(mycart.id), mycart.cart_uid, cart_file.bundle_path) path_created = cart_utils.create_download_path( cart_file, mycart, abs_cart_file_path) #Check size here and make sure enough space is available. enough_space = cart_utils.check_space_requirements( cart_file, mycart, size_needed, True) if path_created and enough_space: try: archive_request.pull_file(cart_file.file_name, abs_cart_file_path) cart_utils.set_file_status(cart_file, mycart, "staged", False) Cart.database_close() except requests.exceptions.RequestException as ex: #if request fails...try a second time, if that fails write error if record_error: error_msg = "Failed to pull with error: " + str(ex) cart_utils.set_file_status( cart_file, mycart, "error", error_msg) Cart.database_close() else: pull_file.delay(file_id, True) Cart.database_close() os.utime(abs_cart_file_path, (int(float(mod_time)), int(float(mod_time))))