def stage_file_task(file_id): """Stage the file from the archive, then call status """ Cart.database_connect() try: cart_file = File.get(File.id == file_id) mycart = cart_file.cart cart_utils = Cartutils() cart_utils.set_file_status(cart_file, mycart, 'staging', False) #make sure cart wasnt deleted before pulling file if mycart.deleted_date: Cart.database_close() return except DoesNotExist: Cart.database_close() return archive_request = ArchiveRequests() try: archive_request.stage_file(cart_file.file_name) except requests.exceptions.RequestException as ex: error_msg = 'Failed to stage with error: ' + str(ex) cart_utils.set_file_status(cart_file, mycart, 'error', error_msg) Cart.database_close() cart_utils.prepare_bundle(mycart.id) return #successful stage so move on to status Cart.database_close() status_file_task.delay(file_id)
def test_prep_bundle_staging(self): """test getting bundle ready with a file in staging state""" data = json.loads(cart_json_helper()) file_ids = data['fileids'] Cart.database_connect() mycart = Cart(cart_uid=343, status='staging') mycart.save() cart_utils = Cartutils() cart_utils.update_cart_files(mycart, file_ids) get_files_locally(mycart.id) for cart_file in File.select().where(File.cart == mycart.id): cart_file.status = 'staging' cart_file.save() cart_utils.prepare_bundle( mycart.id) #hitting more coverage, set files to staged for cart_file in File.select().where(File.cart == mycart.id): cart_file.status = 'staged' cart_file.save() cart_utils.prepare_bundle(mycart.id) #call again after file update status = mycart.status cartid = mycart.id while status == 'staging': mycart = Cart.get(Cart.id == cartid) status = mycart.status Cart.database_close() self.assertEqual(status, 'ready')
def test_get_path_size(self): """test to see if the path size of a directory is returned""" cart_utils = Cartutils() path = os.path.dirname(os.path.realpath(__file__)) rtn = cart_utils.get_path_size(path + '/../') self.assertNotEqual(rtn, 0)
def test_check_file_size_needed(self): """test that the file size returned from the archive is parsed right""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): response = """{ "bytes_per_level": "(24L, 0L, 0L, 0L, 0L)", "ctime": "1444938166", "file": "/myemsl-dev/bundle/file.1", "file_storage_media": "disk", "filesize": "24", "message": "File was found", "mtime": "1444938166" }""" test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() file_size = cart_utils.check_file_size_needed( response, test_file, test_cart) self.assertEqual(file_size, 24) self.assertNotEqual(test_file.status, 'error') #now check for an error by sending a bad response file_size = cart_utils.check_file_size_needed( '', test_file, test_cart) self.assertEqual(file_size, -1) self.assertEqual(test_file.status, 'error')
def test_create_bundle_directories(self): """test the creation of directories where files will be saved""" directory_name = '/tmp/fakedir/' cart_utils = Cartutils() cart_utils.create_bundle_directories(directory_name) self.assertEqual(os.path.isdir(directory_name), True) os.rmdir(directory_name) self.assertEqual(os.path.isdir(directory_name), False)
def test_fix_absolute_path(self): """test the correct creation of paths by removing absolute paths""" cart_utils = Cartutils() return_one = cart_utils.fix_absolute_path('tmp/foo.text') return_two = cart_utils.fix_absolute_path('/tmp/foo.text') return_three = cart_utils.fix_absolute_path('/tmp/foo.text') self.assertEqual(return_one, 'tmp/foo.text') self.assertEqual(return_two, 'tmp/foo.text') self.assertNotEqual(return_three, '/tmp/foo.text')
def test_bad_create_download_path(self, mock_create_bundle): """test the creation of the download path for a cart file""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() mock_create_bundle.side_effect = OSError(mock.Mock(), 'Error') success = cart_utils.create_download_path(test_file, test_cart, test_file.bundle_path) self.assertEqual(success, False)
def test_delete_cart_bundle(self): """test that trys to delete a cart bundle""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging', bundle_path='/tmp/1/') cart_utils = Cartutils() os.makedirs(test_cart.bundle_path, 0o777) deleted = cart_utils.delete_cart_bundle(test_cart) self.assertEqual(test_cart.status, 'deleted') self.assertEqual(deleted, True) self.assertEqual(os.path.isdir(test_cart.bundle_path), False)
def test_check_space_bad_path(self, mock_disk_usage): """test that the error when a bad path""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() mock_disk_usage.side_effect = psutil.Error(mock.Mock()) rtn = cart_utils.check_space_requirements(test_file, test_cart, 10, False) self.assertEqual(rtn, False) self.assertEqual(test_file.status, 'error')
def status(self, env, start_response): """Get the status of a carts tar file""" resp = cart_interface_responses.Responses() uid = fix_cart_uid(env['PATH_INFO']) is_valid = is_valid_uid(uid) if not is_valid: self._response = resp.invalid_uid_error_response( start_response, uid) return self.return_response() cart_utils = Cartutils() status = cart_utils.cart_status(uid) self._response = resp.cart_status_response(start_response, status) return self.return_response()
def test_delete_cart_bundle_fail(self, mock_rmtree): """test that trys to delete a cart bundle but fails""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging', bundle_path='/tmp/1/') cart_utils = Cartutils() os.makedirs(test_cart.bundle_path, 0o777) mock_rmtree.side_effect = OSError(mock.Mock(), 'Error') deleted = cart_utils.delete_cart_bundle(test_cart) self.assertNotEqual(test_cart.status, 'deleted') self.assertEqual(deleted, False) self.assertEqual(os.path.isdir(test_cart.bundle_path), True)
def test_set_file_status(self): """test that trys to set a specific files status""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging', bundle_path='/tmp/1/') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() cart_utils.set_file_status(test_file, test_cart, 'error', 'fake error') self.assertEqual(test_file.status, 'error') self.assertEqual(test_file.error, 'fake error')
def stage_files(file_ids, mycart_id): """Tell the files to be staged on the backend system """ Cart.database_connect() #with update or new, need to add in files mycart = Cart.get(Cart.id == mycart_id) cart_utils = Cartutils() file_id_error = cart_utils.update_cart_files(mycart, file_ids) if not file_id_error: get_files_locally.delay(mycart.id) else: mycart.status = 'error' mycart.error = 'Error parsing file Ids with error: ' + str( file_id_error) mycart.updated_date = datetime.datetime.now() mycart.save() Cart.database_close()
def test_create_download_path(self): """test the creation of the download path for a cart file""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() success = cart_utils.create_download_path(test_file, test_cart, test_file.bundle_path) directory_name = os.path.dirname(test_file.bundle_path) self.assertEqual(success, True) self.assertEqual(os.path.isdir(directory_name), True) os.rmdir(directory_name) self.assertEqual(os.path.isdir(directory_name), False)
def test_cart_no_hash_passed(self): """test error with cart with no hash passed""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging', bundle_path='/tmp/1/') cart_utils = Cartutils() data = json.loads( '{"fileids": [{"id":"foo.txt", "path":"1/2/3/foo.txt", "hashtype":"md5"}]}' ) file_ids = data['fileids'] retval = cart_utils.update_cart_files(test_cart, file_ids) self.assertNotEqual(retval, None)
def test_status_details_fail(self): """test status details fail""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging', bundle_path='/tmp/1/') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() #say file is way to big retval = cart_utils.check_status_details( test_cart, test_file, 99999999999999999999999999999, 1) self.assertEqual(retval, -1)
def delete_cart(self, env, start_response): """Delete a cart that has been created""" resp = cart_interface_responses.Responses() uid = fix_cart_uid(env['PATH_INFO']) is_valid = is_valid_uid(uid) if not is_valid: self._response = resp.invalid_uid_error_response( start_response, uid) return self.return_response() cart_utils = Cartutils() message = cart_utils.remove_cart(uid) if message is False: self._response = resp.cart_not_found(start_response) else: self._response = resp.cart_delete_response(start_response, message) return self.return_response()
def test_stage_files(self): """test getting bundle files ready""" data = json.loads(cart_json_helper()) file_ids = data['fileids'] Cart.database_connect() mycart = Cart(cart_uid=747, status='staging') mycart.save() cart_utils = Cartutils() cart_utils.update_cart_files(mycart, file_ids) stage_files(file_ids, mycart.id) cart_utils.prepare_bundle(mycart.id) status = mycart.status cartid = mycart.id while status == 'staging': mycart = Cart.get(Cart.id == cartid) status = mycart.status Cart.database_close() self.assertEqual(status, 'ready')
def test_cart_deleted_date(self): """test getting bundle ready with a file in staging state""" data = json.loads(cart_json_helper()) file_ids = data['fileids'] Cart.database_connect() mycart = Cart(cart_uid=444, status='staging') mycart.save() cart_utils = Cartutils() cart_utils.update_cart_files(mycart, file_ids) get_files_locally(mycart.id) mycart.status = 'deleted' mycart.deleted_date = datetime.datetime.now() mycart.save() status = mycart.status for cart_file in File.select().where(File.cart == mycart.id): pull_file(cart_file.id, '/tmp/some/Path', '1111', False) Cart.database_close() self.assertEqual(status, 'deleted')
def test_check_space_requirements(self): """test that there is enough space on the volume for the file""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() rtn = cart_utils.check_space_requirements(test_file, test_cart, 10, False) self.assertEqual(rtn, True) self.assertNotEqual(test_file.status, 'error') #now check for an error by sending a way to large size needed number rtn = cart_utils.check_space_requirements(test_file, test_cart, 9999999999999999999999, True) self.assertEqual(rtn, False) self.assertEqual(test_file.status, 'error')
def test_lru_cart_delete(self): """test that trys to delete a cart""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging', bundle_path='/tmp/1/') test_cart2 = Cart.create(cart_uid='2', status='staging', bundle_path='/tmp/2/', updated_date=1) cart_utils = Cartutils() os.makedirs(test_cart2.bundle_path, 0o777) retval = cart_utils.lru_cart_delete(test_cart) self.assertEqual(retval, True) test_c2 = Cart.get(Cart.id == test_cart2.id) self.assertEqual(test_c2.status, 'deleted') #also hit error block when nothing to delete retval = cart_utils.lru_cart_delete(test_cart) self.assertEqual(retval, False)
def pull_file(file_id, filepath, modtime, record_error): """Pull a file from the archive """ Cart.database_connect() try: cart_file = File.get(File.id == file_id) mycart = cart_file.cart cart_utils = Cartutils() #make sure cart wasnt deleted before pulling file if mycart.deleted_date: Cart.database_close() return except DoesNotExist: Cart.database_close() return archive_request = ArchiveRequests() try: archive_request.pull_file(cart_file.file_name, filepath, cart_file.hash_value, cart_file.hash_type) cart_utils.set_file_status(cart_file, mycart, 'staged', False) os.utime(filepath, (int(float(modtime)), int(float(modtime)))) Cart.database_close() except requests.exceptions.RequestException as ex: #if request fails...try a second time, if that fails write error if record_error: error_msg = 'Failed to pull with error: ' + str(ex) cart_utils.set_file_status(cart_file, mycart, 'error', error_msg) Cart.database_close() cart_utils.prepare_bundle(mycart.id) else: pull_file.delay(file_id, filepath, modtime, True) Cart.database_close() except ValueError as ex: error_msg = 'Failed to pull with error: ' + str(ex) cart_utils.set_file_status(cart_file, mycart, 'error', error_msg) Cart.database_close() cart_utils.prepare_bundle(mycart.id) cart_utils.prepare_bundle(mycart.id)
def test_prep_bundle_error(self): """test getting bundle ready with a file in error state""" data = json.loads(cart_json_helper()) file_ids = data['fileids'] Cart.database_connect() mycart = Cart(cart_uid=343, status='staging') mycart.save() cart_utils = Cartutils() cart_utils.update_cart_files(mycart, file_ids) get_files_locally(mycart.id) for cart_file in File.select().where(File.cart == mycart.id): cart_file.status = 'error' cart_file.save() cart_utils.prepare_bundle(mycart.id) status = mycart.status cartid = mycart.id while status == 'staging': mycart = Cart.get(Cart.id == cartid) status = mycart.status Cart.database_close() self.assertEqual(status, 'error')
def test_check_file_not_ready_pull(self): """test that checks to see if a file is not ready to pull by checking the archive response""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): response = """{ "bytes_per_level": "(0L, 24L, 0L, 0L, 0L)", "ctime": "1444938166", "file": "/myemsl-dev/bundle/file.1", "file_storage_media": "tape", "filesize": "24", "message": "File was found", "mtime": "1444938166" }""" resp_bad = """{ "bytes_per_level": "(0L, 33L, 33L, 0L, 0L)", "ctime": "1444938177", "file": "/myemsl-dev/bundle/file.2", "filesize": "33", "message": "File was found", "mtime": "1444938133" }""" test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() ready = cart_utils.check_file_ready_pull(response, test_file, test_cart) self.assertEqual(ready, False) #now check for an error by sending a bad response ready = cart_utils.check_file_ready_pull('', test_file, test_cart) self.assertEqual(ready, -1) self.assertEqual(test_file.status, 'error') #now check for an error with storage media ready = cart_utils.check_file_ready_pull(resp_bad, test_file, test_cart) self.assertEqual(ready, -1) self.assertEqual(test_file.status, 'error')
def test_check_file_ready_pull(self): """test that checks to see if a file is ready to pull by checking the archive response""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): response = """{ "bytes_per_level": "(24L, 0L, 0L, 0L, 0L)", "ctime": "1444938166", "file": "/myemsl-dev/bundle/file.1", "file_storage_media": "disk", "filesize": "24", "message": "File was found", "mtime": "1444938166" }""" test_cart = Cart.create(cart_uid='1', status='staging') test_file = File.create(cart=test_cart, file_name='1.txt', bundle_path='/tmp/1/1.txt') cart_utils = Cartutils() ready = cart_utils.check_file_ready_pull(response, test_file, test_cart) self.assertEqual(ready['enough_space'], True) self.assertNotEqual(test_file.status, 'error')
def test_bad_stage(self, mock_delete_cart): """test the bad stage of a archive file""" with test_database(SqliteDatabase(':memory:'), (Cart, File)): test_cart = Cart.create(cart_uid='1', status='staging') def fake_database_connect(cls_name): """no error""" return cls_name def fake_database_close(cls_name): """no error""" return cls_name cart.cart_orm.CartBase.database_connect = MethodType( fake_database_connect, cart.cart_orm.CartBase) cart.cart_orm.CartBase.database_close = MethodType( fake_database_close, cart.cart_orm.CartBase) cart.cart_orm.CartBase.throw_error = False mock_delete_cart.return_value = False cart_util = Cartutils() return_val = cart_util.remove_cart(test_cart.id) self.assertEqual(return_val, None)
def status_file_task(file_id): """Status a file from the archive. If ready then pull the file""" Cart.database_connect() cart_file = File.get(File.id == file_id) mycart = cart_file.cart cart_utils = Cartutils() #make sure cart wasnt deleted before pulling file if mycart.deleted_date: Cart.database_close() return #check to see if file is available to pull from archive interface archive_request = ArchiveRequests() try: response = archive_request.status_file(cart_file.file_name) except requests.exceptions.RequestException as ex: error_msg = 'Failed to status file with error: ' + str(ex) cart_utils.set_file_status(cart_file, mycart, 'error', error_msg) Cart.database_close() cart_utils.prepare_bundle(mycart.id) return ready = cart_utils.check_file_ready_pull(response, cart_file, mycart) #Check to see if ready to pull. If not recall this to check again # error on less then 0. No coverage on recall since it just calls the method again if ready < 0: Cart.database_close() cart_utils.prepare_bundle(mycart.id) return elif not ready: # pragma: no cover Cart.database_close() status_file_task.delay(file_id) return #ready so try to pull file pull_file.delay(file_id, ready['filepath'], ready['modtime'], False)
def get(self, env, start_response): """Download the tar file created by the cart""" resp = cart_interface_responses.Responses() rtn_name = None if 'filename' in parse_qs(env['QUERY_STRING']): rtn_name = os.path.basename( parse_qs(env['QUERY_STRING'])['filename'][0]) else: rtn_name = "data_" + datetime.now().strftime( '%Y_%m_%d_%H_%M_%S') + ".tar" uid = fix_cart_uid(env['PATH_INFO']) is_valid = is_valid_uid(uid) if not is_valid: self._response = resp.invalid_uid_error_response( start_response, uid) return self.return_response() #get the bundle path if available cart_utils = Cartutils() cart_path = cart_utils.available_cart(uid) if cart_path is False: #cart not ready self._response = resp.unready_cart(start_response) elif cart_path is None: #cart not found self._response = resp.cart_not_found(start_response) else: if os.path.isdir(cart_path): #give back bundle here stderr.flush() try: #want to stream the tar file out (rpipe, wpipe) = os.pipe() cpid = os.fork() # the fork screws up coverage testing... :( if cpid == 0: # pragma: no cover # we are the child process #write the data to the pipe os.close(rpipe) wfd = os.fdopen(wpipe, 'wb') mytar = TarFile.open(fileobj=wfd, mode='w|') mytar.add(cart_path, arcname=rtn_name.replace('.tar', '')) mytar.close() #to exit from the fork child without killing the parent #we need to use_exit. Disabling the pylint for this #so that it doesnt throw an error # pylint: disable=protected-access os._exit(0) # pylint: enable=protected-access # we are the parent os.close(wpipe) #open the pipe as a file rfd = os.fdopen(rpipe, 'rb') start_response( '200 OK', [('Content-Type', 'application/octet-stream'), ('Content-Disposition', 'attachment; filename=' + str(rtn_name))]) if 'wsgi.file_wrapper' in env: return env['wsgi.file_wrapper'](rfd, BLOCK_SIZE) return iter(lambda: rfd.read(BLOCK_SIZE), '') except IOError: self._response = resp.bundle_doesnt_exist(start_response) else: self._response = resp.bundle_doesnt_exist(start_response) return self.return_response() return self.return_response()
def test_bad_makedirs(self, mock_makedirs): """test a error return from a file not ready to pull""" mock_makedirs.side_effect = OSError(mock.Mock(), 'Error') c_util = Cartutils() self.assertRaises(OSError, c_util.create_bundle_directories, "fakepath")
def test_tar_invalid_cart(self): """test pulling a file id that doesnt exist""" cart_utils = Cartutils() cart_utils.tar_files('8765', True) #no action happens on invalid cart to tar, so no assertion to check self.assertEqual(True, True)