def setUp(self):
self.staging_order1 = StagingOrder(id=1,
source='/test/this',
staging_target='/foo',
status=StagingStatus.pending)
self.mock_general_project_repo = mock.MagicMock()
stdout_mimicing_rsync = """
Number of files: 1 (reg: 1)
Number of created files: 0
Number of deleted files: 0
Number of regular files transferred: 1
Total file size: 207,707,566 bytes
Total transferred file size: 207,707,566 bytes
Literal data: 207,707,566 bytes
Matched data: 0 bytes
File list size: 0
File list generation time: 0.001 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 207,758,378
Total bytes received: 35
sent 207,758,378 bytes received 35 bytes 138,505,608.67 bytes/sec
total size is 207,707,566 speedup is 1.00
"""
mock_process = mock.MagicMock()
mock_execution = Execution(pid=random.randint(1, 1000),
process_obj=mock_process)
self.mock_external_runner_service = mock.create_autospec(
ExternalProgramService)
self.mock_external_runner_service.run.return_value = mock_execution
@coroutine
def wait_as_coroutine(x):
return ExecutionResult(stdout=stdout_mimicing_rsync,
stderr="",
status_code=0)
self.mock_external_runner_service.wait_for_execution = wait_as_coroutine
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = self.staging_order1
mock_staging_repo.create_staging_order.return_value = self.staging_order1
self.mock_runfolder_repo = mock.MagicMock()
mock_db_session_factory = mock.MagicMock()
self.staging_service = StagingService(
staging_dir="/tmp",
external_program_service=self.mock_external_runner_service,
staging_repo=mock_staging_repo,
runfolder_repo=self.mock_runfolder_repo,
session_factory=mock_db_session_factory,
project_dir_repo=self.mock_general_project_repo)
self.staging_service.io_loop_factory = MockIOLoop
super(TestStagingService, self).setUp()
def setUp(self):
self.staging_order1 = StagingOrder(id=1,
source='/test/this',
status=StagingStatus.pending)
mock_external_runner_service = self.MockExternalRunnerService()
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = self.staging_order1
mock_staging_repo.create_staging_order.return_value = self.staging_order1
self.mock_runfolder_repo = mock.MagicMock()
mock_db_session_factory = mock.MagicMock()
self.staging_service = StagingService("/tmp",
mock_external_runner_service,
mock_staging_repo,
self.mock_runfolder_repo,
mock_db_session_factory)
def compose_application(config):
"""
Instantiates all service, repos, etc which are then used by the application.
The resulting dictionary can then be passed on to routes which instantiates the
http handlers.
:param config: a configuration instance
:return: a dictionary with references to any relevant resources
"""
runfolder_repo = FileSystemBasedRunfolderRepository(
config["monitored_directory"])
external_program_service = ExternalProgramService()
db_connection_string = config["db_connection_string"]
engine = create_engine(db_connection_string, echo=False)
create_and_migrate_db(engine, db_connection_string)
session_factory = scoped_session(sessionmaker())
session_factory.configure(bind=engine)
staging_repo = DatabaseBasedStagingRepository(
session_factory=session_factory)
staging_service = StagingService(
external_program_service=external_program_service,
runfolder_repo=runfolder_repo,
staging_repo=staging_repo,
staging_dir=config["staging_directory"],
session_factory=session_factory)
delivery_repo = DatabaseBasedDeliveriesRepository(
session_factory=session_factory)
delivery_service = MoverDeliveryService(
external_program_service=external_program_service,
staging_service=staging_service,
delivery_repo=delivery_repo)
return dict(config=config,
runfolder_repo=runfolder_repo,
external_program_service=external_program_service,
staging_service=staging_service,
delivery_service=delivery_service)
def compose_application(config):
"""
Instantiates all service, repos, etc which are then used by the application.
The resulting dictionary can then be passed on to routes which instantiates the
http handlers.
:param config: a configuration instance
:return: a dictionary with references to any relevant resources
"""
def _assert_is_dir(directory):
if not FileSystemService.isdir(directory):
raise AssertionError("{} is not a directory".format(
os.path.abspath(directory)))
staging_dir = config['staging_directory']
_assert_is_dir(staging_dir)
runfolder_dir = config["runfolder_directory"]
_assert_is_dir(runfolder_dir)
project_links_directory = config["project_links_directory"]
_assert_is_dir(project_links_directory)
runfolder_repo = FileSystemBasedRunfolderRepository(runfolder_dir)
project_repository = UnorganisedRunfolderProjectRepository(
sample_repository=RunfolderProjectBasedSampleRepository())
unorganised_runfolder_repo = FileSystemBasedUnorganisedRunfolderRepository(
runfolder_dir, project_repository=project_repository)
general_project_dir = config['general_project_directory']
_assert_is_dir(general_project_dir)
general_project_repo = GeneralProjectRepository(
root_directory=general_project_dir)
external_program_service = ExternalProgramService()
db_connection_string = config["db_connection_string"]
engine = create_engine(db_connection_string, echo=False)
alembic_path = config["alembic_path"]
create_and_migrate_db(engine, alembic_path, db_connection_string)
session_factory = scoped_session(sessionmaker())
session_factory.configure(bind=engine)
staging_repo = DatabaseBasedStagingRepository(
session_factory=session_factory)
staging_service = StagingService(
external_program_service=external_program_service,
runfolder_repo=runfolder_repo,
project_dir_repo=general_project_repo,
staging_repo=staging_repo,
staging_dir=staging_dir,
project_links_directory=project_links_directory,
session_factory=session_factory)
delivery_repo = DatabaseBasedDeliveriesRepository(
session_factory=session_factory)
path_to_mover = config['path_to_mover']
mover_delivery_service = MoverDeliveryService(
external_program_service=external_program_service,
staging_service=staging_service,
delivery_repo=delivery_repo,
session_factory=session_factory,
path_to_mover=path_to_mover)
dds_conf = config['dds_conf']
dds_project_repo = DDSProjectRepository(session_factory=session_factory)
dds_service = DDSService(external_program_service=external_program_service,
staging_service=staging_service,
staging_dir=staging_dir,
delivery_repo=delivery_repo,
dds_project_repo=dds_project_repo,
session_factory=session_factory,
dds_conf=dds_conf)
delivery_sources_repo = DatabaseBasedDeliverySourcesRepository(
session_factory=session_factory)
runfolder_service = RunfolderService(runfolder_repo)
delivery_service = DeliveryService(
mover_service=mover_delivery_service,
staging_service=staging_service,
delivery_sources_repo=delivery_sources_repo,
general_project_repo=general_project_repo,
runfolder_service=runfolder_service,
project_links_directory=project_links_directory)
best_practice_analysis_service = BestPracticeAnalysisService(
general_project_repo)
organise_service = OrganiseService(
runfolder_service=RunfolderService(unorganised_runfolder_repo))
return dict(config=config,
runfolder_repo=runfolder_repo,
external_program_service=external_program_service,
staging_service=staging_service,
mover_delivery_service=mover_delivery_service,
dds_service=dds_service,
delivery_service=delivery_service,
general_project_repo=general_project_repo,
best_practice_analysis_service=best_practice_analysis_service,
organise_service=organise_service)
class TestStagingService(unittest.TestCase):
class MockExternalRunnerService():
def __init__(self, return_status=0, throw=False):
self.return_status = return_status
self.throw = throw
def run(self, cmd):
if self.throw:
raise Exception("Test the exception handling...")
mock_process = mock.MagicMock()
execution = Execution(pid=random.randint(1, 1000),
process_obj=mock_process)
return execution
def wait_for_execution(self, execution):
time.sleep(0.1)
return ExecutionResult(status_code=self.return_status,
stderr="stderr",
stdout="stdout")
def setUp(self):
self.staging_order1 = StagingOrder(id=1,
source='/test/this',
status=StagingStatus.pending)
mock_external_runner_service = self.MockExternalRunnerService()
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = self.staging_order1
mock_staging_repo.create_staging_order.return_value = self.staging_order1
self.mock_runfolder_repo = mock.MagicMock()
mock_db_session_factory = mock.MagicMock()
self.staging_service = StagingService("/tmp",
mock_external_runner_service,
mock_staging_repo,
self.mock_runfolder_repo,
mock_db_session_factory)
# A StagingService should be able to:
# - Stage a staging order
def test_stage_order(self):
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status,
StagingStatus.staging_successful)
# - Set status to failed if rsyncing is not successful
def test_unsuccessful_staging_order(self):
mock_external_runner_service = self.MockExternalRunnerService(
return_status=1)
self.staging_service.external_program_service = mock_external_runner_service
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status,
StagingStatus.staging_failed)
# - Set status to failed if there is an exception is not successful
def test_exception_in_staging_order(self):
mock_external_runner_service = self.MockExternalRunnerService(
throw=True)
self.staging_service.external_program_service = mock_external_runner_service
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status,
StagingStatus.staging_failed)
# - Reject staging order if it has invalid state
def test_stage_order_non_valid_state(self):
with self.assertRaises(InvalidStatusException):
staging_order_in_progress = StagingOrder(
source='/test/this', status=StagingStatus.staging_in_progress)
self.staging_service.stage_order(
stage_order=staging_order_in_progress)
# - Be able to stage a existing runfolder
def test_stage_runfolder(self):
class MockStagingRepo:
orders_state = []
def get_staging_order_by_id(self, identifier, custom_session=None):
return filter(lambda x: x.id == identifier,
self.orders_state)[0]
def create_staging_order(self, source, status, staging_target_dir):
order = StagingOrder(id=len(self.orders_state) + 1,
source=source,
status=status,
staging_target=staging_target_dir)
self.orders_state.append(order)
return order
runfolder1 = FAKE_RUNFOLDERS[0]
self.mock_runfolder_repo.get_runfolder.return_value = runfolder1
mock_staging_repo = MockStagingRepo()
self.staging_service.staging_repo = mock_staging_repo
result = self.staging_service.stage_runfolder(
runfolder_id=runfolder1.name, projects_to_stage=[])
self.assertEqual(result,
map(lambda x: x.id, mock_staging_repo.orders_state))
# - Reject staging a runfolder which does not exist runfolder
def test_stage_runfolder_does_not_exist(self):
with self.assertRaises(RunfolderNotFoundException):
self.mock_runfolder_repo.get_runfolder.return_value = None
self.staging_service.stage_runfolder(runfolder_id='foo_runfolder',
projects_to_stage=[])
# - Be able to get the status of a stage order
def test_get_status_or_stage_order(self):
# Returns correctly for existing order
actual = self.staging_service.get_status_of_stage_order(
self.staging_order1.id)
self.assertEqual(actual, self.staging_order1.status)
# Returns None for none existent order
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = None
self.staging_service.staging_repo = mock_staging_repo
actual_not_there = self.staging_service.get_status_of_stage_order(1337)
self.assertIsNone(actual_not_there)
# - Be able to kill a ongoing staging process
@mock.patch('delivery.services.staging_service.os')
def test_kill_stage_order(self, mock_os):
# If the status is in progress it should be possible to kill it.
self.staging_order1.status = StagingStatus.staging_in_progress
self.staging_order1.pid = 1337
actual = self.staging_service.kill_process_of_staging_order(
self.staging_order1.id)
mock_os.kill.assert_called_with(self.staging_order1.pid,
signal.SIGTERM)
self.assertTrue(actual)
# It should handle if kill raises a OSError gracefully
self.staging_order1.status = StagingStatus.staging_in_progress
self.staging_order1.pid = 1337
mock_os.kill.side_effect = OSError
actual = self.staging_service.kill_process_of_staging_order(
self.staging_order1.id)
mock_os.kill.assert_called_with(self.staging_order1.pid,
signal.SIGTERM)
self.assertFalse(actual)
# If the status is not in progress it should not be possible to kill it.
self.staging_order1.status = StagingStatus.staging_successful
actual = self.staging_service.kill_process_of_staging_order(
self.staging_order1.id)
mock_os.kill.assert_not_called()
self.assertFalse(actual)
class TestStagingService(AsyncTestCase):
class MockStagingRepo:
def __init__(self):
self.orders_state = []
def get_staging_order_by_id(self, identifier, custom_session=None):
return list(filter(lambda x: x.id == identifier, self.orders_state))[0]
def create_staging_order(self, source, status, staging_target_dir):
order = StagingOrder(id=len(self.orders_state) + 1,
source=source,
status=status,
staging_target=staging_target_dir)
self.orders_state.append(order)
return order
def setUp(self):
self.staging_order1 = StagingOrder(id=1,
source='/test/this',
staging_target='/foo',
status=StagingStatus.pending)
self.mock_general_project_repo = mock.MagicMock()
stdout_mimicing_rsync = """
Number of files: 1 (reg: 1)
Number of created files: 0
Number of deleted files: 0
Number of regular files transferred: 1
Total file size: 207,707,566 bytes
Total transferred file size: 207,707,566 bytes
Literal data: 207,707,566 bytes
Matched data: 0 bytes
File list size: 0
File list generation time: 0.001 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 207,758,378
Total bytes received: 35
sent 207,758,378 bytes received 35 bytes 138,505,608.67 bytes/sec
total size is 207,707,566 speedup is 1.00
"""
mock_process = mock.MagicMock()
mock_execution = Execution(pid=random.randint(1, 1000), process_obj=mock_process)
self.mock_external_runner_service = mock.create_autospec(ExternalProgramService)
self.mock_external_runner_service.run.return_value = mock_execution
self.mock_file_system_service = mock.create_autospec(FileSystemService)
@coroutine
def wait_as_coroutine(x):
return ExecutionResult(stdout=stdout_mimicing_rsync, stderr="", status_code=0)
self.mock_external_runner_service.wait_for_execution = wait_as_coroutine
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = self.staging_order1
mock_staging_repo.create_staging_order.return_value = self.staging_order1
self.mock_runfolder_repo = mock.MagicMock()
mock_db_session_factory = mock.MagicMock()
self.staging_service = StagingService(staging_dir="/tmp",
project_links_directory="/tmp",
external_program_service=self.mock_external_runner_service,
staging_repo=mock_staging_repo,
runfolder_repo=self.mock_runfolder_repo,
session_factory=mock_db_session_factory,
project_dir_repo=self.mock_general_project_repo,
file_system_service=self.mock_file_system_service)
self.staging_service.io_loop_factory = MockIOLoop
super(TestStagingService, self).setUp()
# A StagingService should be able to:
# - Stage a staging order
@tornado.testing.gen_test
def test_stage_order(self):
res = yield self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
def _get_staging_size():
return self.staging_order1.size
assert_eventually_equals(self, 1, _get_stating_status, StagingStatus.staging_successful)
self.assertEqual(self.staging_order1.size, 207707566)
# - Set status to failed if rsyncing is not successful
@tornado.testing.gen_test
def test_unsuccessful_staging_order(self):
@coroutine
def wait_as_coroutine(x):
return ExecutionResult(stdout="", stderr="", status_code=1)
self.mock_external_runner_service.wait_for_execution = wait_as_coroutine
yield self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status, StagingStatus.staging_failed)
# - Set status to failed if there is an exception is not successful
def test_exception_in_staging_order(self):
def raise_exception(x):
raise Exception
self.mock_external_runner_service.wait_for_execution = raise_exception
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status, StagingStatus.staging_failed)
# - Reject staging order if it has invalid state
@tornado.testing.gen_test
def test_stage_order_non_valid_state(self):
with self.assertRaises(InvalidStatusException):
staging_order_in_progress = StagingOrder(source='/test/this',
status=StagingStatus.staging_in_progress)
res = yield self.staging_service.stage_order(stage_order=staging_order_in_progress)
# - Be able to get the status of a stage order
def test_get_status_or_stage_order(self):
# Returns correctly for existing order
actual = self.staging_service.get_status_of_stage_order(self.staging_order1.id)
self.assertEqual(actual, self.staging_order1.status)
# Returns None for none existent order
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = None
self.staging_service.staging_repo = mock_staging_repo
actual_not_there = self.staging_service.get_status_of_stage_order(1337)
self.assertIsNone(actual_not_there)
# - Be able to kill a ongoing staging process
@mock.patch('delivery.services.staging_service.os')
def test_kill_stage_order(self, mock_os):
# If the status is in progress it should be possible to kill it.
self.staging_order1.status = StagingStatus.staging_in_progress
self.staging_order1.pid = 1337
actual = self.staging_service.kill_process_of_staging_order(self.staging_order1.id)
mock_os.kill.assert_called_with(self.staging_order1.pid, signal.SIGTERM)
self.assertTrue(actual)
# It should handle if kill raises a OSError gracefully
self.staging_order1.status = StagingStatus.staging_in_progress
self.staging_order1.pid = 1337
mock_os.kill.side_effect = OSError
actual = self.staging_service.kill_process_of_staging_order(self.staging_order1.id)
mock_os.kill.assert_called_with(self.staging_order1.pid, signal.SIGTERM)
self.assertFalse(actual)
@mock.patch('delivery.services.staging_service.os')
def test_kill_stage_order_not_valid_state(self, mock_os):
# If the status is not in progress it should not be possible to kill it.
self.staging_order1.status = StagingStatus.staging_successful
actual = self.staging_service.kill_process_of_staging_order(self.staging_order1.id)
mock_os.kill.assert_not_called()
self.assertFalse(actual)
def compose_application(config):
"""
Instantiates all service, repos, etc which are then used by the application.
The resulting dictionary can then be passed on to routes which instantiates the
http handlers.
:param config: a configuration instance
:return: a dictionary with references to any relevant resources
"""
def _assert_is_dir(directory):
if not FileSystemService.isdir(directory):
raise AssertionError("{} is not a directory".format(directory))
staging_dir = config['staging_directory']
_assert_is_dir(staging_dir)
runfolder_dir = config["runfolder_directory"]
_assert_is_dir(runfolder_dir)
runfolder_repo = FileSystemBasedRunfolderRepository(runfolder_dir)
general_project_dir = config['general_project_directory']
_assert_is_dir(general_project_dir)
general_project_repo = GeneralProjectRepository(
root_directory=general_project_dir)
external_program_service = ExternalProgramService()
db_connection_string = config["db_connection_string"]
engine = create_engine(db_connection_string, echo=False)
alembic_path = config["alembic_path"]
create_and_migrate_db(engine, alembic_path, db_connection_string)
session_factory = scoped_session(sessionmaker())
session_factory.configure(bind=engine)
staging_repo = DatabaseBasedStagingRepository(
session_factory=session_factory)
staging_service = StagingService(
external_program_service=external_program_service,
runfolder_repo=runfolder_repo,
project_dir_repo=general_project_repo,
staging_repo=staging_repo,
staging_dir=staging_dir,
session_factory=session_factory)
delivery_repo = DatabaseBasedDeliveriesRepository(
session_factory=session_factory)
path_to_mover = config['path_to_mover']
delivery_service = MoverDeliveryService(
external_program_service=external_program_service,
staging_service=staging_service,
delivery_repo=delivery_repo,
session_factory=session_factory,
path_to_mover=path_to_mover)
return dict(config=config,
runfolder_repo=runfolder_repo,
external_program_service=external_program_service,
staging_service=staging_service,
delivery_service=delivery_service)