def test_deliver_by_staging_id(self):
source = '/foo/bar'
staging_target = '/staging/dir/bar'
staging_order = StagingOrder(source=source, staging_target=staging_target)
staging_order.status = StagingStatus.staging_successful
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
self.mock_staging_service.get_delivery_order_by_id.return_value = self.delivery_order
with patch('shutil.rmtree') as mock_rmtree:
res = yield self.dds_service.deliver_by_staging_id(
staging_id=1,
delivery_project='snpseq00001',
token_path='token_path',
md5sum_file='md5sum_file')
mock_rmtree.assert_called_once_with(staging_target)
def _get_delivery_order():
return self.delivery_order.delivery_status
assert_eventually_equals(self, 1, _get_delivery_order, DeliveryStatus.delivery_successful)
self.mock_mover_runner.run.assert_called_with([
'dds',
'--token-path', 'token_path',
'--log-file', '/foo/bar/log',
'--no-prompt',
'data', 'put',
'--mount-dir', '/foo/bar/staging_dir',
'--source', '/staging/dir/bar',
'--project', 'snpseq00001',
'--silent'
])
def create_staging_order(self, source, status, staging_target_dir):
"""
Create a StatingOrder and commit it to the database
:param source: the directory or file to stage
:param status: the initial StatingStatus to assign to the StatingORder
:param staging_target_dir: the directory to which the StagingOrder should transfer the source
:return:
"""
order = StagingOrder(source=source, status=status)
self.session.add(order)
self.session.commit()
if self.file_system_service.isfile(order.source):
log.debug("Order source is a file")
source_base_name = self.file_system_service.basename(order.source)
elif self.file_system_service.isdir(order.source):
log.debug("Order source is a dir")
source_base_name = self.file_system_service.basename(
self.file_system_service.abspath(order.source))
else:
raise NotImplementedError(
"Could not parse a valid type from: {}, valid types"
" are directory and file.".format(order.source))
staging_target = os.path.join(
staging_target_dir, "{}_{}".format(order.id, source_base_name))
log.debug("Set the staging target to: {}".format(staging_target))
order.staging_target = staging_target
self.session.commit()
return order
def test_get_staging_path(self):
staging_order = StagingOrder(
source='/staging/source/data',
staging_target='/staging/target/data',
status=StagingStatus.pending,
)
self.assertEquals(staging_order.get_staging_path(),
'/staging/target/data')
def test_deliver_by_staging_id_raises_on_non_successful_stage_id(self):
staging_order = StagingOrder()
staging_order.status = StagingStatus.staging_failed
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
with self.assertRaises(InvalidStatusException):
yield self.mover_delivery_service.deliver_by_staging_id(staging_id=1,
delivery_project='foo',
md5sum_file='md5sum_file')
def test_deliver_by_staging_id(self):
staging_order = StagingOrder(source='/foo/bar', staging_target='/staging/dir/bar')
staging_order.status = StagingStatus.staging_successful
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
self.mock_staging_service.get_delivery_order_by_id.return_value = self.delivery_order
res = yield self.mover_delivery_service.deliver_by_staging_id(staging_id=1,
delivery_project='xyz123',
md5sum_file='md5sum_file')
def _get_delivery_order():
return self.delivery_order.delivery_status
assert_eventually_equals(self, 1, _get_delivery_order, DeliveryStatus.delivery_in_progress)
self.mock_mover_runner.run.assert_called_once_with(['/foo/bar/to_outbox', '/foo', 'TestProj'])
def test_possible_to_delivery_by_staging_id_and_skip_mover(self):
staging_order = StagingOrder(source='/foo/bar', staging_target='/staging/dir/bar')
staging_order.status = StagingStatus.staging_successful
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
self.mock_staging_service.get_delivery_order_by_id.return_value = self.delivery_order
self.mover_delivery_service.deliver_by_staging_id(staging_id=1,
delivery_project='xyz123',
md5sum_file='md5sum_file',
skip_mover=True)
def _get_delivery_order():
return self.delivery_order.delivery_status
assert_eventually_equals(self, 1, _get_delivery_order, DeliveryStatus.delivery_skipped)
def setUp(self):
# NOTE setting echo to true is very useful to se which sql statements get
# executed, but since it fills up the logs a lot it's been disabled by
# default here.
engine = create_engine('sqlite:///:memory:', echo=False)
SQLAlchemyBase.metadata.create_all(engine)
# Throw some data into the in-memory db
session_factory = sessionmaker()
session_factory.configure(bind=engine)
self.session = session_factory()
self.staging_order_1 = StagingOrder(source='foo',
status=StagingStatus.pending,
staging_target='foo/target')
self.session.add(self.staging_order_1)
self.session.commit()
mock_file_system_service = create_autospec(FileSystemService)
mock_file_system_service.basename.return_value = 'foo'
# Prep the repo
self.staging_repo = DatabaseBasedStagingRepository(
session_factory, mock_file_system_service)
def test_stage_order_non_valid_state(self):
with self.assertRaises(InvalidStatusException):
staging_order_in_progress = StagingOrder(source='/test/this',
status=StagingStatus.staging_in_progress)
res = yield self.staging_service.stage_order(stage_order=staging_order_in_progress)
def test_deliver_arbitrary_directory_project(self):
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=self.general_project.path,
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024
)
general_project_repo_mock = mock.create_autospec(GeneralProjectRepository)
general_project_repo_mock.get_project.return_value = self.general_project
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = False
delivery_sources_repo_mock.create_source.return_value = DeliverySource(project_name="ABC_123",
source_name=self.general_project.name,
path=self.general_project.path)
self._compose_delivery_service(general_project_repo=general_project_repo_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock)
result = self.delivery_service.deliver_arbitrary_directory_project("ABC_123")
self.assertTrue(result["ABC_123"] == 1)
def test_deliver_single_runfolder_force(self):
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=self.runfolder_projects[0].path,
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024)
runfolder_service_mock = mock.create_autospec(RunfolderService)
def my_project_iterator(runfolder_name, only_these_projects):
yield self.runfolder_projects[0]
runfolder_service_mock.find_projects_on_runfolder = my_project_iterator
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = True
delivery_sources_repo_mock.create_source.return_value = \
DeliverySource(project_name="ABC_123",
source_name="{}/{}".format(
self.runfolder_projects[0].runfolder_name,
self.runfolder_projects[0].name),
path=self.general_project.path)
self._compose_delivery_service(runfolder_service=runfolder_service_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock)
with self.assertRaises(ProjectAlreadyDeliveredException):
self.delivery_service.deliver_single_runfolder(runfolder_name="160930_ST-E00216_0112_BH37CWALXX",
only_these_projects=None,
force_delivery=False)
result = self.delivery_service.deliver_single_runfolder(runfolder_name="160930_ST-E00216_0112_BH37CWALXX",
only_these_projects=None,
force_delivery=True)
self.assertEqual(result["ABC_123"], 1)
def setUp(self):
self.staging_order1 = StagingOrder(id=1,
source='/test/this',
staging_target='/foo',
status=StagingStatus.pending)
self.mock_general_project_repo = mock.MagicMock()
stdout_mimicing_rsync = """
Number of files: 1 (reg: 1)
Number of created files: 0
Number of deleted files: 0
Number of regular files transferred: 1
Total file size: 207,707,566 bytes
Total transferred file size: 207,707,566 bytes
Literal data: 207,707,566 bytes
Matched data: 0 bytes
File list size: 0
File list generation time: 0.001 seconds
File list transfer time: 0.000 seconds
Total bytes sent: 207,758,378
Total bytes received: 35
sent 207,758,378 bytes received 35 bytes 138,505,608.67 bytes/sec
total size is 207,707,566 speedup is 1.00
"""
mock_process = mock.MagicMock()
mock_execution = Execution(pid=random.randint(1, 1000),
process_obj=mock_process)
self.mock_external_runner_service = mock.create_autospec(
ExternalProgramService)
self.mock_external_runner_service.run.return_value = mock_execution
@coroutine
def wait_as_coroutine(x):
return ExecutionResult(stdout=stdout_mimicing_rsync,
stderr="",
status_code=0)
self.mock_external_runner_service.wait_for_execution = wait_as_coroutine
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = self.staging_order1
mock_staging_repo.create_staging_order.return_value = self.staging_order1
self.mock_runfolder_repo = mock.MagicMock()
mock_db_session_factory = mock.MagicMock()
self.staging_service = StagingService(
staging_dir="/tmp",
external_program_service=self.mock_external_runner_service,
staging_repo=mock_staging_repo,
runfolder_repo=self.mock_runfolder_repo,
session_factory=mock_db_session_factory,
project_dir_repo=self.mock_general_project_repo)
self.staging_service.io_loop_factory = MockIOLoop
super(TestStagingService, self).setUp()
def create_staging_order(self, source, status, staging_target_dir):
order = StagingOrder(id=len(self.orders_state) + 1,
source=source,
status=status,
staging_target=staging_target_dir)
self.orders_state.append(order)
return order
def create_staging_order(self, source, status, staging_target_dir):
"""
Create a StatingOrder and commit it to the database
:param source: the directory or file to stage
:param status: the initial StatingStatus to assign to the StatingORder
:param staging_target_dir: the directory to which the StagingOrder should transfer the source
:return:
"""
order = StagingOrder(source=source, status=status)
self.session.add(order)
self.session.commit()
staging_target = os.path.join(
staging_target_dir, "{}_{}".format(order.id,
os.path.basename(order.source)))
order.staging_target = staging_target
self.session.commit()
return order
def test_deliver_all_runfolders_for_project(self):
with tempfile.TemporaryDirectory() as tmpdirname:
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=os.path.join(tmpdirname, "ABC_123", "1"),
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024)
runfolder_service_mock = mock.create_autospec(RunfolderService)
def my_project_iterator(project_name):
for proj in self.runfolder_projects:
yield proj
runfolder_service_mock.find_runfolders_for_project = my_project_iterator
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = False
delivery_sources_repo_mock.find_highest_batch_nbr.return_value = 1
delivery_sources_repo_mock.create_source.return_value = \
DeliverySource(project_name="ABC_123",
source_name="{}/{}".format(
"ABC_123",
"batch1"),
path=self.general_project.path,
batch=1)
self._compose_delivery_service(runfolder_service=runfolder_service_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock,
project_links_dir=tmpdirname)
projects_and_ids, projects = \
self.delivery_service.deliver_all_runfolders_for_project(project_name="ABC_123",
mode=DeliveryMode.CLEAN)
self.assertEqual(projects_and_ids["ABC_123"], 1)
staged_runfolders = list(map(lambda staged_path: staged_path.to_dict(), projects))
paths = []
for item in staged_runfolders:
paths.append(item['path'])
self.assertEqual(paths,["/foo/160930_ST-E00216_0112_BH37CWALXX/Projects/ABC_123",
"/foo/160930_ST-E00216_0111_BH37CWALXX/Projects/ABC_123"])
def setUp(self):
self.staging_order1 = StagingOrder(id=1,
source='/test/this',
status=StagingStatus.pending)
mock_external_runner_service = self.MockExternalRunnerService()
mock_staging_repo = mock.MagicMock()
mock_staging_repo.get_staging_order_by_id.return_value = self.staging_order1
mock_staging_repo.create_staging_order.return_value = self.staging_order1
self.mock_runfolder_repo = mock.MagicMock()
mock_db_session_factory = mock.MagicMock()
self.staging_service = StagingService("/tmp",
mock_external_runner_service,
mock_staging_repo,
self.mock_runfolder_repo,
mock_db_session_factory)