def project_from_dir(d):
project = RunfolderProject(name=os.path.basename(d),
path=os.path.join(projects_base_dir, d),
runfolder_path=runfolder.path,
runfolder_name=runfolder.name)
project.project_files = self.get_report_files(
project, checksums=runfolder.checksums)
project.samples = self.sample_repository.get_samples(
project, runfolder)
return project
def project_from_dir(d):
project = RunfolderProject(name=os.path.basename(d),
path=os.path.join(projects_base_dir, d),
runfolder_path=runfolder.path,
runfolder_name=runfolder.name)
try:
project.project_files = self.get_report_files(
project, checksums=runfolder.checksums)
except ProjectReportNotFoundException as e:
log.warning(e)
project.samples = self.sample_repository.get_samples(
project, runfolder)
return project
def test_get_project(self):
project_name = "ABC_123"
expected_projects = [RunfolderProject(name="ABC_123",
runfolder_path="/foo/160930_ST-E00216_0111_BH37CWALXX",
path="/foo/160930_ST-E00216_0111_BH37CWALXX/Projects/ABC_123",
runfolder_name="160930_ST-E00216_0111_BH37CWALXX"),
RunfolderProject(name="ABC_123",
runfolder_path="/foo/160930_ST-E00216_0112_BH37CWALXX",
path="/foo/160930_ST-E00216_0112_BH37CWALXX/Projects/ABC_123",
runfolder_name="160930_ST-E00216_0112_BH37CWALXX")]
actual_projects = list(self.repo.get_project(project_name))
self.assertEqual(len(actual_projects), 2)
self.assertEqual(actual_projects, expected_projects)
def project_from_dir(d):
return RunfolderProject(
name=os.path.basename(d),
path=os.path.join(projects_base_dir, d),
runfolder_path=runfolder.path,
runfolder_name=runfolder.name
)
def organise_project(self, runfolder, project, organised_projects_path,
lanes):
"""
Organise a project on a runfolder into its own directory and into a standard structure. If the project has
already been organised, a ProjectAlreadyOrganisedException will be raised, unless force is True. If force is
True, the existing project path will be renamed with a unique suffix.
:param runfolder: a Runfolder instance representing the runfolder on which the project belongs
:param project: a Project instance representing the project to be organised
:param lanes: if not None, only samples on any of the specified lanes will be organised
:param force: if True, a previously organised project will be renamed with a unique suffix
:raises ProjectAlreadyOrganisedException: if project has already been organised and force is False
:return: a Project instance representing the project after organisation
"""
# symlink the samples
organised_project_path = os.path.join(organised_projects_path,
project.name)
organised_project_runfolder_path = os.path.join(
organised_project_path, runfolder.name)
organised_samples = []
for sample in project.samples:
organised_samples.append(
self.organise_sample(sample, organised_project_runfolder_path,
lanes))
# symlink the project files
organised_project_files = []
if project.project_files:
project_file_base = self.file_system_service.dirname(
project.project_files[0].file_path)
for project_file in project.project_files:
organised_project_files.append(
self.organise_project_file(
project_file,
organised_project_runfolder_path,
project_file_base=project_file_base))
organised_project = RunfolderProject(project.name,
organised_project_path,
runfolder.path,
runfolder.name,
samples=organised_samples)
organised_project_files.append(
self.runfolder_service.dump_project_samplesheet(
runfolder, organised_project))
organised_project.project_files = organised_project_files
self.runfolder_service.dump_project_checksums(organised_project)
return organised_project
def runfolder_project(runfolder,
project_name="ABC_123",
sample_indexes=sample_index_generator(),
lane_numbers=lane_generator(),
project_root="Unaligned",
report_type=report_type_generator()):
project = RunfolderProject(name=project_name,
path=os.path.join(runfolder.path, project_root,
project_name),
runfolder_path=runfolder.path,
runfolder_name=runfolder.name)
project.project_files = project_report_files(project, next(report_type))
sample_names = sample_name_generator()
# a straight-forward sample with files on one lane
lane_number = next(lane_numbers)
samples = [
project_sample(project, next(sample_names), next(sample_indexes),
lane_number)
]
# a sample with files on two lanes
sample_name = next(sample_names)
sample_index = next(sample_indexes)
sample = project_sample(project, sample_name, sample_index, lane_number)
lane_number = next(lane_numbers)
t_sample = project_sample(project, sample_name, sample_index, lane_number)
sample.sample_files.extend(t_sample.sample_files)
samples.append(sample)
# a sample with two preps on two lanes and sample files in subdirectories
sample_name = next(sample_names)
t_samples = [
project_sample(project,
sample_name=sample_name,
sample_index=si,
lane_no=l,
sample_id="{}-{}-{}".format(sample_name, si, l))
for si in [next(sample_indexes),
next(sample_indexes)]
for l in [next(lane_numbers), next(lane_numbers)]
]
samples.extend(t_samples)
project.samples = samples
return project
fake_projects = ["ABC_123", "DEF_456"]
def mock_file_system_service(directories, projects):
mock_file_system_service_instance = MagicMock()
mock_file_system_service_instance.find_runfolder_directories.return_value = directories
mock_file_system_service_instance.find_project_directories.return_value = projects
return mock_file_system_service_instance
_runfolder1 = Runfolder(name="160930_ST-E00216_0111_BH37CWALXX",
path="/foo/160930_ST-E00216_0111_BH37CWALXX")
_runfolder1.projects = [
RunfolderProject(
name="ABC_123",
path="/foo/160930_ST-E00216_0111_BH37CWALXX/Projects/ABC_123",
runfolder_path=_runfolder1.path),
RunfolderProject(
name="DEF_456",
path="/foo/160930_ST-E00216_0111_BH37CWALXX/Projects/DEF_456",
runfolder_path=_runfolder1.path)
]
_runfolder2 = Runfolder(name="160930_ST-E00216_0112_BH37CWALXX",
path="/foo/160930_ST-E00216_0112_BH37CWALXX")
_runfolder2.projects = [
RunfolderProject(
name="ABC_123",
path="/foo/160930_ST-E00216_0112_BH37CWALXX/Projects/ABC_123",
runfolder_path=_runfolder2.path),
class TestDeliveryService(unittest.TestCase):
runfolder_projects = [RunfolderProject(name="ABC_123",
path="/foo/160930_ST-E00216_0112_BH37CWALXX/Projects/ABC_123",
runfolder_path="/foo/160930_ST-E00216_0112_BH37CWALXX",
runfolder_name="160930_ST-E00216_0112_BH37CWALXX"),
RunfolderProject(name="ABC_123",
path="/foo/160930_ST-E00216_0111_BH37CWALXX/Projects/ABC_123",
runfolder_path="/foo/160930_ST-E00216_0111_BH37CWALXX/",
runfolder_name="160930_ST-E00216_0111_BH37CWALXX")]
general_project = GeneralProject(name="ABC_123", path="/foo/bar/ABC_123")
def _compose_delivery_service(self,
mover_delivery_service=mock.create_autospec(MoverDeliveryService),
staging_service=mock.create_autospec(StagingService),
delivery_sources_repo=mock.create_autospec(DatabaseBasedDeliverySourcesRepository),
general_project_repo=mock.create_autospec(GeneralProjectRepository),
runfolder_service=mock.create_autospec(RunfolderService),
project_links_dir=mock.MagicMock()):
mover_delivery_service = mover_delivery_service
self.staging_service = staging_service
delivery_sources_repo = delivery_sources_repo
general_project_repo = general_project_repo
runfolder_service = runfolder_service
self.project_links_dir = project_links_dir
self.delivery_service = DeliveryService(mover_service=mover_delivery_service,
staging_service=self.staging_service,
delivery_sources_repo=delivery_sources_repo,
general_project_repo=general_project_repo,
runfolder_service=runfolder_service,
project_links_directory=self.project_links_dir)
def setUp(self):
self._compose_delivery_service()
def test__create_links_area_for_project_runfolders(self):
with tempfile.TemporaryDirectory() as tmpdirname:
self.delivery_service.project_links_directory = tmpdirname
batch_nbr = 1337
project_link_area = self.delivery_service._create_links_area_for_project_runfolders("ABC_123",
self.runfolder_projects,
batch_nbr)
project_linking_area_base = os.path.join(self.delivery_service.project_links_directory,
"ABC_123",
str(batch_nbr))
self.assertEqual(project_link_area,
project_linking_area_base)
self.assertTrue(
os.path.islink(
os.path.join(
project_linking_area_base,
"160930_ST-E00216_0112_BH37CWALXX")))
self.assertTrue(
os.path.islink(
os.path.join(
project_linking_area_base,
"160930_ST-E00216_0111_BH37CWALXX")))
def test_deliver_arbitrary_directory_project(self):
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=self.general_project.path,
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024
)
general_project_repo_mock = mock.create_autospec(GeneralProjectRepository)
general_project_repo_mock.get_project.return_value = self.general_project
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = False
delivery_sources_repo_mock.create_source.return_value = DeliverySource(project_name="ABC_123",
source_name=self.general_project.name,
path=self.general_project.path)
self._compose_delivery_service(general_project_repo=general_project_repo_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock)
result = self.delivery_service.deliver_arbitrary_directory_project("ABC_123")
self.assertTrue(result["ABC_123"] == 1)
def test_deliver_arbitrary_directory_project_force(self):
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=self.general_project.path,
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024
)
general_project_repo_mock = mock.create_autospec(GeneralProjectRepository)
general_project_repo_mock.get_project.return_value = self.general_project
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = True
delivery_sources_repo_mock.create_source.return_value = DeliverySource(project_name="ABC_123",
source_name=self.general_project.name,
path=self.general_project.path)
self._compose_delivery_service(general_project_repo=general_project_repo_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock)
with self.assertRaises(ProjectAlreadyDeliveredException):
self.delivery_service.deliver_arbitrary_directory_project("ABC_123", force_delivery=False)
result = self.delivery_service.deliver_arbitrary_directory_project("ABC_123", force_delivery=True)
self.assertTrue(result["ABC_123"] == 1)
def test_deliver_single_runfolder(self):
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=self.runfolder_projects[0].path,
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024)
runfolder_service_mock = mock.create_autospec(RunfolderService)
def my_project_iterator(runfolder_name, only_these_projects):
yield self.runfolder_projects[0]
runfolder_service_mock.find_projects_on_runfolder = my_project_iterator
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = False
delivery_sources_repo_mock.create_source.return_value = \
DeliverySource(project_name="ABC_123",
source_name="{}/{}".format(
self.runfolder_projects[0].runfolder_name,
self.runfolder_projects[0].name),
path=self.general_project.path)
self._compose_delivery_service(runfolder_service=runfolder_service_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock)
result = self.delivery_service.deliver_single_runfolder(runfolder_name="160930_ST-E00216_0112_BH37CWALXX",
only_these_projects=None,
force_delivery=False)
self.assertEqual(result["ABC_123"], 1)
def test_deliver_single_runfolder_force(self):
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=self.runfolder_projects[0].path,
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024)
runfolder_service_mock = mock.create_autospec(RunfolderService)
def my_project_iterator(runfolder_name, only_these_projects):
yield self.runfolder_projects[0]
runfolder_service_mock.find_projects_on_runfolder = my_project_iterator
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = True
delivery_sources_repo_mock.create_source.return_value = \
DeliverySource(project_name="ABC_123",
source_name="{}/{}".format(
self.runfolder_projects[0].runfolder_name,
self.runfolder_projects[0].name),
path=self.general_project.path)
self._compose_delivery_service(runfolder_service=runfolder_service_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock)
with self.assertRaises(ProjectAlreadyDeliveredException):
self.delivery_service.deliver_single_runfolder(runfolder_name="160930_ST-E00216_0112_BH37CWALXX",
only_these_projects=None,
force_delivery=False)
result = self.delivery_service.deliver_single_runfolder(runfolder_name="160930_ST-E00216_0112_BH37CWALXX",
only_these_projects=None,
force_delivery=True)
self.assertEqual(result["ABC_123"], 1)
def test_deliver_all_runfolders_for_project(self):
with tempfile.TemporaryDirectory() as tmpdirname:
staging_service_mock = mock.create_autospec(StagingService)
staging_service_mock.create_new_stage_order.return_value = \
StagingOrder(id=1,
source=os.path.join(tmpdirname, "ABC_123", "1"),
status=StagingStatus.pending,
staging_target='/foo/bar',
size=1024)
runfolder_service_mock = mock.create_autospec(RunfolderService)
def my_project_iterator(project_name):
for proj in self.runfolder_projects:
yield proj
runfolder_service_mock.find_runfolders_for_project = my_project_iterator
delivery_sources_repo_mock = mock.create_autospec(DatabaseBasedDeliverySourcesRepository)
delivery_sources_repo_mock.source_exists.return_value = False
delivery_sources_repo_mock.find_highest_batch_nbr.return_value = 1
delivery_sources_repo_mock.create_source.return_value = \
DeliverySource(project_name="ABC_123",
source_name="{}/{}".format(
"ABC_123",
"batch1"),
path=self.general_project.path,
batch=1)
self._compose_delivery_service(runfolder_service=runfolder_service_mock,
delivery_sources_repo=delivery_sources_repo_mock,
staging_service=staging_service_mock,
project_links_dir=tmpdirname)
projects_and_ids, projects = \
self.delivery_service.deliver_all_runfolders_for_project(project_name="ABC_123",
mode=DeliveryMode.CLEAN)
self.assertEqual(projects_and_ids["ABC_123"], 1)
staged_runfolders = list(map(lambda staged_path: staged_path.to_dict(), projects))
paths = []
for item in staged_runfolders:
paths.append(item['path'])
self.assertEqual(paths,["/foo/160930_ST-E00216_0112_BH37CWALXX/Projects/ABC_123",
"/foo/160930_ST-E00216_0111_BH37CWALXX/Projects/ABC_123"])