def test_import_project(self):
"""import_project: check project is imported
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_M00879_0087_000000000-AGEW9', 'miseq', top_dir=self.dirn)
mockdir.create()
# Check that the project is not currently present
ap = AutoProcess(mockdir.dirn)
self.assertFalse(
'NewProj' in [p.name for p in ap.get_analysis_projects()])
self.assertFalse(
'NewProj' in
[p.name for p in ap.get_analysis_projects_from_dirs()])
self.assertFalse(
os.path.exists(os.path.join(ap.analysis_dir, 'NewProj')))
# Import the project
import_project(ap, self.new_project_dir)
self.assertTrue(
'NewProj' in [p.name for p in ap.get_analysis_projects()])
self.assertTrue('NewProj' in
[p.name for p in ap.get_analysis_projects_from_dirs()])
self.assertTrue(
os.path.exists(os.path.join(ap.analysis_dir, 'NewProj')))
# Verify via fresh AutoProcess object
ap2 = AutoProcess(mockdir.dirn)
self.assertTrue(
'NewProj' in [p.name for p in ap2.get_analysis_projects()])
self.assertTrue(
'NewProj' in
[p.name for p in ap2.get_analysis_projects_from_dirs()])
self.assertTrue(
os.path.exists(os.path.join(ap2.analysis_dir, 'NewProj')))
def test_publish_qc_with_projects_with_multiple_fastq_sets(self):
"""publish_qc: projects with multiple Fastq sets
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={ "run_number": 87,
"source": "local",
"instrument_datestamp": "160621" },
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn)
# Add processing report and QC outputs
UpdateAnalysisDir(ap).add_processing_report()
for project in ap.get_analysis_projects():
UpdateAnalysisProject(project).add_qc_outputs()
# Add additional fastq set for first project
multi_fastqs_project = ap.get_analysis_projects()[0]
UpdateAnalysisProject(multi_fastqs_project).add_fastq_set(
"fastqs.extra",
("Alt1.r1.fastq.gz","Alt2.r1.fastq.gz"))
UpdateAnalysisProject(multi_fastqs_project).add_qc_outputs(
fastq_set="fastqs.extra",
qc_dir="qc.extra")
# Make a mock publication area
publication_dir = os.path.join(self.dirn,'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap,location=publication_dir)
# Check outputs
outputs = ["index.html",
"processing_qc.html"]
for project in ap.get_analysis_projects():
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (project.name,
os.path.basename(
ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc,"qc_report.html"))
outputs.append(os.path.join(project_qc,"qc"))
# MultiQC output
outputs.append("multiqc_report.%s.html" % project.name)
# Additional QC for second fastq set in first project
project_qc = "qc.extra_report.%s.%s" % (multi_fastqs_project.name,
os.path.basename(
ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc,"qc.extra_report.html"))
outputs.append(os.path.join(project_qc,"qc.extra"))
# MultiQC output
outputs.append("multiqc.extra_report.%s.html" %
multi_fastqs_project.name)
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f),"Missing %s" % f)
def test_import_project(self):
"""Check AutoProcess.import_project imports a project
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_M00879_0087_000000000-AGEW9',
'miseq',
top_dir=self.dirn)
mockdir.create()
# Check that the project is not currently present
ap = AutoProcess(mockdir.dirn)
self.assertFalse('NewProj' in [p.name
for p in ap.get_analysis_projects()])
self.assertFalse('NewProj' in [p.name
for p in ap.get_analysis_projects_from_dirs()])
self.assertFalse(os.path.exists(os.path.join(ap.analysis_dir,'NewProj')))
# Import the project
ap.import_project(self.new_project_dir)
self.assertTrue('NewProj' in [p.name
for p in ap.get_analysis_projects()])
self.assertTrue('NewProj' in [p.name
for p in ap.get_analysis_projects_from_dirs()])
self.assertTrue(os.path.exists(os.path.join(ap.analysis_dir,'NewProj')))
# Verify via fresh AutoProcess object
ap2 = AutoProcess(mockdir.dirn)
self.assertTrue('NewProj' in [p.name
for p in ap2.get_analysis_projects()])
self.assertTrue('NewProj' in [p.name
for p in ap2.get_analysis_projects_from_dirs()])
self.assertTrue(os.path.exists(os.path.join(ap2.analysis_dir,'NewProj')))
def test_publish_qc_subset_of_projects(self):
"""publish_qc: only publish subset of projects
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={ "run_number": 87,
"source": "local",
"instrument_datestamp": "160621" },
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn)
# Add processing report
UpdateAnalysisDir(ap).add_processing_report()
# Add QC outputs for subset of projects
projects = ap.get_analysis_projects()
missing_projects = projects[1:]
projects = projects[0:1]
for project in ap.get_analysis_projects():
UpdateAnalysisProject(project).add_qc_outputs()
# Make a mock publication area
publication_dir = os.path.join(self.dirn,'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap,location=publication_dir,
projects="AB*")
# Check outputs
outputs = ["index.html",
"processing_qc.html"]
for project in projects:
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (project.name,
os.path.basename(
ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc,"qc_report.html"))
outputs.append(os.path.join(project_qc,"qc"))
# MultiQC output
outputs.append("multiqc_report.%s.html" % project.name)
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f),"Missing %s" % f)
# Check that missing projects weren't copied
for project in missing_projects:
self.assertFalse(os.path.exists(
os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
"qc_report.%s.%s" % (project.name,
os.path.basename(
ap.analysis_dir)))),
"%s exists in final dir, but shouldn't" %
project.name)
def test_publish_qc_with_icell8_outputs(self):
"""publish_qc: project with ICell8 QC outputs
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={ "run_number": 87,
"source": "local",
"instrument_datestamp": "160621" },
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn)
# Add processing report and QC outputs
UpdateAnalysisDir(ap).add_processing_report()
projects = ap.get_analysis_projects()
for project in projects:
UpdateAnalysisProject(project).add_qc_outputs()
# Add ICell8 report for one project
icell8_project = projects[0]
UpdateAnalysisProject(icell8_project).add_icell8_outputs()
# Make a mock publication area
publication_dir = os.path.join(self.dirn,'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap,location=publication_dir)
# Check outputs
outputs = ["index.html",
"processing_qc.html"]
for project in ap.get_analysis_projects():
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (project.name,
os.path.basename(
ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc,"qc_report.html"))
outputs.append(os.path.join(project_qc,"qc"))
# MultiQC output
outputs.append("multiqc_report.%s.html" % project.name)
# ICell8 outputs
icell8_dir = "icell8_processing.%s.%s" % (icell8_project.name,
os.path.basename(
ap.analysis_dir))
outputs.append(icell8_dir)
outputs.append("%s.zip" % icell8_dir)
outputs.append(os.path.join(icell8_dir,"icell8_processing_data"))
outputs.append(os.path.join(icell8_dir,"icell8_processing.html"))
outputs.append(os.path.join(icell8_dir,"stats"))
# Do checks
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f),"Missing %s" % f)
def test_publish_qc_with_projects_no_reports(self):
"""publish_qc: projects with all QC outputs but no reports
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={
"run_number": 87,
"source": "local",
"instrument_datestamp": "160621"
},
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn, settings=self.settings)
# Add processing report and QC outputs
UpdateAnalysisDir(ap).add_processing_report()
for project in ap.get_analysis_projects():
UpdateAnalysisProject(project).add_qc_outputs()
# Remove the QC reports
for project in ap.get_analysis_projects():
qc_reports = []
qc_reports.append(
"qc_report.%s.%s.zip" %
(project.name, os.path.basename(ap.analysis_dir)))
qc_reports.append("qc_report.html")
qc_reports.append("multiqc_report.html")
for f in qc_reports:
os.remove(os.path.join(project.dirn, f))
# Make a mock multiqc
MockMultiQC.create(os.path.join(self.bin, "multiqc"))
os.environ['PATH'] = "%s:%s" % (self.bin, os.environ['PATH'])
# Make a mock publication area
publication_dir = os.path.join(self.dirn, 'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap, location=publication_dir)
# Check outputs
outputs = ["index.html", "processing_qc.html"]
for project in ap.get_analysis_projects():
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (
project.name, os.path.basename(ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc, "qc_report.html"))
outputs.append(os.path.join(project_qc, "qc"))
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f), "Missing %s" % f)
def test_import_project_with_qc(self):
"""import_project: check project with QC outputs is imported
"""
# Make mock multiqc
MockMultiQC.create(os.path.join(self.bin, "multiqc"))
os.environ['PATH'] = "%s:%s" % (self.bin, os.environ['PATH'])
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_M00879_0087_000000000-AGEW9', 'miseq', top_dir=self.dirn)
mockdir.create()
# Add QC outputs to the project to be imported
UpdateAnalysisProject(AnalysisProject(
'NewProj',
self.new_project_dir)).add_qc_outputs(include_multiqc=False)
print(os.listdir(os.path.join(self.dirn, 'NewProj')))
# Check that the project is not currently present
ap = AutoProcess(mockdir.dirn)
self.assertFalse(
'NewProj' in [p.name for p in ap.get_analysis_projects()])
self.assertFalse(
'NewProj' in
[p.name for p in ap.get_analysis_projects_from_dirs()])
self.assertFalse(
os.path.exists(os.path.join(ap.analysis_dir, 'NewProj')))
# Import the project
import_project(ap, self.new_project_dir)
self.assertTrue(
'NewProj' in [p.name for p in ap.get_analysis_projects()])
self.assertTrue('NewProj' in
[p.name for p in ap.get_analysis_projects_from_dirs()])
self.assertTrue(
os.path.exists(os.path.join(ap.analysis_dir, 'NewProj')))
# Verify via fresh AutoProcess object
ap2 = AutoProcess(mockdir.dirn)
self.assertTrue(
'NewProj' in [p.name for p in ap2.get_analysis_projects()])
self.assertTrue(
'NewProj' in
[p.name for p in ap2.get_analysis_projects_from_dirs()])
self.assertTrue(
os.path.exists(os.path.join(ap2.analysis_dir, 'NewProj')))
# Check for QC report and ZIP file
print(os.listdir(os.path.join(ap2.analysis_dir, 'NewProj')))
for f in (
"qc_report.html",
"multiqc_report.html",
"qc_report.NewProj.160621_M00879_0087_000000000-AGEW9.zip",
):
f = os.path.join(ap2.analysis_dir, 'NewProj', f)
self.assertTrue(os.path.exists(f), "Missing %s" % f)
def test_publish_qc_missing_destination(self):
"""publish_qc: raise exception if destination doesn't exist
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={
"run_number": 87,
"source": "local",
"instrument_datestamp": "160621"
},
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn, settings=self.settings)
# Add processing report and QC outputs
UpdateAnalysisDir(ap).add_processing_report()
for project in ap.get_analysis_projects():
UpdateAnalysisProject(project).add_qc_outputs()
# Reference publication area which doesn't exist
publication_dir = os.path.join(self.dirn, 'QC')
self.assertFalse(os.path.exists(publication_dir))
# Publish
self.assertRaises(Exception, publish_qc, ap, location=publication_dir)
self.assertFalse(os.path.exists(publication_dir))
def test_publish_qc_with_project_missing_qc(self):
"""publish_qc: raises exception if project has missing QC
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={
"run_number": 87,
"source": "local",
"instrument_datestamp": "160621"
},
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn, settings=self.settings)
# Add processing report
UpdateAnalysisDir(ap).add_processing_report()
# Add QC outputs for subset of projects
projects = ap.get_analysis_projects()[1:]
for project in projects:
UpdateAnalysisProject(project).add_qc_outputs()
# Make a mock publication area
publication_dir = os.path.join(self.dirn, 'QC')
os.mkdir(publication_dir)
# Publish
self.assertRaises(Exception, publish_qc, ap, location=publication_dir)
def test_publish_qc_use_hierarchy(self):
"""publish_qc: publish using YEAR/PLATFORM hierarchy
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={ "run_number": 87,
"source": "local",
"instrument_datestamp": "160621" },
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn)
# Add processing report and QC outputs
UpdateAnalysisDir(ap).add_processing_report()
for project in ap.get_analysis_projects():
UpdateAnalysisProject(project).add_qc_outputs()
# Make a mock publication area
publication_dir = os.path.join(self.dirn,'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap,location=publication_dir,
use_hierarchy=True)
# Check outputs
final_dir = os.path.join(publication_dir,
"2016",
"hiseq")
self.assertTrue(os.path.exists(final_dir))
outputs = ["index.html",
"processing_qc.html"]
for project in ap.get_analysis_projects():
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (project.name,
os.path.basename(
ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc,"qc_report.html"))
outputs.append(os.path.join(project_qc,"qc"))
# MultiQC output
outputs.append("multiqc_report.%s.html" % project.name)
for item in outputs:
f = os.path.join(final_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f),"Missing %s" % f)
def test_import_project_with_comment(self):
"""import_project: check comment is appended
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_M00879_0087_000000000-AGEW9', 'miseq', top_dir=self.dirn)
mockdir.create()
# Check that the project is not currently present
ap = AutoProcess(mockdir.dirn)
self.assertFalse(
'NewProj' in [p.name for p in ap.get_analysis_projects()])
self.assertFalse(
'NewProj' in
[p.name for p in ap.get_analysis_projects_from_dirs()])
self.assertFalse(
os.path.exists(os.path.join(ap.analysis_dir, 'NewProj')))
# Import the project and append a comment
import_project(ap,
self.new_project_dir,
comment="imported from elsewhere")
self.assertTrue(
'NewProj' in [p.name for p in ap.get_analysis_projects()])
self.assertTrue('NewProj' in
[p.name for p in ap.get_analysis_projects_from_dirs()])
self.assertTrue(
os.path.exists(os.path.join(ap.analysis_dir, 'NewProj')))
# Verify via fresh AutoProcess object
ap2 = AutoProcess(mockdir.dirn)
self.assertTrue(
'NewProj' in [p.name for p in ap2.get_analysis_projects()])
self.assertTrue(
'NewProj' in
[p.name for p in ap2.get_analysis_projects_from_dirs()])
self.assertTrue(
os.path.exists(os.path.join(ap2.analysis_dir, 'NewProj')))
# Check the comment has been updated
with open(os.path.join(mockdir.dirn,'NewProj','README.info'),'rt') \
as fp:
self.assertTrue(
"Comments\t1% PhiX spike in; imported from elsewhere" in
fp.read())
def test_publish_qc_exclude_zip_files(self):
"""publish_qc: exclude ZIP files from publication
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={
"run_number": 87,
"source": "local",
"instrument_datestamp": "160621"
},
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn, settings=self.settings)
# Add processing report and QC outputs
UpdateAnalysisDir(ap).add_processing_report()
projects = ap.get_analysis_projects()
for project in projects:
UpdateAnalysisProject(project).add_qc_outputs()
# Add ICell8 report for one project
icell8_project = projects[0]
UpdateAnalysisProject(icell8_project).add_icell8_outputs()
# Add cellranger count output for one project
tenxgenomics_project = projects[-1]
UpdateAnalysisProject(
tenxgenomics_project).add_cellranger_count_outputs()
# Make a mock publication area
publication_dir = os.path.join(self.dirn, 'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap, location=publication_dir, exclude_zip_files=True)
# Check outputs
outputs = ["index.html", "processing_qc.html"]
zip_files = []
for project in ap.get_analysis_projects():
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (
project.name, os.path.basename(ap.analysis_dir))
outputs.append(project_qc)
outputs.append(os.path.join(project_qc, "qc_report.html"))
outputs.append(os.path.join(project_qc, "qc"))
zip_files.append("%s.zip" % project_qc)
# ICell8 outputs
icell8_dir = "icell8_processing.%s.%s" % (
icell8_project.name, os.path.basename(ap.analysis_dir))
outputs.append(icell8_dir)
outputs.append(os.path.join(icell8_dir, "icell8_processing_data"))
outputs.append(os.path.join(icell8_dir, "icell8_processing.html"))
outputs.append(os.path.join(icell8_dir, "stats"))
zip_files.append("%s.zip" % icell8_dir)
# Cellranger count outputs
cellranger_count_dir = "cellranger_count_report.%s.%s" % (
tenxgenomics_project.name, os.path.basename(ap.analysis_dir))
outputs.append(cellranger_count_dir)
outputs.append(
os.path.join(cellranger_count_dir, "cellranger_count_report.html"))
outputs.append(os.path.join(cellranger_count_dir, "cellranger_count"))
for sample in tenxgenomics_project.samples:
outputs.append(
os.path.join(cellranger_count_dir, "cellranger_count",
sample.name, "outs", "web_summary.html"))
zip_files.append("%s.zip" % cellranger_count_dir)
# Do checks
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f), "Missing %s" % f)
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f), "Missing %s" % f)
# Check the ZIP files were excluded
for zip_file in zip_files:
self.assertFalse(
os.path.exists(
os.path.join(
publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
zip_file)),
"ZIP file '%s' exists, but shouldn't" % zip_file)
def test_publish_qc_with_cellranger_counts(self):
"""publish_qc: project with cellranger count output
"""
# Make an auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'160621_K00879_0087_000000000-AGEW9',
'hiseq',
metadata={
"run_number": 87,
"source": "local",
"instrument_datestamp": "160621"
},
top_dir=self.dirn)
mockdir.create()
ap = AutoProcess(mockdir.dirn, settings=self.settings)
# Add processing and cellranger QC reports
UpdateAnalysisDir(ap).add_processing_report()
UpdateAnalysisDir(ap).add_cellranger_qc_output()
# Add QC outputs
projects = ap.get_analysis_projects()
for project in projects:
UpdateAnalysisProject(project).add_qc_outputs(
protocol="singlecell")
# Add cellranger count output for one project
tenxgenomics_project = projects[0]
UpdateAnalysisProject(
tenxgenomics_project).add_cellranger_count_outputs()
# Make a mock publication area
publication_dir = os.path.join(self.dirn, 'QC')
os.mkdir(publication_dir)
# Publish
publish_qc(ap, location=publication_dir)
# Check outputs
outputs = [
"index.html", "processing_qc.html", "cellranger_qc_summary.html"
]
for project in ap.get_analysis_projects():
# Standard QC outputs
project_qc = "qc_report.%s.%s" % (
project.name, os.path.basename(ap.analysis_dir))
outputs.append(project_qc)
outputs.append("%s.zip" % project_qc)
outputs.append(os.path.join(project_qc, "qc_report.html"))
outputs.append(os.path.join(project_qc, "qc"))
# Cellranger count outputs
cellranger_count_dir = "cellranger_count_report.%s.%s" % (
tenxgenomics_project.name, os.path.basename(ap.analysis_dir))
outputs.append(cellranger_count_dir)
outputs.append("%s.zip" % cellranger_count_dir)
outputs.append(
os.path.join(cellranger_count_dir, "cellranger_count_report.html"))
outputs.append(os.path.join(cellranger_count_dir, "cellranger_count"))
for sample in tenxgenomics_project.samples:
outputs.append(
os.path.join(cellranger_count_dir, "cellranger_count",
sample.name, "outs", "web_summary.html"))
# Do checks
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f), "Missing %s" % f)
# Do checks
for item in outputs:
f = os.path.join(publication_dir,
"160621_K00879_0087_000000000-AGEW9_analysis",
item)
self.assertTrue(os.path.exists(f), "Missing %s" % f)
def test_archive_to_final_multiple_fastq_sets_read_only_fastqs(self):
"""archive: test copying multiple fastq sets to final archive dir (read-only Fastqs)
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
# Add additional fastq set for first project
multi_fastqs_project = ap.get_analysis_projects()[0]
UpdateAnalysisProject(multi_fastqs_project).add_fastq_set(
"fastqs.extra",
("Alt1.r1.fastq.gz","Alt2.r1.fastq.gz"))
# Do archiving op
status = archive(ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=True,
final=True)
self.assertEqual(status,0)
# Check that final dir exists
final_archive_dir = os.path.join(
final_dir,
"170901_M00879_0087_000000000-AGEW9_analysis")
self.assertTrue(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Check contents
dirs = ("AB","CDE","logs","undetermined")
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info",
"custom_SampleSheet.csv",
"metadata.info",
"projects.info",
"SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir,f)
self.assertTrue(os.path.exists(f))
# Check that Fastqs are not writable
for project in ("AB","CDE","undetermined"):
fq_dir = os.path.join(final_archive_dir,
project,
"fastqs")
self.assertTrue(os.path.exists(fq_dir))
fqs = os.listdir(fq_dir)
self.assertTrue(len(fqs) > 0)
for fq in fqs:
fq = os.path.join(fq_dir,fq)
self.assertTrue(os.access(fq,os.R_OK))
self.assertFalse(os.access(fq,os.W_OK))
# Check additional Fastqs are not writable
fq_dir = os.path.join(final_archive_dir,
multi_fastqs_project.name,
"fastqs.extra")
self.assertTrue(os.path.exists(fq_dir))
fqs = os.listdir(fq_dir)
self.assertTrue(len(fqs) > 0)
for fq in fqs:
fq = os.path.join(fq_dir,fq)
self.assertTrue(os.access(fq,os.R_OK))
self.assertFalse(os.access(fq,os.W_OK))
