def test_archive_to_final_via_staging(self):
"""archive: test copying to staging then final archive dir
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={"instrument_datestamp": "170901"},
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn, "archive")
final_dir = os.path.join(archive_dir, "2017", "miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)), 0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn)
ap.set_metadata("source", "testing")
ap.set_metadata("run_number", "87")
# Do staging archiving op
status = archive(ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=False)
self.assertEqual(status, 0)
# Check that staging dir exists
staging_dir = os.path.join(
final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending")
final_archive_dir = os.path.join(
final_dir, "170901_M00879_0087_000000000-AGEW9_analysis")
self.assertTrue(os.path.exists(staging_dir))
self.assertFalse(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)), 1)
# Do final archiving op
status = archive(ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=True)
self.assertEqual(status, 0)
self.assertFalse(os.path.exists(staging_dir))
self.assertTrue(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)), 1)
# Check contents
dirs = ("AB", "CDE", "logs", "undetermined")
for d in dirs:
d = os.path.join(final_archive_dir, d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info", "custom_SampleSheet.csv",
"metadata.info", "projects.info", "SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir, f)
self.assertTrue(os.path.exists(f))
def test_archive_force_no_projects(self):
"""archive: force final archiving of run with no projects
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create(no_project_dirs=True)
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
# Do archiving op to staging
status = archive(ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=True,
final=True,
force=True)
self.assertEqual(status,0)
# Check that final dir exists
final_archive_dir = os.path.join(
final_dir,
"170901_M00879_0087_000000000-AGEW9_analysis")
self.assertTrue(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Check contents
dirs = ("bcl2fastq","logs",)
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info",
"custom_SampleSheet.csv",
"metadata.info",
"projects.info",
"SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir,f)
self.assertTrue(os.path.exists(f))
def test_archive_force_fails_for_no_data(self):
"""archive: force archiving fails if run has no data
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create(no_project_dirs=True)
# Remove the 'bcl2fastq' subdirectory
shutil.rmtree(os.path.join(
self.dirn,
'170901_M00879_0087_000000000-AGEW9_analysis',
'bcl2fastq'))
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
# Do archiving op to staging
self.assertRaises(Exception,
archive,
ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=True,
final=False,
force=True)
# Check that staging dir exists
staging_dir = os.path.join(
final_dir,
"__170901_M00879_0087_000000000-AGEW9_analysis.pending")
self.assertFalse(os.path.exists(staging_dir))
def test_archive_to_staging_set_group(self):
"""archive: test copying to staging archive dir and set group
"""
# Find groups for current user
current_user = pwd.getpwuid(os.getuid()).pw_name
groups = [g.gr_gid for g in grp.getgrall() if current_user in g.gr_mem]
if len(groups) < 2:
raise unittest.SkipTest("user '%s' must be in at least "
"two groups for this test" % current_user)
# Find a group to set archived files to
current_gid = os.stat(self.dirn).st_gid
new_group = None
for gid in groups:
if gid != current_gid:
new_group = gid
break
self.assertTrue(new_group is not None)
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={"instrument_datestamp": "170901"},
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn, "archive")
final_dir = os.path.join(archive_dir, "2017", "miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)), 0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn)
ap.set_metadata("source", "testing")
ap.set_metadata("run_number", "87")
# Do archiving op
status = archive(ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
group=new_group,
final=False)
self.assertEqual(status, 0)
# Check that staging dir exists
staging_dir = os.path.join(
final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending")
self.assertTrue(os.path.exists(staging_dir))
self.assertEqual(len(os.listdir(final_dir)), 1)
# Check group of staging dir
self.assertEqual(os.stat(staging_dir).st_gid, new_group)
# Check contents
dirs = ("AB", "CDE", "logs", "undetermined")
for d in dirs:
d = os.path.join(staging_dir, d)
self.assertTrue(os.path.exists(d))
self.assertEqual(os.stat(d).st_gid, new_group)
files = ("auto_process.info", "custom_SampleSheet.csv",
"metadata.info", "projects.info", "SampleSheet.orig.csv")
for f in files:
f = os.path.join(staging_dir, f)
self.assertTrue(os.path.exists(f))
self.assertEqual(os.stat(f).st_gid, new_group)
def test_archive_oserror_if_destination_doesnt_exist(self):
"""archive: test archiving raises OSError if destination doesn't exist
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={"instrument_datestamp": "170901"},
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn, "archive")
os.makedirs(archive_dir)
final_dir = os.path.join(archive_dir, "2017", "miseq")
self.assertFalse(os.path.isdir(final_dir))
staging_dir = os.path.join(
final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending")
final_archive_dir = os.path.join(
final_dir, "170901_M00879_0087_000000000-AGEW9_analysis")
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn)
ap.set_metadata("source", "testing")
ap.set_metadata("run_number", "87")
# Staging attempt should fail
self.assertRaises(OSError,
archive,
ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=False)
self.assertFalse(os.path.exists(staging_dir))
# Final archiving attempt should fail
self.assertRaises(OSError,
archive,
ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=True)
self.assertFalse(os.path.exists(final_archive_dir))
# Make "YEAR" level in archive dir
os.makedirs(os.path.join(archive_dir, "2017"))
# Staging attempt should fail
self.assertRaises(OSError,
archive,
ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=False)
self.assertFalse(os.path.exists(staging_dir))
# Final archiving attempt should fail
self.assertRaises(OSError,
archive,
ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=True)
self.assertFalse(os.path.exists(final_archive_dir))
def test_archive_staging_to_final(self):
"""archive: test archiving directly from staging dir
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={"instrument_datestamp": "170901"},
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn, "archive")
final_dir = os.path.join(archive_dir, "2017", "miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)), 0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn)
ap.set_metadata("source", "testing")
ap.set_metadata("run_number", "87")
ap.save_metadata()
# Move to the archive area as a "pending" directory
os.rename(
mockdir.dirn,
os.path.join(
archive_dir, "2017", "miseq",
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
# Load pending dir into a new autoprocess instance
ap = AutoProcess(analysis_dir=os.path.join(
archive_dir, "2017", "miseq",
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
# Staging archiving attempt should fail
self.assertRaises(Exception,
archive,
ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=False)
staging_dir = os.path.join(
final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending")
final_archive_dir = os.path.join(
final_dir, "170901_M00879_0087_000000000-AGEW9_analysis")
self.assertFalse(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)), 1)
# Copy to final should work
status = archive(ap,
archive_dir=archive_dir,
year='2017',
platform='miseq',
read_only_fastqs=False,
final=True)
self.assertEqual(status, 0)
self.assertTrue(os.path.exists(final_archive_dir))
self.assertFalse(os.path.exists(staging_dir))
self.assertEqual(len(os.listdir(final_dir)), 1)
# Check contents
dirs = ("AB", "CDE", "logs", "undetermined")
for d in dirs:
d = os.path.join(final_archive_dir, d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info", "custom_SampleSheet.csv",
"metadata.info", "projects.info", "SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir, f)
self.assertTrue(os.path.exists(f))
except ValueError:
logging.error("Can't process '%s'" % options.key_value)
d.save_parameters(force=True)
else:
d.print_params()
elif cmd == 'metadata':
# Update the metadata associated with the analysis
if options.update:
d.update_metadata()
if options.key_value is not None:
for key_value in options.key_value:
try:
i = key_value.index('=')
key = key_value[:i]
value = key_value[i+1:].strip("'").strip('"')
d.set_metadata(key,value)
except ValueError:
logging.error("Can't process '%s'" % options.key_value)
if options.key_value or options.update:
# Save updates to file
d.save_metadata(force=True)
else:
# Only print values
d.print_metadata()
elif cmd == 'readme':
if options.init:
d.init_readme()
elif options.edit:
d.edit_readme()
else:
if d.readme_file is not None:
def test_archive_to_final_ignores_bak_projects(self):
"""archive: check final archiving ignores .bak directories
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
# Add a .bak project directory
shutil.copytree(os.path.join(mockdir.dirn,"AB"),
os.path.join(mockdir.dirn,"AB.bak"))
# Do archiving op
status = archive(ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=True,
final=True)
self.assertEqual(status,0)
# Check that final dir exists
final_archive_dir = os.path.join(
final_dir,
"170901_M00879_0087_000000000-AGEW9_analysis")
self.assertTrue(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Check contents
dirs = ("AB","CDE","logs","undetermined")
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info",
"custom_SampleSheet.csv",
"metadata.info",
"projects.info",
"SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir,f)
self.assertTrue(os.path.exists(f))
# Check that Fastqs are not writable
for project in ("AB","CDE","undetermined"):
fq_dir = os.path.join(final_archive_dir,
project,
"fastqs")
self.assertTrue(os.path.exists(fq_dir))
fqs = os.listdir(fq_dir)
self.assertTrue(len(fqs) > 0)
for fq in fqs:
fq = os.path.join(fq_dir,fq)
self.assertTrue(os.access(fq,os.R_OK))
self.assertFalse(os.access(fq,os.W_OK))
# Check .bak directory wasn't copied
dirs = ("AB.bak",)
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertFalse(os.path.exists(d))
def test_archive_to_final_multiple_fastq_sets_read_only_fastqs(self):
"""archive: test copying multiple fastq sets to final archive dir (read-only Fastqs)
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
# Add additional fastq set for first project
multi_fastqs_project = ap.get_analysis_projects()[0]
UpdateAnalysisProject(multi_fastqs_project).add_fastq_set(
"fastqs.extra",
("Alt1.r1.fastq.gz","Alt2.r1.fastq.gz"))
# Do archiving op
status = archive(ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=True,
final=True)
self.assertEqual(status,0)
# Check that final dir exists
final_archive_dir = os.path.join(
final_dir,
"170901_M00879_0087_000000000-AGEW9_analysis")
self.assertTrue(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Check contents
dirs = ("AB","CDE","logs","undetermined")
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info",
"custom_SampleSheet.csv",
"metadata.info",
"projects.info",
"SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir,f)
self.assertTrue(os.path.exists(f))
# Check that Fastqs are not writable
for project in ("AB","CDE","undetermined"):
fq_dir = os.path.join(final_archive_dir,
project,
"fastqs")
self.assertTrue(os.path.exists(fq_dir))
fqs = os.listdir(fq_dir)
self.assertTrue(len(fqs) > 0)
for fq in fqs:
fq = os.path.join(fq_dir,fq)
self.assertTrue(os.access(fq,os.R_OK))
self.assertFalse(os.access(fq,os.W_OK))
# Check additional Fastqs are not writable
fq_dir = os.path.join(final_archive_dir,
multi_fastqs_project.name,
"fastqs.extra")
self.assertTrue(os.path.exists(fq_dir))
fqs = os.listdir(fq_dir)
self.assertTrue(len(fqs) > 0)
for fq in fqs:
fq = os.path.join(fq_dir,fq)
self.assertTrue(os.access(fq,os.R_OK))
self.assertFalse(os.access(fq,os.W_OK))