def test_archive_to_final_via_staging(self): """archive: test copying to staging then final archive dir """ # Make a mock auto-process directory mockdir = MockAnalysisDirFactory.bcl2fastq2( '170901_M00879_0087_000000000-AGEW9', 'miseq', metadata={"instrument_datestamp": "170901"}, top_dir=self.dirn) mockdir.create() # Make a mock archive directory archive_dir = os.path.join(self.dirn, "archive") final_dir = os.path.join(archive_dir, "2017", "miseq") os.makedirs(final_dir) self.assertTrue(os.path.isdir(final_dir)) self.assertEqual(len(os.listdir(final_dir)), 0) # Make autoprocess instance and set required metadata ap = AutoProcess(analysis_dir=mockdir.dirn) ap.set_metadata("source", "testing") ap.set_metadata("run_number", "87") # Do staging archiving op status = archive(ap, archive_dir=archive_dir, year='2017', platform='miseq', read_only_fastqs=False, final=False) self.assertEqual(status, 0) # Check that staging dir exists staging_dir = os.path.join( final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending") final_archive_dir = os.path.join( final_dir, "170901_M00879_0087_000000000-AGEW9_analysis") self.assertTrue(os.path.exists(staging_dir)) self.assertFalse(os.path.exists(final_archive_dir)) self.assertEqual(len(os.listdir(final_dir)), 1) # Do final archiving op status = archive(ap, archive_dir=archive_dir, year='2017', platform='miseq', read_only_fastqs=False, final=True) self.assertEqual(status, 0) self.assertFalse(os.path.exists(staging_dir)) self.assertTrue(os.path.exists(final_archive_dir)) self.assertEqual(len(os.listdir(final_dir)), 1) # Check contents dirs = ("AB", "CDE", "logs", "undetermined") for d in dirs: d = os.path.join(final_archive_dir, d) self.assertTrue(os.path.exists(d)) files = ("auto_process.info", "custom_SampleSheet.csv", "metadata.info", "projects.info", "SampleSheet.orig.csv") for f in files: f = os.path.join(final_archive_dir, f) self.assertTrue(os.path.exists(f))
def test_archive_force_no_projects(self): """archive: force final archiving of run with no projects """ # Make a mock auto-process directory mockdir = MockAnalysisDirFactory.bcl2fastq2( '170901_M00879_0087_000000000-AGEW9', 'miseq', metadata={ "instrument_datestamp": "170901" }, top_dir=self.dirn) mockdir.create(no_project_dirs=True) # Make a mock archive directory archive_dir = os.path.join(self.dirn,"archive") final_dir = os.path.join(archive_dir, "2017", "miseq") os.makedirs(final_dir) self.assertTrue(os.path.isdir(final_dir)) self.assertEqual(len(os.listdir(final_dir)),0) # Make autoprocess instance and set required metadata ap = AutoProcess(analysis_dir=mockdir.dirn, settings=self.settings) ap.set_metadata("source","testing") ap.set_metadata("run_number","87") # Do archiving op to staging status = archive(ap, archive_dir=archive_dir, year='2017',platform='miseq', read_only_fastqs=True, final=True, force=True) self.assertEqual(status,0) # Check that final dir exists final_archive_dir = os.path.join( final_dir, "170901_M00879_0087_000000000-AGEW9_analysis") self.assertTrue(os.path.exists(final_archive_dir)) self.assertEqual(len(os.listdir(final_dir)),1) # Check contents dirs = ("bcl2fastq","logs",) for d in dirs: d = os.path.join(final_archive_dir,d) self.assertTrue(os.path.exists(d)) files = ("auto_process.info", "custom_SampleSheet.csv", "metadata.info", "projects.info", "SampleSheet.orig.csv") for f in files: f = os.path.join(final_archive_dir,f) self.assertTrue(os.path.exists(f))
def test_archive_to_staging_set_group(self): """archive: test copying to staging archive dir and set group """ # Find groups for current user current_user = pwd.getpwuid(os.getuid()).pw_name groups = [g.gr_gid for g in grp.getgrall() if current_user in g.gr_mem] if len(groups) < 2: raise unittest.SkipTest("user '%s' must be in at least " "two groups for this test" % current_user) # Find a group to set archived files to current_gid = os.stat(self.dirn).st_gid new_group = None for gid in groups: if gid != current_gid: new_group = gid break self.assertTrue(new_group is not None) # Make a mock auto-process directory mockdir = MockAnalysisDirFactory.bcl2fastq2( '170901_M00879_0087_000000000-AGEW9', 'miseq', metadata={"instrument_datestamp": "170901"}, top_dir=self.dirn) mockdir.create() # Make a mock archive directory archive_dir = os.path.join(self.dirn, "archive") final_dir = os.path.join(archive_dir, "2017", "miseq") os.makedirs(final_dir) self.assertTrue(os.path.isdir(final_dir)) self.assertEqual(len(os.listdir(final_dir)), 0) # Make autoprocess instance and set required metadata ap = AutoProcess(analysis_dir=mockdir.dirn) ap.set_metadata("source", "testing") ap.set_metadata("run_number", "87") # Do archiving op status = archive(ap, archive_dir=archive_dir, year='2017', platform='miseq', read_only_fastqs=False, group=new_group, final=False) self.assertEqual(status, 0) # Check that staging dir exists staging_dir = os.path.join( final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending") self.assertTrue(os.path.exists(staging_dir)) self.assertEqual(len(os.listdir(final_dir)), 1) # Check group of staging dir self.assertEqual(os.stat(staging_dir).st_gid, new_group) # Check contents dirs = ("AB", "CDE", "logs", "undetermined") for d in dirs: d = os.path.join(staging_dir, d) self.assertTrue(os.path.exists(d)) self.assertEqual(os.stat(d).st_gid, new_group) files = ("auto_process.info", "custom_SampleSheet.csv", "metadata.info", "projects.info", "SampleSheet.orig.csv") for f in files: f = os.path.join(staging_dir, f) self.assertTrue(os.path.exists(f)) self.assertEqual(os.stat(f).st_gid, new_group)
def test_archive_staging_to_final(self): """archive: test archiving directly from staging dir """ # Make a mock auto-process directory mockdir = MockAnalysisDirFactory.bcl2fastq2( '170901_M00879_0087_000000000-AGEW9', 'miseq', metadata={"instrument_datestamp": "170901"}, top_dir=self.dirn) mockdir.create() # Make a mock archive directory archive_dir = os.path.join(self.dirn, "archive") final_dir = os.path.join(archive_dir, "2017", "miseq") os.makedirs(final_dir) self.assertTrue(os.path.isdir(final_dir)) self.assertEqual(len(os.listdir(final_dir)), 0) # Make autoprocess instance and set required metadata ap = AutoProcess(analysis_dir=mockdir.dirn) ap.set_metadata("source", "testing") ap.set_metadata("run_number", "87") ap.save_metadata() # Move to the archive area as a "pending" directory os.rename( mockdir.dirn, os.path.join( archive_dir, "2017", "miseq", "__170901_M00879_0087_000000000-AGEW9_analysis.pending")) # Load pending dir into a new autoprocess instance ap = AutoProcess(analysis_dir=os.path.join( archive_dir, "2017", "miseq", "__170901_M00879_0087_000000000-AGEW9_analysis.pending")) # Staging archiving attempt should fail self.assertRaises(Exception, archive, ap, archive_dir=archive_dir, year='2017', platform='miseq', read_only_fastqs=False, final=False) staging_dir = os.path.join( final_dir, "__170901_M00879_0087_000000000-AGEW9_analysis.pending") final_archive_dir = os.path.join( final_dir, "170901_M00879_0087_000000000-AGEW9_analysis") self.assertFalse(os.path.exists(final_archive_dir)) self.assertEqual(len(os.listdir(final_dir)), 1) # Copy to final should work status = archive(ap, archive_dir=archive_dir, year='2017', platform='miseq', read_only_fastqs=False, final=True) self.assertEqual(status, 0) self.assertTrue(os.path.exists(final_archive_dir)) self.assertFalse(os.path.exists(staging_dir)) self.assertEqual(len(os.listdir(final_dir)), 1) # Check contents dirs = ("AB", "CDE", "logs", "undetermined") for d in dirs: d = os.path.join(final_archive_dir, d) self.assertTrue(os.path.exists(d)) files = ("auto_process.info", "custom_SampleSheet.csv", "metadata.info", "projects.info", "SampleSheet.orig.csv") for f in files: f = os.path.join(final_archive_dir, f) self.assertTrue(os.path.exists(f))
def test_archive_to_final_ignores_bak_projects(self): """archive: check final archiving ignores .bak directories """ # Make a mock auto-process directory mockdir = MockAnalysisDirFactory.bcl2fastq2( '170901_M00879_0087_000000000-AGEW9', 'miseq', metadata={ "instrument_datestamp": "170901" }, top_dir=self.dirn) mockdir.create() # Make a mock archive directory archive_dir = os.path.join(self.dirn,"archive") final_dir = os.path.join(archive_dir, "2017", "miseq") os.makedirs(final_dir) self.assertTrue(os.path.isdir(final_dir)) self.assertEqual(len(os.listdir(final_dir)),0) # Make autoprocess instance and set required metadata ap = AutoProcess(analysis_dir=mockdir.dirn, settings=self.settings) ap.set_metadata("source","testing") ap.set_metadata("run_number","87") # Add a .bak project directory shutil.copytree(os.path.join(mockdir.dirn,"AB"), os.path.join(mockdir.dirn,"AB.bak")) # Do archiving op status = archive(ap, archive_dir=archive_dir, year='2017',platform='miseq', read_only_fastqs=True, final=True) self.assertEqual(status,0) # Check that final dir exists final_archive_dir = os.path.join( final_dir, "170901_M00879_0087_000000000-AGEW9_analysis") self.assertTrue(os.path.exists(final_archive_dir)) self.assertEqual(len(os.listdir(final_dir)),1) # Check contents dirs = ("AB","CDE","logs","undetermined") for d in dirs: d = os.path.join(final_archive_dir,d) self.assertTrue(os.path.exists(d)) files = ("auto_process.info", "custom_SampleSheet.csv", "metadata.info", "projects.info", "SampleSheet.orig.csv") for f in files: f = os.path.join(final_archive_dir,f) self.assertTrue(os.path.exists(f)) # Check that Fastqs are not writable for project in ("AB","CDE","undetermined"): fq_dir = os.path.join(final_archive_dir, project, "fastqs") self.assertTrue(os.path.exists(fq_dir)) fqs = os.listdir(fq_dir) self.assertTrue(len(fqs) > 0) for fq in fqs: fq = os.path.join(fq_dir,fq) self.assertTrue(os.access(fq,os.R_OK)) self.assertFalse(os.access(fq,os.W_OK)) # Check .bak directory wasn't copied dirs = ("AB.bak",) for d in dirs: d = os.path.join(final_archive_dir,d) self.assertFalse(os.path.exists(d))
def test_archive_to_final_multiple_fastq_sets_read_only_fastqs(self): """archive: test copying multiple fastq sets to final archive dir (read-only Fastqs) """ # Make a mock auto-process directory mockdir = MockAnalysisDirFactory.bcl2fastq2( '170901_M00879_0087_000000000-AGEW9', 'miseq', metadata={ "instrument_datestamp": "170901" }, top_dir=self.dirn) mockdir.create() # Make a mock archive directory archive_dir = os.path.join(self.dirn,"archive") final_dir = os.path.join(archive_dir, "2017", "miseq") os.makedirs(final_dir) self.assertTrue(os.path.isdir(final_dir)) self.assertEqual(len(os.listdir(final_dir)),0) # Make autoprocess instance and set required metadata ap = AutoProcess(analysis_dir=mockdir.dirn, settings=self.settings) ap.set_metadata("source","testing") ap.set_metadata("run_number","87") # Add additional fastq set for first project multi_fastqs_project = ap.get_analysis_projects()[0] UpdateAnalysisProject(multi_fastqs_project).add_fastq_set( "fastqs.extra", ("Alt1.r1.fastq.gz","Alt2.r1.fastq.gz")) # Do archiving op status = archive(ap, archive_dir=archive_dir, year='2017',platform='miseq', read_only_fastqs=True, final=True) self.assertEqual(status,0) # Check that final dir exists final_archive_dir = os.path.join( final_dir, "170901_M00879_0087_000000000-AGEW9_analysis") self.assertTrue(os.path.exists(final_archive_dir)) self.assertEqual(len(os.listdir(final_dir)),1) # Check contents dirs = ("AB","CDE","logs","undetermined") for d in dirs: d = os.path.join(final_archive_dir,d) self.assertTrue(os.path.exists(d)) files = ("auto_process.info", "custom_SampleSheet.csv", "metadata.info", "projects.info", "SampleSheet.orig.csv") for f in files: f = os.path.join(final_archive_dir,f) self.assertTrue(os.path.exists(f)) # Check that Fastqs are not writable for project in ("AB","CDE","undetermined"): fq_dir = os.path.join(final_archive_dir, project, "fastqs") self.assertTrue(os.path.exists(fq_dir)) fqs = os.listdir(fq_dir) self.assertTrue(len(fqs) > 0) for fq in fqs: fq = os.path.join(fq_dir,fq) self.assertTrue(os.access(fq,os.R_OK)) self.assertFalse(os.access(fq,os.W_OK)) # Check additional Fastqs are not writable fq_dir = os.path.join(final_archive_dir, multi_fastqs_project.name, "fastqs.extra") self.assertTrue(os.path.exists(fq_dir)) fqs = os.listdir(fq_dir) self.assertTrue(len(fqs) > 0) for fq in fqs: fq = os.path.join(fq_dir,fq) self.assertTrue(os.access(fq,os.R_OK)) self.assertFalse(os.access(fq,os.W_OK))