def setUp(self): pip_install_vtam_for_tests() self.test_path = PathManager.get_test_path() self.outdir_path = os.path.join(self.test_path, 'outdir') pathlib.Path(self.outdir_path).mkdir(exist_ok=True, parents=True)
def setUpClass(cls): ######################################################################## # # These tests need the vtam command in the path # ######################################################################## pip_install_vtam_for_tests() cls.package_path = os.path.join(PathManager.get_package_path()) cls.test_path = os.path.join(PathManager.get_test_path()) cls.outdir_path = os.path.join(cls.test_path, 'outdir') cls.outdir_data_path = os.path.join(cls.outdir_path, 'data') shutil.rmtree(cls.outdir_path, ignore_errors=True) pathlib.Path(cls.outdir_data_path).mkdir(parents=True, exist_ok=True) ############################################################################################ # # Download sorted reads dataset (Updated Oct 10, 2020) # ############################################################################################ sorted_tar_path = os.path.join(cls.outdir_data_path, "sorted.tar.gz") # Test first in local dir, otherwise in the remote URLs if not os.path.isfile(sorted_tar_path) or pathlib.Path( sorted_tar_path).stat().st_size < 1000000: try: # urllib.request.urlretrieve(sorted_tar_gz_url1, sorted_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(sorted_tar_path)) urllib.request.urlretrieve(sorted_tar_gz_url1, sorted_tar_path, reporthook=tqdm_hook(t)) except Exception: try: # urllib.request.urlretrieve(sorted_tar_gz_url2, sorted_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(sorted_tar_path)) urllib.request.urlretrieve(sorted_tar_gz_url2, sorted_tar_path, reporthook=tqdm_hook(t)) except Exception: # urllib.request.urlretrieve(sorted_tar_gz_url3, sorted_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(sorted_tar_path)) urllib.request.urlretrieve(sorted_tar_gz_url3, sorted_tar_path, reporthook=tqdm_hook(t)) tar = tarfile.open(sorted_tar_path, "r:gz") tar.extractall(path=cls.outdir_data_path) tar.close()
def setUpClass(cls): # vtam needs to be in the tsv_path pip_install_vtam_for_tests() cls.test_path = os.path.join(PathManager.get_test_path()) cls.outdir_path = os.path.join(cls.test_path, 'outdir') cls.args = {} cls.args['taxonomy'] = os.path.join(cls.outdir_path, "taxonomy.tsv") cls.args['coi_blast_db_dir'] = os.path.join(cls.outdir_path, "coi_blast_db_dir") pathlib.Path(cls.args['coi_blast_db_dir']).mkdir(exist_ok=True, parents=True) ############################################################################################ # # Run 'vtam taxonomy' # ############################################################################################ cmd = "vtam taxonomy --output {taxonomy} --precomputed".format(**cls.args) if sys.platform.startswith("win"): args = cmd else: args = shlex.split(cmd) subprocess.run(args=args) ############################################################################################ # # Run 'vtam coi_blast_db' # ############################################################################################ cmd = "vtam coi_blast_db --blastdbdir {coi_blast_db_dir} --blastdbname coi_blast_db_20200420 ".format(**cls.args) # if not (os.path.isfile(os.path.join(cls.args['coi_blast_db_dir'], "coi_blast_db_20200420.nhr"))): if sys.platform.startswith("win"): args = cmd else: args = shlex.split(cmd) subprocess.run(args=args)
def setUpClass(cls): ######################################################################## # # These tests need the vtam command in the path # ######################################################################## pip_install_vtam_for_tests() # vtam needs to be in the path cls.package_path = PathManager.get_package_path() cls.test_path = PathManager.get_test_path() cls.outdir_path = os.path.join(cls.test_path, 'outdir') shutil.rmtree(cls.outdir_path, ignore_errors=True) cls.outdir_data_path = os.path.join(cls.outdir_path, 'data') pathlib.Path(cls.outdir_data_path).mkdir(parents=True, exist_ok=True) cls.outdir_download_path = os.path.join(cls.test_path, 'outdir_download') pathlib.Path(cls.outdir_download_path).mkdir(parents=True, exist_ok=True) cls.snakefile_tuto_data = os.path.join( cls.package_path, "data/snake.tuto.data_makeknownoccurrences.yml") ############################################################################################ # # Set command args # ############################################################################################ cls.args = {} cls.args['package_path'] = cls.package_path cls.args['snake_tuto_data'] = cls.snakefile_tuto_data ############################################################################################ # # Download fastq test dataset # ############################################################################################ fastq_tar_path = os.path.join(cls.outdir_download_path, "fastq.tar.gz") # Test first in local dir, otherwise in the remote URLs if not os.path.isfile(fastq_tar_path) or pathlib.Path( fastq_tar_path).stat().st_size < 1000000: try: # urllib.request.urlretrieve(fastq_tar_gz_url1, fastq_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(fastq_tar_path)) urllib.request.urlretrieve(fastq_tar_gz_url1, fastq_tar_path, reporthook=tqdm_hook(t)) except Exception: try: # urllib.request.urlretrieve(fastq_tar_gz_url2, fastq_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(fastq_tar_path)) urllib.request.urlretrieve(fastq_tar_gz_url2, fastq_tar_path, reporthook=tqdm_hook(t)) except Exception: # urllib.request.urlretrieve(fastq_tar_gz_url3, fastq_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(fastq_tar_path)) urllib.request.urlretrieve(fastq_tar_gz_url3, fastq_tar_path, reporthook=tqdm_hook(t)) tar = tarfile.open(fastq_tar_path, "r:gz") tar.extractall(path=cls.outdir_path) tar.close() ############################################################################################ # # Copy data to directory tree # ############################################################################################ cmd = "snakemake --cores 1 -s {snake_tuto_data} --config MARKER=mfzr " \ "PROJECT=asper1 PACKAGE_PATH={package_path} --until all_one_marker_makeknownoccurrences".format(**cls.args) if sys.platform.startswith("win"): args = cmd else: args = shlex.split(cmd) subprocess.run(args=args, check=True, cwd=cls.outdir_path)
def setUp(self): pip_install_vtam_for_tests() self.test_path = PathManager.get_test_path() self.package_path = PathManager.get_package_path() self.outdir_path = os.path.join(self.test_path, 'outdir') shutil.rmtree(self.outdir_path, ignore_errors=True) pathlib.Path(self.outdir_path).mkdir(parents=True, exist_ok=True) self.args = {} self.args['runmarker'] = os.path.join(self.package_path, "data", "example", "pool_run_marker.tsv") self.args['db'] = os.path.join(self.outdir_path, "db.sqlite") ############################################################################################ # # Init DB # ############################################################################################ filter_codon_stop_path = os.path.join(self.test_path, "test_files_dryad.f40v5_small", "run1_mfzr_zfzr", "filter_codon_stop.tsv") variant_path = os.path.join(self.test_path, "test_files_dryad.f40v5_small", "run1_mfzr_zfzr", "variant_filter_codon_stop.tsv") sample_information_path = os.path.join(self.test_path, "test_files_dryad.f40v5_small", "run1_mfzr_zfzr", "sample_information.tsv") self.engine = sqlalchemy.create_engine('sqlite:///{}'.format( self.args['db']), echo=False) sample_information_df = pandas.read_csv(sample_information_path, sep="\t", header=0) sample_information_df.to_sql(name=SampleInformation.__tablename__, con=self.engine.connect(), if_exists='replace') run_df = pandas.DataFrame({'name': ['run1']}, index=range(1, 2)) run_df.to_sql(name=Run.__tablename__, con=self.engine.connect(), index_label='id', if_exists='replace') marker_df = pandas.DataFrame({'name': ['MFZR', 'ZFZR']}, index=range(1, 3)) marker_df.to_sql(name=Marker.__tablename__, con=self.engine.connect(), index_label='id', if_exists='replace') sample_df = pandas.DataFrame( {'name': ['tpos1_run1', 'tnegtag_run1', '14ben01', '14ben02']}, index=range(1, 5)) sample_df.to_sql(name=Sample.__tablename__, con=self.engine.connect(), index_label='id', if_exists='replace') variant_df = pandas.read_csv(variant_path, sep="\t", header=0, index_col='id') variant_df.to_sql(name=Variant.__tablename__, con=self.engine.connect(), index_label='id', if_exists='replace') filter_codon_stop_df = pandas.read_csv(filter_codon_stop_path, sep="\t", header=0) filter_codon_stop_df.to_sql(name=FilterCodonStop.__tablename__, con=self.engine.connect(), if_exists='replace') filter_chimera_borderline_path = os.path.join( self.test_path, "test_files_dryad.f40v5_small", "run1_mfzr_zfzr", "filter_chimera_borderline_and_filter_codon_stop.tsv") filter_chimera_borderline_db = pandas.read_csv( filter_chimera_borderline_path, sep="\t", header=0) filter_chimera_borderline_db.to_sql( name=FilterChimeraBorderline.__tablename__, con=self.engine.connect(), if_exists='replace') self.sample_list = ['tpos1_run1', 'tnegtag_run1', '14ben01', '14ben02']
def setUpClass(cls): pip_install_vtam_for_tests() # vtam needs to be in the path
def setUpClass(cls): pip_install_vtam_for_tests() # vtam needs to be in the path cls.package_path = PathManager.get_package_path() cls.test_path = os.path.join(PathManager.get_test_path()) cls.outdir_path = os.path.join(cls.test_path, 'outdir') cls.outdir_data_path = os.path.join(cls.outdir_path, 'data') shutil.rmtree(cls.outdir_path, ignore_errors=True ) # during development of the test, this prevents errors pathlib.Path(cls.outdir_data_path).mkdir(parents=True, exist_ok=True) ############################################################################################ # # Download fastq test dataset # ############################################################################################ fastq_tar_path = os.path.join(cls.outdir_data_path, "fastq.tar.gz") # Test first in local dir, otherwise in the remote URLs if not os.path.isfile(fastq_tar_path) or pathlib.Path( fastq_tar_path).stat().st_size < 1000000: try: # urllib.request.urlretrieve(fastq_tar_gz_url1, fastq_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(fastq_tar_path)) urllib.request.urlretrieve(fastq_tar_gz_url1, fastq_tar_path, reporthook=tqdm_hook(t)) except Exception: try: # urllib.request.urlretrieve(fastq_tar_gz_url2, fastq_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(fastq_tar_path)) urllib.request.urlretrieve(fastq_tar_gz_url2, fastq_tar_path, reporthook=tqdm_hook(t)) except Exception: # urllib.request.urlretrieve(fastq_tar_gz_url3, fastq_tar_path, MyProgressBar()) with tqdm(...) as t: t.set_description(os.path.basename(fastq_tar_path)) urllib.request.urlretrieve(fastq_tar_gz_url3, fastq_tar_path, reporthook=tqdm_hook(t)) tar = tarfile.open(fastq_tar_path, "r:gz") tar.extractall(path=cls.outdir_path) tar.close() # Set test paths cls.fastqinfo_path = os.path.join(PathManager.get_package_path(), "data/example/fastqinfo.tsv") cls.fastqdir_path = os.path.join(cls.outdir_path, "fastq") cls.fastainfo_path = os.path.join(cls.outdir_path, "fastainfo.tsv") cls.fastadir_path = os.path.join(cls.outdir_path, "merged") cls.sorted_dir_path = os.path.join(cls.outdir_path, "sorted") cls.sortedinfo_path = os.path.join(cls.sorted_dir_path, "sortedinfo.tsv") cls.log_path = os.path.join(cls.outdir_path, "vtam.log") cls.asvtable_path = os.path.join(cls.outdir_path, "asvtable_default.tsv") cls.args = {} cls.args['fastqinfo'] = cls.fastqinfo_path cls.args['fastqdir'] = cls.fastqdir_path cls.args['fastainfo'] = cls.fastainfo_path cls.args['fastadir'] = cls.fastadir_path cls.args['sorted'] = cls.sorted_dir_path cls.args['db'] = os.path.join(cls.outdir_path, "db.sqlite") cls.args['sortedinfo'] = cls.sortedinfo_path cls.args['sorteddir'] = cls.sorted_dir_path cls.args['asvtable'] = cls.asvtable_path cls.args['log'] = cls.log_path ################################################################################################################ # # Command Merge # ################################################################################################################ cmd = "vtam merge --fastqinfo {fastqinfo} --fastqdir {fastqdir} --fastainfo {fastainfo} --fastadir {fastadir} " \ "-v --log {log}".format(**cls.args) if sys.platform.startswith("win"): args = cmd else: args = shlex.split(cmd) subprocess.run(args=args)
def setUp(self): pip_install_vtam_for_tests() self.test_path = PathManager.get_test_path() self.package_path = PathManager.get_package_path() self.outdir_path = os.path.join(self.test_path, 'outdir') shutil.rmtree(self.outdir_path, ignore_errors=True) pathlib.Path(self.outdir_path).mkdir(exist_ok=True, parents=True) marker_str = "id name\n1 IIICBR" run_str = "id name\n1 TAS2" sample_str = "id name\n1 S21" variant_str = """id sequence 1 ATTGTCAGACACTCCGTACCATTAGGGTGCTGCAGTCGACTAGTCTATTTTAAGCTTACACGTAGCCGGAATTAGTTCATTACTGGGGTCAATTAATATCATAACAACGATCATTAACTAGAGGGCCCCAGGAATGACCTGGGAGAACTTACCGTTATTCGTGTGGGCTGTATTTATTACAGCGTGGTTACTTGTACTGTCTTTACCAGTACTAGCTGGTGCGATTACCATGCTGCTAACAGATAGGAACTAGAATACTAGTTTCTACGACCCGAACGGAGGAGGAGATCCTCTGCTATACCAGCATCTATTC""" filter_indel_str = """id run_id marker_id variant_id sample_id replicate read_count filter_delete 1 1 1 1 1 1 50 0""" from sqlalchemy import create_engine db_path = os.path.join(self.outdir_path, 'db.sqlite') self.engine = create_engine('sqlite:///{}'.format(db_path), echo=True) from wopmars.Base import Base Session = sqlalchemy.orm.sessionmaker(bind=self.engine) self.session = Session() Base.metadata.create_all(self.engine) from io import StringIO run_df = pandas.read_csv(StringIO(run_str), sep=" ") run_df.to_sql(name=Run.__tablename__, con=self.engine.connect(), if_exists='append', index=False) marker_df = pandas.read_csv(StringIO(marker_str), sep=" ") marker_df.to_sql(name=Marker.__tablename__, con=self.engine.connect(), if_exists='append', index=False) sample_df = pandas.read_csv(StringIO(sample_str), sep=" ") sample_df.to_sql(name=Sample.__tablename__, con=self.engine.connect(), if_exists='append', index=False) filter_indel_df = pandas.read_csv(StringIO(filter_indel_str), sep=" ") filter_indel_df.to_sql(name=FilterIndel.__tablename__, con=self.engine.connect(), if_exists='append', index=False) variant_df = pandas.read_csv(StringIO(variant_str), sep=" ") variant_df.to_sql(name=Variant.__tablename__, con=self.engine.connect(), if_exists='append', index=False) pathlib.Path(os.path.join(self.outdir_path, "params.yml")).touch() sortereadinfo_str = """run marker sample replicate sortedfasta TAS2 IIICBR S21 1 TAS2-R1_S1_L001_R1_001_000.fasta""" with open(os.path.join(self.outdir_path, "sortedinfo.tsv"), 'w') as fout: fout.write(sortereadinfo_str)