def test_decompress(self): out_file = os.path.join(tempfile.gettempdir(), 'gtf.txt') ftp_down(self.remote_1, out_file) ftl = run_child('file %s' % out_file) os.remove(out_file) ft = parse_file_line(ftl) self.assertEqual(ft, 'ascii')
def fetch_bovine(): """Download cow genome and annotations.""" target_dir = os.path.join(DB_DIR, 'bovine') try: os.mkdir(target_dir) except FileExistsError: pass os.chdir(target_dir) try: os.mkdir('fasta') except FileExistsError: pass os.chdir('fasta') chromosomes = ['chr%d' % chrom for chrom in range(1, 30)] chromosomes.extend(['chrMT', 'chrX', 'unplaced']) # Y IS MISSING logging.info('Downloading bovine genome') local_file_name = os.path.join(target_dir, 'fasta', 'bt_ref_Bos_taurus_UMD_3.1.1.fasta') if os.path.exists(local_file_name): os.remove(local_file_name) for chrom in chromosomes: logging.debug('Downloading bovine chromosome %s', chrom) fasta_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/Bos_taurus/Assembled_chromosomes/seq/bt_ref_Bos_taurus_UMD_3.1.1_%s.fa.gz' % chrom download_handle = ftp_down(fasta_url, local_file_name) download_handle.close() logging.debug('Downloaded bovine chromosome %s', chrom) run_child('bgzip %s' % local_file_name) logging.info('Downloading gff annotation file') gff3_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/Bos_taurus/GFF/ref_Bos_taurus_UMD_3.1.1_top_level.gff3.gz' download_handle = ftp_down(gff3_url) download_handle.close()
def fetch_bovine(): """Download cow genome and annotations.""" target_dir = os.path.join(DB_DIR, 'bovine') try: os.mkdir(target_dir) except FileExistsError: pass os.chdir(target_dir) try: os.mkdir('fasta') except FileExistsError: pass os.chdir('fasta') chromosomes = ['chr%d' % chrom for chrom in range(1, 30)] chromosomes.extend(['chrMT', 'chrX', 'unplaced']) # Y IS MISSING logging.info('Downloading bovine genome') local_file_name = os.path.join(target_dir, 'fasta', 'bt_ref_Bos_taurus_UMD_3.1.1.fasta') if os.path.exists(local_file_name): os.remove(local_file_name) for chrom in chromosomes: logging.debug('Downloading bovine chromosome %s', chrom) fasta_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/Bos_taurus/Assembled_chromosomes/seq/bt_ref_Bos_taurus_UMD_3.1.1_%s.fa.gz' % chrom download_handle = ftp_down(fasta_url, local_file_name) download_handle.close() logging.debug('Downloaded bovine chromosome %s', chrom) run_child('bgzip %s' % local_file_name) logging.info('Downloading gff annotation file') gff3_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/Bos_taurus/GFF/ref_Bos_taurus_UMD_3.1.1_top_level.gff3.gz' download_handle = ftp_down(gff3_url) download_handle.close()
def test_append(self): out_file = os.path.join(tempfile.gettempdir(), 'README.TXT') try: os.remove(out_file) except FileNotFoundError: pass ftp_down(self.remote_2, out_file) with open(out_file) as f: n_lines_1 = sum(1 for _ in f) ftp_down(self.remote_2, out_file) with open(out_file) as f: n_lines_2 = sum(1 for _ in f) self.assertEqual(n_lines_2, 2 * n_lines_1)
def fetch_viral(viral_mode): """Download nucleotide or protein database.""" # define the search nuccore/protein if viral_mode == 'n': logging.info('downloading viral nuccore sequences') target_dir = os.path.join(DB_DIR, 'viral_nuccore') cml_search = viral_query('n') elif viral_mode == 'p': logging.info('downloaded viral protein sequences') target_dir = os.path.join(DB_DIR, 'viral_protein') cml_search = viral_query('p') # run the search and download os.chdir(target_dir) run_child(cml_search) cml_fetch_fasta = 'efetch -format fasta < ncbi_search > viral_database.fasta' run_child(cml_fetch_fasta) cml_efetch_xtract = 'efetch -format docsum < ncbi_search | xtract' cml_efetch_xtract += ' -pattern DocumentSummary -element Caption TaxId Slen Organism Title > viral_seqs_info.tsv' run_child(cml_efetch_xtract) logging.info('downloaded viral seqs info in %s', target_dir) logging.info('saving viral taxonomy') # viral_seqs_info.tsv contains Accn TaxId cml = 'cut -f 1,2 viral_seqs_info.tsv > viral_accn_taxid.dmp' run_child(cml) accs_1 = set(get_accs('viral_database.fasta')) accs_2 = set([l.split()[0] for l in open('viral_accn_taxid.dmp')]) assert accs_1 == accs_2, accs_1 ^ accs_2 logging.info('taxonomy and fasta sequences match') os.chdir(DB_DIR) logging.info('downloading taxonomy databases') download_handle = ftp_down( 'ftp://ftp.ncbi.nlm.nih.gov/blast/db/taxdb.tar.gz') download_handle.close() run_child('tar xvfz taxdb.tar.gz') os.remove('taxdb.tar.gz') download_handle = ftp_down( 'ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz') download_handle.close() run_child('tar xvfz taxdump.tar.gz') for ftd in [ 'taxdump.tar.gz', 'merged.dmp', 'gencode.dmp', 'division.dmp', 'delnodes.dmp', 'citations.dmp' ]: try: os.remove(ftd) except OSError: logging.warning('Could not find file %s', ftd)
def fetch_viral(viral_mode): """Download nucleotide or protein database.""" # define the search nuccore/protein if viral_mode == 'n': logging.info('downloading viral nuccore sequences') target_dir = os.path.join(DB_DIR, 'viral_nuccore') cml_search = viral_query('n') elif viral_mode == 'p': logging.info('downloaded viral protein sequences') target_dir = os.path.join(DB_DIR, 'viral_protein') cml_search = viral_query('p') # run the search and download os.chdir(target_dir) run_child(cml_search) cml_fetch_fasta = 'efetch -format fasta < ncbi_search > viral_database.fasta' run_child(cml_fetch_fasta) cml_efetch_xtract = 'efetch -format docsum < ncbi_search | xtract' cml_efetch_xtract += ' -pattern DocumentSummary -element Caption TaxId Slen Organism Title > viral_seqs_info.tsv' run_child(cml_efetch_xtract) logging.info('downloaded viral seqs info in %s', target_dir) logging.info('saving viral taxonomy') # viral_seqs_info.tsv contains Accn TaxId cml = 'cut -f 1,2 viral_seqs_info.tsv > viral_accn_taxid.dmp' run_child(cml) accs_1 = set(get_accs('viral_database.fasta')) accs_2 = set([l.split()[0] for l in open('viral_accn_taxid.dmp')]) assert accs_1 == accs_2, accs_1 ^ accs_2 logging.info('taxonomy and fasta sequences match') os.chdir(DB_DIR) logging.info('downloading taxonomy databases') download_handle = ftp_down('ftp://ftp.ncbi.nlm.nih.gov/blast/db/taxdb.tar.gz') download_handle.close() run_child('tar xvfz taxdb.tar.gz') os.remove('taxdb.tar.gz') download_handle = ftp_down('ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz') download_handle.close() run_child('tar xvfz taxdump.tar.gz') for ftd in ['taxdump.tar.gz', 'merged.dmp', 'gencode.dmp', 'division.dmp', 'delnodes.dmp', 'citations.dmp']: try: os.remove(ftd) except OSError: logging.warning('Could not find file %s', ftd)
def fetch_bovine(): """Download cow genome and annotations.""" target_dir = os.path.join(DB_DIR, 'bovine') try: os.mkdir(target_dir) except FileExistsError: pass os.chdir(target_dir) try: os.mkdir('fasta') except FileExistsError: pass os.chdir('fasta') chromosomes = ['chr%d' % chrom for chrom in range(1, 30)] chromosomes.extend(['chrX']) # chrY is missing logging.info('Downloading bovine genome') local_file_name = os.path.join( target_dir, 'fasta', 'ref_Bos_taurus_GCF_002263795.1_ARS-UCD1.2.fasta') if os.path.exists(local_file_name): os.remove(local_file_name) for chrom in chromosomes: logging.debug('Downloading bovine chromosome %s', chrom) fasta_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Bos_taurus/latest_assembly_versions/GCF_002263795.1_ARS-UCD1.2/GCF_002263795.1_ARS-UCD1.2_assembly_structure/Primary_Assembly/assembled_chromosomes/FASTA/%s.fna.gz' % chrom download_handle = ftp_down(fasta_url, local_file_name) download_handle.close() logging.debug('Downloaded bovine chromosome %s', chrom) fasta_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Bos_taurus/latest_assembly_versions/GCF_002263795.1_ARS-UCD1.2/GCF_002263795.1_ARS-UCD1.2_assembly_structure/non-nuclear/assembled_chromosomes/FASTA/chrMT.fna.gz' download_handle = ftp_down(fasta_url, local_file_name) download_handle.close() logging.debug('Downloaded bovine chromosome MT') fasta_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Bos_taurus/latest_assembly_versions/GCF_002263795.1_ARS-UCD1.2/GCF_002263795.1_ARS-UCD1.2_assembly_structure/Primary_Assembly/unplaced_scaffolds/FASTA/unplaced.scaf.fna.gz' download_handle = ftp_down(fasta_url, local_file_name) download_handle.close() logging.debug('Downloaded bovine chromosome unplaced') run_child('bgzip %s' % local_file_name) logging.info('Downloading gff annotation file') gff_url = 'ftp://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Bos_taurus/latest_assembly_versions/GCF_002263795.1_ARS-UCD1.2/GCF_002263795.1_ARS-UCD1.2_genomic.gff.gz' download_handle = ftp_down(gff_url) download_handle.close()
def fetch_human(): """Download human genome and annotations.""" target_dir = os.path.join(DB_DIR, 'human') try: os.mkdir(target_dir) except FileExistsError: pass os.chdir(target_dir) try: os.mkdir('fasta') except FileExistsError: pass os.chdir('fasta') fasta_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_24/GRCh38.primary_assembly.genome.fa.gz' gtf_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_24/gencode.v24.primary_assembly.annotation.gtf.gz' logging.info('Downloading human annotation') download_handle = ftp_down(gtf_url) download_handle.close() logging.info('Downloading human genome and bgzip compressing') if os.path.exists('GRCh38.fasta'): os.remove('GRCh38.fasta') download_handle = ftp_down(fasta_url, 'GRCh38.fasta') download_handle.close() run_child('bgzip GRCh38.fasta')
def fetch_human(): """Download human genome and annotations.""" target_dir = os.path.join(DB_DIR, 'human') try: os.mkdir(target_dir) except FileExistsError: pass os.chdir(target_dir) try: os.mkdir('fasta') except FileExistsError: pass os.chdir('fasta') fasta_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_24/GRCh38.primary_assembly.genome.fa.gz' gtf_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_24/gencode.v24.primary_assembly.annotation.gtf.gz' logging.info('Downloading human annotation') download_handle = ftp_down(gtf_url) download_handle.close() logging.info('Downloading human genome and bgzip compressing') if os.path.exists('GRCh38.fasta'): os.remove('GRCh38.fasta') download_handle = ftp_down(fasta_url, 'GRCh38.fasta') download_handle.close() run_child('bgzip GRCh38.fasta')