def test_get_summary(monkeypatch, req, tmpdir): """Test getting the assembly summary file.""" cache_dir = tmpdir.mkdir('cache') monkeypatch.setattr(core, 'CACHE_DIR', str(cache_dir)) cache_file = cache_dir.join('refseq_bacteria_assembly_summary.txt') req.get( 'https://ftp.ncbi.nih.gov/genomes/refseq/bacteria/assembly_summary.txt', text='test') ret = core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), False) assert ret.read() == 'test' assert not cache_file.check() ret = core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), True) assert ret.read() == 'test' assert cache_file.check() req.get( 'https://ftp.ncbi.nih.gov/genomes/refseq/bacteria/assembly_summary.txt', text='never read') ret = core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), True) assert ret.read() == 'test'
def test_filter_entries(): """Test filter_entries.""" config = NgdConfig() with open(_get_file('assembly_status.txt'), 'r') as fh: entries = list(core.parse_summary(fh)) assert core.filter_entries(entries, config) == entries expected = entries[-1:] config.assembly_accessions = "GCF_000203835.1" assert core.filter_entries(entries, config) == expected
def test_filter_entries(): """Test filter_entries.""" config = NgdConfig() with open(_get_file('assembly_status.txt'), 'r') as fh: entries = list(core.parse_summary(fh)) assert core.filter_entries(entries, config) == entries expected = entries[-1:] config.assembly_accessions = "GCF_000203835.1" assert core.filter_entries(entries, config) == expected
def prepare_create_downloadjob(req, tmpdir, format_map=NgdConfig._FORMATS, human_readable=False, create_local_file=False): # Set up test env entry = { 'assembly_accession': 'FAKE0.1', 'organism_name': 'Example species', 'infraspecific_name': 'strain=ABC 1234', 'ftp_path': 'https://fake/genomes/FAKE0.1' } config = NgdConfig() outdir = tmpdir.mkdir('output') download_jobs = [] config.output = str(outdir) config.human_readable = human_readable checksum_file_content = '' for key, val in format_map.items(): seqfile = tmpdir.join('fake{}'.format(val)) seqfile.write(key) checksum = core.md5sum(str(seqfile)) filename = path.basename(str(seqfile)) full_url = 'https://fake/genomes/FAKE0.1/{}'.format(filename) local_file = outdir.join('refseq', 'bacteria', 'FAKE0.1', filename) if create_local_file: local_file.write(seqfile.read(), ensure=True) symlink_path = None if human_readable: symlink_path = str( outdir.join('human_readable', 'refseq', 'bacteria', 'Example', 'species', 'ABC_1234', filename)) download_jobs.append( core.DownloadJob(full_url, str(local_file), checksum, symlink_path)) checksum_file_content += '{}\t./{}\n'.format(checksum, filename) req.get(full_url, text=seqfile.read()) req.get('https://fake/genomes/FAKE0.1/md5checksums.txt', text=checksum_file_content) return entry, config, download_jobs
def test_get_summary(monkeypatch, req, tmpdir): """Test getting the assembly summary file.""" cache_dir = tmpdir.mkdir('cache') monkeypatch.setattr(core, 'CACHE_DIR', str(cache_dir)) cache_file = cache_dir.join('refseq_bacteria_assembly_summary.txt') req.get('https://ftp.ncbi.nih.gov/genomes/refseq/bacteria/assembly_summary.txt', text='test') ret = core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), False) assert ret.read() == 'test' assert not cache_file.check() ret = core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), True) assert ret.read() == 'test' assert cache_file.check() req.get('https://ftp.ncbi.nih.gov/genomes/refseq/bacteria/assembly_summary.txt', text='never read') ret = core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), True) assert ret.read() == 'test'
def test_get_summary_error_handling(monkeypatch, mocker, req, tmpdir): """Test get_summary error handling.""" cache_dir = tmpdir.join('cache') monkeypatch.setattr(core, 'CACHE_DIR', str(cache_dir)) req.get('https://ftp.ncbi.nih.gov/genomes/refseq/bacteria/assembly_summary.txt', text='test') fake_makedirs = mocker.MagicMock(side_effect=OSError(13, "Permission denied")) monkeypatch.setattr(os, 'makedirs', fake_makedirs) with pytest.raises(OSError): core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), True)
def test_get_summary_error_handling(monkeypatch, mocker, req, tmpdir): """Test get_summary error handling.""" cache_dir = tmpdir.join('cache') monkeypatch.setattr(core, 'CACHE_DIR', str(cache_dir)) req.get('https://ftp.ncbi.nih.gov/genomes/refseq/bacteria/assembly_summary.txt', text='test') fake_makedirs = mocker.MagicMock(side_effect=OSError(13, "Permission denied")) monkeypatch.setattr(os, 'makedirs', fake_makedirs) with pytest.raises(OSError): core.get_summary('refseq', 'bacteria', NgdConfig.get_default('uri'), True)
def prepare_create_downloadjob(req, tmpdir, format_map=NgdConfig._FORMATS, human_readable=False, create_local_file=False): # Set up test env entry = { 'assembly_accession': 'FAKE0.1', 'organism_name': 'Example species', 'infraspecific_name': 'strain=ABC 1234', 'ftp_path': 'https://fake/genomes/FAKE0.1' } config = NgdConfig() outdir = tmpdir.mkdir('output') download_jobs = [] config.output = str(outdir) config.human_readable = human_readable checksum_file_content = '' for key, val in format_map.items(): seqfile = tmpdir.join('fake{}'.format(val)) seqfile.write(key) checksum = core.md5sum(str(seqfile)) filename = path.basename(str(seqfile)) full_url = 'https://fake/genomes/FAKE0.1/{}'.format(filename) local_file = outdir.join('refseq', 'bacteria', 'FAKE0.1', filename) if create_local_file: local_file.write(seqfile.read(), ensure=True) symlink_path = None if human_readable: symlink_path = str( outdir.join('human_readable', 'refseq', 'bacteria', 'Example', 'species', 'ABC_1234', filename)) download_jobs.append(core.DownloadJob(full_url, str(local_file), checksum, symlink_path)) checksum_file_content += '{}\t./{}\n'.format(checksum, filename) req.get(full_url, text=seqfile.read()) req.get('https://fake/genomes/FAKE0.1/md5checksums.txt', text=checksum_file_content) return entry, config, download_jobs
def test_get_name_and_checksum(): class TestData: def __init__(self, checksums, end, filename, md5sum): self.checksums = checksums self.end = end self.filename = filename self.md5sum = md5sum regular_filenames = ( {'checksum': 'd76c643ec4bbc34d2935eb0664156d99', 'file': 'GCF_000009605.1_ASM960v1_cds_from_genomic.fna.gz'}, {'checksum': '42c1bb1447aea2512a17aeb3645b55e9', 'file': 'GCF_000009605.1_ASM960v1_genomic.fna.gz'}, {'checksum': '8a685d49d826c4f0ad05152e906f3250', 'file': 'GCF_000009605.1_ASM960v1_genomic.gbff.gz'}, {'checksum': 'e2d9e1cfa085cb462a73d3d2d2c22be5', 'file': 'GCF_000009605.1_ASM960v1_genomic.gff.gz'}, ) weird_filenames = ( {'checksum': '4d5f39ceb7e113ad461f8370aaac4e41', 'file': 'GCF_003583405.1_CHULA_Jazt_1.1_for_version_1.1_of_the_Jishengella_sp._nov._AZ1-13_genome_from_a_lab_in_CHULA_cds_from_genomic.fna.gz'}, {'checksum': 'e77c1e8bf0df2c353ce6a4899ae0cb5e', 'file': 'GCF_003583405.1_CHULA_Jazt_1.1_for_version_1.1_of_the_Jishengella_sp._nov._AZ1-13_genome_from_a_lab_in_CHULA_genomic.fna.gz'}, {'checksum': 'c93ba924075c8b22210ac283d41207ad', 'file': 'GCF_003583405.1_CHULA_Jazt_1.1_for_version_1.1_of_the_Jishengella_sp._nov._AZ1-13_genome_from_a_lab_in_CHULA_genomic.gbff.gz'}, {'checksum': 'd8394d0aff594ae962c88e1192238413', 'file': 'GCF_003583405.1_CHULA_Jazt_1.1_for_version_1.1_of_the_Jishengella_sp._nov._AZ1-13_genome_from_a_lab_in_CHULA_rna_from_genomic.fna.gz'}, ) test_table = ( TestData(regular_filenames, NgdConfig.get_fileending('genbank'), regular_filenames[2]['file'], regular_filenames[2]['checksum']), TestData(regular_filenames, NgdConfig.get_fileending('fasta'), regular_filenames[1]['file'], regular_filenames[1]['checksum']), TestData(regular_filenames, NgdConfig.get_fileending('cds-fasta'), regular_filenames[0]['file'], regular_filenames[0]['checksum']), TestData(weird_filenames, NgdConfig.get_fileending('genbank'), weird_filenames[2]['file'], weird_filenames[2]['checksum']), TestData(weird_filenames, NgdConfig.get_fileending('fasta'), weird_filenames[1]['file'], weird_filenames[1]['checksum']), TestData(weird_filenames, NgdConfig.get_fileending('cds-fasta'), weird_filenames[0]['file'], weird_filenames[0]['checksum']), ) for test in test_table: filename, checksum = core.get_name_and_checksum(test.checksums, test.end) assert filename == test.filename assert checksum == test.md5sum
"-n", "name", help="input the phylum name or other. use ; to separate multiple ") @click.option( "-t", "taxons", help= "input the taxon id. It will retrieve all the genomes desceding to the provided taxon; to separate multiple " ) @click.option( "-F", "formats", help='Which formats to download (default: %(default)s).' 'A comma-separated list of formats is also possible. For example: "fasta,assembly-report". ' 'Choose from: {choices}'.format( choices=NgdConfig.get_choices('file_formats')), default='fasta') @click.option( "-o", "odir", help= f"Create output hierarchy in specified folder (default: {NgdConfig.get_default('output')})", default=NgdConfig.get_default('output')) @click.option("-size", "size_of_batch", help=f"The size of each batch.", default=20) @click.option("-p", "parallel", help=f"Run N downloads in parallel (default: 10)", default=5)