def search(self, term): """ Search for term in genome names, descriptions and taxonomy ID. The search is case-insensitive. Parameters ---------- term : str Search term, case-insensitive. Can be assembly name (e.g. hg38), (part of a) scientific name (Danio rerio) or taxonomy id (722). Yields ------ tuples with name and metadata """ genomes = self.genomes term = safe(str(term)) if term.startswith("GCA_") and self.name != "NCBI": for row in self._search_accessions(term): yield (row) elif is_number(term): for name in genomes: if self._search_taxids(genomes[name], term): yield self._genome_info_tuple(name) else: term = term.lower() for name in genomes: if term in safe(name).lower() or self._search_descriptions( genomes[name], term): yield self._genome_info_tuple(name)
def get_genomes(rest_url): logger.info("Downloading assembly summaries from Ensembl") genomes = {} divisions = retry(request_json, 3, rest_url, "info/divisions?") for division in divisions: if division == "EnsemblBacteria": continue division_genomes = retry(request_json, 3, rest_url, f"info/genomes/division/{division}?") # filter summaries to these keys (to reduce the size of the cached data) summary_keys_to_keep = [ "assembly_name", "assembly_accession", "taxonomy_id", "name", "scientific_name", "url_name", "display_name", "genebuild", "division", "base_count", ] for genome in division_genomes: name = safe(genome["assembly_name"]) genomes[name] = {k: genome[k] for k in summary_keys_to_keep} genomes = add_grch37(genomes) return genomes
def get_genome_download_link(self, name, mask="soft", **kwargs): """ Return NCBI ftp link to top-level genome sequence Parameters ---------- name : str Genome name. Current implementation will fail if exact name is not found. mask : str , optional Masking level. Options: soft, hard or none. Default is soft. Returns ------ str with the http/ftp download link. """ genome = self.genomes[safe(name)] # only soft masked genomes available. can be (un)masked in _post _process_download link = genome["ftp_path"] link = link.replace("ftp://", "https://") link += "/" + link.split("/")[-1] + "_genomic.fna.gz" if check_url(link, 2): return link raise GenomeDownloadError( f"Could not download genome {name} from {self.name}.\n" "URL is broken. Select another genome or provider.\n" f"Broken URL: {link}")
def search(term, provider=None): """ Search for a genome. If provider is specified, search only that specific provider, else search all providers. Both the name and description are used for the search. Search term is case-insensitive. Parameters ---------- term : str or int Search term, case-insensitive. provider : str , optional Provider name Yields ------ tuple genome information (name/identifier and description) """ term = safe(str(term)) providers = _providers(provider) for p in providers: for row in p.search(term): yield [ x.encode("utf-8") for x in list(row[:1]) + [p.name] + list(row[1:]) ]
def __init__(self, name, genomes_dir=None, *args, **kwargs): self.name = safe(os.path.basename(re.sub(r"\.fa(\.gz)?$", "", name))) "genome name" self.genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) "path to the genomepy genomes directory" self.filename = self._parse_filename(name) super(Genome, self).__init__(self.filename, *args, **kwargs) # file paths self.genome_file = self.filename "path to the genome fasta" self.genome_dir = os.path.dirname(self.filename) "path to the genome directory" self.index_file = self.genome_file + ".fai" "path to the genome index" self.sizes_file = self._check_support_file("sizes") "path to the chromosome sizes file" self.gaps_file = self._check_support_file("gaps") "path to the chromosome gaps file" self.annotation_gtf_file = self._check_annotation_file("gtf") "path to the gene annotation GTF file" self.annotation_bed_file = self._check_annotation_file("bed") "path to the gene annotation BED file" self.readme_file = os.path.join(self.genome_dir, "README.txt") "path to the README file" # genome attributes metadata, _ = read_readme(self.readme_file) self.tax_id = metadata["tax_id"] "genome taxonomy identifier" self.assembly_accession = metadata["assembly_accession"] "genome assembly accession"
def search(term, provider: str = None, size=False): """ Search for a genome. If provider is specified, search only that specific provider, else search all providers. Both the name and description are used for the search. Search term is case-insensitive. Parameters ---------- term : str Search term, case-insensitive. provider : str , optional Only search the specified provider (faster). size : bool, optional Show absolute genome size. Yields ------ list genome name, provider and metadata """ term = safe(str(term)) for p in online_providers(provider): for row in p.search(term, size): ret = list(row[:1]) + [p.name] + list(row[1:]) yield ret
def get_annotation_download_links(self, name, **kwargs): """ Retrieve functioning gene annotation download link(s). Parameters ---------- name : str genome name **kwargs: dict, optional: version : Ensembl version to use. By default the latest version is used Returns ------- list http link(s) """ genome = self.genomes[safe(name)] division, is_vertebrate = get_division(genome) # base directory of the genome ftp = "http://ftp.ensemblgenomes.org" if is_vertebrate: ftp = "http://ftp.ensembl.org" version = self.get_version(is_vertebrate, kwargs.get("version")) div_path = "" if is_vertebrate else f"/{division}" lwr_name = genome["url_name"].lower() ftp_directory = f"{ftp}/pub/release-{version}{div_path}/gtf/{lwr_name}" # some entries don't use url_name in their url... -,- # examples: # - EnsemblVertebrates: mus_musculus_nzohlltj # - EnsemblMetazoa: caenorhabditis_elegans if not check_url(ftp_directory, 2): lwr_name = genome["name"] ftp_directory = f"{ftp}/pub/release-{version}{div_path}/gtf/{lwr_name}" # specific gtf file cap_name = lwr_name.capitalize() asm_name = re.sub(r"\.p\d+$", "", safe(genome["assembly_name"])) ftp_file = f"{cap_name}.{asm_name}.{version}.gtf.gz" # combine link = f"{ftp_directory}/{ftp_file}" if name == "GRCh37": link = genome["annotation"].format(version) return [link] if check_url(link, max_tries=2) else []
def annotation_links(self, name: str, **kwargs) -> List[str]: """ Return available gene annotation links (http/ftp) for a genome Parameters ---------- name: str genome name Returns ------ list Gene annotation links """ if "annotations" not in self.genomes[safe(name)]: links = self.get_annotation_download_links(name, **kwargs) self.genomes[safe(name)]["annotations"] = links return self.genomes[safe(name)]["annotations"]
def _check_name(self, name): """check if genome name can be found for provider""" name = safe(name) if name in self.genomes: return name raise GenomeDownloadError( f"Could not download genome {name} from {self.name}.\n\n" "Check for typos or try\n" f" genomepy search {name} -p {self.name}")
def _ftp_or_html_link(self, name, file_suffix, skip_check=False): """ NCBI's files are accessible over FTP and HTTPS Try HTTPS first and return the first functioning link """ genome = self.genomes[safe(name)] ftp_link = genome["ftp_path"] html_link = ftp_link.replace("ftp://", "https://") for link in [html_link, ftp_link]: link += "/" + link.split("/")[-1] + file_suffix if skip_check or check_url(link, max_tries=2, timeout=10): return link
def _get_genomes(self, rest_url): sys.stderr.write("Downloading assembly summaries from Ensembl\n") genomes = {} divisions = retry(self._request_json, 3, rest_url, "info/divisions?") for division in divisions: if division == "EnsemblBacteria": continue division_genomes = retry(self._request_json, 3, rest_url, f"info/genomes/division/{division}?") for genome in division_genomes: genomes[safe(genome["assembly_name"])] = genome return genomes
def get_genomes(assembly_url): """Parse genomes from assembly summary txt files.""" logger.info( "Downloading assembly summaries from NCBI, this will take a while...") def load_summary(url): """ lazy loading of the url so we can parse while downloading """ for row in urlopen(url): yield row genomes = {} # order is important as asm_name can repeat (overwriting the older name) names = [ "assembly_summary_genbank_historical.txt", "assembly_summary_refseq_historical.txt", "assembly_summary_genbank.txt", "assembly_summary_refseq.txt", ] # filter summaries to these keys (to reduce the size of the cached data) summary_keys_to_keep = [ 0, # 'assembly_accession', 5, # 'taxid', 6, # 'species_taxid', 7, # 'organism_name', 16, # 'submitter', 17, # 'gbrs_paired_asm', 18, # 'paired_asm_comp', 19, # 'ftp_path', ] for fname in names: lines = load_summary(f"{assembly_url}/{fname}") # line 0 = comment _ = next(lines) # line 1 = header header = next(lines).decode("utf-8").strip("# ").strip("\n").split( "\t") header = [header[n] for n in summary_keys_to_keep] for line in tqdm(lines, desc=fname[17:-4], unit_scale=1, unit=" genomes"): line = line.decode("utf-8").strip("\n").split("\t") name = safe(line[15]) # overwrites older asm_names if line[19] != "na": # ftp_path must exist line = [line[n] for n in summary_keys_to_keep] genomes[name] = dict(zip(header, line)) return genomes
def _get_name_and_dir(name, genomes_dir=None): """ Returns the name and directory of the genome. """ fname = cleanpath(name) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) if os.path.isfile(fname): exts = ["gtf", "GTF", "bed", "BED", "fa"] if not any(ext in fname for ext in exts): raise NotImplementedError( "Only (gzipped) bed, gtf or fasta files are supported!") genome_dir = os.path.dirname(fname) name = safe(os.path.basename(fname)) # remove suffices any_ext = "(" + ")|(".join(exts) + ")" name = re.sub(fr"(\.annotation)?\.({any_ext})(\.gz)?$", "", name) elif os.path.isdir(fname): genome_dir = fname name = safe(os.path.basename(fname)) elif name in os.listdir(genomes_dir): genome_dir = os.path.join(genomes_dir, name) else: raise FileNotFoundError(f"Could not find {name}") return name, genome_dir
def get_url(level="toplevel"): masks = { "soft": "dna_sm.{}", "hard": "dna_rm.{}", "none": "dna.{}" } pattern = masks[mask].format(level) asm_url = "{}/{}.{}.{}.fa.gz".format( url, genome["url_name"].capitalize(), re.sub(r"\.p\d+$", "", safe(genome["assembly_name"])), pattern, ) return asm_url
def get_annotation_download_link(self, name, **kwargs): """ Parse and test the link to the NCBI annotation file. Parameters ---------- name : str Genome name """ genome = self.genomes[safe(name)] link = genome["ftp_path"] link = link.replace("ftp://", "https://") link += "/" + link.split("/")[-1] + "_genomic.gff.gz" if check_url(link, 2): return link
def _update_metadata(self, metadata): """check if there is missing info that can be updated""" print("Updating metadata in README.txt", file=sys.stderr) if metadata.get("provider", "na") == "na": self._update_provider(metadata) known_provider = metadata["provider"] in ["Ensembl", "UCSC", "NCBI"] name = safe(metadata.get("original name", "")) missing_info = any(key not in metadata for key in ["tax_id", "assembly_accession"]) p = genome = None if known_provider and name and missing_info: p = ProviderBase.create(metadata["provider"]) genome = p.genomes.get(name) if "tax_id" not in metadata: self._update_tax_id(metadata, p, genome) if "assembly_accession" not in metadata: self._update_assembly_accession(metadata, p, genome)
def get_annotation_download_link(self, name, **kwargs): """ Parse and test the link to the Ensembl annotation file. Parameters ---------- name : str Genome name kwargs: dict , optional: Provider specific options. version : int , optional Ensembl version. By default the latest version is used. """ genome = self.genomes[safe(name)] division = genome["division"].lower().replace("ensembl", "") ftp_site = "ftp://ftp.ensemblgenomes.org/pub" if division == "vertebrates": ftp_site = "ftp://ftp.ensembl.org/pub" # Ensembl release version version = kwargs.get("version") if version is None: version = self.get_version(self.rest_url, division == "vertebrates") if division != "vertebrates": ftp_site += f"/{division}" # Get the GTF URL base_url = ftp_site + "/release-{}/gtf/{}/{}.{}.{}.gtf.gz" safe_name = re.sub(r"\.p\d+$", "", name) link = base_url.format( version, genome["url_name"].lower(), genome["url_name"].capitalize(), safe_name, version, ) if check_url(link, 2): return link
def _get_genomes(assembly_url): """Parse genomes from assembly summary txt files.""" sys.stderr.write( "Downloading assembly summaries from NCBI, this will take a while...\n" ) genomes = {} # order is important as asm_name can repeat (overwriting the older name) names = [ "assembly_summary_refseq_historical.txt", "assembly_summary_genbank.txt", "assembly_summary_refseq.txt", ] for fname in names: urlcleanup() with urlopen(os.path.join(assembly_url, fname)) as response: lines = response.read().decode("utf-8").splitlines() header = lines[1].strip("# ").split("\t") for line in lines[2:]: vals = line.strip("# ").split("\t") # overwrites older asm_names genomes[safe(vals[15])] = dict(zip(header, vals)) return genomes
def download_genome( self, name, genomes_dir=None, localname=None, mask="soft", keep_alt=False, regex=None, invert_match=False, bgzip=None, **kwargs, ): """ Download a (gzipped) genome file to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install genome localname : str , optional Custom name for your genome mask: str , optional Masking, soft, hard or none (all other strings) keep_alt : bool , optional Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. """ name = safe(name) self.check_name(name) link = self.get_genome_download_link(name, mask=mask, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) if not os.path.exists(out_dir): mkdir_p(out_dir) sys.stderr.write( f"Downloading genome from {self.name}.\nTarget URL: {link}...\n") # download to tmp dir. Move genome on completion. # tmp dir is in genome_dir to prevent moving the genome between disks tmp_dir = mkdtemp(dir=out_dir) fname = os.path.join(tmp_dir, f"{localname}.fa") urlcleanup() download_file(link, fname) sys.stderr.write( "Genome download successful, starting post processing...\n") # unzip genome if link.endswith(".tar.gz"): tar_to_bigfile(fname, fname) elif link.endswith(".gz"): os.rename(fname, fname + ".gz") ret = sp.check_call(["gunzip", "-f", fname]) if ret != 0: raise Exception(f"Error gunzipping genome {fname}") def regex_filer(_fname, _regex, _v): infa = _fname + "_to_regex" os.rename(_fname, infa) # filter the fasta and store the output's keys keys_out = filter_fasta(infa, outfa=_fname, regex=_regex, v=_v, force=True).keys() keys_in = Fasta(infa).keys() return [k for k in keys_in if k not in keys_out] not_included = [] # remove alternative regions if not keep_alt: not_included.extend(regex_filer(fname, "alt", True)) # keep/remove user defined regions if regex: not_included.extend(regex_filer(fname, regex, invert_match)) # process genome (e.g. masking) if hasattr(self, "_post_process_download"): self._post_process_download(name=name, localname=localname, out_dir=tmp_dir, mask=mask) # bgzip genome if requested if bgzip or config.get("bgzip"): # bgzip to stdout, track progress, and output to file fsize = int(os.path.getsize(fname) * 10**-6) cmd = ( f"bgzip -fc {fname} | " f"tqdm --bytes --desc Bgzipping {fsize}MB fasta --log ERROR | " f"cat > {fname}.gz") ret = sp.check_call(cmd, shell=True) if ret != 0: raise Exception(f"Error bgzipping {name}. Is tabix installed?") fname += ".gz" # transfer the genome from the tmpdir to the genome_dir src = fname dst = os.path.join(genomes_dir, localname, os.path.basename(fname)) shutil.move(src, dst) rm_rf(tmp_dir) sys.stderr.write("\n") sys.stderr.write("name: {}\n".format(name)) sys.stderr.write("local name: {}\n".format(localname)) sys.stderr.write("fasta: {}\n".format(dst)) # Create readme with information readme = os.path.join(genomes_dir, localname, "README.txt") metadata = { "name": localname, "provider": self.name, "original name": name, "original filename": os.path.split(link)[-1], "assembly_accession": self.assembly_accession(self.genomes.get(name)), "tax_id": self.genome_taxid(self.genomes.get(name)), "mask": mask, "genome url": link, "annotation url": "na", "date": time.strftime("%Y-%m-%d %H:%M:%S"), } lines = [] if not keep_alt or regex: regex_line = "regex: " if not keep_alt: regex_line += "'alt' (inverted match)" if not keep_alt and regex: regex_line += " and " if regex: regex_line += f"'{regex}'" if invert_match: regex_line += " (inverted match)" lines += ["", regex_line, "sequences that were excluded:"] for seq in not_included: lines.append(f"\t{seq}") write_readme(readme, metadata, lines)
def install_genome( name: str, provider: Optional[str] = None, genomes_dir: Optional[str] = None, localname: Optional[str] = None, mask: Optional[str] = "soft", keep_alt: Optional[bool] = False, regex: Optional[str] = None, invert_match: Optional[bool] = False, bgzip: Optional[bool] = None, # None -> check config. False -> dont check. annotation: Optional[bool] = False, only_annotation: Optional[bool] = False, skip_matching: Optional[bool] = False, skip_filter: Optional[bool] = False, threads: Optional[int] = 1, force: Optional[bool] = False, **kwargs: Optional[dict], ) -> Genome: """ Install a genome (& gene annotation). Parameters ---------- name : str Genome name provider : str , optional Provider name. will try Ensembl, UCSC and NCBI (in that order) if not specified. genomes_dir : str , optional Where to create the output folder. localname : str , optional Custom name for this genome. mask : str , optional Genome masking of repetitive sequences. Options: hard/soft/none, default is soft. keep_alt : bool , optional Some genomes contain alternative regions. These regions cause issues with sequence alignment, as they are inherently duplications of the consensus regions. Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that *don't* match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip, and gene annotation will be compressed with gzip. threads : int , optional Build genome index using multithreading (if supported). Default: lowest of 8/all threads. force : bool , optional Set to True to overwrite existing files. annotation : bool , optional If set to True, download gene annotation in BED and GTF format. only_annotation : bool , optional If set to True, only download the gene annotation files. skip_matching : bool , optional If set to True, contigs in the annotation not matching those in the genome will not be corrected. skip_filter : bool , optional If set to True, the gene annotations will not be filtered to match the genome contigs. kwargs : dict , optional Provider specific options. toplevel : bool , optional Ensembl only: Always download the toplevel genome. Ignores potential primary assembly. version : int , optional Ensembl only: Specify release version. Default is latest. to_annotation : text , optional URL only: direct link to annotation file. Required if this is not the same directory as the fasta. Returns ------- Genome Genome class with the installed genome """ name = safe(name) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) genome_file = os.path.join(out_dir, f"{localname}.fa") provider = _provider_selection(name, localname, genomes_dir, provider) # check which files need to be downloaded genome_found = _is_genome_dir(out_dir) download_genome = ( genome_found is False or force is True ) and only_annotation is False annotation_found = bool(glob_ext_files(out_dir, "annotation.gtf")) and bool( glob_ext_files(out_dir, "annotation.bed") ) download_annotation = (annotation_found is False or force is True) and any( [ annotation, only_annotation, skip_matching, skip_filter, kwargs.get("to_annotation"), kwargs.get("path_to_annotation"), kwargs.get("ucsc_annotation_type"), ] ) genome = None genome_downloaded = False if download_genome: if force: _delete_extensions(out_dir, ["fa", "fai"]) provider.download_genome( name, genomes_dir, mask=mask, localname=localname, **kwargs, ) genome_found = True genome_downloaded = True # Filter genome _filter_genome(genome_file, regex, invert_match, keep_alt) # Generates a Fasta object and the genome index, gaps and sizes files genome = Genome(localname, genomes_dir=genomes_dir) # Download the NCBI assembly report asm_report = os.path.join(out_dir, "assembly_report.txt") asm_acc = genome.assembly_accession if not os.path.exists(asm_report) and asm_acc != "na": download_assembly_report(asm_acc, asm_report) # Export installed genome(s) generate_env(genomes_dir=genomes_dir) annotation_downloaded = False if download_annotation: if force: _delete_extensions(out_dir, ["annotation.gtf", "annotation.bed"]) provider.download_annotation(name, genomes_dir, localname=localname, **kwargs) annotation_downloaded = bool( glob_ext_files(out_dir, "annotation.gtf") ) and bool(glob_ext_files(out_dir, "annotation.bed")) if annotation_downloaded: annotation = Annotation(localname, genomes_dir=genomes_dir) if genome_found and not (skip_matching and skip_filter): annotation.sanitize(not skip_matching, not skip_filter, True) # Run active plugins (also if the genome was downloaded earlier) if genome_found: genome = genome if genome else Genome(localname, genomes_dir=genomes_dir) for plugin in get_active_plugins(): plugin.after_genome_download(genome, threads, force) # zip files downloaded now if bgzip is True or (bgzip is None and config.get("bgzip")): if genome_downloaded: bgzip_and_name(genome.filename) if annotation_downloaded: gzip_and_name(annotation.annotation_gtf_file) gzip_and_name(annotation.annotation_bed_file) return genome
def install_genome( name, provider=None, genomes_dir=None, localname=None, mask="soft", keep_alt=False, regex=None, invert_match=False, bgzip=None, annotation=False, only_annotation=False, skip_sanitizing=False, threads=1, force=False, **kwargs, ): """ Install a genome. Parameters ---------- name : str Genome name provider : str , optional Provider name. will try Ensembl, UCSC and NCBI (in that order) if not specified. genomes_dir : str , optional Where to store the fasta files localname : str , optional Custom name for this genome. mask : str , optional Default is 'soft', choices 'hard'/'soft/'none' for respective masking level. keep_alt : bool , optional Some genomes contain alternative regions. These regions cause issues with sequence alignment, as they are inherently duplications of the consensus regions. Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. threads : int , optional Build genome index using multithreading (if supported). Default: lowest of 8/all threads force : bool , optional Set to True to overwrite existing files. annotation : bool , optional If set to True, download gene annotation in BED and GTF format. only_annotation : bool , optional If set to True, only download the annotation files. skip_sanitizing : bool , optional If set to True, downloaded annotation files whose sequence names do not match with the (first header fields of) the genome.fa will not be corrected. kwargs : dict , optional Provider specific options. toplevel : bool , optional Ensembl only: Always download the toplevel genome. Ignores potential primary assembly. version : int , optional Ensembl only: Specify release version. Default is latest. to_annotation : text , optional URL only: direct link to annotation file. Required if this is not the same directory as the fasta. """ name = safe(name) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) # Check if genome already exists, or if downloading is forced genome_found = _is_genome_dir(out_dir) if (not genome_found or force) and not only_annotation: # Download genome from provider p = _provider_selection(name, localname, genomes_dir, provider) p.download_genome( name, genomes_dir, mask=mask, keep_alt=keep_alt, regex=regex, invert_match=invert_match, localname=localname, bgzip=bgzip, **kwargs, ) genome_found = True # Export installed genome(s) generate_env(genomes_dir=genomes_dir) # Generates a Fasta object, index, gaps and sizes file g = None if genome_found: g = Genome(localname, genomes_dir=genomes_dir) if force: # overwrite previous versions generate_fa_sizes(g.genome_file, g.sizes_file) generate_gap_bed(g.genome_file, g.gaps_file) # Check if any annotation flags are given, if annotation already exists, or if downloading is forced if any([ annotation, only_annotation, skip_sanitizing, kwargs.get("to_annotation"), kwargs.get("ucsc_annotation_type"), ]): annotation = True annotation_found = bool(glob_ext_files(out_dir, "gtf")) if (not annotation_found or force) and annotation: # Download annotation from provider p = _provider_selection(name, localname, genomes_dir, provider) p.download_annotation(name, genomes_dir, localname=localname, **kwargs) # Sanitize annotation if needed (requires genome) annotation_found = bool(glob_ext_files(out_dir, "gtf")) if genome_found and annotation_found and not skip_sanitizing: sanitize_annotation(g) if genome_found: # Run all active plugins (requires genome) for plugin in get_active_plugins(): plugin.after_genome_download(g, threads, force)
def get_genome_download_link(self, name, mask="soft", **kwargs): """ Return http link to the genome sequence Parameters ---------- name : str Genome name. Current implementation will fail if exact name is not found. mask : str , optional Masking level. Options: soft, hard or none. Default is soft. Returns ------ str with the http download link. """ genome = self.genomes[safe(name)] division, is_vertebrate = get_division(genome) # base directory of the genome ftp = "http://ftp.ensemblgenomes.org" if is_vertebrate: ftp = "http://ftp.ensembl.org" version = self.get_version(is_vertebrate, kwargs.get("version")) div_path = "" if is_vertebrate else f"/{division}" lwr_name = genome["url_name"].lower() ftp_directory = f"{ftp}/pub/release-{version}{div_path}/fasta/{lwr_name}/dna" # some entries don't use url_name in their url... -,- # examples: # - EnsemblVertebrates: mus_musculus_nzohlltj # - EnsemblMetazoa: caenorhabditis_elegans if not check_url(ftp_directory, 2): lwr_name = genome["name"] ftp_directory = f"{ftp}/pub/release-{version}{div_path}/fasta/{lwr_name}/dna" # this assembly has its own directory if name == "GRCh37": ftp_directory = genome["genome"].format(version) # specific fasta file cap_name = lwr_name.capitalize() asm_name = re.sub(r"\.p\d+$", "", safe(genome["assembly_name"])) mask_lvl = {"soft": "_sm", "hard": "_rm", "none": ""}[mask] asm_lvl = "toplevel" if kwargs.get("toplevel") else "primary_assembly" version_tag = "" if int(version) > 30 else f".{version}" ftp_file = f"{cap_name}.{asm_name}{version_tag}.dna{mask_lvl}.{asm_lvl}.fa.gz" # combine link = f"{ftp_directory}/{ftp_file}" if check_url(link, 2): return link # primary assemblies do not always exist if asm_lvl == "primary_assembly": link = link.replace("primary_assembly", "toplevel") if check_url(link, 2): return link raise GenomeDownloadError( f"Could not download genome {name} from {self.name}.\n" "URL is broken. Select another genome or provider.\n" f"Broken URL: {link}")
def _post_process_download(self, name, localname, out_dir, mask="soft"): """ Replace accessions with sequence names in fasta file. Applies masking. Parameters ---------- name : str NCBI genome name localname : str Custom name for your genome out_dir : str Output directory mask : str , optional masking level: soft/hard/none, default=soft """ # Create mapping of accessions to names genome = self.genomes[safe(name)] url = genome["ftp_path"] url += f"/{url.split('/')[-1]}_assembly_report.txt" url = url.replace("ftp://", "https://") tr = {} urlcleanup() with urlopen(url) as response: for line in response.read().decode("utf-8").splitlines(): if line.startswith("#"): continue vals = line.strip().split("\t") tr[vals[6]] = vals[0] # mask sequence if required if mask == "soft": def mask_cmd(txt): return txt elif mask == "hard": sys.stderr.write( "\nNCBI genomes are softmasked by default. Hard masking...\n") def mask_cmd(txt): return re.sub("[actg]", "N", txt) else: sys.stderr.write( "\nNCBI genomes are softmasked by default. Unmasking...\n") def mask_cmd(txt): return txt.upper() # apply mapping and masking fa = os.path.join(out_dir, f"{localname}.fa") old_fa = os.path.join(out_dir, f"old_{localname}.fa") os.rename(fa, old_fa) with open(old_fa) as old, open(fa, "w") as new: for line in old: if line.startswith(">"): desc = line.strip()[1:] name = desc.split(" ")[0] new.write(">{} {}\n".format(tr.get(name, name), desc)) else: new.write(mask_cmd(line))
def get_genome_download_link(self, name, mask="soft", **kwargs): """ Return Ensembl http or ftp link to the genome sequence Parameters ---------- name : str Genome name. Current implementation will fail if exact name is not found. mask : str , optional Masking level. Options: soft, hard or none. Default is soft. Returns ------ str with the http/ftp download link. """ genome = self.genomes[safe(name)] # parse the division division = genome["division"].lower().replace("ensembl", "") if division == "bacteria": raise NotImplementedError( "bacteria from ensembl not yet supported") ftp_site = "ftp://ftp.ensemblgenomes.org/pub" if division == "vertebrates": ftp_site = "ftp://ftp.ensembl.org/pub" # Ensembl release version version = kwargs.get("version") if version is None: version = self.get_version(self.rest_url, division == "vertebrates") # division dependent url format ftp_dir = "{}/release-{}/fasta/{}/dna".format( division, version, genome["url_name"].lower()) if division == "vertebrates": ftp_dir = "release-{}/fasta/{}/dna".format( version, genome["url_name"].lower()) url = f"{ftp_site}/{ftp_dir}" # masking and assembly level def get_url(level="toplevel"): masks = { "soft": "dna_sm.{}", "hard": "dna_rm.{}", "none": "dna.{}" } pattern = masks[mask].format(level) asm_url = "{}/{}.{}.{}.fa.gz".format( url, genome["url_name"].capitalize(), re.sub(r"\.p\d+$", "", safe(genome["assembly_name"])), pattern, ) return asm_url # try to get the (much smaller) primary assembly, # unless specified otherwise link = get_url("primary_assembly") if kwargs.get("toplevel") or not check_url(link, 2): link = get_url() if check_url(link, 2): return link raise GenomeDownloadError( f"Could not download genome {name} from {self.name}.\n" "URL is broken. Select another genome or provider.\n" f"Broken URL: {link}")
def download_genome( self, name, genomes_dir=None, localname=None, mask="soft", keep_alt=False, regex=None, invert_match=False, bgzip=None, **kwargs, ): """ Download a (gzipped) genome file to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install genome localname : str , optional Custom name for your genome mask: str , optional Masking, soft, hard or none (all other strings) keep_alt : bool , optional Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. """ name = safe(name) self.check_name(name) link = self.get_genome_download_link(name, mask=mask, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) if not os.path.exists(out_dir): mkdir_p(out_dir) sys.stderr.write( f"Downloading genome from {self.name}.\nTarget URL: {link}...\n") # download to tmp dir. Move genome on completion. # tmp dir is in genome_dir to prevent moving the genome between disks with TemporaryDirectory(dir=out_dir) as tmp_dir: fname = os.path.join(tmp_dir, f"{localname}.fa") # actual download urlcleanup() with urlopen(link) as response: # check available memory vs file size. available_memory = int(virtual_memory().available) file_size = int(response.info()["Content-Length"]) # download file in chunks if >75% of memory would be used cutoff = int(available_memory * 0.75) chunk_size = None if file_size < cutoff else cutoff with open(fname, "wb") as f_out: shutil.copyfileobj(response, f_out, chunk_size) sys.stderr.write( "Genome download successful, starting post processing...\n") # unzip genome if link.endswith(".tar.gz"): tar_to_bigfile(fname, fname) elif link.endswith(".gz"): os.rename(fname, fname + ".gz") ret = sp.check_call(["gunzip", "-f", fname]) if ret != 0: raise Exception(f"Error gunzipping genome {fname}") def regex_filer(_fname, _regex, _v): os.rename(_fname, _fname + "_to_regex") infa = _fname + "_to_regex" outfa = _fname filter_fasta(infa, outfa, regex=_regex, v=_v, force=True) return [ k for k in Fasta(infa).keys() if k not in Fasta(outfa).keys() ] not_included = [] # remove alternative regions if not keep_alt: not_included.extend(regex_filer(fname, "alt", True)) # keep/remove user defined regions if regex: not_included.extend(regex_filer(fname, regex, invert_match)) # process genome (e.g. masking) if hasattr(self, "_post_process_download"): self._post_process_download(name=name, localname=localname, out_dir=tmp_dir, mask=mask) # bgzip genome if requested if bgzip or config.get("bgzip"): ret = sp.check_call(["bgzip", "-f", fname]) if ret != 0: raise Exception( f"Error bgzipping {name}. Is tabix installed?") fname += ".gz" # transfer the genome from the tmpdir to the genome_dir src = fname dst = os.path.join(genomes_dir, localname, os.path.basename(fname)) shutil.move(src, dst) sys.stderr.write("\n") sys.stderr.write("name: {}\n".format(name)) sys.stderr.write("local name: {}\n".format(localname)) sys.stderr.write("fasta: {}\n".format(dst)) # Create readme with information readme = os.path.join(genomes_dir, localname, "README.txt") metadata = { "name": localname, "provider": self.name, "original name": name, "original filename": os.path.split(link)[-1], "assembly_accession": self.assembly_accession(self.genomes.get(name)), "tax_id": self.genome_taxid(self.genomes.get(name)), "mask": mask, "genome url": link, "annotation url": "na", "date": time.strftime("%Y-%m-%d %H:%M:%S"), } lines = [] if regex: regex_line = f"regex: {regex}" if invert_match: regex_line += " (inverted match)" lines += ["", regex_line, "sequences that were excluded:"] for seq in not_included: lines.append(f"\t{seq}") write_readme(readme, metadata, lines)
def _search_descriptions(self, genome, term): """check if search term corresponds to the provider's description field(s)""" for field in self.description_fields: if term in safe(genome[field].lower()): return True