def download_annotation(self, name, genomes_dir=None, localname=None, **kwargs): """ Download annotation file to to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install annotation localname : str , optional Custom name for your genome """ self.check_name(name) link = self.get_annotation_download_link(name, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) self.attempt_and_report(name, localname, link, genomes_dir)
def _post_process_download(self, name, localname, out_dir, mask="soft"): """ Replace accessions with sequence names in fasta file. Parameters ---------- name : str NCBI genome name out_dir : str Output directory """ # Get the FTP url for this specific genome and download # the assembly report for genome in self.genomes: if name in [genome["asm_name"], genome["asm_name"].replace(" ", "_")]: url = genome["ftp_path"] url += "/" + url.split("/")[-1] + "_assembly_report.txt" url = url.replace("ftp://", "https://") break # Create mapping of accessions to names tr = {} urlcleanup() with urlopen(url) as response: for line in response.read().decode("utf-8").splitlines(): if line.startswith("#"): continue vals = line.strip().split("\t") tr[vals[6]] = vals[0] localname = get_localname(name, localname) # Check of the original genome fasta exists fa = os.path.join(out_dir, "{}.fa".format(localname)) if not os.path.exists(fa): raise Exception("Genome fasta file not found, {}".format(fa)) # Use a tmp file and replace the names new_fa = os.path.join(out_dir, ".process.{}.fa".format(localname)) if mask != "soft": sys.stderr.write( "NCBI genomes are softmasked by default. Changing mask...\n" ) with open(fa) as old: with open(new_fa, "w") as new: for line in old: if line.startswith(">"): desc = line.strip()[1:] name = desc.split(" ")[0] new.write(">{} {}\n".format(tr.get(name, name), desc)) elif mask == "hard": new.write(re.sub("[actg]", "N", line)) elif mask not in ["hard", "soft"]: new.write(line.upper()) else: new.write(line) # Rename tmp file to real genome file shutil.move(new_fa, fa)
def download_annotation(self, url, genomes_dir=None, localname=None, **kwargs): """ Attempts to download a gtf or gff3 file from the same location as the genome url Parameters ---------- url : str url of where to download genome from genomes_dir : str Directory to install annotation localname : str , optional Custom name for your genome kwargs: dict , optional: Provider specific options. to_annotation : str , optional url to annotation file (only required if this not located in the same directory as the fasta) """ name = get_localname(url) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) if kwargs.get("to_annotation"): links = [self.get_annotation_download_link(None, **kwargs)] else: # can return multiple possible hits links = self.search_url_for_annotations(url, name) for link in links: try: self.attempt_and_report(name, localname, link, genomes_dir) break except GenomeDownloadError as e: if not link == links[-1]: sys.stdout.write( "\nOne of the potential annotations was incompatible with genomepy." + "\nAttempting another...\n\n") continue return e
def search_url_for_annotations(url: str) -> list: """Attempts to find gtf or gff3 files in the same location as the genome url""" name = get_localname(url) urldir = url[:url.rfind("/")] logger.info("You have requested the gene annotation to be downloaded. " "Genomepy will check the remote directory: " f"{urldir} for annotation files...") # try to find a GTF or GFF3 file dirty_list = read_url(urldir).split("\n") fnames = fuzzy_annotation_search(name, dirty_list) links = [urldir + "/" + fname for fname in fnames] if not links: logger.warning("Could not parse the remote directory. " "Please supply a URL using --URL-to-annotation.\n") return links
def download_annotation(self, name, genomes_dir=None, localname=None, **kwargs): """ Download annotation file to to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install annotation localname : str , optional Custom name for your genome """ name = self._check_name(name) link = self.get_annotation_download_link(name, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) logger.info( f"Downloading annotation from {self.name}. Target URL: {link}...") try: # download exact assembly report to rename the scaffolds acc = self.assembly_accession(name) fname = os.path.join(genomes_dir, localname, "assembly_report.txt") download_assembly_report(acc, fname) download_annotation(genomes_dir, link, localname) logger.info("Annotation download successful") except Exception as e: raise GenomeDownloadError( f"An error occured while installing the gene annotation for {name} from {self.name}.\n" "If you think the annotation should be there, please file a bug report at: " "https://github.com/vanheeringen-lab/genomepy/issues\n\n" f"Error: {e.args[0]}") # Add annotation URL to readme readme = os.path.join(genomes_dir, localname, "README.txt") update_readme(readme, updated_metadata={"annotation url": link})
def search_url_for_annotation(url): """Attempts to find a gtf or gff3 file in the same location as the genome url""" urldir = os.path.dirname(url) sys.stderr.write( "You have requested gene annotation to be downloaded.\n" "Genomepy will check the remote directory:\n" f"{urldir} for annotation files...\n") # try to find a GTF or GFF3 file name = get_localname(url) with urlopen(urldir) as f: for urlline in f.readlines(): urlstr = str(urlline) if any(substring in urlstr.lower() for substring in [".gtf", name + ".gff"]): break # retrieve the filename from the HTML line fname = "" for split in re.split('>|<|><|/|"', urlstr): if split.lower().endswith(( ".gtf", ".gtf.gz", name + ".gff", name + ".gff.gz", name + ".gff3", name + ".gff3.gz", )): fname = split break else: raise FileNotFoundError( "Could not parse the remote directory. " "Please supply a URL using --url-to-annotation.\n") # set variables for downloading link = urldir + "/" + fname if check_url(link): return link
def _post_process_download(self, name, localname, out_dir, mask="soft"): """ Unmask a softmasked genome if required Parameters ---------- name : str UCSC genome name out_dir : str Output directory """ if mask not in ["hard", "soft"]: localname = get_localname(name, localname) # Check of the original genome fasta exists fa = os.path.join(out_dir, "{}.fa".format(localname)) if not os.path.exists(fa): raise Exception("Genome fasta file not found, {}".format(fa)) sys.stderr.write("UCSC genomes are softmasked by default. Unmasking...\n") # Use a tmp file and replace the names new_fa = os.path.join( out_dir, localname, ".process.{}.fa".format(localname) ) with open(fa) as old: with open(new_fa, "w") as new: for line in old: if not line.startswith(">"): new.write(line.upper()) else: new.write(line) # Rename tmp file to real genome file shutil.move(new_fa, fa)
def download_annotation(self, name, genomes_dir=None, localname=None, **kwargs): """ Download the UCSC genePred via their MySQL database, and convert to annotations. """ name = self._check_name(name) annot = self.get_annotation_download_link(name, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) logger.info( f"Downloading the {annot} annotation from the UCSC MySQL database." ) try: download_annotation(name, annot, genomes_dir, localname) logger.info("Annotation download successful") except Exception as e: raise GenomeDownloadError( f"An error occured while installing the gene annotation for {name} from {self.name}.\n" "If you think the annotation should be there, please file a bug report at: " "https://github.com/vanheeringen-lab/genomepy/issues\n\n" f"Error: {e.args[0]}") # Add annotation URL to readme readme = os.path.join(genomes_dir, localname, "README.txt") update_readme( readme, updated_metadata={ "annotation url": f"UCSC MySQL database: {name}, table: {annot}" }, )
def download_annotation(self, name, genome_dir, localname=None, **kwargs): """ Download Ensembl annotation file to to a specific directory Parameters ---------- name : str Ensembl genome name. genome_dir : str Genome directory. kwargs: dict , optional: Provider specific options. version : int , optional Ensembl version. By default the latest version is used. """ sys.stderr.write("Downloading gene annotation...\n") localname = get_localname(name, localname) genome_info = self._get_genome_info(name) # parse the division division = genome_info["division"].lower().replace("ensembl", "") if division == "bacteria": raise NotImplementedError("bacteria from ensembl not yet supported") # Get the base link depending on division ftp_site = "ftp://ftp.ensemblgenomes.org/pub" if division == "vertebrates": ftp_site = "https://ftp.ensembl.org/pub" version = self.version if kwargs.get("version", None): version = kwargs.get("version") elif not version: version = self.get_version(ftp_site) if division != "vertebrates": ftp_site += "/{}".format(division) # Get the GTF URL base_url = ftp_site + "/release-{}/gtf/{}/{}.{}.{}.gtf.gz" safe_name = name.replace(" ", "_") safe_name = re.sub(r"\.p\d+$", "", safe_name) ftp_link = base_url.format( version, genome_info["url_name"].lower(), genome_info["url_name"].capitalize(), safe_name, version, ) out_dir = os.path.join(genome_dir, localname) if not os.path.exists(out_dir): os.mkdir(out_dir) # download to tmp dir. Move genome on completion. with TemporaryDirectory(dir=out_dir) as tmpdir: try: # actual download sys.stderr.write("Using {}\n".format(ftp_link)) with urlopen(ftp_link) as response: gtf_file = os.path.join(tmpdir, localname + ".annotation.gtf.gz") with open(gtf_file, "wb") as f: f.write(response.read()) bed_file = gtf_file.replace("gtf.gz", "bed") cmd = ( "gtfToGenePred {0} /dev/stdout | " "genePredToBed /dev/stdin {1} && gzip -f {1}" ) sp.check_call(cmd.format(gtf_file, bed_file), shell=True) # transfer the genome from the tmpdir to the genome_dir for f in [gtf_file, bed_file + ".gz"]: src = f dst = os.path.join(out_dir, os.path.basename(f)) shutil.move(src, dst) readme = os.path.join(out_dir, "README.txt") with open(readme, "a") as f: f.write("annotation url: {}\n".format(ftp_link)) except Exception: sys.stderr.write("\nCould not download {}\n".format(ftp_link)) raise
def download_genome( self, name, genome_dir, localname=None, mask="soft", regex=None, invert_match=False, bgzip=None, **kwargs ): """ Download a (gzipped) genome file to a specific directory Parameters ---------- name : str Genome / species name genome_dir : str Directory to install genome localname : str , optional Custom name for your genome mask: str , optional Masking, soft, hard or none (all other strings) regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. """ genome_dir = os.path.expanduser(genome_dir) if not os.path.exists(genome_dir): os.makedirs(genome_dir) dbname, link = self.get_genome_download_link(name, mask=mask, **kwargs) myname = get_localname(dbname, localname) if not os.path.exists(os.path.join(genome_dir, myname)): os.makedirs(os.path.join(genome_dir, myname)) sys.stderr.write("Downloading genome from {}...\n".format(link)) # download to tmp dir. Move genome on completion. # tmp dir is in genome_dir to prevent moving the genome between disks with TemporaryDirectory(dir=os.path.join(genome_dir, myname)) as tmpdir: fname = os.path.join(tmpdir, myname + ".fa") # actual download urlcleanup() with urlopen(link) as response: # check available memory vs file size. available_memory = int(virtual_memory().available) file_size = int(response.info()["Content-Length"]) # download file in chunks if >75% of memory would be used cutoff = int(available_memory * 0.75) chunk_size = None if file_size < cutoff else cutoff with open(fname, "wb") as f_out: shutil.copyfileobj(response, f_out, chunk_size) # unzip genome if link.endswith("tar.gz"): self.tar_to_bigfile(fname, fname) elif link.endswith(".gz"): # gunzip will only work with files ending with ".gz" os.rename(fname, fname + ".gz") ret = sp.check_call(["gunzip", "-f", fname]) if ret != 0: raise Exception("Error gunzipping genome {}".format(fname)) # process genome (e.g. masking) if hasattr(self, "_post_process_download"): self._post_process_download(name, localname, tmpdir, mask) if regex: os.rename(fname, fname + "_to_regex") infa = fname + "_to_regex" outfa = fname filter_fasta(infa, outfa, regex=regex, v=invert_match, force=True) not_included = [ k for k in Fasta(infa).keys() if k not in Fasta(outfa).keys() ] # bgzip genome if requested if bgzip is None: bgzip = config.get("bgzip", False) if bgzip: ret = sp.check_call(["bgzip", "-f", fname]) if ret != 0: raise Exception( "Error bgzipping {}. ".format(fname) + "Is tabix installed?" ) fname += ".gz" # transfer the genome from the tmpdir to the genome_dir src = fname dst = os.path.join(genome_dir, myname, os.path.basename(fname)) shutil.move(src, dst) sys.stderr.write("name: {}\n".format(dbname)) sys.stderr.write("local name: {}\n".format(myname)) sys.stderr.write("fasta: {}\n".format(dst)) # Create readme with information readme = os.path.join(genome_dir, myname, "README.txt") with open(readme, "w") as f: f.write("name: {}\n".format(myname)) f.write("original name: {}\n".format(dbname)) f.write("original filename: {}\n".format(os.path.split(link)[-1])) f.write("url: {}\n".format(link)) f.write("mask: {}\n".format(mask)) f.write("date: {}\n".format(time.strftime("%Y-%m-%d %H:%M:%S"))) if regex: if invert_match: f.write("regex: {} (inverted match)\n".format(regex)) else: f.write("regex: {}\n".format(regex)) f.write("sequences that were excluded:\n") for seq in not_included: f.write("\t{}\n".format(seq))
def download_genome( self, name: str, genomes_dir: str = None, localname: str = None, mask: str = "soft", **kwargs, ): """ Download a (gzipped) genome file to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install genome localname : str , optional Custom name for your genome mask: str , optional Masking, soft, hard or none (all other strings) """ name = self._check_name(name) link = self.get_genome_download_link(name, mask=mask, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) mkdir_p(out_dir) logger.info( f"Downloading genome from {self.name}. Target URL: {link}...") # download to tmp dir. Move genome on completion. # tmp dir is in genome_dir to prevent moving the genome between disks get_file = shutil.copyfile if os.path.exists(link) else download_file with TemporaryDirectory(dir=out_dir) as tmp_dir: tmp_fname = os.path.join(tmp_dir, link.split("/")[-1]) fname = os.path.join(tmp_dir, f"{localname}.fa") get_file(link, tmp_fname) logger.info( "Genome download successful, starting post processing...") # unzip genome _, is_compressed = get_file_info(link) if is_compressed: extract_archive(tmp_fname, outfile=fname, concat=True) else: shutil.move(tmp_fname, fname) # process genome (e.g. masking) if hasattr(self, "_post_process_download"): self._post_process_download(name=name, fname=fname, out_dir=out_dir, mask=mask) # transfer the genome from the tmpdir to the genome_dir src = fname dst = os.path.join(out_dir, f"{localname}.fa") shutil.move(src, dst) logger.info("name: {}".format(name)) logger.info("local name: {}".format(localname)) logger.info("fasta: {}".format(dst)) # Create readme with information readme = os.path.join(genomes_dir, localname, "README.txt") asm_acc = self.assembly_accession(name) tax_id = self.genome_taxid(name) metadata = { "name": localname, "provider": self.name, "original name": name, "original filename": os.path.split(link)[-1], "assembly_accession": asm_acc if asm_acc else "na", "tax_id": tax_id if tax_id else "na", "mask": mask, "genome url": link, "genomepy version": __version__, "date": time.strftime("%Y-%m-%d %H:%M:%S"), } update_readme(readme, metadata)
def download_annotation(self, name, genome_dir, localname=None, version=None): """ Download annotation file to to a specific directory Parameters ---------- name : str Genome / species name genome_dir : str Directory to install annotation """ sys.stderr.write("Downloading annotation...\n") if not self.genomes: self.genomes = self._get_genomes() for genome in self.genomes: if genome["asm_name"] == name: url = genome["ftp_path"] url += "/" + url.split("/")[-1] + "_genomic.gff.gz" localname = get_localname(name, localname) out_dir = os.path.join(genome_dir, localname) if not os.path.exists(out_dir): os.mkdir(out_dir) # Download the file try: response = urlopen(url) gff_file = out_dir + "/" + localname + ".annotation.gff.gz" with open(gff_file, "wb") as f: f.write(response.read()) except Exception: sys.stderr.write( "WARNING: Could not download annotation from NCBI, " + "skipping.\n") sys.stderr.write("URL: {}\n".format(url)) sys.stderr.write("If you think the annotation should be there, " + "please file a bug report at:\n") sys.stderr.write("https://github.com/simonvh/genomepy/issues\n") return cmd = "gff3ToGenePred {0} /dev/stdout | wc -l" out = sp.check_output(cmd.format(gff_file), shell=True) if out.strip() == b"0": sys.stderr.write( "WARNING: annotation from NCBI contains no genes, " + "skipping.\n") else: # Convert to BED file bed_file = gff_file.replace("gff.gz", "bed") cmd = ("gff3ToGenePred -rnaNameAttr=gene {0} /dev/stdout | " "genePredToBed /dev/stdin {1} && gzip {1}") sp.check_call(cmd.format(gff_file, bed_file), shell=True) # Convert to GTF file gtf_file = gff_file.replace("gff.gz", "gtf") cmd = ("gff3ToGenePred -geneNameAttr=gene {0} /dev/stdout | " + "genePredToGtf file /dev/stdin {1} && gzip {1}") sp.check_call(cmd.format(gff_file, gtf_file), shell=True) readme = os.path.join(genome_dir, name, "README.txt") with open(readme, "a") as f: f.write("annotation url: {}\n".format(url)) return out_dir
def download_annotation(self, name, genome_dir, localname=None, **kwargs): """ Download NCBI annotation file to to a specific directory Parameters ---------- name : str Genome / species name genome_dir : str Directory to install annotation """ sys.stderr.write("Downloading gene annotation...\n") localname = get_localname(name, localname) if not self.genomes: self.genomes = self._get_genomes() for genome in self.genomes: if name in [genome["asm_name"], genome["asm_name"].replace(" ", "_")]: url = genome["ftp_path"] url = url.replace("ftp://", "https://") url += "/" + url.split("/")[-1] + "_genomic.gff.gz" out_dir = os.path.join(genome_dir, localname) if not os.path.exists(out_dir): os.mkdir(out_dir) # download to tmp dir. Move genome on completion. with TemporaryDirectory(dir=out_dir) as tmpdir: try: # actual download sys.stderr.write("Using {}\n".format(url)) gff_file = os.path.join(tmpdir, localname + ".annotation.gff.gz") with urlopen(url) as response: with open(gff_file, "wb") as f: f.write(response.read()) # check gff for genes cmd = "gff3ToGenePred {0} /dev/stdout | wc -l" out = sp.check_output(cmd.format(gff_file), shell=True) if out.strip() == b"0": sys.stderr.write( "WARNING: annotation from NCBI contains no genes, " + "skipping.\n" ) return else: # Convert to BED file bed_file = gff_file.replace("gff.gz", "bed") cmd = ( "gff3ToGenePred -rnaNameAttr=gene {0} /dev/stdout | " "genePredToBed /dev/stdin {1} && gzip -f {1}" ) sp.check_call(cmd.format(gff_file, bed_file), shell=True) # Convert to GTF file gtf_file = gff_file.replace("gff.gz", "gtf") cmd = ( "gff3ToGenePred -geneNameAttr=gene {0} /dev/stdout | " + "genePredToGtf file /dev/stdin {1} && gzip -f {1}" ) sp.check_call(cmd.format(gff_file, gtf_file), shell=True) # transfer the genome from the tmpdir to the genome_dir for f in [gtf_file + ".gz", bed_file + ".gz"]: src = f dst = os.path.join(out_dir, os.path.basename(f)) shutil.move(src, dst) readme = os.path.join(genome_dir, localname, "README.txt") with open(readme, "a") as f: f.write("annotation url: {}\n".format(url)) except Exception: sys.stderr.write( "WARNING: Could not download annotation from NCBI, " + "skipping.\n" ) sys.stderr.write("URL: {}\n".format(url)) sys.stderr.write( "If you think the annotation should be there, " + "please file a bug report at:\n" ) sys.stderr.write("https://github.com/simonvh/genomepy/issues\n")
def download_genome( self, name, genomes_dir=None, localname=None, mask="soft", keep_alt=False, regex=None, invert_match=False, bgzip=None, **kwargs, ): """ Download a (gzipped) genome file to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install genome localname : str , optional Custom name for your genome mask: str , optional Masking, soft, hard or none (all other strings) keep_alt : bool , optional Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. """ name = safe(name) self.check_name(name) link = self.get_genome_download_link(name, mask=mask, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) if not os.path.exists(out_dir): mkdir_p(out_dir) sys.stderr.write( f"Downloading genome from {self.name}.\nTarget URL: {link}...\n") # download to tmp dir. Move genome on completion. # tmp dir is in genome_dir to prevent moving the genome between disks tmp_dir = mkdtemp(dir=out_dir) fname = os.path.join(tmp_dir, f"{localname}.fa") urlcleanup() download_file(link, fname) sys.stderr.write( "Genome download successful, starting post processing...\n") # unzip genome if link.endswith(".tar.gz"): tar_to_bigfile(fname, fname) elif link.endswith(".gz"): os.rename(fname, fname + ".gz") ret = sp.check_call(["gunzip", "-f", fname]) if ret != 0: raise Exception(f"Error gunzipping genome {fname}") def regex_filer(_fname, _regex, _v): infa = _fname + "_to_regex" os.rename(_fname, infa) # filter the fasta and store the output's keys keys_out = filter_fasta(infa, outfa=_fname, regex=_regex, v=_v, force=True).keys() keys_in = Fasta(infa).keys() return [k for k in keys_in if k not in keys_out] not_included = [] # remove alternative regions if not keep_alt: not_included.extend(regex_filer(fname, "alt", True)) # keep/remove user defined regions if regex: not_included.extend(regex_filer(fname, regex, invert_match)) # process genome (e.g. masking) if hasattr(self, "_post_process_download"): self._post_process_download(name=name, localname=localname, out_dir=tmp_dir, mask=mask) # bgzip genome if requested if bgzip or config.get("bgzip"): # bgzip to stdout, track progress, and output to file fsize = int(os.path.getsize(fname) * 10**-6) cmd = ( f"bgzip -fc {fname} | " f"tqdm --bytes --desc Bgzipping {fsize}MB fasta --log ERROR | " f"cat > {fname}.gz") ret = sp.check_call(cmd, shell=True) if ret != 0: raise Exception(f"Error bgzipping {name}. Is tabix installed?") fname += ".gz" # transfer the genome from the tmpdir to the genome_dir src = fname dst = os.path.join(genomes_dir, localname, os.path.basename(fname)) shutil.move(src, dst) rm_rf(tmp_dir) sys.stderr.write("\n") sys.stderr.write("name: {}\n".format(name)) sys.stderr.write("local name: {}\n".format(localname)) sys.stderr.write("fasta: {}\n".format(dst)) # Create readme with information readme = os.path.join(genomes_dir, localname, "README.txt") metadata = { "name": localname, "provider": self.name, "original name": name, "original filename": os.path.split(link)[-1], "assembly_accession": self.assembly_accession(self.genomes.get(name)), "tax_id": self.genome_taxid(self.genomes.get(name)), "mask": mask, "genome url": link, "annotation url": "na", "date": time.strftime("%Y-%m-%d %H:%M:%S"), } lines = [] if not keep_alt or regex: regex_line = "regex: " if not keep_alt: regex_line += "'alt' (inverted match)" if not keep_alt and regex: regex_line += " and " if regex: regex_line += f"'{regex}'" if invert_match: regex_line += " (inverted match)" lines += ["", regex_line, "sequences that were excluded:"] for seq in not_included: lines.append(f"\t{seq}") write_readme(readme, metadata, lines)
def download_annotation(self, name, genome_dir, localname=None, version=None): """ Download gene annotation from Ensembl based on genome name. Parameters ---------- name : str Ensembl genome name. genome_dir : str Genome directory. version : str , optional Ensembl version. By default the latest version is used. """ localname = get_localname(name, localname) genome_info = self._get_genome_info(name) # parse the division division = genome_info["division"].lower().replace("ensembl", "") if division == "bacteria": raise NotImplementedError( "bacteria from ensembl not yet supported") # Get the base link depending on division ftp_site = "ftp://ftp.ensemblgenomes.org/pub" if division == 'vertebrates': ftp_site = "https://ftp.ensembl.org/pub" if not version: version = self.get_version(ftp_site) if division != "vertebrates": ftp_site += "/{}".format(division) # Get the GTF URL base_url = ftp_site + "/release-{}/gtf/{}/{}.{}.{}.gtf.gz" safe_name = name.replace(" ", "_") safe_name = re.sub('\.p\d+$', '', safe_name) ftp_link = base_url.format(version, genome_info["url_name"].lower(), genome_info["url_name"].capitalize(), safe_name, version) out_dir = os.path.join(genome_dir, localname) if not os.path.exists(out_dir): os.mkdir(out_dir) # Download the file try: response = urlopen(ftp_link) gtf_file = out_dir + "/" + localname + ".annotation.gtf.gz" with open(gtf_file, "wb") as f: f.write(response.read()) bed_file = gtf_file.replace("gtf.gz", "bed") cmd = ("gtfToGenePred {0} /dev/stdout | " "genePredToBed /dev/stdin {1} && gzip {1}") sp.check_call(cmd.format(gtf_file, bed_file), shell=True) readme = os.path.join(genome_dir, localname, "README.txt") with open(readme, "a") as f: f.write("annotation url: {}\n".format(ftp_link)) except Exception: sys.stderr.write("\nCould not download {}\n".format(ftp_link)) raise return out_dir
def download_genome( self, name, genomes_dir=None, localname=None, mask="soft", keep_alt=False, regex=None, invert_match=False, bgzip=None, **kwargs, ): """ Download a (gzipped) genome file to a specific directory Parameters ---------- name : str Genome / species name genomes_dir : str , optional Directory to install genome localname : str , optional Custom name for your genome mask: str , optional Masking, soft, hard or none (all other strings) keep_alt : bool , optional Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. """ name = safe(name) self.check_name(name) link = self.get_genome_download_link(name, mask=mask, **kwargs) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) if not os.path.exists(out_dir): mkdir_p(out_dir) sys.stderr.write( f"Downloading genome from {self.name}.\nTarget URL: {link}...\n") # download to tmp dir. Move genome on completion. # tmp dir is in genome_dir to prevent moving the genome between disks with TemporaryDirectory(dir=out_dir) as tmp_dir: fname = os.path.join(tmp_dir, f"{localname}.fa") # actual download urlcleanup() with urlopen(link) as response: # check available memory vs file size. available_memory = int(virtual_memory().available) file_size = int(response.info()["Content-Length"]) # download file in chunks if >75% of memory would be used cutoff = int(available_memory * 0.75) chunk_size = None if file_size < cutoff else cutoff with open(fname, "wb") as f_out: shutil.copyfileobj(response, f_out, chunk_size) sys.stderr.write( "Genome download successful, starting post processing...\n") # unzip genome if link.endswith(".tar.gz"): tar_to_bigfile(fname, fname) elif link.endswith(".gz"): os.rename(fname, fname + ".gz") ret = sp.check_call(["gunzip", "-f", fname]) if ret != 0: raise Exception(f"Error gunzipping genome {fname}") def regex_filer(_fname, _regex, _v): os.rename(_fname, _fname + "_to_regex") infa = _fname + "_to_regex" outfa = _fname filter_fasta(infa, outfa, regex=_regex, v=_v, force=True) return [ k for k in Fasta(infa).keys() if k not in Fasta(outfa).keys() ] not_included = [] # remove alternative regions if not keep_alt: not_included.extend(regex_filer(fname, "alt", True)) # keep/remove user defined regions if regex: not_included.extend(regex_filer(fname, regex, invert_match)) # process genome (e.g. masking) if hasattr(self, "_post_process_download"): self._post_process_download(name=name, localname=localname, out_dir=tmp_dir, mask=mask) # bgzip genome if requested if bgzip or config.get("bgzip"): ret = sp.check_call(["bgzip", "-f", fname]) if ret != 0: raise Exception( f"Error bgzipping {name}. Is tabix installed?") fname += ".gz" # transfer the genome from the tmpdir to the genome_dir src = fname dst = os.path.join(genomes_dir, localname, os.path.basename(fname)) shutil.move(src, dst) sys.stderr.write("\n") sys.stderr.write("name: {}\n".format(name)) sys.stderr.write("local name: {}\n".format(localname)) sys.stderr.write("fasta: {}\n".format(dst)) # Create readme with information readme = os.path.join(genomes_dir, localname, "README.txt") metadata = { "name": localname, "provider": self.name, "original name": name, "original filename": os.path.split(link)[-1], "assembly_accession": self.assembly_accession(self.genomes.get(name)), "tax_id": self.genome_taxid(self.genomes.get(name)), "mask": mask, "genome url": link, "annotation url": "na", "date": time.strftime("%Y-%m-%d %H:%M:%S"), } lines = [] if regex: regex_line = f"regex: {regex}" if invert_match: regex_line += " (inverted match)" lines += ["", regex_line, "sequences that were excluded:"] for seq in not_included: lines.append(f"\t{seq}") write_readme(readme, metadata, lines)
def install_genome( name, provider, version=None, genome_dir=None, localname=None, mask="soft", regex=None, invert_match=False, annotation=False, bgzip=None, ): """ Install a genome. Parameters ---------- name : str Genome name provider : str Provider name version : str Version (only for Ensembl) genome_dir : str , optional Where to store the fasta files localname : str , optional Custom name for this genome. mask : str , optional Default is 'soft', specify 'hard' for hard masking. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. annotation : bool , optional If set to True, download gene annotation in BED and GTF format. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. """ if not genome_dir: genome_dir = config.get("genome_dir", None) if not genome_dir: raise norns.exceptions.ConfigError("Please provide or configure a genome_dir") genome_dir = os.path.expanduser(genome_dir) localname = get_localname(name, localname) # Download genome from provider p = ProviderBase.create(provider) p.download_genome( name, genome_dir, version=version, mask=mask, localname=localname, regex=regex, invert_match=invert_match, bgzip=bgzip, ) if annotation: # Download annotation from provider p.download_annotation(name, genome_dir, localname=localname, version=version) g = Genome(localname, genome_dir=genome_dir) for plugin in get_active_plugins(): plugin.after_genome_download(g) generate_env()
def download_annotation(self, name, genome_dir, localname=None, **kwargs): """ Download UCSC annotation file to to a specific directory. Will check UCSC, Ensembl and RefSeq annotation. Parameters ---------- name : str UCSC genome name. genome_dir : str Genome directory. """ sys.stderr.write("Downloading gene annotation...\n") localname = get_localname(name, localname) UCSC_GENE_URL = "http://hgdownload.cse.ucsc.edu/goldenPath/{}/database/" ANNOS = ["knownGene.txt.gz", "ensGene.txt.gz", "refGene.txt.gz"] pred = "genePredToBed" tmp = NamedTemporaryFile(delete=False, suffix=".gz") anno = [] p = re.compile(r"\w+.Gene.txt.gz") with urlopen(UCSC_GENE_URL.format(name)) as f: for line in f.readlines(): m = p.search(line.decode()) if m: anno.append(m.group(0)) url = "" for a in ANNOS: if a in anno: url = UCSC_GENE_URL.format(name) + a break out_dir = os.path.join(genome_dir, localname) if not os.path.exists(out_dir): os.mkdir(out_dir) # download to tmp dir. Move genome on completion. with TemporaryDirectory(dir=out_dir) as tmpdir: try: if url == "": raise Exception sys.stderr.write("Using {}\n".format(url)) urlretrieve(url, tmp.name) with gzip.open(tmp.name) as f: cols = f.readline().decode(errors="ignore").split("\t") start_col = 1 for i, col in enumerate(cols): if col == "+" or col == "-": start_col = i - 1 break end_col = start_col + 10 # Convert to BED file bed_file = os.path.join(tmpdir, localname + ".annotation.bed") cmd = "zcat {} | cut -f{}-{} | {} /dev/stdin {} && gzip -f {}" sp.call( cmd.format(tmp.name, start_col, end_col, pred, bed_file, bed_file), shell=True, ) # Convert to GTF file gtf_file = bed_file.replace(".bed", ".gtf") cmd = ( "bedToGenePred {0}.gz /dev/stdout | " "genePredToGtf file /dev/stdin /dev/stdout -utr -honorCdsStat | " "sed 's/.dev.stdin/UCSC/' > {1} && gzip -f {1}" ) sp.check_call(cmd.format(bed_file, gtf_file), shell=True) # transfer the genome from the tmpdir to the genome_dir for f in [gtf_file + ".gz", bed_file + ".gz"]: src = f dst = os.path.join(out_dir, os.path.basename(f)) shutil.move(src, dst) readme = os.path.join(genome_dir, localname, "README.txt") with open(readme, "a") as f: f.write("annotation url: {}\n".format(url)) except Exception: sys.stderr.write("No annotation found!")
def install_genome( name, provider=None, genomes_dir=None, localname=None, mask="soft", keep_alt=False, regex=None, invert_match=False, bgzip=None, annotation=False, only_annotation=False, skip_sanitizing=False, threads=1, force=False, **kwargs, ): """ Install a genome. Parameters ---------- name : str Genome name provider : str , optional Provider name. will try Ensembl, UCSC and NCBI (in that order) if not specified. genomes_dir : str , optional Where to store the fasta files localname : str , optional Custom name for this genome. mask : str , optional Default is 'soft', choices 'hard'/'soft/'none' for respective masking level. keep_alt : bool , optional Some genomes contain alternative regions. These regions cause issues with sequence alignment, as they are inherently duplications of the consensus regions. Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. threads : int , optional Build genome index using multithreading (if supported). Default: lowest of 8/all threads force : bool , optional Set to True to overwrite existing files. annotation : bool , optional If set to True, download gene annotation in BED and GTF format. only_annotation : bool , optional If set to True, only download the annotation files. skip_sanitizing : bool , optional If set to True, downloaded annotation files whose sequence names do not match with the (first header fields of) the genome.fa will not be corrected. kwargs : dict , optional Provider specific options. toplevel : bool , optional Ensembl only: Always download the toplevel genome. Ignores potential primary assembly. version : int , optional Ensembl only: Specify release version. Default is latest. to_annotation : text , optional URL only: direct link to annotation file. Required if this is not the same directory as the fasta. """ name = safe(name) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) # Check if genome already exists, or if downloading is forced genome_found = _is_genome_dir(out_dir) if (not genome_found or force) and not only_annotation: # Download genome from provider p = _provider_selection(name, localname, genomes_dir, provider) p.download_genome( name, genomes_dir, mask=mask, keep_alt=keep_alt, regex=regex, invert_match=invert_match, localname=localname, bgzip=bgzip, **kwargs, ) genome_found = True # Export installed genome(s) generate_env(genomes_dir=genomes_dir) # Generates a Fasta object, index, gaps and sizes file g = None if genome_found: g = Genome(localname, genomes_dir=genomes_dir) if force: # overwrite previous versions generate_fa_sizes(g.genome_file, g.sizes_file) generate_gap_bed(g.genome_file, g.gaps_file) # Check if any annotation flags are given, if annotation already exists, or if downloading is forced if any([ annotation, only_annotation, skip_sanitizing, kwargs.get("to_annotation"), kwargs.get("ucsc_annotation_type"), ]): annotation = True annotation_found = bool(glob_ext_files(out_dir, "gtf")) if (not annotation_found or force) and annotation: # Download annotation from provider p = _provider_selection(name, localname, genomes_dir, provider) p.download_annotation(name, genomes_dir, localname=localname, **kwargs) # Sanitize annotation if needed (requires genome) annotation_found = bool(glob_ext_files(out_dir, "gtf")) if genome_found and annotation_found and not skip_sanitizing: sanitize_annotation(g) if genome_found: # Run all active plugins (requires genome) for plugin in get_active_plugins(): plugin.after_genome_download(g, threads, force)
def install_genome( name: str, provider: Optional[str] = None, genomes_dir: Optional[str] = None, localname: Optional[str] = None, mask: Optional[str] = "soft", keep_alt: Optional[bool] = False, regex: Optional[str] = None, invert_match: Optional[bool] = False, bgzip: Optional[bool] = None, # None -> check config. False -> dont check. annotation: Optional[bool] = False, only_annotation: Optional[bool] = False, skip_matching: Optional[bool] = False, skip_filter: Optional[bool] = False, threads: Optional[int] = 1, force: Optional[bool] = False, **kwargs: Optional[dict], ) -> Genome: """ Install a genome (& gene annotation). Parameters ---------- name : str Genome name provider : str , optional Provider name. will try Ensembl, UCSC and NCBI (in that order) if not specified. genomes_dir : str , optional Where to create the output folder. localname : str , optional Custom name for this genome. mask : str , optional Genome masking of repetitive sequences. Options: hard/soft/none, default is soft. keep_alt : bool , optional Some genomes contain alternative regions. These regions cause issues with sequence alignment, as they are inherently duplications of the consensus regions. Set to true to keep these alternative regions. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that *don't* match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip, and gene annotation will be compressed with gzip. threads : int , optional Build genome index using multithreading (if supported). Default: lowest of 8/all threads. force : bool , optional Set to True to overwrite existing files. annotation : bool , optional If set to True, download gene annotation in BED and GTF format. only_annotation : bool , optional If set to True, only download the gene annotation files. skip_matching : bool , optional If set to True, contigs in the annotation not matching those in the genome will not be corrected. skip_filter : bool , optional If set to True, the gene annotations will not be filtered to match the genome contigs. kwargs : dict , optional Provider specific options. toplevel : bool , optional Ensembl only: Always download the toplevel genome. Ignores potential primary assembly. version : int , optional Ensembl only: Specify release version. Default is latest. to_annotation : text , optional URL only: direct link to annotation file. Required if this is not the same directory as the fasta. Returns ------- Genome Genome class with the installed genome """ name = safe(name) localname = get_localname(name, localname) genomes_dir = get_genomes_dir(genomes_dir, check_exist=False) out_dir = os.path.join(genomes_dir, localname) genome_file = os.path.join(out_dir, f"{localname}.fa") provider = _provider_selection(name, localname, genomes_dir, provider) # check which files need to be downloaded genome_found = _is_genome_dir(out_dir) download_genome = ( genome_found is False or force is True ) and only_annotation is False annotation_found = bool(glob_ext_files(out_dir, "annotation.gtf")) and bool( glob_ext_files(out_dir, "annotation.bed") ) download_annotation = (annotation_found is False or force is True) and any( [ annotation, only_annotation, skip_matching, skip_filter, kwargs.get("to_annotation"), kwargs.get("path_to_annotation"), kwargs.get("ucsc_annotation_type"), ] ) genome = None genome_downloaded = False if download_genome: if force: _delete_extensions(out_dir, ["fa", "fai"]) provider.download_genome( name, genomes_dir, mask=mask, localname=localname, **kwargs, ) genome_found = True genome_downloaded = True # Filter genome _filter_genome(genome_file, regex, invert_match, keep_alt) # Generates a Fasta object and the genome index, gaps and sizes files genome = Genome(localname, genomes_dir=genomes_dir) # Download the NCBI assembly report asm_report = os.path.join(out_dir, "assembly_report.txt") asm_acc = genome.assembly_accession if not os.path.exists(asm_report) and asm_acc != "na": download_assembly_report(asm_acc, asm_report) # Export installed genome(s) generate_env(genomes_dir=genomes_dir) annotation_downloaded = False if download_annotation: if force: _delete_extensions(out_dir, ["annotation.gtf", "annotation.bed"]) provider.download_annotation(name, genomes_dir, localname=localname, **kwargs) annotation_downloaded = bool( glob_ext_files(out_dir, "annotation.gtf") ) and bool(glob_ext_files(out_dir, "annotation.bed")) if annotation_downloaded: annotation = Annotation(localname, genomes_dir=genomes_dir) if genome_found and not (skip_matching and skip_filter): annotation.sanitize(not skip_matching, not skip_filter, True) # Run active plugins (also if the genome was downloaded earlier) if genome_found: genome = genome if genome else Genome(localname, genomes_dir=genomes_dir) for plugin in get_active_plugins(): plugin.after_genome_download(genome, threads, force) # zip files downloaded now if bgzip is True or (bgzip is None and config.get("bgzip")): if genome_downloaded: bgzip_and_name(genome.filename) if annotation_downloaded: gzip_and_name(annotation.annotation_gtf_file) gzip_and_name(annotation.annotation_bed_file) return genome
def install_genome(name, provider, genome_dir=None, localname=None, mask="soft", regex=None, invert_match=False, bgzip=None, annotation=False, force=False, **kwargs): """ Install a genome. Parameters ---------- name : str Genome name provider : str Provider name genome_dir : str , optional Where to store the fasta files localname : str , optional Custom name for this genome. mask : str , optional Default is 'soft', choices 'hard'/'soft/'none' for respective masking level. regex : str , optional Regular expression to select specific chromosome / scaffold names. invert_match : bool , optional Set to True to select all chromosomes that don't match the regex. bgzip : bool , optional If set to True the genome FASTA file will be compressed using bgzip. If not specified, the setting from the configuration file will be used. annotation : bool , optional If set to True, download gene annotation in BED and GTF format. force : bool , optional Set to True to overwrite existing files. kwargs : dict, optional Provider specific options. Ensembl: toplevel : bool , optional Ensembl only: Always download the toplevel genome. Ignores potential primary assembly. version : int, optional Ensembl only: Specify release version. Default is latest. """ if not genome_dir: genome_dir = config.get("genome_dir", None) if not genome_dir: raise norns.exceptions.ConfigError( "Please provide or configure a genome_dir") genome_dir = os.path.expanduser(genome_dir) localname = get_localname(name, localname) out_dir = os.path.join(genome_dir, localname) # Check if genome already exists, or if downloading is forced no_genome_found = not any( os.path.exists(fname) for fname in glob_ext_files(out_dir, "fa")) if no_genome_found or force: # Download genome from provider p = ProviderBase.create(provider) p.download_genome(name, genome_dir, mask=mask, regex=regex, invert_match=invert_match, localname=localname, bgzip=bgzip, **kwargs) # If annotation is requested, check if annotation already exists, or if downloading is forced no_annotation_found = not any( os.path.exists(fname) for fname in glob_ext_files(out_dir, "gtf")) if annotation and (no_annotation_found or force): # Download annotation from provider p = ProviderBase.create(provider) p.download_annotation(name, genome_dir, localname=localname, **kwargs) # generates a Fasta object and the index file g = Genome(localname, genome_dir=genome_dir) # Run all active plugins for plugin in get_active_plugins(): plugin.after_genome_download(g, force) # Generate gap file if not found or if generation is forced gap_file = os.path.join(out_dir, localname + ".gaps.bed") if not os.path.exists(gap_file) or force: generate_gap_bed(glob_ext_files(out_dir, "fa")[0], gap_file) generate_env()
def _parse_name(name): """extract a safe name from file path, url or regular names""" return os.path.basename(re.sub(".fa(.gz)?$", "", get_localname(name)))
def download_annotation(self, name, genome_dir, localname=None, version=None): """ Download gene annotation from UCSC based on genomebuild. Will check UCSC, Ensembl and RefSeq annotation. Parameters ---------- genomebuild : str UCSC genome name. genome_dir : str Genome directory. """ localname = get_localname(name, localname) UCSC_GENE_URL = "http://hgdownload.cse.ucsc.edu/goldenPath/{}/database/" ANNOS = ["knownGene.txt.gz", "ensGene.txt.gz", "refGene.txt.gz"] pred = "genePredToBed" tmp = NamedTemporaryFile(delete=False, suffix=".gz") anno = [] f = urlopen(UCSC_GENE_URL.format(name)) p = re.compile(r'\w+.Gene.txt.gz') for line in f.readlines(): m = p.search(line.decode()) if m: anno.append(m.group(0)) sys.stderr.write("Retrieving gene annotation for {}\n".format(name)) url = "" for a in ANNOS: if a in anno: url = UCSC_GENE_URL.format(name) + a break if url: sys.stderr.write("Using {}\n".format(url)) urlretrieve(url, tmp.name) with gzip.open(tmp.name) as f: cols = f.readline().decode(errors='ignore').split("\t") start_col = 1 for i, col in enumerate(cols): if col == "+" or col == "-": start_col = i - 1 break end_col = start_col + 10 localname = localname.replace(" ", "_") path = os.path.join(genome_dir, localname) if not os.path.exists(path): os.mkdir(path) bed_file = os.path.join(genome_dir, localname, localname + ".annotation.bed") cmd = "zcat {} | cut -f{}-{} | {} /dev/stdin {} && gzip {}" sp.call(cmd.format(tmp.name, start_col, end_col, pred, bed_file, bed_file), shell=True) gtf_file = bed_file.replace(".bed", ".gtf") cmd = ( "bedToGenePred {0}.gz /dev/stdout | " "genePredToGtf file /dev/stdin /dev/stdout -utr -honorCdsStat | " "sed 's/.dev.stdin/UCSC/' > {1} && gzip {1}") sp.check_call(cmd.format(bed_file, gtf_file), shell=True) readme = os.path.join(genome_dir, localname, "README.txt") with open(readme, "a") as f: f.write("annotation url: {}\n".format(url)) else: sys.stderr.write("No annotation found!")