def _upgrade_snpeff_data(galaxy_dir, args, remotes): """Install or upgrade snpEff databases, localized to reference directory. """ snpeff_version = effects.snpeff_version(args) if not snpeff_version: return for dbkey, ref_file in genome.get_builds(galaxy_dir): resource_file = os.path.join(os.path.dirname(ref_file), "%s-resources.yaml" % dbkey) if os.path.exists(resource_file): with open(resource_file) as in_handle: resources = yaml.load(in_handle) snpeff_db, snpeff_base_dir = effects.get_db({"genome_resources": resources, "reference": {"fasta": {"base": ref_file}}}) if snpeff_db: snpeff_db_dir = os.path.join(snpeff_base_dir, snpeff_db) if os.path.exists(snpeff_db_dir) and _is_old_database(snpeff_db_dir, args): shutil.rmtree(snpeff_db_dir) if not os.path.exists(snpeff_db_dir): print("Installing snpEff database %s in %s" % (snpeff_db, snpeff_base_dir)) dl_url = remotes["snpeff_dl_url"].format( snpeff_ver=snpeff_version.replace(".", "_"), genome=snpeff_db) dl_file = os.path.basename(dl_url) with utils.chdir(snpeff_base_dir): subprocess.check_call(["wget", "-c", "-O", dl_file, dl_url]) subprocess.check_call(["unzip", dl_file]) os.remove(dl_file) dl_dir = os.path.join(snpeff_base_dir, "data", snpeff_db) shutil.move(dl_dir, snpeff_db_dir) os.rmdir(os.path.join(snpeff_base_dir, "data"))
def _upgrade_genome_resources(galaxy_dir, base_url): """Retrieve latest version of genome resource YAML configuration files. """ for dbkey, ref_file in genome.get_builds(galaxy_dir): # Check for a remote genome resources file remote_url = base_url % dbkey requests.packages.urllib3.disable_warnings() r = requests.get(remote_url, verify=False) if r.status_code == requests.codes.ok: local_file = os.path.join(os.path.dirname(ref_file), os.path.basename(remote_url)) if os.path.exists(local_file): with open(local_file) as in_handle: local_config = yaml.load(in_handle) remote_config = yaml.load(r.text) needs_update = remote_config["version"] > local_config.get( "version", 0) if needs_update: shutil.move( local_file, local_file + ".old%s" % local_config.get("version", 0)) else: needs_update = True if needs_update: print("Updating %s genome resources configuration" % dbkey) with open(local_file, "w") as out_handle: out_handle.write(r.text)
def _upgrade_snpeff_data(galaxy_dir, args, remotes): """Install or upgrade snpEff databases, localized to reference directory. """ for dbkey, ref_file in genome.get_builds(galaxy_dir): resource_file = os.path.join(os.path.dirname(ref_file), "%s-resources.yaml" % dbkey) if os.path.exists(resource_file): with open(resource_file) as in_handle: resources = yaml.load(in_handle) snpeff_db, snpeff_base_dir = effects.get_db({ "genome_resources": resources, "reference": { "fasta": { "base": ref_file } } }) if snpeff_db: snpeff_db_dir = os.path.join(snpeff_base_dir, snpeff_db) if not os.path.exists(snpeff_db_dir): print("Installing snpEff database %s in %s" % (snpeff_db, snpeff_base_dir)) dl_url = remotes["snpeff_dl_url"].format( snpeff_ver=_get_snpeff_version(args), genome=snpeff_db) dl_file = os.path.basename(dl_url) with utils.chdir(snpeff_base_dir): subprocess.check_call( ["wget", "-c", "-O", dl_file, dl_url]) subprocess.check_call(["unzip", dl_file]) os.remove(dl_file) dl_dir = os.path.join(snpeff_base_dir, "data", snpeff_db) os.rename(dl_dir, snpeff_db_dir) os.rmdir(os.path.join(snpeff_base_dir, "data"))
def _upgrade_snpeff_data(galaxy_dir, args, remotes): """Install or upgrade snpEff databases, localized to reference directory. """ for dbkey, ref_file in genome.get_builds(galaxy_dir): resource_file = os.path.join(os.path.dirname(ref_file), "%s-resources.yaml" % dbkey) with open(resource_file) as in_handle: resources = yaml.load(in_handle) snpeff_db, snpeff_base_dir = effects.get_db(ref_file, resources) if snpeff_db: snpeff_db_dir = os.path.join(snpeff_base_dir, snpeff_db) if not os.path.exists(snpeff_db_dir): print("Installing snpEff database %s in %s" % (snpeff_db, snpeff_base_dir)) tooldir = args.tooldir or get_defaults()["tooldir"] config = {"resources": {"snpeff": {"jvm_opts": ["-Xms500m", "-Xmx1g"], "dir": os.path.join(tooldir, "share", "java", "snpeff")}}} raw_version = programs.java_versioner("snpeff", "snpEff", stdout_flag="snpEff version SnpEff")(config) snpeff_version = "".join([x for x in raw_version if x in set(string.digits + ".")]).replace(".", "_") dl_url = remotes["snpeff_dl_url"].format(snpeff_ver=snpeff_version, genome=snpeff_db) dl_file = os.path.basename(dl_url) with utils.chdir(snpeff_base_dir): subprocess.check_call(["wget", "-c", "-O", dl_file, dl_url]) subprocess.check_call(["unzip", dl_file]) os.remove(dl_file) dl_dir = os.path.join(snpeff_base_dir, "data", snpeff_db) os.rename(dl_dir, snpeff_db_dir) os.rmdir(os.path.join(snpeff_base_dir, "data"))
def _upgrade_snpeff_data(galaxy_dir, args, remotes): """Install or upgrade snpEff databases, localized to reference directory. """ snpeff_version = effects.snpeff_version(args) if not snpeff_version: return for dbkey, ref_file in genome.get_builds(galaxy_dir): resource_file = os.path.join(os.path.dirname(ref_file), "%s-resources.yaml" % dbkey) if os.path.exists(resource_file): with open(resource_file) as in_handle: resources = yaml.safe_load(in_handle) snpeff_db, snpeff_base_dir = effects.get_db({"genome_resources": resources, "reference": {"fasta": {"base": ref_file}}}) if snpeff_db: snpeff_db_dir = os.path.join(snpeff_base_dir, snpeff_db) if os.path.exists(snpeff_db_dir) and _is_old_database(snpeff_db_dir, args): shutil.rmtree(snpeff_db_dir) if not os.path.exists(snpeff_db_dir): print("Installing snpEff database %s in %s" % (snpeff_db, snpeff_base_dir)) dl_url = remotes["snpeff_dl_url"].format( snpeff_ver=snpeff_version.replace(".", "_"), genome=snpeff_db) dl_file = os.path.basename(dl_url) with utils.chdir(snpeff_base_dir): subprocess.check_call(["wget", "--no-check-certificate", "-c", "-O", dl_file, dl_url]) subprocess.check_call(["unzip", dl_file]) os.remove(dl_file) dl_dir = os.path.join(snpeff_base_dir, "data", snpeff_db) shutil.move(dl_dir, snpeff_db_dir) os.rmdir(os.path.join(snpeff_base_dir, "data")) if args.cwl: create.directory_tarball(snpeff_db_dir)
def _upgrade_snpeff_data(galaxy_dir, args, remotes): """Install or upgrade snpEff databases, localized to reference directory. """ for dbkey, ref_file in genome.get_builds(galaxy_dir): resource_file = os.path.join(os.path.dirname(ref_file), "%s-resources.yaml" % dbkey) if os.path.exists(resource_file): with open(resource_file) as in_handle: resources = yaml.load(in_handle) snpeff_db, snpeff_base_dir = effects.get_db({"genome_resources": resources, "reference": {"fasta": {"base": ref_file}}}) if snpeff_db: snpeff_db_dir = os.path.join(snpeff_base_dir, snpeff_db) if not os.path.exists(snpeff_db_dir): print("Installing snpEff database %s in %s" % (snpeff_db, snpeff_base_dir)) tooldir = args.tooldir or get_defaults()["tooldir"] config = {"resources": {"snpeff": {"jvm_opts": ["-Xms500m", "-Xmx1g"], "dir": os.path.join(tooldir, "share", "java", "snpeff")}}} raw_version = programs.java_versioner("snpeff", "snpEff", stdout_flag="snpEff version SnpEff")(config) snpeff_version = "".join([x for x in raw_version if x in set(string.digits + ".")]).replace(".", "_") dl_url = remotes["snpeff_dl_url"].format(snpeff_ver=snpeff_version, genome=snpeff_db) dl_file = os.path.basename(dl_url) with utils.chdir(snpeff_base_dir): subprocess.check_call(["wget", "-c", "-O", dl_file, dl_url]) subprocess.check_call(["unzip", dl_file]) os.remove(dl_file) dl_dir = os.path.join(snpeff_base_dir, "data", snpeff_db) os.rename(dl_dir, snpeff_db_dir) os.rmdir(os.path.join(snpeff_base_dir, "data"))
def upgrade_vcfanno_data(galaxy_dir): vcfanno_files = _get_vcfanno_files() vcfanno_remote = REMOTES["vcfanno"] cmd = "wget --no-check-certificate -O {out_file} {remote_url}" for dbkey, ref_file in genome.get_builds(galaxy_dir): build_dir = os.path.abspath( os.path.join(os.path.dirname(ref_file), os.pardir)) vcfanno_dir = os.path.join(build_dir, "vcfanno") utils.safe_makedir(vcfanno_dir) build_files = [x for x in vcfanno_files if x.startswith(dbkey)] for annofile in build_files: out_file = os.path.join(vcfanno_dir, annofile) remote_url = vcfanno_remote + annofile print cmd.format(**locals()) subprocess.check_call(cmd.format(**locals()), shell=True)
def upgrade_vcfanno_data(galaxy_dir): vcfanno_files = _get_vcfanno_files() vcfanno_remote = REMOTES["vcfanno"] cmd = "wget --no-check-certificate -O {out_file} {remote_url}" for dbkey, ref_file in genome.get_builds(galaxy_dir): build_dir = os.path.abspath(os.path.join(os.path.dirname(ref_file), os.pardir)) vcfanno_dir = os.path.join(build_dir, "vcfanno") utils.safe_makedir(vcfanno_dir) build_files = [x for x in vcfanno_files if x.startswith(dbkey)] for annofile in build_files: out_file = os.path.join(vcfanno_dir, annofile) remote_url = vcfanno_remote + annofile print cmd.format(**locals()) subprocess.check_call(cmd.format(**locals()), shell=True)
def _upgrade_snpeff_data(galaxy_dir, args): """Install or upgrade snpEff databases, localized to reference directory. """ for dbkey, ref_file in genome.get_builds(galaxy_dir): resource_file = os.path.join(os.path.dirname(ref_file), "%s-resources.yaml" % dbkey) with open(resource_file) as in_handle: resources = yaml.load(in_handle) snpeff_db, snpeff_base_dir = effects.get_db(ref_file, resources) if snpeff_db: snpeff_db_dir = os.path.join(snpeff_base_dir, snpeff_db) if not os.path.exists(snpeff_db_dir): print("Installing snpEff database %s in %s" % (snpeff_db, snpeff_base_dir)) tooldir = args.tooldir or get_defaults()["tooldir"] config = {"resources": {"snpeff": {"jvm_opts": ["-Xms500m", "-Xmx1g"], "dir": os.path.join(tooldir, "share", "java", "snpeff")}}} snpeff_cmd = effects.get_cmd("download", snpeff_base_dir, config) subprocess.check_call("%s %s" % (snpeff_cmd, snpeff_db), shell=True)
def _upgrade_genome_resources(galaxy_dir, base_url): """Retrieve latest version of genome resource YAML configuration files. """ for dbkey, ref_file in genome.get_builds(galaxy_dir): # Check for a remote genome resources file remote_url = base_url % dbkey r = requests.get(remote_url) if r.status_code == requests.codes.ok: local_file = os.path.join(os.path.dirname(ref_file), os.path.basename(remote_url)) if os.path.exists(local_file): with open(local_file) as in_handle: local_config = yaml.load(in_handle) remote_config = yaml.load(r.text) needs_update = remote_config["version"] > local_config.get("version", 0) if needs_update: shutil.move(local_file, local_file + ".old%s" % local_config.get("version", 0)) else: needs_update = True if needs_update: print("Updating %s genome resources configuration" % dbkey) with open(local_file, "w") as out_handle: out_handle.write(r.text)
def _upgrade_vep_data(galaxy_dir, tooldir): for dbkey, ref_file in genome.get_builds(galaxy_dir): effects.prep_vep_cache(dbkey, ref_file, tooldir)