def download_snp_history(url=SNP_HISTORY_URL, outpath="SNPHistory.bcp.gz"): with tqdm(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url, filename=outpath, reporthook=tqdm_hook(t), data=None) return outpath
def download_ebi_catalog(url, outpath): with tqdm(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url, filename=outpath, reporthook=tqdm_hook(t), data=None) return outpath
def download_merge_arch(url=RS_MERGE_ARCH_URL, outpath="RsMergeArch.bcp.gz"): with tqdm(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url, filename=outpath, reporthook=tqdm_hook(t), data=None) return outpath
def download_dbsnp_vcf(dbsnp_build=None,genome_build=None,url=None,outpath=None): """ Download the NCBI dbSNP VCF for a given human genome build and dbSNP build Args: dbsnp_build: b147 genome_build: GRCh37p13 url: Direct URL to file, e.g. ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606_b147_GRCh37p13/VCF/00-All.vcf.gz outpath: Constructed from genome_build and dbsnp_build. If not given, a random filename will be generated. Returns: Name of file into which we saved the data (will be constructed from builds, or random name) """ if url is None: if not genome_build.startswith("GRC"): raise ValueError("Genome build should begin with GRC") if not dbsnp_build.startswith("b"): raise ValueError("dbSNP build should look like b147, b148, etc.") url = NCBI_VCF_TEMPLATE_URL.format(dbsnp_build,genome_build) if outpath is None: if genome_build is None or dbsnp_build is None: outpath = "dbsnp.vcf.gz" else: outpath = "human_9606_{}_{}_All.vcf.gz".format(dbsnp_build,genome_build) with tqdm(unit='B',unit_scale=True,miniters=1,desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url,filename=outpath,reporthook=tqdm_hook(t),data=None) return outpath
def download_dbsnp_vcf(dbsnp_build=None, genome_build=None, url=None, outpath=None): """ Download the NCBI dbSNP VCF for a given human genome build and dbSNP build Args: dbsnp_build: b147 genome_build: GRCh37p13 url: Direct URL to file, e.g. ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606_b147_GRCh37p13/VCF/00-All.vcf.gz outpath: Constructed from genome_build and dbsnp_build. If not given, a random filename will be generated. Returns: Name of file into which we saved the data (will be constructed from builds, or random name) """ if url is None: if not genome_build.startswith("GRC"): raise ValueError("Genome build should begin with GRC") if not dbsnp_build.startswith("b"): raise ValueError("dbSNP build should look like b147, b148, etc.") url = NCBI_VCF_TEMPLATE_URL.format(dbsnp_build, genome_build) if outpath is None: if genome_build is None or dbsnp_build is None: outpath = "dbsnp.vcf.gz" else: outpath = "human_9606_{}_{}_All.vcf.gz".format( dbsnp_build, genome_build) with tqdm(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url, filename=outpath, reporthook=tqdm_hook(t), data=None) return outpath
def download_tile(tile, url, pid, version, username, password): grass.debug("Download tile: %s" % tile, debug=1) local_tile = "NASADEM_HGT_" + str(tile) + ".zip" urllib2.urlcleanup() remote_tile = str(url) + "/" + version + "/2000.02.11/" + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password( None, "https://urs.earthdata.nasa.gov", username, password ) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar), ) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl
def get_file(filename): '''Downloads filename from ChEBI FTP site''' destination = __DOWNLOAD_PARAMS['path'] filepath = os.path.join(destination, filename) if not __is_current(filepath): if not os.path.exists(destination): os.makedirs(destination) url = 'ftp://ftp.ebi.ac.uk/pub/databases/chebi/' + \ 'Flat_file_tab_delimited/' urlretrieve(urlparse.urljoin(url, filename), filepath) urlcleanup() if filepath.endswith('.zip'): zfile = zipfile.ZipFile(filepath, 'r') filepath = os.path.join(destination, zfile.namelist()[0]) zfile.extractall(destination) elif filepath.endswith('.gz'): unzipped_filepath = filepath[:-len('.gz')] if os.path.exists(unzipped_filepath) \ and __is_current(unzipped_filepath): filepath = unzipped_filepath else: input_file = gzip.open(filepath, 'rb') filepath = os.path.join(destination, input_file.name[:-len('.gz')]) output_file = open(filepath, 'wb') for line in input_file: output_file.write(line) input_file.close() output_file.close() return filepath
def download_tile(tile, url, pid, srtmv3, one, username, password): grass.debug("Download tile: %s" % tile, debug = 1) output = tile + '.r.in.srtm.tmp.' + str(pid) if srtmv3: if one: local_tile = str(tile) + '.SRTMGL1.hgt.zip' else: local_tile = str(tile) + '.SRTMGL3.hgt.zip' else: local_tile = str(tile) + '.hgt.zip' urllib2.urlcleanup() if srtmv3: remote_tile = str(url) + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, "https://urs.earthdata.nasa.gov", username, password) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), #urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see #urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar)) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, 'w+b') fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl # SRTM subdirs: Africa, Australia, Eurasia, Islands, North_America, South_America for srtmdir in ('Africa', 'Australia', 'Eurasia', 'Islands', 'North_America', 'South_America'): remote_tile = str(url) + str(srtmdir) + '/' + local_tile goturl = 1 try: response = urllib2.urlopen(request) fo = open(local_tile, 'w+b') fo.write(response.read()) fo.close time.sleep(0.5) # does not work: #urllib.urlretrieve(remote_tile, local_tile, data = None) except: goturl = 0 pass if goturl == 1: return 1 return 0
def download_snp_history(url=SNP_HISTORY_URL,outpath="SNPHistory.bcp.gz"): with tqdm(unit='B',unit_scale=True,miniters=1,desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url,filename=outpath,reporthook=tqdm_hook(t),data=None) return outpath
def download_merge_arch(url=RS_MERGE_ARCH_URL,outpath="RsMergeArch.bcp.gz"): with tqdm(unit='B',unit_scale=True,miniters=1,desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url,filename=outpath,reporthook=tqdm_hook(t),data=None) return outpath
def download_ebi_catalog(url,outpath): with tqdm(unit='B',unit_scale=True,miniters=1,desc=url.split('/')[-1]) as t: urlcleanup() urlretrieve(url,filename=outpath,reporthook=tqdm_hook(t),data=None) return outpath