def main(args1, args2): url = sys.argv[1] file_name = sys.argv[2] downloads_dir = create_downloads_dir() download_path = download_file(url, downloads_dir) image_exists(download_path) download_path = simple_download(url, downloads_dir, file_name) image_exists(download_path)
def download_from_url(argv: t.List[str]): """ Main module function that accepts command-line arguments and downloads a single file from specified url. The command-line arguments are passed to a 'download_file' utility function. """ url: str = DEFAULT_URL directory: str = DEFAULT_DEST_DIR filename: str = None try: opts, args = getopt.getopt(argv, "hu:d:f:", ["url=", "dir=", "fname="]) except getopt.GetoptError: print( f"GetoptError: \n{os.path.basename(os.path.abspath(__file__))} -u <url> -d <directory> -f <filename>" ) sys.exit(2) for opt, arg in opts: if opt == "-h": print( f"usage: {os.path.basename(os.path.abspath(__file__))} -u <url> -d <directory> -f <filename>" ) sys.exit() elif opt in ("-u", "--url"): url = arg elif opt in ("-d", "--directory"): directory = arg elif opt in ("-f", "--filename"): filename = arg # Call our util function download_file with these updated args dl_filepath: str = download_file(url, directory, filename) print(f"Number of args passed: {len(argv)} arguments.") print(f"url: {url}") print(f"directory: {directory}") print(f"filename: {filename}") print(f"Final download filepath: {dl_filepath}")
import os import requests from download_util import download_file THIS_FILE_PATH = os.path.abspath(__file__) BASE_DIR = os.path.dirname(THIS_FILE_PATH) DOWNLOADS_DIR = os.path.join(BASE_DIR, 'downloads') os.makedirs(DOWNLOADS_DIR, exist_ok=True) downloaded_img_path = os.path.join(DOWNLOADS_DIR, 'test.jpg') url = "https://q-cf.bstatic.com/images/hotel/max1024x768/211/211636126.jpg" # a smallish item r = requests.get(url, stream=True) r.raise_for_status() # 200 with open(downloaded_img_path, 'wb') as f: f.write(r.content) download_file(url, DOWNLOADS_DIR)
from download_util import download_file import os import requests THIS_FILE_PATH = os.path.abspath(__file__) BASE_DIR = os.path.dirname(THIS_FILE_PATH) DOWNLOAD_DIR = os.path.join(BASE_DIR, "downloads") DLED_IMGPATH = os.path.join(DOWNLOAD_DIR, "1.jpg") os.makedirs(DOWNLOAD_DIR, exist_ok=True) image_url1 = "https://media.wired.com/photos/5d09594a62bcb0c9752779d9/4:3/w_2000,h_1500,c_limit/Transpo_G70_TA-518126.jpg" DL_FILENAME = os.path.basename(image_url1) NEW_DLED_PATH = os.path.join(DOWNLOAD_DIR, DL_FILENAME) # r = requests.get(image_url1, stream = True) # with open(DLED_IMGPATH, "wb") as f: # f.write(r.content) download_file(image_url1, DOWNLOAD_DIR)
if not (os.path.exists(download_folder)): os.makedirs(download_folder) # 1. 1000 Genomes Phase III SNPs KG_folder = os.path.join('data', 'hg', 'raw1000G') base_url = 'ftp://ftp-trace.ncbi.nih.gov/1000genomes/ftp/release/20130502/' # Create download directory if not os.path.exists(KG_folder): os.makedirs(KG_folder) # Download somatic chromosomes for chrom in range(1, 23): filename = 'ALL.chr{}.phase3_shapeit2_mvncall_integrated_v5a.20130502.' \ 'genotypes.vcf.gz'.format(chrom) download_file(base_url, filename, KG_folder) # Download sex chromosomes sex_chrm = [ 'ALL.chrX.phase3_shapeit2_mvncall_integrated_v1b.20130502.' 'genotypes.vcf.gz', 'ALL.chrY.phase3_integrated_v1b.20130502.genotypes.vcf.gz' ] for chrom in sex_chrm: download_file(base_url, chrom, KG_folder) # 2. hg19 Gencode Genes base_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_19/' filename = 'gencode.v19.annotation.gtf.gz' download_file(base_url, filename, download_folder)
import os from download_util import download_file, download_file_slower BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DOWNLOAD_DIR = os.path.join(BASE_DIR, 'downloads') os.makedirs(DOWNLOAD_DIR, exist_ok=True) url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/db/Classic_view_of_a_cloudfree_Peyto_Lake%2C_Banff_National_Park%2C_Alberta%2C_Canada_%284110933448%29.jpg/240px-Classic_view_of_a_cloudfree_Peyto_Lake%2C_Banff_National_Park%2C_Alberta%2C_Canada_%284110933448%29.jpg' file_name = download_file(url=url, directory=DOWNLOAD_DIR, fname='downlaod1.jpg') print(f'{file_name} saved') file_name = download_file_slower(url=url, directory=DOWNLOAD_DIR, fname='download2.jpg') print(file_name)
import os import shutil import subprocess from download_util import download_file, process_repeats, untar_mouse_domain # Make new folder if it doesn't exist already genome = 'mm' download_folder = os.path.join('data', 'mm') if not (os.path.exists(download_folder)): os.makedirs(download_folder) # 1. Mouse Genome Project SNPs v2 base_url = 'ftp://ftp-mouse.sanger.ac.uk/REL-1211-SNPs_Indels/' filename = 'mgp.v2.snps.annot.reformat.vcf.gz' download_file(base_url, filename, download_folder) # 2. Gencode Mouse base_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_mouse/release_M9/' filename = 'gencode.vM9.annotation.gtf.gz' download_file(base_url, filename, download_folder) # 3. RepeatMasker mm9 Repeat Elements base_url = 'http://www.repeatmasker.org/genomes/mm9/RepeatMasker-rm328' \ '-db20090604/' filename = 'mm9.fa.out.gz' download_file(base_url, filename, download_folder) process_repeats(download_folder, filename, genome) # 4. TAD Domain Boundaries base_url = 'http://chromosome.sdsc.edu/mouse/hi-c/'
import requests import os import shutil from download_util import download_file BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DOWNLOADS_DIR = os.path.join(BASE_DIR, "downloads") downloaded_img_path = os.path.join(DOWNLOADS_DIR, '1.jpg') os.makedirs(DOWNLOADS_DIR, exist_ok=True) url = "https://i.hurimg.com/i/hdn/75/0x0/5dcd630a0f25441794d60a1c.jpg" # Small item we used requests lib #r = requests.get(url, stream = True) #r.raise_for_status() #with open(downloaded_img_path, 'wb') as f: #f.write(r.content) # bigger item we used shutil lib #dl_filename=os.path.basename(url) #new_dl_path = os.path.join(DOWNLOADS_DIR, dl_filename) #with requests.get(url, stream = True) as r: # with open(new_dl_path, 'wb') as file_obj: # shutil.copyfileobj(r.raw, file_obj) download_file(url, DOWNLOADS_DIR, 'test.jpg')
from download_util import download_file BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DOWNLOADS_DIR = os.path.join(BASE_DIR, 'downloads') os.makedirs(DOWNLOADS_DIR, exist_ok=True) downloaded_img_path = os.path.join(DOWNLOADS_DIR, '1.jpg') url = 'https://images2.minutemediacdn.com/image/upload/c_fill\ ,g_auto,/h_1248,w_2220/v1555922701/shape/mentalfloss/\ istock_000008977856_small.jpg' # smallish item r1 = requests.get(url) r1.raise_for_status with open(downloaded_img_path, 'wb') as f: f.write(r1.content) # better method but cannot reuse it as it is not a function # dl_filename = os.path.basename(url) # new_dl_filename = os.path.join(DOWNLOADS_DIR, dl_filename) # with requests.get(url, stream=True) as r2: # with open(new_dl_filename, 'wb') as file_obj: # shutil.copyfileobj(r2.raw, file_obj) download_file( url, DOWNLOADS_DIR, ) download_file(url, DOWNLOADS_DIR, 'img+from+web.jpg')
import os import requests from download_util import download_file THIS_FILE_PATH = os.path.abspath(__file__) BASE_DIR = os.path.dirname(THIS_FILE_PATH) DOWNLOADS_DIR = os.path.join(BASE_DIR, "downloads") os.makedirs(DOWNLOADS_DIR, exist_ok=True) url = "https://media.timeout.com/images/105236338/image.jpg" r = requests.get(url, stream=True) r.raise_for_status() downloaded_image_path = os.path.join(DOWNLOADS_DIR, 'cali.jpg') # smallish items with open(downloaded_image_path, 'wb') as f: f.write(r.content) download_file(url, DOWNLOADS_DIR, fname="memento.jpg")