def main(args1, args2):
url = sys.argv[1]
file_name = sys.argv[2]
downloads_dir = create_downloads_dir()
download_path = download_file(url, downloads_dir)
image_exists(download_path)
download_path = simple_download(url, downloads_dir, file_name)
image_exists(download_path)
def download_from_url(argv: t.List[str]):
"""
Main module function that accepts command-line arguments and
downloads a single file from specified url.
The command-line arguments are passed to a 'download_file'
utility function.
"""
url: str = DEFAULT_URL
directory: str = DEFAULT_DEST_DIR
filename: str = None
try:
opts, args = getopt.getopt(argv, "hu:d:f:", ["url=", "dir=", "fname="])
except getopt.GetoptError:
print(
f"GetoptError: \n{os.path.basename(os.path.abspath(__file__))} -u <url> -d <directory> -f <filename>"
)
sys.exit(2)
for opt, arg in opts:
if opt == "-h":
print(
f"usage: {os.path.basename(os.path.abspath(__file__))} -u <url> -d <directory> -f <filename>"
)
sys.exit()
elif opt in ("-u", "--url"):
url = arg
elif opt in ("-d", "--directory"):
directory = arg
elif opt in ("-f", "--filename"):
filename = arg
# Call our util function download_file with these updated args
dl_filepath: str = download_file(url, directory, filename)
print(f"Number of args passed: {len(argv)} arguments.")
print(f"url: {url}")
print(f"directory: {directory}")
print(f"filename: {filename}")
print(f"Final download filepath: {dl_filepath}")
import os
import requests
from download_util import download_file
THIS_FILE_PATH = os.path.abspath(__file__)
BASE_DIR = os.path.dirname(THIS_FILE_PATH)
DOWNLOADS_DIR = os.path.join(BASE_DIR, 'downloads')
os.makedirs(DOWNLOADS_DIR, exist_ok=True)
downloaded_img_path = os.path.join(DOWNLOADS_DIR, 'test.jpg')
url = "https://q-cf.bstatic.com/images/hotel/max1024x768/211/211636126.jpg"
# a smallish item
r = requests.get(url, stream=True)
r.raise_for_status() # 200
with open(downloaded_img_path, 'wb') as f:
f.write(r.content)
download_file(url, DOWNLOADS_DIR)
from download_util import download_file import os import requests THIS_FILE_PATH = os.path.abspath(__file__) BASE_DIR = os.path.dirname(THIS_FILE_PATH) DOWNLOAD_DIR = os.path.join(BASE_DIR, "downloads") DLED_IMGPATH = os.path.join(DOWNLOAD_DIR, "1.jpg") os.makedirs(DOWNLOAD_DIR, exist_ok=True) image_url1 = "https://media.wired.com/photos/5d09594a62bcb0c9752779d9/4:3/w_2000,h_1500,c_limit/Transpo_G70_TA-518126.jpg" DL_FILENAME = os.path.basename(image_url1) NEW_DLED_PATH = os.path.join(DOWNLOAD_DIR, DL_FILENAME) # r = requests.get(image_url1, stream = True) # with open(DLED_IMGPATH, "wb") as f: # f.write(r.content) download_file(image_url1, DOWNLOAD_DIR)
if not (os.path.exists(download_folder)):
os.makedirs(download_folder)
# 1. 1000 Genomes Phase III SNPs
KG_folder = os.path.join('data', 'hg', 'raw1000G')
base_url = 'ftp://ftp-trace.ncbi.nih.gov/1000genomes/ftp/release/20130502/'
# Create download directory
if not os.path.exists(KG_folder):
os.makedirs(KG_folder)
# Download somatic chromosomes
for chrom in range(1, 23):
filename = 'ALL.chr{}.phase3_shapeit2_mvncall_integrated_v5a.20130502.' \
'genotypes.vcf.gz'.format(chrom)
download_file(base_url, filename, KG_folder)
# Download sex chromosomes
sex_chrm = [
'ALL.chrX.phase3_shapeit2_mvncall_integrated_v1b.20130502.'
'genotypes.vcf.gz',
'ALL.chrY.phase3_integrated_v1b.20130502.genotypes.vcf.gz'
]
for chrom in sex_chrm:
download_file(base_url, chrom, KG_folder)
# 2. hg19 Gencode Genes
base_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_19/'
filename = 'gencode.v19.annotation.gtf.gz'
download_file(base_url, filename, download_folder)
import os
from download_util import download_file, download_file_slower
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DOWNLOAD_DIR = os.path.join(BASE_DIR, 'downloads')
os.makedirs(DOWNLOAD_DIR, exist_ok=True)
url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/db/Classic_view_of_a_cloudfree_Peyto_Lake%2C_Banff_National_Park%2C_Alberta%2C_Canada_%284110933448%29.jpg/240px-Classic_view_of_a_cloudfree_Peyto_Lake%2C_Banff_National_Park%2C_Alberta%2C_Canada_%284110933448%29.jpg'
file_name = download_file(url=url,
directory=DOWNLOAD_DIR,
fname='downlaod1.jpg')
print(f'{file_name} saved')
file_name = download_file_slower(url=url,
directory=DOWNLOAD_DIR,
fname='download2.jpg')
print(file_name)
import os
import shutil
import subprocess
from download_util import download_file, process_repeats, untar_mouse_domain
# Make new folder if it doesn't exist already
genome = 'mm'
download_folder = os.path.join('data', 'mm')
if not (os.path.exists(download_folder)):
os.makedirs(download_folder)
# 1. Mouse Genome Project SNPs v2
base_url = 'ftp://ftp-mouse.sanger.ac.uk/REL-1211-SNPs_Indels/'
filename = 'mgp.v2.snps.annot.reformat.vcf.gz'
download_file(base_url, filename, download_folder)
# 2. Gencode Mouse
base_url = 'ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_mouse/release_M9/'
filename = 'gencode.vM9.annotation.gtf.gz'
download_file(base_url, filename, download_folder)
# 3. RepeatMasker mm9 Repeat Elements
base_url = 'http://www.repeatmasker.org/genomes/mm9/RepeatMasker-rm328' \
'-db20090604/'
filename = 'mm9.fa.out.gz'
download_file(base_url, filename, download_folder)
process_repeats(download_folder, filename, genome)
# 4. TAD Domain Boundaries
base_url = 'http://chromosome.sdsc.edu/mouse/hi-c/'
import requests import os import shutil from download_util import download_file BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DOWNLOADS_DIR = os.path.join(BASE_DIR, "downloads") downloaded_img_path = os.path.join(DOWNLOADS_DIR, '1.jpg') os.makedirs(DOWNLOADS_DIR, exist_ok=True) url = "https://i.hurimg.com/i/hdn/75/0x0/5dcd630a0f25441794d60a1c.jpg" # Small item we used requests lib #r = requests.get(url, stream = True) #r.raise_for_status() #with open(downloaded_img_path, 'wb') as f: #f.write(r.content) # bigger item we used shutil lib #dl_filename=os.path.basename(url) #new_dl_path = os.path.join(DOWNLOADS_DIR, dl_filename) #with requests.get(url, stream = True) as r: # with open(new_dl_path, 'wb') as file_obj: # shutil.copyfileobj(r.raw, file_obj) download_file(url, DOWNLOADS_DIR, 'test.jpg')
from download_util import download_file
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DOWNLOADS_DIR = os.path.join(BASE_DIR, 'downloads')
os.makedirs(DOWNLOADS_DIR, exist_ok=True)
downloaded_img_path = os.path.join(DOWNLOADS_DIR, '1.jpg')
url = 'https://images2.minutemediacdn.com/image/upload/c_fill\
,g_auto,/h_1248,w_2220/v1555922701/shape/mentalfloss/\
istock_000008977856_small.jpg'
# smallish item
r1 = requests.get(url)
r1.raise_for_status
with open(downloaded_img_path, 'wb') as f:
f.write(r1.content)
# better method but cannot reuse it as it is not a function
# dl_filename = os.path.basename(url)
# new_dl_filename = os.path.join(DOWNLOADS_DIR, dl_filename)
# with requests.get(url, stream=True) as r2:
# with open(new_dl_filename, 'wb') as file_obj:
# shutil.copyfileobj(r2.raw, file_obj)
download_file(
url,
DOWNLOADS_DIR,
)
download_file(url, DOWNLOADS_DIR, 'img+from+web.jpg')
import os
import requests
from download_util import download_file
THIS_FILE_PATH = os.path.abspath(__file__)
BASE_DIR = os.path.dirname(THIS_FILE_PATH)
DOWNLOADS_DIR = os.path.join(BASE_DIR, "downloads")
os.makedirs(DOWNLOADS_DIR, exist_ok=True)
url = "https://media.timeout.com/images/105236338/image.jpg"
r = requests.get(url, stream=True)
r.raise_for_status()
downloaded_image_path = os.path.join(DOWNLOADS_DIR, 'cali.jpg')
# smallish items
with open(downloaded_image_path, 'wb') as f:
f.write(r.content)
download_file(url, DOWNLOADS_DIR, fname="memento.jpg")
