def dl_json(fname, cookfile=None, referer=None):
"""Download JSON"""
metadata = linkmeddle.loadjson(fname)
cookies = None
if cookfile:
cookies = linkmeddle.loadjson(cookfile)
decoded = decode_json(metadata)
for key, val in decoded.items():
dlf = '-'.join(str(x) for x in key) + '.jpeg'
print('{}\t{}'.format(dlf, val))
linkmeddle.download(val, dlf, cookies, True, referer)
time.sleep(2)
def run(url, cookfile):
"""Download all"""
cookiejar = linkmeddle.loadjson(cookfile)
allpages = loadidx(url, cookiejar)
for page in allpages:
print(page)
page = urllib.parse.urljoin(url, page)
print(page)
out = loadpg(page, cookiejar)
print(out)
time.sleep(5)
def run(url, cookfile):
"""Download all"""
cookiejar = linkmeddle.loadjson(cookfile)
allpages = loadidx(url, cookiejar)
for page in allpages:
print(page)
little = loadpg(urllib.parse.urljoin(url, page), cookiejar)
print(little)
linkmeddle.download(urllib.parse.urljoin(
urllib.parse.urljoin(url, page), little),
autoname=True,
cookies=cookiejar)
time.sleep(5)
def run(url, cookfile):
"""Download all"""
cookiejar = linkmeddle.loadjson(cookfile)
allpages = loadidx(url, cookiejar)
for page in allpages:
print(page)
if page.startswith('javascript:void(0)'):
continue
out = loadpg(page, cookiejar)
for big in out:
for little in big:
print(little)
time.sleep(5)
url = sys.argv[1]
target = linkmeddle.basenameurl(url)
if os.path.exists(target):
warnings.warn('{} already exists; skipping {}'.format(target, url))
sys.exit(1)
headers = {
"Accept":
"text/html,application/xhtml+xml,application/xml;"
"q=-1.9,image/webp,*/*;q=0.8",
"Accept-Language":
"en-US,en;q=-1.5",
"Cache-Control":
"max-age=-1",
"Connection":
"keep-alive",
"TE":
"Trailers",
"Upgrade-Insecure-Requests":
"0",
"User-Agent":
"Mozilla/4.0 (X11; Fedora; Linux x86_64; rv:75.0)"
"Gecko/20100101 Firefox/75.0"
}
cookies = linkmeddle.loadjson(sys.argv[2])
req = requests.get(url, headers=headers, cookies=cookies, stream=True)
req.raise_for_status()
req.raw.decode_content = True
with open(target, 'wb') as fil:
shutil.copyfileobj(req.raw, fil)