def test_download(output_directory, concurrent_connections, multiply_urls,
urls):
def _iterator():
os.makedirs(output_directory, exist_ok=True)
for i, url in enumerate(urls):
for j in range(int(multiply_urls)):
ij = str(i) + '.' + str(j)
yield {
'ij':
ij,
'url':
url,
'output_filename':
os.path.join(output_directory, ij + '.output'),
'header_filename':
os.path.join(output_directory, ij + '.header'),
}
for _ in range(5):
yield None
def _save_result(urlobj, response_code=None, errno=None, errmsg=None):
print('%s %s: response_code=%s errno=%s errmsg=%s' %
(urlobj['ij'], urlobj['url'], response_code, errno, errmsg))
download(int(concurrent_connections), _iterator(), _save_result)
def get_link(link):
content = _get_content(link)[0]
for item in content:
logger.debug("Downloading " + str(item))
download.download(
url=item["downloadUrl"],
fname=os.path.join(item["fdir"], item["fname"])
)
def get_soup(soup):
logger.debug('Getting Yandi.sk links..')
links = []
for link in soup.findAll('a'):
this_link = link.get('href')
if any(pattern in str(this_link).lower() for pattern in url_patterns):
links.append(this_link)
links = list(dict.fromkeys(links))
logger.debug('Found yandex links: ' + str(links))
for link in links:
download.download(_get_real_direct_link(link))
def get_link(link):
logger.debug('Getting ' + __file__ + ' link..')
download.download(link)
def get_link(link):
download.download(_get_real_direct_link(link))
def get_link(link):
download.download(link, fname=_get_fname(link))
def get_link(link):
logger.debug('Getting ' + __file__ + ' link..')
if "redgifs.com" in link:
_dl_redgifs(link)
else:
download.download(link)
def get_link(link):
download.download(link)