async def main():
download_dir = setup_download_dir()
async with aiohttp.ClientSession(conn_timeout=3, read_timeout=3) as session:
links = get_links(CLIENT_ID)
logger.info("There has {} images".format(links.__len__()))
tasks = [(async_download_link(session, download_dir, l)) for l in links]
await asyncio.gather(*tasks, return_exceptions=True)
def main():
ts = time.time()
thumbnail_nail_size = partial(thumbnail_nail, (128, 128))
with ProcessPoolExecutor() as executor:
executor.map(thumbnail_nail_size, Path("images").iterdir())
logger.info("Took {} seconds".format(timedelta(seconds=time.time() - ts)))
def main():
ts = time.time()
img_count = 0
for img_path in Path("images").iterdir():
img_count += 1
thumbnail_nail((128, 128), img_path)
logger.info("There has {} images".format(img_count))
logger.info("Took {} seconds".format(timedelta(seconds=time.time()-ts)))
async def async_download_link(session, directory, link):
download_path = directory / os.path.basename(link)
async with session.get(link) as response:
with download_path.open("wb") as f:
while True:
chunk = await response.content.read(1024)
if not chunk:
break
f.write(chunk)
logger.info("Download {} image".format(link))
def main():
ts = time.time()
download_dir = setup_download_dir()
links = get_links(CLIENT_ID)
logger.info("There has {} images".format(links.__len__()))
with ThreadPoolExecutor(max_workers=20) as exectutor:
download = partial(download_link, download_dir)
exectutor.map(download, links, timeout=30)
logger.info("Took {} seconds".format(timedelta(seconds=time.time() - ts)))
def main():
ts = time.time()
download_dir = setup_download_dir()
links = get_links(CLIENT_ID)
logger.info("They has {} images".format(len(links)))
for link in links:
download_link(download_dir, link)
# worker
logger.info("Took {} seconds".format(timedelta(seconds=time.time() - ts)))
def main():
ts = time.time()
links = get_links(CLIENT_ID)
logger.info("They has {} images".format(len(links)))
download_dir = setup_download_dir()
pool = Pool(4)
download = partial(download_link, download_dir)
with pool as p:
p.map(download, links)
logger.info("Took {} seconds".format(timedelta(seconds=time.time() - ts)))
def main():
ts = time.time()
download_dir = setup_download_dir()
links = get_links(CLIENT_ID)
logger.info("They has {} images".format(len(links)))
queue = Queue()
# worker
for x in range(8):
worker = DownloadWorker(queue)
worker.daemon = True
worker.start()
# producer
for link in links:
logger.info('Queueing {}'.format(link))
queue.put((download_dir, link))
queue.join()
logger.info("Took {} seconds".format(timedelta(seconds=time.time() - ts)))
download_path = directory / os.path.basename(link)
async with session.get(link) as response:
with download_path.open("wb") as f:
while True:
chunk = await response.content.read(1024)
if not chunk:
break
f.write(chunk)
logger.info("Download {} image".format(link))
async def main():
download_dir = setup_download_dir()
async with aiohttp.ClientSession(conn_timeout=3, read_timeout=3) as session:
links = get_links(CLIENT_ID)
logger.info("There has {} images".format(links.__len__()))
tasks = [(async_download_link(session, download_dir, l)) for l in links]
await asyncio.gather(*tasks, return_exceptions=True)
if __name__ == '__main__':
ts = time.time()
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
finally:
loop.close()
logger.info("Took {} seconds".format(timedelta(seconds=time.time() - ts)))