-
Notifications
You must be signed in to change notification settings - Fork 1
/
process_toutiao.py
37 lines (31 loc) · 1022 Bytes
/
process_toutiao.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
from functools import partial
from multiprocessing.pool import Pool
from itertools import chain
from time import time
from download import setup_download_dir, get_links, download_link
def main():
ts = time()
url1 = 'http://www.toutiao.com/a6333981316853907714'
url2 = 'http://www.toutiao.com/a6334459308533350658'
url3 = 'http://www.toutiao.com/a6313664289211924737'
url4 = 'http://www.toutiao.com/a6334337170774458625'
url5 = 'http://www.toutiao.com/a6334486705982996738'
download_dir = setup_download_dir('process_imgs')
links = list(chain(
get_links(url1),
get_links(url2),
get_links(url3),
get_links(url4),
get_links(url5),
))
download = partial(download_link, download_dir)
with Pool(8) as p:
p.map(download, links)
print('一共下载了 {} 张图片'.format(len(links)))
print('Took {}s'.format(time() - ts))
if __name__ == '__main__':
main()
"""
一共下载了 253 张图片
Took 58.69389724731445s
"""