def main():
'''
save access
'''
accesslog_path="""/tmp/sohu.log"""
e = queue.loadDataSpring ()
urlset=e.job_exectt()
f = open(accesslog_path, 'wb')
for i in urlset:
uri="http://image.ganjistatic1.com"+i
f.write(str(uri) + '\n')
f.flush()
f.close()
# must use Manager queue here, or will not work
manager = mp.Manager()
q = manager.Queue()
pool = mp.Pool(mp.cpu_count() + 2)
# put listener to work first
watcher = pool.apply_async(listener, (q,))
# fire off workers
jobs = []
f = open(accesslog_path, 'rb')
try:
for i in range(80):
sohu_uri=f.readline()
job = pool.apply_async(worker2, (i,sohu_uri, q))
jobs.append(job)
# collect results from the workers through the pool result queue
for job in jobs:
job.get()
except Exception as inst:
print inst
q.put('kill123')
f.close()
pool.close()
# now we are done, kill the listener
q.put('kill')
f.close()
pool.close()
#!/usr/local/bin/python2.7
# encoding: utf-8
'''
Created on 2015年7月12日
@author: a11
'''
import HttpHelper
import queue
if __name__ == '__main__':
accesslog_path="""/tmp/sohu.log"""
e = queue.loadDataSpring ()
urlset=e.job_exectt()
f = open(accesslog_path, 'ab')
for i in urlset:
uri="http://image.ganjistatic1.com"+i
f.write(str(uri) + '\n')
f.flush()
f.close()
client = HttpHelper.HttpHelper()
f = open(accesslog_path, 'rb')
with open(accesslog_path) as f:
for ganji_uri in f:
