def worker_process_html(gearman_worker, gearman_job):
url=gearman_job.data
if is_processed(url):
print url,'is skipped as it was processed already'
return
print 'processing',url
try:
html_doc=myhttplib.urlopen(url)[2]
except HTTPError,e:
print 'http error,',e.code
set_processed(url,e.code)
return
def worker_process_pic(gearman_worker, gearman_job):
root_dir,title,url=gearman_job.data
if is_processed(url):
print url,'is skipped as it was processed already'
return
print 'processing',url
try:
content=myhttplib.urlopen(url)[2]
except HTTPError,e:
print 'http error:',e.code
set_processed(url,e.code)
return
