} </script> ''' picId = urlsplit(url).path.split('/')[-1] userId = parse_qs(urlsplit(url).query)['u'][0] albumId = json.loads(soup.find('script', id='pix-json-set-info').string)['albumId'] ajax_url = 'http://wantu.taobao.com/ajax/PicDetailAjax.do?picId=%s&userId=%s&albumId=%s&t=1365154666759&_method=read' ajax_url = ajax_url % (picId, userId, albumId) resp = json.loads(httplib.urlopen(ajax_url)[2].decode('gbk')) picture = resp['data']['models'][0]['picPath'] description = httplib.html_unescape(resp['data']['models'][0]['desc']) return (picture, description) #used for pipeline work gm_client = GearmanClient([config.job_server]) gm_client.data_encoder = JSONDataEncoder def submit_html_job(url): func_name = 'worker_process_html' job_req = gm_client.submit_job(func_name, url, unique=md5sum(url), background=True, wait_until_complete=False) return job_req.job.unique def submit_pic_job(url): func_name = 'worker_process_pic' job_req = gm_client.submit_job(func_name, url, unique=md5sum(url), background=True, wait_until_complete=False) return job_req.job.unique def worker_process_html(gearman_worker, gearman_job): url = gearman_job.data
def submit(url): gm_client = GearmanClient([config.job_server]) gm_client.data_encoder = JSONDataEncoder job_req = gm_client.submit_job('worker_process_html', url) print ns.url[0], 'is submitted' return job_req