def init_task(mq, link): lines = csv.reader(file(link, 'rb')) nd = {} cd = {} for line in lines: url = line[-1] if not cd.has_key(url): cd[url] = 0 else: continue filename = 'japan' + '/' + line[2] if not nd.has_key(filename): nd[filename] = 0 else: nd[filename] += 1 filename = filename + '_' + str(nd[filename]) task = etask([url, filename, 0]).get_task() mq._cqs.put(task) while 1: try: task = mq._cqs.get(True, 10) except Exception, e: logger.info('...get cqs task out of time...') else: logger.info('...run a task...') result = paser_page.apply_async(args=[task], queue='machine1', routing_key='machine1') #result = result.get() #print result try: mq._cqx.put(result) except Exception, e: logger.error('...put cqx result error...')
def getline(url): mc = MC() p = get_proxy('XXXX') if not p: logger.error('get proxy error ... %s',str(p)) mc.set_proxy(p) page = mc.req( 'get', url, html_flag = True) return page
def product_pooltask(self,poolres): pooltask= poolres spi_lst = pooltask['spi_act'] exp_lst = pooltask['exp_act'] oldinfo = pooltask['oldinfo'] newinfo = pooltask['newinfo'] file_new = [] for file in newinfo: on = oldinfo[1] n = file[1] if self.__name_dict.has_key(on+n): self.__name_dict[on+n] += 1 else: self.__name_dict[on+n] = 0 if self.__name_dict[on+n] != 0: n = n+'_'+str(self.__name_dict[on+n]) newurl = file[0] oldurl = oldinfo[0] nus = newurl.split('/') if 'http' in newurl: pass elif nus[0] == '..': for fl in nus: if fl == '..': nus=nus[1:]#remove('..') oldurl = '/'.join(oldurl.split('/')[:-2])+'/' else: break newurl = oldurl+'/'.join(nus) else: oldurl = '/'.join(oldurl.split('/')[:-1])+'/' if nus[0] == '.': newurl = oldurl+'/'.join(nus[1:]) else: newurl = oldurl+'/'.join(nus) if oldinfo == []: file_new.append([newurl,n,file[2]]) else: file_new.append([newurl,oldinfo[1]+'__'+n,file[2]]) for info in file_new: if info[2] == -1: continue newtask = {} newtask['info'] = info if type(info[0]) == type(()): for i in range(len(info[0])): spi_lst[info[2]]['post_data'].replace('miaoji@'+str(i),info[0][i]) newtask['spi_act'] = spi_lst newtask['exp_act'] = exp_lst try: self._cqs.put(newtask) except Exception, e: logger.error('(file:%s)requestqueue...%s',__name__,traceback.format_exc(e))
def load_by_mc(self, spi, url, name): refer = spi['refer'] mode = spi['mode'] spi_url = spi['spi_url'] post_type = spi['post_type'] post_data = spi['post_data'] post_url = spi['post_url'] isproxy = spi['isproxy'] debug = spi['debug'] key = name.split('__')[0] if self.objmc.has_key(key): mc = self.objmc[key] else: mc = MC() mc.set_debug(debug) self.objmc[key] = mc if isproxy != '': p = get_proxy(source='citytraffic') if not p: logger.error('get proxy error ... %s', str(p)) else: logger.info('this proxy is ... %s', str(p)) mc.set_proxy(p) if refer != '': mc.add_referer(self.refer) if spi_url != '': mc.req(self.mode, self.spi_url) try: if mode == 'post': page = mc.req(mode, post_url, paras=post_data, paras_type=post_type, html_flag=True) else: page = mc.req(mode, url, html_flag=True) except Exception, e: #traceback.print_exc(e) logger.error('load by mc ...<-!error::%s!-> <-!proxy::%s!->', traceback.format_exc(e), str(p))
def thread_handle(self): logger.info('...thread_handle start...') while True: try: res = self._cqx.get(True, self.__resq_timeout) except Exception ,e: '''队列开始空了''' logger.info('result queue cqx is empty') else: #logger.info('running task') try: ged = res.get(timeout = 1) except: try: self._cqx.put(res) except: logger.error('time out task put cqx again error ...') else: self.res_handle(ged)
def req(self, method, url_base, paras={}, paras_type=1, html_flag=False, time_out=60): if method.lower() == 'get': url = url_base + urllib.urlencode(paras) elif method.lower() == 'post': url = url_base else: logger.error('req, wrong method(post or get)') sys.exit(-1) html = '' try: request = urllib2.Request(url) request.add_headers = [( 'User-agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0' )] if self.header != {}: for key, value in self.header.items(): request.add_header(key, value) self.header = {} if self.p != '': request.set_proxy(self.p, 'http') request.set_proxy(self.p, 'https') if self.refer != {}: for key, value in self.refer.items(): request.add_header(key, value) self.refer = {} opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self.cookie), self.httpHandler) urllib2.install_opener(opener) if method.lower() == 'get': resp = urllib2.urlopen(request, timeout=time_out) # resp = self.br.open(url, timeout=time_out) else: if paras_type == 1: paras = json.dumps(paras) elif paras_type == 0: paras = urllib.urlencode(paras) elif paras_type == 2: pass else: logger.error('req, wrong paras type( 0 or 1)') #resp = self.br.open(url, paras, timeout=time_out) #安装opener,此后调用urlopen()时都会使用安装过的opener对象 resp = opener.open(request, paras) if html_flag: html = resp.read() except Exception, e: if self.debug: logger.error(traceback.format_exc(e))
def load_by_request(self, spi, url, name): refer = spi['refer'] mode = spi['mode'] spi_url = spi['spi_url'] post_type = spi['post_type'] post_data = spi['post_data'] isproxy = spi['isproxy'] debug = spi['debug'] p = get_proxy('XXXX') if not p: logger.error('get proxy error ... %s', str(p)) key = name.split('__')[0] if self.objreq.has_key(key): req = self.objreq[key] else: req = urllibCrawler() self.objreq[key] = req req.set_debug(debug) if isproxy != '' and p: req.set_proxy(isproxy) if refer != '': req.add_referer(self.refer) if spi_url != '': req.req(self.mode, self.spi_url) try: if mode == 'post': page = req.req(mode, post_url, paras=post_data, paras_type=post_type, html_flag=True) else: page = req.req(mode, url, html_flag=True) except Exception, e: #traceback.print_exc(e) logger.error('loading by urllib :: %s', traceback.format_exc(e))
def paser_page( kwds): realtime = [] spacetime = [] lname = [] sname = [] type = [] url_res = [] coordinates = [] dl = download() task = kwds key_l = { 'lname':[],\ 'sname':[],\ 'type':[],\ 'coordinates':[],\ 'realtime':[],\ 'spacetime':[],\ 'url_res':[]\ } task_url = task['info'][0] path = task['info'][1] filename = path.split('/')[0]+'/'+md5(path.split('/')[1]) city_name = path.split('/')[0] step = task['info'][2] exp_act = task['exp_act'][step] spi_act = task['spi_act'][step] temp = dl.temp count = 0 flag = True p = '0.0.0.0:0' for key, value in exp_act.items(): if count == 0: i = 0 while key_l[key] == []: i += 1 if i > MAX: logger.info('a task fail ::%s',str(task['info'])) flag = False break try: if os.path.exists(filename) and Islocal: with open(filename,'r') as file : page = file.read() else: if spi_act['way'].lower() == 'req': page,p = dl.load_by_request(spi_act,task_url,filename) elif spi_act['way'].lower() == 'mc': page,p = dl.load_by_mc(spi_act,task_url,filename) exec(key+'='+value) except Exception, e: logger.error('a error spider :: <-!error::%s!-> <-!task::%s!-> <-!proxy::%s!->',traceback.format_exc(e),str(task['info']),str(p)) else: logger.info('task success :: <-!task::%s!-> <-!proxy::%s!->',str(task['info']),str(p)) if key == 'lname': key_l[key] = lname elif key == 'sname': key_l[key] = sname elif key == 'type': key_l[key] = type elif key == 'spacetime': key_l[key] = spacetime elif key == 'realtime': key_l[key] = realtime elif key == 'coordinates': key_l[key] = coordinates elif key == 'url_res': key_l[key] = url_res else: if i >= MAX: break try: exec(key+'='+value) except Exception, e: logger.error('not first validate error :: <-!error::%s!-> <-!task::%s!->',traceback.format_exc(e),str(task['info']))