def make_url(asin, url_type='goods', urltitle='', sessionId=''): url_tuple = tuple() if url_type == 'goods': url, referer = GoodsParser.make_goods_url(asin, urltitle=urltitle, sessionId=sessionId) url_tuple = (url, referer) if url_type == 'reviews': url = ReviewsParser.make_reviews_url(asin, urltitle=urltitle) url_tuple = (url, ) if url_type == 'tosell': url = TosellParser.make_tosell_url(asin) url_tuple = (url, ) return url_tuple
def download(self, ip, asin_or_kw, url_dict): url_type = self.url_type asin = asin_or_kw monitor_type = url_dict.get('mtp') or 1 print('url type: ', url_type) url_md5key = url_dict.get('md5') or '' if not url_md5key: url_md5key = self.get_md5_key(asin + url_type) startTime = return_PST().strftime("%Y-%m-%d %H:%M:%S") time_now = lambda: time.time() time1 = time_now() ua = self.get_ua() self.debug_log.debug('goods ua: %s' % (ua)) # value_str = ip + ua # self.debug_log.debug('goods value_str: %s' % (value_str)) # cookMd5key = self.get_md5_key(value_str) cookMd5key = '' cookie = self.get_cookie(cookMd5key) print('\ngoodsCookie: ', cookie) # url_title = urlQ.get_urlTitle_from_string(asin) url_title = '' sessionId = '' # if cookie: # sessionId = cookie.get('session-id') retry = False old_dnum = url_dict.get('dnum') or 0 if old_dnum > 3: retry = True url, referer = GoodsParser.make_goods_url(asin, urltitle=url_title, sessionId=sessionId, retry=retry) cookies = cookie print('goods referer: %s' % (referer)) print('[ip %s] 工作中... [%s]' % (ip, url)) if url: print('goods_url: ', url) html, cookiesObj, is_error = self.get_html(url, ua, ip, cookies, referer, url_type=url_type, asin=asin) print('这是HTML') # with open('data/devtest/6_1.html', 'w', encoding='utf8') as f: # f.write(html) durl = url_dict.get('durl') or [] durl.append(url) url_dict['durl'] = list(set(durl)) url_dict['dnum'] = old_dnum + 1 if is_error: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) msgInt = 6 proxyInfo = 'get Html error' self.record_log(asin, time1, msgInt, url_type, startTime, ip, proxyInfo) else: analyze = self.analyze_html(html, cookie, cookiesObj, ip, asin_or_kw, url_dict, cookMd5key, time1, startTime, html_type=url_type) if analyze and analyze != 404: # 获取url_title 并保存 # self.save_url_title(asin, html) cook = cookie if not cookie: cook = cookiesObj result, is_error = self.parser(html, html_type=url_type, asin=asin, ip=ip, ua=ua, debug_log=self.debug_log, monitor_type=monitor_type, cookie=cook, url=url) if is_error: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) msgInt = 3 proxyInfo = 'get data error' self.record_log(asin, time1, msgInt, url_type, startTime, ip, proxyInfo) else: if not result: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) msgInt = 2 proxyInfo = 'get data defeated' self.record_log(asin, time1, msgInt, url_type, startTime, ip, proxyInfo) else: goods_datas = result[0] if goods_datas: print('goods_datas:', goods_datas) qty, qtydt = inv_start(asin, html, cook, ua, goods_datas[asin]) print('qty, qtydt:', qty, qtydt) goods_datas[asin]['quantity'] = qty goods_datas[asin]['qtydt'] = qtydt print('goods_datas.add(qty, qtydt):', goods_datas) bsr_data = result[1] data_bytes = pickle.dumps(goods_datas) if bsr_data: # print('bsr_data1', bsr_data) bsrData_bytes = pickle.dumps(bsr_data) # print('bsrData_bytes', bsrData_bytes) self.dataQ.add_bsrData_to_queue( bsrData_bytes) # print('bsr data ok', bsr_data) # from pprint import pprint # pprint(goods_datas) result1 = self.dataQ.add_goods_data_to_queue( data_bytes) print(result1) if not result1: sys.exit() # self.dataQ.record_data_ok_times() # self.dataQ.record_goods_ok_times() self.save_success_asin_keyword( asin, url_type=url_type) msgInt = 1 proxyInfo = 'get data success' self.record_log(asin, time1, msgInt, url_type, startTime, ip, proxyInfo) else: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) time.sleep(1) else: print(url_type, '没有url') self.the_url_is_discard(asin, url_dict, url_type, url_md5key) time.sleep(1)
def download(self, asin_or_kw, url_dict): print(asin_or_kw, url_dict) url_type = self.url_type print(url_type) asin = asin_or_kw monitor_type = url_dict.get('mtp') or 1 print('url type: ', url_type) url_md5key = url_dict.get('md5') or '' if not url_md5key: url_md5key = self.get_md5_key(asin + url_type) startTime = return_PST().strftime("%Y-%m-%d %H:%M:%S") time_now = lambda: time.time() time1 = time_now() retry = False old_dnum = url_dict.get('dnum') or 0 if old_dnum > 3: retry = True url, referer = GoodsParser.make_goods_url(asin, retry=retry) if url: print('goods_url: ', url) html, cookiesObj, is_error = self.get_html(url, referer=referer, url_type=url_type, asin=asin) print('is_error:', is_error) durl = url_dict.get('durl') or [] durl.append(url) url_dict['durl'] = list(set(durl)) url_dict['dnum'] = old_dnum + 1 if is_error: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) msgInt = 6 proxyInfo = 'get Html error' self.record_log(asin, time1, msgInt, url_type, startTime, proxyInfo) else: analyze = self.analyze_html(html, asin_or_kw, url_dict, time1, startTime, html_type=url_type) if analyze and analyze != 404: result, is_error = self.parser(html, html_type=url_type, asin=asin, debug_log=self.debug_log, monitor_type=monitor_type, url=url) if is_error: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) msgInt = 3 proxyInfo = 'get data error' self.record_log(asin, time1, msgInt, url_type, startTime, proxyInfo) else: if not result: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) msgInt = 2 proxyInfo = 'get data defeated' self.record_log(asin, time1, msgInt, url_type, startTime, proxyInfo) else: goods_datas = result[0] if goods_datas: cookies, headers = cookiesObj user_anget = headers.get('User-Agent') print(user_anget) from pprint import pprint pprint(cookies) pprint(goods_datas) msgInt = 1 proxyInfo = 'get data success' log_param = (asin, time1, msgInt, url_type, startTime, proxyInfo) start(asin=asin, goods_datas=goods_datas, user_anget=user_anget, url_dict=url_dict, goods_html=html, cookies=cookies, log_param=log_param, crawler_obj=self) else: self.the_url_is_discard(asin, url_dict, url_type, url_md5key) time.sleep(1) else: print(url_type, '没有url') self.the_url_is_discard(asin, url_dict, url_type, url_md5key) time.sleep(1)