def do_execute(self, context):
"""
:param context:
:return:
"""
timestamps = int(datetime.now().timestamp() * 1000)
data = self.__build_data(context=context)
s_data = data.replace('+', ' ')
token = self.__build_token(context=context)
sign = get_sign(token=token,
timestamps=str(timestamps),
appKey='12574478',
data=s_data)
# data = data.replace(':', '%3A')
sale_url = SpiderUrls.get_taobao_serarch_url(
type=TaobaoSearchType.sale,
timestamps=timestamps,
sign=sign,
data=data)
sale_http_request = HttpRequest(url=sale_url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_SALE_LIST_HTTP_REQUEST,
sale_http_request)
http_request = context.get(Context.KEY_TAOBAO_SALE_LIST_HTTP_REQUEST)
response = self.execute_in_retry(context=context,
http_request=http_request)
self.unmarshal(context=context, response=response)
cookies = context.get(Context.KEY_COOKIES)
account = context.get(Context.KEY_CURRENT_TASK_ACCOUNT)
self.__cookie_service.dump(cookies, account)
return True
def _login(self, account, force, risk=False, proxy=None):
requests_cookie_jar = self._cookie_service.load(account)
if not requests_cookie_jar or force:
cookies, origin_cookies = spider_qt5_bootstrap(url=SpiderUrls.get_sycm_login_url(), account=account,
risk=risk, proxy=proxy)
self._cookie_service.dump(cookies, account)
self._cookie_service.dump(cookies=origin_cookies, account=account, type_=PickleFileType.origin_cookie)
def __execute_sycm_product_actions(self, task):
raw_data = task.raw_data
cate_id = raw_data['cateId']
cate_name = raw_data['cateName']
account = raw_data['account']
date_str = raw_data['dateStr']
context = Context()
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_IS_UPDATE_COOKIES, False)
referer = 'https://sycm.taobao.com/mc/mq/product_insight'
context.attach(Context.KEY_HEADERS, SpiderHttp.get_sycm_headers(referer))
requests_cookie_jar = self._cookie_service.load(account)
if not requests_cookie_jar:
raise CookieNotFoundException('cookie not found')
# cookies = spider_qt5_bootstrap(url=SpiderUrls.get_sycm_login_url(), account=account)
# requests_cookie_jar = cookies
# self._cookie_service.dump(cookies, account)
context.attach(Context.KEY_COOKIES, requests_cookie_jar)
context.attach(Context.KEY_IS_UPDATE_COOKIES, False)
context.attach(Context.KEY_CURRENT_TASK_ACCOUNT, account)
good = Good()
good.set_category_id(cate_id)
good.set_category_name(cate_name)
good.set_flag(str(int(GoodDataType.initial)))
# yesterday_date_str = yesterday().strftime("%Y-%m-%d")
# yesterday_date_str = day_before_yesterday().strftime("%Y-%m-%d")
yesterday_date_str = date_str
good.set_date(yesterday_date_str)
context.attach(Context.KEY_GOOD_DICT, good)
sycm_brands_url = SpiderUrls.get_sycm_product_get_brands_url(cate_id)
token = self._cookie_service.load(account=account, type_=PickleFileType.legality_token)
sycm_prod_hot_rank_url = SpiderUrls.get_sycm_product_prod_hot_rank(start_data=yesterday_date_str,
end_date=yesterday_date_str, cate_id=cate_id,
token=token)
sycm_brands_http_request = HttpRequest(url=sycm_brands_url, method=HttpMethod.GET)
sycm_prod_hot_rank_http_request = HttpRequest(url=sycm_prod_hot_rank_url, method=HttpMethod.GET)
context.attach(Context.KEY_SYCM_PRODUCT_GET_BRANDS_HTTP_REQUEST, sycm_brands_http_request)
context.attach(Context.KEY_SYCM_PRODUCT_PROD_HOT_RANK_HTTP_REQUEST, sycm_prod_hot_rank_http_request)
for action in self.get_sycm_product_actions():
action.execute(context=context)
return
def test_app_login(self):
url = SpiderUrls.get_app_taobao_login_url()
# account = global_config.s_accounts[1]
# proxy = __proxy_service.get_origin_static_proxy(account['username'])
account = global_config.s_accounts[0]
proxy = None
cookies, origin_cookies = spider_qt5_bootstrap(url, account, True, proxy)
self.__cookie_service.dump(cookies=cookies, account=account)
print(cookies)
def test(self):
url = SpiderUrls.get_taobao_presearch_url(u'三星s9')
http_request = HttpRequest(url=url, method=HttpMethod.GET)
context = Context()
context.attach(Context.KEY_TAOBAO_PRESEARCH_HTTP_REQUEST, http_request)
context.attach(Context.KEY_IS_UPDATE_COOKIES, False)
context.attach(
Context.KEY_HEADERS,
SpiderHttp.get_taobao_headers('https://s.m.taobao.com/h5'))
context.attach(Context.KEY_COOKIES, RequestsCookieJar())
action = TaobaoPresearchAction()
action.execute(context=context)
def do_execute(self, context):
"""
:param context:
:return:
"""
http_request = HttpRequest(SpiderUrls.get_sycm_home_htm_url(),
HttpMethod.GET)
context.attach(Context.KEY_SYCM_HOME_HTM_TOKEN_HTTP_REQUEST,
http_request)
http_request = context.get(
Context.KEY_SYCM_HOME_HTM_TOKEN_HTTP_REQUEST)
response = self.execute_in_retry(context=context,
http_request=http_request)
result = self.unmarshal(context=context, response=response)
context.attach(Context.KEY_SYCM_HOME_HTM_TOKEN_RESULT, result)
return True
def _relogin(self, account, cate_id, cate_name):
for i in range(0, 3):
try:
cookies, origin_cookies = spider_qt5_bootstrap(url=SpiderUrls.get_sycm_login_url(), account=account)
self._execute_legality_token_actions(cookies=cookies, account=account)
self._cookie_service.dump(cookies, account)
return
except CookieExpiredException as e:
logger.warning('relogin cookie is expired,cate_id:%s,cate_name:%s,account:%s', cate_id, cate_name,
account['username'])
if i == 2:
raise ExitException('exit when try login util max 3 times retry')
except Exception as e:
logger.error('retry error,retry times,%s,%s', i, e)
if i == 2:
raise ExitException('exit when try login util max 3 times retry')
time.sleep(5)
def do_execute(self, context):
"""
:param context:
:return:
"""
good = context.get(Context.KEY_GOOD_DICT)
# brand_name = good.get_brand_name()
# model_name = good.get_model_name()
query = good.get_query()
url = SpiderUrls.get_taobao_presearch_url(query=query)
http_request = HttpRequest(url=url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_PRESEARCH_HTTP_REQUEST, http_request)
http_request = context.get(Context.KEY_TAOBAO_PRESEARCH_HTTP_REQUEST)
response = self.execute_in_retry(context=context, http_request=http_request)
result = self.unmarshal(context=context, response=response)
context.attach(Context.KEY_TAOBAO_PRESEARCH_RESULT, result)
return True
def do_execute(self, context):
"""
:param context:
:return:
"""
timestamps = int(datetime.now().timestamp() * 1000)
data = self.__build_data(context=context)
token = self.__build_token(context=context)
sign = get_sign(token=token,
timestamps=str(timestamps),
appKey='12574478',
data=data)
# data = data.replace(':', '%3A')
sale_url = SpiderUrls.get_taobao_baichuan_url(timestamps=timestamps,
sign=sign)
sale_http_request = HttpRequest(url=sale_url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_BAICHUAN_HTTP_REQUEST,
sale_http_request)
http_request = context.get(Context.KEY_TAOBAO_BAICHUAN_HTTP_REQUEST)
response = self.execute_in_retry(context=context,
http_request=http_request)
self.unmarshal(context=context, response=response)
return True
def __execut_taobao_detail_actions(self, task, proxy=None):
raw_data = task.raw_data
good_result = Good(raw_data['goodResult'])
model_name = good_result.get_model_name()
cate_id = good_result.get_category_id()
integrate_infos = raw_data['integrateInfos']
sale_infos = raw_data['saleInfos']
i = 0
j = 1
length = min(len(integrate_infos), len(sale_infos))
is_success = False
context = Context()
context.attach(Context.KEY_GOOD_DICT, good_result)
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_CURRENT_PROXY, proxy)
for x in range(0, length):
is_need_retry = False
if i < len(sale_infos):
sale_info = sale_infos[i]
sale_item_id = sale_info['itemId']
sale_title = sale_info['title']
sale_cate_id = sale_info['category']
sale_price = sale_info['price']
# str(sale_title).upper()
# if str(sale_title).upper().find(str(model_name).upper()) != -1 and str(cate_id) == str(sale_cate_id):
if str(sale_title).upper().find(str(model_name).upper()) != -1 and Category.check_cate_id(cate_id,
sale_cate_id):
actions = self.get_taobao_detail_actions()
is_success = True
price_info = [{
'skuId': '-1',
'price': yuan_2_cent(sale_price)
}]
good_result.set_price_info(price_info=price_info)
good_result.set_flag(str(int(GoodDataType.success)))
for action in actions:
action.execute(context=context)
break
elif i < 5:
actions = self.get_taobao_http_detail_actions()
timestamps = int(datetime.now().timestamp() * 1000)
# sign = get_sign('414804c1e894540b7f18f703c74346cf', str(timestamps), '12574478',
# '{"itemNumId":"%s"' % (sale_item_id))
sale_detail_url = SpiderUrls.get_taobao_detail_url(timestamps, '', sale_item_id)
context = Context()
context.attach(Context.KEY_GOOD_DICT, good_result)
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_COOKIES, RequestsCookieJar())
context.attach(Context.KEY_IS_UPDATE_COOKIES, True)
context.attach(Context.KEY_CURRENT_PROXY, proxy)
detail_m_url = SpiderUrls.get_detail_m_url(sale_info['userType'], sale_item_id)
detail_m_http_request = HttpRequest(detail_m_url, method=HttpMethod.GET)
context.attach(Context.KEY_DETAIL_M_HTTP_REQUEST, detail_m_http_request)
sale_http_request = HttpRequest(url=sale_detail_url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_DETAIL_HTTP_REQUEST, sale_http_request)
context.attach(Context.KEY_HEADERS, SpiderHttp.get_taobao_headers(detail_m_url))
try:
for action in actions:
action.execute(context=context)
is_success = True
break
except RetryException as e:
logger.error(e)
time.sleep(5)
except InterruptException as e:
logger.exception(e)
time.sleep(10)
is_need_retry = True
# raise e
# if is_success:
# break
if j < len(integrate_infos):
integrate_info = integrate_infos[j]
integrate_item_id = integrate_info['itemId']
integrate_title = integrate_info['title']
integrate_cate_id = integrate_info['category']
integrate_price = integrate_info['price']
# if str(integrate_title).upper().find(
# str(model_name).upper()) != -1 and str(cate_id) == str(integrate_cate_id):
if str(integrate_title).upper().find(
str(model_name).upper()) != -1 and Category.check_cate_id(cate_id, integrate_cate_id):
actions = self.get_taobao_detail_actions()
is_success = True
price_info = [{
'skuId': '-2',
'price': yuan_2_cent(integrate_price)
}]
good_result.set_price_info(price_info=price_info)
good_result.set_flag(str(int(GoodDataType.success)))
for action in actions:
action.execute(context=context)
break
elif j < 6:
actions = self.get_taobao_http_detail_actions()
timestamps = int(datetime.now().timestamp() * 1000)
integrate_detail_url = SpiderUrls.get_taobao_detail_url(timestamps, '', integrate_item_id)
context = Context()
# referer = 'https://s.m.taobao.com/h5'
# context.attach(Context.KEY_HEADERS, SpiderHttp.get_taobao_headers(referer))
context.attach(Context.KEY_GOOD_DICT, good_result)
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_COOKIES, RequestsCookieJar())
context.attach(Context.KEY_IS_UPDATE_COOKIES, True)
context.attach(Context.KEY_CURRENT_PROXY, proxy)
detail_m_url = SpiderUrls.get_detail_m_url(integrate_info['userType'], integrate_item_id)
detail_m_http_request = HttpRequest(detail_m_url, method=HttpMethod.GET)
context.attach(Context.KEY_DETAIL_M_HTTP_REQUEST, detail_m_http_request)
integrate_http_request = HttpRequest(url=integrate_detail_url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_DETAIL_HTTP_REQUEST, integrate_http_request)
context.attach(Context.KEY_HEADERS, SpiderHttp.get_taobao_headers(detail_m_url))
try:
for action in actions:
action.execute(context=context)
is_success = True
break
except RetryException as e:
logger.exception(e)
time.sleep(5)
except InterruptException as e:
logger.exception(e)
time.sleep(10)
is_need_retry = True
# raise e
if not is_need_retry:
i += 1
j += 1
if not is_success:
actions = self.get_taobao_detail_actions()
good_result.set_flag(str(int(GoodDataType.not_found)))
for action in actions:
action.execute(context=context)
def test_login(self):
cookies, origin_cookies = spider_qt5_bootstrap(url=SpiderUrls.get_sycm_login_url(), account=self.account)
self.__cookie_service.dump(cookies, self.account)