def _execute_sycm_product_actions(self, task):
raw_data = task.raw_data
account = raw_data['account']
cate_id = raw_data['cateId']
cate_name = raw_data['cateName']
try:
for i in range(0, 3):
try:
self.__execute_sycm_product_actions(task)
return
except CookieNotFoundException as e:
logger.warning('cookie is not exist,cate_id:%s,cate_name:%s,account:%s', cate_id, cate_name,
account['username'])
self._relogin(account=account, cate_id=cate_id, cate_name=cate_name)
if i == 2:
raise e
except CookieExpiredException as e:
logger.warning('cookie is expired,cate_id:%s,cate_name:%s,account:%s', cate_id, cate_name,
account['username'])
self._relogin(account=account, cate_id=cate_id, cate_name=cate_name)
if i == 2:
raise e
except Exception as e:
logger.exception(e)
if i == 2:
raise e
time.sleep(5)
except Exception as e:
logger.exception(e)
raise e
def process_bind_param(self, value, dialect):
try:
value = json.dumps(value, ensure_ascii=False, indent=2)
except BaseException as e:
logger.exception(e)
value = ''
return value
def _execut_taobao_detail_actions(self, task, proxy=None):
try:
self.__execut_taobao_detail_actions(task, proxy)
except ProxyException as e:
# logger.exception(e)
return proxy
except Exception as e:
logger.exception(e)
def __call__(self, *args, **kwargs):
_tries, _delay = self.tries, self.delay
while _tries:
try:
return self.func(*args, **kwargs)
except self.exceptions:
logger.exception(u'Call func %s failed, retry it.' %
self.func.__name__)
_tries -= 1
if not _tries:
raise
time.sleep(_delay)
if isinstance(self.wait_delta, tuple):
_delay += random.uniform(*self.wait_delta)
else:
_delay += self.wait_delta
if self.max_delay is not None:
_delay = min(_delay, self.max_delay)
def _execute_taobao_integrate_list_actions(self, task, account, proxy):
try:
self.__execute_taobao_integrate_list_actions(task, account, proxy)
except ProxyException as e:
logger.exception(e)
# return proxy
except CookieExpiredException as e:
logger.exception(e)
# raw_data = task.raw_data
# default_account = global_config.accounts[0]
# account = raw_data.get('account', default_account)
return account, True, True
except InterruptException as e:
logger.exception(e)
# raw_data = task.raw_data
# default_account = global_config.accounts[0]
# account = raw_data.get('account', default_account)
return account, True, False
except Exception as e:
logger.exception(e)
return account, False, False
def __execut_taobao_detail_actions(self, task, proxy=None):
raw_data = task.raw_data
good_result = Good(raw_data['goodResult'])
model_name = good_result.get_model_name()
cate_id = good_result.get_category_id()
integrate_infos = raw_data['integrateInfos']
sale_infos = raw_data['saleInfos']
i = 0
j = 1
length = min(len(integrate_infos), len(sale_infos))
is_success = False
context = Context()
context.attach(Context.KEY_GOOD_DICT, good_result)
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_CURRENT_PROXY, proxy)
for x in range(0, length):
is_need_retry = False
if i < len(sale_infos):
sale_info = sale_infos[i]
sale_item_id = sale_info['itemId']
sale_title = sale_info['title']
sale_cate_id = sale_info['category']
sale_price = sale_info['price']
# str(sale_title).upper()
# if str(sale_title).upper().find(str(model_name).upper()) != -1 and str(cate_id) == str(sale_cate_id):
if str(sale_title).upper().find(str(model_name).upper()) != -1 and Category.check_cate_id(cate_id,
sale_cate_id):
actions = self.get_taobao_detail_actions()
is_success = True
price_info = [{
'skuId': '-1',
'price': yuan_2_cent(sale_price)
}]
good_result.set_price_info(price_info=price_info)
good_result.set_flag(str(int(GoodDataType.success)))
for action in actions:
action.execute(context=context)
break
elif i < 5:
actions = self.get_taobao_http_detail_actions()
timestamps = int(datetime.now().timestamp() * 1000)
# sign = get_sign('414804c1e894540b7f18f703c74346cf', str(timestamps), '12574478',
# '{"itemNumId":"%s"' % (sale_item_id))
sale_detail_url = SpiderUrls.get_taobao_detail_url(timestamps, '', sale_item_id)
context = Context()
context.attach(Context.KEY_GOOD_DICT, good_result)
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_COOKIES, RequestsCookieJar())
context.attach(Context.KEY_IS_UPDATE_COOKIES, True)
context.attach(Context.KEY_CURRENT_PROXY, proxy)
detail_m_url = SpiderUrls.get_detail_m_url(sale_info['userType'], sale_item_id)
detail_m_http_request = HttpRequest(detail_m_url, method=HttpMethod.GET)
context.attach(Context.KEY_DETAIL_M_HTTP_REQUEST, detail_m_http_request)
sale_http_request = HttpRequest(url=sale_detail_url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_DETAIL_HTTP_REQUEST, sale_http_request)
context.attach(Context.KEY_HEADERS, SpiderHttp.get_taobao_headers(detail_m_url))
try:
for action in actions:
action.execute(context=context)
is_success = True
break
except RetryException as e:
logger.error(e)
time.sleep(5)
except InterruptException as e:
logger.exception(e)
time.sleep(10)
is_need_retry = True
# raise e
# if is_success:
# break
if j < len(integrate_infos):
integrate_info = integrate_infos[j]
integrate_item_id = integrate_info['itemId']
integrate_title = integrate_info['title']
integrate_cate_id = integrate_info['category']
integrate_price = integrate_info['price']
# if str(integrate_title).upper().find(
# str(model_name).upper()) != -1 and str(cate_id) == str(integrate_cate_id):
if str(integrate_title).upper().find(
str(model_name).upper()) != -1 and Category.check_cate_id(cate_id, integrate_cate_id):
actions = self.get_taobao_detail_actions()
is_success = True
price_info = [{
'skuId': '-2',
'price': yuan_2_cent(integrate_price)
}]
good_result.set_price_info(price_info=price_info)
good_result.set_flag(str(int(GoodDataType.success)))
for action in actions:
action.execute(context=context)
break
elif j < 6:
actions = self.get_taobao_http_detail_actions()
timestamps = int(datetime.now().timestamp() * 1000)
integrate_detail_url = SpiderUrls.get_taobao_detail_url(timestamps, '', integrate_item_id)
context = Context()
# referer = 'https://s.m.taobao.com/h5'
# context.attach(Context.KEY_HEADERS, SpiderHttp.get_taobao_headers(referer))
context.attach(Context.KEY_GOOD_DICT, good_result)
context.attach(Context.KEY_CURRENT_TASK, task)
context.attach(Context.KEY_COOKIES, RequestsCookieJar())
context.attach(Context.KEY_IS_UPDATE_COOKIES, True)
context.attach(Context.KEY_CURRENT_PROXY, proxy)
detail_m_url = SpiderUrls.get_detail_m_url(integrate_info['userType'], integrate_item_id)
detail_m_http_request = HttpRequest(detail_m_url, method=HttpMethod.GET)
context.attach(Context.KEY_DETAIL_M_HTTP_REQUEST, detail_m_http_request)
integrate_http_request = HttpRequest(url=integrate_detail_url, method=HttpMethod.GET)
context.attach(Context.KEY_TAOBAO_DETAIL_HTTP_REQUEST, integrate_http_request)
context.attach(Context.KEY_HEADERS, SpiderHttp.get_taobao_headers(detail_m_url))
try:
for action in actions:
action.execute(context=context)
is_success = True
break
except RetryException as e:
logger.exception(e)
time.sleep(5)
except InterruptException as e:
logger.exception(e)
time.sleep(10)
is_need_retry = True
# raise e
if not is_need_retry:
i += 1
j += 1
if not is_success:
actions = self.get_taobao_detail_actions()
good_result.set_flag(str(int(GoodDataType.not_found)))
for action in actions:
action.execute(context=context)