def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.post(
url=search_url,
headers=headers,
data=json.dumps({
"Keyword": keyword,
"SaleCityId": "1",
"Tab": 64
}),
)
content = response.content
city_list = json.loads(content)['Data']
suggest['suggest'] = content
db = client['SuggestName']
db.CtripCitySuggestion.save(suggest)
self.task.error_code = 0
return {'搜索到的city数量': len(city_list)}
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.get(url=search_url,
headers=headers,
data={
'Jsoncallback': 'jQuery',
'keyword': keyword
})
json_data = json.loads(response.content[7:-1])
suggest['suggest'] = json_data
db = client['SuggestName']
db.CtripPoiSDK.save(suggest)
self.task.error_code = 0
count = 1
if isinstance(json_data, list):
count = len(json_data)
return {'搜索到的suggest数量': count}
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.get(url=search_url,
headers=headers,
params={
'searchType': 'InCity',
'applyGrouping': True,
'isWebRequest': True,
'searchTerm': keyword
})
content = response.content
root = html.fromstring(content)
city_list = root.xpath('//city')
suggest['suggest'] = content
db = client['SuggestName']
db.MarriottCitySuggest.save(suggest)
self.task.error_code = 0
return {'搜索到的city数量': len(city_list)}
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
with MySession(need_proxies=True,
need_cache=True,
auto_update_host=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.post(url=search_url,
headers=headers,
data={'searchText': keyword})
json_data = json.loads(response.content)
suggest['suggest'] = json_data
db = client['SuggestName']
db.AccorCitySuggest.save(suggest)
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
self.task.error_code = 0
return {'搜索到的suggest数量': json_data['TotalItemsCount']}
def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.post(url=search_url,
headers=headers,
params={
'r':
'search/search/searchSugguestV2',
'query': keyword,
'format': 'json'
})
content = response.content
suggest['suggest'] = content
db = client['SuggestName']
db.TuniuCitySuggestion.save(suggest)
self.task.error_code = 0
return {'搜索到的city数量': 1}
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.get(url=search_url,
headers=headers,
params={
'country': 'cn',
'language': 'zh',
'brand': 'ihg',
'query': keyword
})
json_data = json.loads(response.content)
suggest['suggest'] = json_data
db = client['SuggestName']
db.IhgCitySuggest.save(suggest)
self.task.error_code = 0
return {'搜索到的suggest数量': json_data['preFilterCount']}
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
table_name = self.task.kwargs['table_name']
source = self.task.kwargs['source']
sid = self.task.kwargs['sid']
other_info = self.task.kwargs['other_info']
address = json.loads(other_info).get('address').encode('utf8')
if not address:
execute_sql(update_status % (2, source, sid, table_name))
raise ServiceStandardError(
error_code=ServiceStandardError.KEY_WORDS_FILTER,
wrapped_exception=Exception(u'address 为空'))
map_info = google_get_map_info(address)
if not map_info:
execute_sql(update_status % (2, source, sid, table_name))
raise ServiceStandardError(
error_code=ServiceStandardError.KEY_WORDS_FILTER,
wrapped_exception=Exception(u'mapinfo 为空'))
sql = update_map_info % (table_name, map_info, source, sid)
typ2 = table_name.split('_')[1]
sql = sql.format(field='source_id' if typ2 == 'hotel' else 'id')
execute_sql(sql)
execute_sql(update_status % (1, source, sid, table_name))
self.task.error_code = 0
return source, sid
def _execute(self, **kwargs):
table_name = self.task.kwargs['task_name']
source = self.task.kwargs['source']
sid = self.task.kwargs['sid']
url = self.task.kwargs['url']
typ2 = table_name.split('_')[1]
try:
source_id = re.compile(r'd(\d+)').findall(url)[0]
if not source_id:
raise ServiceStandardError(
error_code=ServiceStandardError.PARSE_ERROR)
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.PARSE_ERROR,
wrapped_exception=Exception(
'can not find source_id, url %s' % url))
if typ2 == 'attr':
img_url = attr_image_parser(source_id)
elif typ2 == 'rest':
img_url = rest_image_parser(source_id)
else:
img_url = None
if not img_url:
raise ServiceStandardError(
error_code=ServiceStandardError.EMPTY_TICKET)
sql = update_imgurl % (table_name, img_url, source, sid)
execute_sql(sql)
self.task.error_code = 0
return source, sid
def _execute(self, **kwargs):
dept_info = self.task.kwargs['dept_info']
dest_info = self.task.kwargs['dest_info']
source = self.task.kwargs['source']
error_code, result, page_store_key = GT_to_database(
tid=self.task.task_id,
used_times=self.task.used_times,
vacation_type=self.task.kwargs['vacation_type'],
source=source,
ticket=self.task.kwargs,
need_cache=self.task.used_times == 0)
db[source + 'GT_list'].save({
'collections': self.task.collection,
'task_id': self.task.task_id,
'used_times': self.task.used_times[0],
'stored_page_keys': page_store_key,
'dept_info': dept_info,
'dest_info': dest_info,
'result': result,
'source': self.task.kwargs['source'],
'insert_time': datetime.datetime.now()
})
self.task.error_code = error_code
sql = SQL.format(self.task.task_name)
data = []
if source == 'ctrip':
for res in result:
data.append(
(source, res['pid_3rd'], dept_info['id'], dest_info['id'],
json.dumps(res)))
elif source == 'tuniu':
for res in result:
data.append(
(source, res['id'], dept_info['id'], dest_info['id'],
json.dumps(res)))
try:
service_platform_conn = service_platform_pool.connection()
cursor = service_platform_conn.cursor()
_res = cursor.executemany(sql, data)
service_platform_conn.commit()
cursor.close()
service_platform_conn.close()
self.task.get_data_per_times = len(data)
self.task.list_task_insert_db_count = _res
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
if error_code == 27 or len(data) > 0:
self.task.error_code = 0
else:
raise ServiceStandardError(
error_code=ServiceStandardError.EMPTY_TICKET)
return len(result), error_code
def _execute(self, **kwargs):
sql = SQL.format(table_name=self.task.task_name)
poi_type = self.task.kwargs['poi_type']
code, result, others_info, page_store_key = hotel_list_database(
self.task.kwargs['source'],
self.task.kwargs['url'],
type_dict[poi_type],
spider_name[poi_type],
need_cache=self.task.used_times == 0)
self.logger.info('spider %s %s' % (str(code), str(result)))
self.task.error_code = code
try:
collections.save({
'collections': self.task.collection,
'task_id': self.task.task_id,
'used_times': self.task.used_times[0],
'others_info': others_info,
'stored_page_keys': page_store_key,
'result': result,
'insert_time': datetime.datetime.now()
})
except Exception as e:
self.logger.exception(msg=traceback.format_exc(e))
data = []
try:
for one in result:
for key, view in one.items():
data.append(
(self.task.kwargs['source'], view['source_id'],
self.task.kwargs['city_id'],
self.task.kwargs['country_id'], view['view_url']))
logger.info('%s' % str(
(self.task.kwargs['source'], view['source_id'],
self.task.kwargs['city_id'],
self.task.kwargs['country_id'], view['view_url'])))
logger.info('%s %s' % (sql, str(data)))
res = insert(sql, data)
self.task.list_task_insert_db_count = res
self.task.get_data_per_times = len(data)
except Exception as e:
self.logger.exception(msg="[insert db error]", exc_info=e)
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
# 由于错误都是 raise 的，
# 所以当出现此种情况是，return 的内容均为正确内容
# 对于抓取平台来讲，当出现此中情况时，数据均应该入库
# 用 res_data 判断，修改 self.error_code 的值
if len(data) > 0:
self.task.error_code = 0
else:
raise ServiceStandardError(
error_code=ServiceStandardError.EMPTY_TICKET)
return self.task.error_code, self.task.kwargs['url']
def _execute(self, **kwargs):
city_id = self.task.kwargs['city_id']
country_id = self.task.kwargs['country_id']
check_in = self.task.kwargs['check_in']
city_url = self.task.kwargs['city_url']
error_code, result, page_store_key, types_result_num = qyer_list_to_database(
tid=self.task.task_id,
used_times=self.task.used_times,
source=self.task.kwargs['source'],
city_id=city_id,
check_in=check_in,
city_url=city_url,
need_cache=self.task.used_times == 0)
collections.save({
'collections': self.task.collection,
'task_id': self.task.task_id,
'used_times': self.task.used_times[0],
'total_num': types_result_num,
'stored_page_keys': page_store_key,
'result': result,
'insert_time': datetime.datetime.now()
})
self.task.error_code = error_code
sql = SQL.format(self.task.task_name)
data = []
for sid, url, page_id, branch, tag in result:
data.append(('qyer', sid, city_id, country_id, url))
try:
service_platform_conn = service_platform_pool.connection()
cursor = service_platform_conn.cursor()
_res = cursor.executemany(sql, data)
service_platform_conn.commit()
cursor.close()
service_platform_conn.close()
self.task.get_data_per_times = len(data)
self.task.list_task_insert_db_count = _res
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
if len(data) > 0:
self.task.error_code = 0
else:
raise ServiceStandardError(
error_code=ServiceStandardError.EMPTY_TICKET)
return result, error_code
def _execute(self, **kwargs):
with MySession(need_cache=True, need_proxies=True) as session:
keyword = self.task.kwargs['keyword']
page = session.get(search_url.format(keyword),
headers=headers,
timeout=240)
city_count = 0
try:
json_data = json.loads(page.content)
client = pymongo.MongoClient(**mongo_config)
db = client['SuggestName']
db.QyerRawSuggest.save({'suggest': json_data})
city_list = []
citys = json_data.get('data', {}).get('list')
for city in citys:
if city.get('type_name') == 'city':
city_count += 1
city_list.append(city)
db.QyerCity.save({'city': city_list})
client.close()
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
self.task.error_code = 0
return '抓取到的城市数量:%s' % city_count
def _execute(self, **kwargs):
with MySession(need_cache=True, need_proxies=True) as session:
keyword = self.task.kwargs['keyword']
page_info = {}
response = session.get(url=search_url,
params={
'ie': 'utf-8',
'tn': 'baidu',
'wd': keyword,
'rqlang': 'cn'
},
headers=headers)
try:
content = response.content
root = html.fromstring(content)
page_info['keyword'] = keyword
page_info['content'] = content
city_url = []
city_list = root.xpath(
'//a[contains(text(),"place.qyer.com")]/text()')
for city in city_list:
url_str = urljoin('http:', city)
url_str = url_str.strip('.').strip('')
if not city_url or url_str not in city_url:
city_url.append(url_str)
page_info['city_url'] = city_url
client = pymongo.MongoClient(**mongo_config)
db = client['SuggestName']
db.BaiDuSuggest.save(page_info)
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
self.task.error_code = 0
return page_info
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
if not exc_type:
self.update_proxy(0)
elif exc_type in (SSLError, ProxyError):
self.browser_log()
self.update_proxy(22)
raise ServiceStandardError(22,
"代理异常 from Browser [proxy: {}]".format(
self.p_r_o_x_y),
wrapped_exception=exc_type)
elif exc_type in (ConnectionError, ConnectTimeout):
self.browser_log()
self.update_proxy(23)
raise ServiceStandardError(23,
"代理被禁 from Browser [proxy: {}]".format(
self.p_r_o_x_y),
wrapped_exception=exc_type)
if self.need_cache:
# store page check
if exc_type is None:
# save any not stored cache
for k, v in self.md5_resp.items():
if not proj.my_lib.Common.RespStore.has_cache(k):
logger.info('[保存缓存][md5: {}]'.format(k))
proj.my_lib.Common.RespStore.put_by_md5(k, v)
else:
# store debug page
for k, v in self.md5_resp.items():
debug_key = "debug_{}".format(k)
logger.info('[保存 debug 缓存][md5: {}]'.format(debug_key))
if not proj.my_lib.Common.RespStore.has_cache(debug_key):
proj.my_lib.Common.RespStore.put_by_md5(debug_key, v)
# don't store page or delete the page
if not self.do_not_delete_cache:
for each_md5 in self.md5:
if proj.my_lib.Common.RespStore.has_cache(each_md5):
logger.info('[删除缓存][md5: {}]'.format(each_md5))
proj.my_lib.Common.RespStore.delete_cache(each_md5)
else:
for each_md5 in self.md5:
logger.info(
'[出现异常不清楚并不清除缓存][md5: {}]'.format(each_md5))
def _execute(self, **kwargs):
error_code, result = hilton_to_database(
tid=self.task.task_id,
used_times=self.task.used_times,
source='hilton',
keyword=self.task.kwargs['keyword'],
extra=self.task.kwargs['extra'],
spider_tag = 'hiltonSuggest',
need_cache=self.task.used_times == 0
)
conn = pymysql.connect(**config)
cursor = conn.cursor()
save_result = []
self.task.error_code = error_code
# for sid, url, page_id in result:
# data.append(('hilton', sid, city_id, country_id, url))
# try:
# service_platform_conn = service_platform_pool.connection()
# cursor = service_platform_conn.cursor()
# _res = cursor.executemany(sql, data)
# service_platform_conn.commit()
# cursor.close()
# service_platform_conn.close()
# self.task.get_data_per_times = len(data)
# self.task.list_task_insert_db_count = _res
# except Exception as e:
# raise ServiceStandardError(error_code=ServiceStandardError.MYSQL_ERROR, wrapped_exception=e)
print(result)
self.logger.info('this time, result count of keyword {} is {}'.format(self.task.kwargs['keyword'], len(result)))
if len(result) > 0:
try:
for i in result:
# source, sid, sid_md5, s_city, s_country, suggest_type, suggest
# save_result.append(
# (str(i['source']), str(i['sid']), str(i['sid_md5']), str(i['s_city']), str(i['s_country']),
# int(i['suggest_type']), str(i['suggest'])))
sql = SQL.format(str(i['source']), str(i['sid']), str(i['sid_md5']), str(i['s_city']), str(i['s_country']),
i['suggest_type'], str(i['suggest']))
cursor.execute(sql)
# cursor.executemany(SQL, save_result)
except Exception as e:
conn.rollback()
traceback.print_exc(e)
finally:
conn.commit()
conn.close()
self.task.error_code = 0
else:
raise ServiceStandardError(error_code=ServiceStandardError.EMPTY_TICKET)
return result, error_code
def hotel_list_insert_daodao(table_name, res_data):
try:
service_platform_conn = service_platform_pool.connection()
cursor = service_platform_conn.cursor()
sel_sql = 'select id, localtion_id, source_list from {} where source_id = %s'.format(table_name)
rep_sql = "replace into {} (id, name, name_en, city_id, country_id, `from`, source_id, localtion_id, status, source_list) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)".format(
table_name)
ins_sql = "insert into {} (name, name_en, city_id, country_id, `from`, source_id, localtion_id, status, source_list) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)".format(
table_name)
cursor.execute(sel_sql, source_id)
exists_localtion = {}
for id, localtion_id, source_list in cursor.fetchall():
print id, localtion_id, source_list
exists_localtion[localtion_id] = (id, json.loads(source_list or "{}"))
new_hotels_and_not_id = []
if exists_localtion:
for _i, line in enumerate(res_data):
new_hotels = {}
mylocaltion_id, myhotels = line[6], line[8]
if exists_localtion.has_key(mylocaltion_id):
yourid, yourhotels = exists_localtion[mylocaltion_id]
mykeys = myhotels.keys()
yourkeys = yourhotels.keys()
if len(mykeys) > len(yourkeys):
for k in mykeys:
new_hotels[k] = myhotels.get(k) or yourhotels.get(k)
else:
for k in yourkeys:
new_hotels[k] = yourhotels.get(k) or myhotels.get(k)
line[8] = json.dumps(new_hotels)
line.insert(0, yourid)
else:
line[8] = json.dumps(line[8] or {})
new_hotels_and_not_id.append(line)
res_data = filter(lambda x:len(x)==10, res_data)
cursor.executemany(rep_sql, res_data)
service_platform_conn.commit()
if new_hotels_and_not_id:
cursor.executemany(ins_sql, new_hotels_and_not_id)
service_platform_conn.commit()
if not exists_localtion:
for line in res_data:
line[8] = json.dumps(line[8])
cursor.executemany(ins_sql, res_data)
service_platform_conn.commit()
cursor.close()
service_platform_conn.close()
except Exception as e:
self.logger.exception(msg="[mysql error]", exc_info=e)
raise ServiceStandardError(error_code=ServiceStandardError.MYSQL_ERROR, wrapped_exception=e)
def hotel_list_insert_db(table_name, res_data):
try:
service_platform_conn = service_platform_pool.connection()
cursor = service_platform_conn.cursor()
sql = "replace into {} (name, name_en, city_id, country_id, `from`, status, source_list) VALUES (%s,%s,%s,%s,%s,%s,%s)".format(
table_name)
cursor.executemany(sql, res_data)
service_platform_conn.commit()
cursor.close()
service_platform_conn.close()
except Exception as e:
self.logger.exception(msg="[mysql error]", exc_info=e)
raise ServiceStandardError(error_code=ServiceStandardError.MYSQL_ERROR, wrapped_exception=e)
def _execute(self, **kwargs):
with MySession(need_cache=True, need_proxies=True) as session:
iata_code = self.task.kwargs['iata_code']
request_body = {
"union": "",
"maker": "",
"isStop": "0",
"isDomestic": "1",
"isCross": "1",
"queryDate2": "",
"ftype": "",
"queryDate1": "",
"dep": iata_code,
"isShare": "0",
"depType": "1",
}
response = session.post(
url="http://map.variflight.com/___api/SuXAvAQ0qWkchQuUUqHN/de1",
headers=headers,
data=request_body
)
try:
data = json.loads(response.text)
if int(data['code']) != 0:
raise ServiceStandardError(error_code=ServiceStandardError.PROXY_FORBIDDEN)
data_collections.save(
{
'iata_code': iata_code,
'data': data
}
)
except Exception as e:
raise ServiceStandardError(error_code=ServiceStandardError.MYSQL_ERROR, wrapped_exception=e)
self.task.error_code = 0
return data
def get_content(self, source, line):
try:
res = self._get_content(source, line)
if res:
return res[1]
else:
return None
except ReportException as e:
add_content_report(source, str(e.type))
return None
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.post(url=search_url,
headers=headers,
data={
'action': 'API',
'uiOrigin': 'PTPT-dest',
'types': 'geo,dest',
'hglt': True,
'global': True,
'legacy_format': True,
'_ignoreMinCount': True,
'query': keyword
})
json_data = json.loads(response.content)
suggest['suggest'] = json_data
db = client['SuggestName']
db.DaoDaoCitySuggest.save(suggest)
self.task.error_code = 0
count = 1
if isinstance(json_data, list):
count = len(json_data)
return {'搜索到的suggest数量': count}
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
city_id = self.task.kwargs['city_id']
source_id = self.task.kwargs['source_id']
check_in = self.task.kwargs['check_in']
error_code, result, page_store_key = hilton_to_database(
tid=self.task.task_id,
used_times=self.task.used_times,
source=self.task.kwargs['source'],
source_id=source_id,
check_in=check_in,
city_id=city_id,
need_cache=self.task.used_times == 0
)
collections.save({
'collections': self.task.collection,
'task_id': self.task.task_id,
'used_times': self.task.used_times[0],
'stored_page_keys': page_store_key,
'check_in': self.task.kwargs['check_in'],
'result': result,
'insert_time': datetime.datetime.now()
})
self.task.error_code = error_code
sql = SQL.format(self.task.task_name)
data = json.dumps(result, ensure_ascii=False)
# for sid, url, page_id in result:
# data.append(('hilton', sid, city_id, country_id, url))
# try:
# service_platform_conn = service_platform_pool.connection()
# cursor = service_platform_conn.cursor()
# _res = cursor.executemany(sql, data)
# service_platform_conn.commit()
# cursor.close()
# service_platform_conn.close()
# self.task.get_data_per_times = len(data)
# self.task.list_task_insert_db_count = _res
# except Exception as e:
# raise ServiceStandardError(error_code=ServiceStandardError.MYSQL_ERROR, wrapped_exception=e)
if len(data) > 0:
self.task.error_code = 0
else:
raise ServiceStandardError(error_code=ServiceStandardError.EMPTY_TICKET)
return result, error_code
def _execute(self, **kwargs):
url = self.task.kwargs['url']
flag = self.task.kwargs['flag']
table_name = self.task.kwargs['table_name']
md5_url = encode(url)
with MySession(need_proxies=True, need_cache=True) as session:
page = session.get(url, timeout=240)
page.encoding = 'utf8'
if len(page.text) == 0:
raise ServiceStandardError(
error_code=ServiceStandardError.PROXY_FORBIDDEN)
else:
content = page.text
j_data = json.loads(content)
if j_data['status'] not in ['OK', 'ZERO_RESULTS']:
raise ServiceStandardError(
error_code=ServiceStandardError.PROXY_FORBIDDEN)
data = (md5_url, url, content, flag)
conn = pymysql.connect(host='10.10.231.105',
user='******',
passwd='hourong',
db='crawled_html',
charset="utf8")
try:
with conn as cursor:
sql = 'insert ignore into crawled_html.{0}(`md5`,`url`,`content`,`flag`) values (%s,%s,%s,%s)'.format(
table_name)
print(cursor.execute(sql, data))
except Exception as e:
raise ServiceStandardError(
error_code=ServiceStandardError.PROXY_FORBIDDEN,
wrapped_exception=e)
self.task.error_code = 0
return 'OK', url
def hotel_list_insert_db():
try:
service_platform_conn = service_platform_pool.connection()
cursor = service_platform_conn.cursor()
sql = "INSERT IGNORE INTO {} (source, source_id, city_id, country_id, hotel_url) VALUES (%s,%s,%s,%s,%s)".format(
self.task.task_name)
_res = cursor.executemany(sql, res_data)
service_platform_conn.commit()
cursor.close()
service_platform_conn.close()
self.task.list_task_insert_db_count = _res
self.task.get_data_per_times = len(res_data)
except Exception as e:
self.logger.exception(msg="[mysql error]", exc_info=e)
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
source = self.task.kwargs['source']
t1 = time.time()
error_code, result, page_store_key = GTdetail_to_database(
tid=self.task.task_id,
used_times=self.task.used_times,
source=source,
ticket=self.task.kwargs,
need_cache=self.task.used_times == 0)
t2 = time.time()
self.logger.info('抓取耗时： {}'.format(t2 - t1))
if source == 'ctrip':
my_collections = ctrip_collections
elif source == 'tuniu':
my_collections = tuniu_collections
else:
raise Exception(u'未知的源')
my_collections.save({
'source': source,
'collections': self.task.collection,
'task_id': self.task.task_id,
'used_times': self.task.used_times[0],
'stored_page_keys': page_store_key,
'result': result,
'args': self.task.kwargs,
'insert_time': datetime.datetime.now()
})
t3 = time.time()
self.logger.info('入库耗时： {}'.format(t3 - t2))
# -- detail 2 mysql --
if len(result) > 0:
self.task.error_code = 0
elif len(result) == 0:
self.task.error_code = 29
# elif len(result) == 0:
# raise ServiceStandardError(ServiceStandardError.EMPTY_TICKET)
else:
raise ServiceStandardError(error_code=error_code)
return self.task.error_code, len(result)
def _execute(self, **kwargs):
# init task val
source = self.task.kwargs['source']
source_id = self.task.kwargs['source_id']
target_url = self.task.kwargs['target_url']
bucket_name = self.task.kwargs['bucket_name']
file_prefix = self.task.kwargs['file_prefix']
is_poi_task = self.task.kwargs.get('is_poi_task', True)
need_insert_db = self.task.kwargs.get('need_insert_db', True)
special_file_name = self.task.kwargs.get('special_file_name', '')
# /album/user/2225/43/Q0tXRx4EY00/index/980x576
if 'qyer.com' in target_url and source == 'qyer':
if target_url.endswith('/index'):
target_url += '/980x576'
elif target_url.endswith('/index/'):
target_url += '980x576'
if 'ahstatic.com' in target_url and source == 'accor':
if not target_url.startswith('http://'):
target_url = 'http://' + target_url
if source == 'ihg':
if target_url.endswith('4x3?fmt=png-alpha'):
target_url += '&wid=800&hei=600'
flag = None
h = None
w = None
file_name = ''
with MySession(need_cache=True) as session:
@func_time_logger
def img_file_get():
_page = session.get(target_url, timeout=(10800, 10800))
return _page
page = img_file_get()
f_stream = StringIO(page.content)
if f_stream.len > 10485760:
# 大于 10MB 的图片信息不入库
raise ServiceStandardError(
error_code=ServiceStandardError.IMG_TOO_LARGE)
file_md5 = get_stream_md5(f_stream)
flag, h, w = is_complete_scale_ok(f_stream)
try:
suffix = target_url.rsplit('.', 1)[1]
# 对于 qyer 的图片特殊处理，无文件后缀
if len(suffix) > 16:
suffix = ''
except IndexError as e:
suffix = page.headers['Content-Type'].split('/')[1]
# 无文件后缀名图片直接 md5
if suffix:
file_name = hashlib.md5(target_url).hexdigest() + '.' + suffix
else:
file_name = hashlib.md5(target_url).hexdigest()
if flag in [1, 2]:
raise ServiceStandardError(
error_code=ServiceStandardError.IMG_INCOMPLETE)
else:
# get img p hash
_p_hash = img_p_hash(StringIO(page.content))
# save file stream
r2 = True
if bucket_name != 'mioji-wanle':
r1 = upload_ks_file_stream(bucket_name,
file_name,
StringIO(page.content),
page.headers['Content-Type'],
hash_check=file_md5)
else:
r1 = upload_ks_file_stream(bucket_name,
'{}/'.format(file_prefix) +
file_name,
StringIO(page.content),
page.headers['Content-Type'],
hash_check=file_md5)
if bucket_name == 'mioji-attr':
r2 = upload_ks_file_stream('mioji-shop',
file_name,
StringIO(page.content),
page.headers['Content-Type'],
hash_check=file_md5)
if not (r1 and r2):
raise ServiceStandardError(
ServiceStandardError.IMG_UPLOAD_ERROR)
use_flag = 1 if flag == 0 else 0
size = str((h, w))
# 更新 file name
if special_file_name != '':
file_name = special_file_name
# bucket_name = file_path.split('_')[1] + '_bucket' if is_poi_task else ''
data = (
source, # source
source_id, # source_id
target_url, # pic_url
file_name, # pic_md5
self.task.task_name[-9:], # part
size, # size
use_flag, # poi use , hotel flag
file_md5, # file_md5
bucket_name, # poi rest attr shop
json.dumps({"p_hash":
_p_hash}), # img phash for check duplicate
)
try:
table_name = self.task.task_name
if need_insert_db:
if is_poi_task:
poi_make_kw(data, table_name)
else:
hotel_make_kw(data, table_name)
# 设置标识位
self.task.error_code = 0
except exc.SQLAlchemyError as err:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=err)
except IOError as err:
raise ServiceStandardError(
ServiceStandardError.IMG_UPLOAD_ERROR,
wrapped_exception=err)
# 被过滤的图片返回错误码不为 0
if flag in [3, 4, 5]:
raise ServiceStandardError(ServiceStandardError.IMG_SIZE_FILTER)
self.task.error_code = 0
return flag, h, w, self.task.error_code, bucket_name, file_name, self.task.task_name
def _execute(self, **kwargs):
target_url = self.task.kwargs['target_url']
city_id = self.task.kwargs['city_id']
poi_type = self.task.kwargs['poi_type']
target_url = target_url.replace('.com.hk', '.cn')
with MySession(need_cache=True) as session:
page = session.get(target_url, timeout=120)
page.encoding = 'utf8'
parser = parser_type[poi_type]
result = parser(page.content, target_url, city_id=city_id)
if result == 'Error':
raise ServiceStandardError(ServiceStandardError.PARSE_ERROR)
result['city_id'] = city_id
# result['utime'] = datetime.datetime.now()
sql_key = result.keys()
name = result['name']
# if name.find('停业') > -1:
# raise ServiceStandardError(error_code=ServiceStandardError.TARGET_CLOSED)
name_en = result['name_en']
map_info = result['map_info']
address = result['address']
map_info_is_legal = True
try:
lon, lat = map_info.split(',')
if float(lon) == 0.0 and float(lat) == 0.0:
map_info_is_legal = False
except Exception as e:
map_info_is_legal = False
logger.exception(msg="[map info is not legal]", exc_info=e)
if not key_is_legal(map_info) or not map_info_is_legal:
if not key_is_legal(address):
pass
# raise TypeCheckError(
# 'Error map_info and address NULL with parser %ss url %s' % (
# parser.func_name, target_url))
google_map_info = google_get_map_info(address)
if not key_is_legal(google_map_info):
pass
# raise TypeCheckError(
# 'Error google_map_info NULL with [parser: {}][url: {}][address: {}][map_info: {}]'.format(
# parser.func_name, target_url, address, map_info)
# )
result['map_info'] = google_map_info
if key_is_legal(name) or key_is_legal(
name_en) or map_info_is_legal or key_is_legal(
result.introduction):
logger.info(name + ' ---------- ' + name_en)
else:
raise TypeCheckError(
'Error All Keys is None with parser %s url %s' %
(parser.func_name, target_url))
try:
session = DBSession()
session.execute(
text(
text_2_sql(sql_key).format(
table_name=self.task.task_name)), [result])
session.commit()
session.close()
except Exception as e:
logger.exception(e)
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
self.task.error_code = 0
return self.task.error_code
def _execute(self, **kwargs):
with MySession(need_cache=True, need_proxies=True) as session:
city_id = self.task.kwargs['city_id']
target_url = self.task.kwargs['target_url']
headers = {'Host': 'place.qyer.com'}
page = session.get(target_url, headers=headers, timeout=240)
page.encoding = 'utf8'
content = page.text
if '请输入验证码' in content:
raise Exception("请输入验证码")
result = page_parser(content=content, target_url=target_url)
result.city_id = city_id
name = result.name
name_en = result.name_en
map_info = result.map_info
address = result.address
map_info_is_legal = True
try:
lon, lat = map_info.split(',')
if float(lon) == 0.0 and float(lat) == 0.0:
map_info_is_legal = False
except Exception as e:
map_info_is_legal = False
logger.exception(msg="[map info is not legal]", exc_info=e)
if not key_is_legal(map_info) or not map_info_is_legal:
if not key_is_legal(address):
# todo 临时注释
pass
# raise TypeCheckError(
# 'Error map_info and address NULL with parser %ss url %s' % (
# page_parser.func_name, target_url))
google_map_info = google_get_map_info(address)
if not key_is_legal(google_map_info):
# todo 临时注释
pass
# raise TypeCheckError(
# 'Error google_map_info NULL with [parser: {}][url: {}][address: {}][map_info: {}]'.format(
# page_parser.func_name, target_url, address, map_info)
# )
result.map_info = google_map_info
if key_is_legal(name) or key_is_legal(
name_en) or map_info_is_legal or key_is_legal(
result.introduction):
logger.info(name + ' ---------- ' + name_en)
else:
# raise TypeCheckError(
# 'Error name and name_en Both NULL with parser %s url %s' % (
# page_parser.func_name, target_url))
raise TypeCheckError("All Available Key is Null")
sql_result = result.__dict__
sql_key = sql_result.keys()
if '_sa_instance_state' in sql_key:
sql_key.remove('_sa_instance_state')
try:
session = DBSession()
session.execute(
text(
text_2_sql(sql_key).format(
table_name=self.task.task_name)), [sql_result])
session.commit()
session.close()
except Exception as e:
self.logger.exception(msg="[mysql exec err]", exc_info=e)
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
self.task.error_code = 0
return self.task.error_code
def _execute(self, **kwargs):
with MySession(need_proxies=True, need_cache=True) as session:
keyword = self.task.kwargs['keyword']
suggest = {}
try:
response = session.get(url=search_url.format(keyword),
headers=headers)
#response = requests.get(search_url.format(keyword))
res = response.content
root = html.fromstring(res.decode('utf-8'))
dests = root.xpath("//div[@class='breadbar_v1 cf']/ul/li")
dest = ''
try:
for de in dests[2:-1]:
if dest != '':
dest += '|'
dest += de.xpath("a/text()")[0]
except:
pass
print dest
tag = {}
try:
tags = root.xpath("//ul[@class='map_tab cf']/li")
for ta in tags:
t = ta.xpath('a/span/text()')[0]
tt = ta.xpath('a/text()')[-1].strip()
tag[t] = tt
except:
pass
print tag
map_info = ''
try:
map_info = re.findall('centerGeo: ({.+})',
res)[0].replace('\'', '\"')
except:
pass
print map_info
db = client['SuggestName']
db.CtripPoiSDK_detail.save({
'name':
self.task.kwargs['name'],
'dest_name':
self.task.kwargs['dest_name'],
'keyword':
keyword,
'dest':
dest,
'tag_info':
tag,
'map_info':
map_info
})
self.task.error_code = 0
return 'OK'
except requests.exceptions.RequestException as e:
raise ServiceStandardError(ServiceStandardError.PROXY_INVALID,
wrapped_exception=e)
except pymongo.errors.PyMongoError as e:
raise ServiceStandardError(ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
except Exception as e:
raise ServiceStandardError(ServiceStandardError.UNKNOWN_ERROR,
wrapped_exception=e)
def _execute(self, **kwargs):
url = self.task.kwargs['url']
source = self.task.kwargs['source']
source_id = self.task.kwargs['source_id']
city_id = self.task.kwargs['city_id']
country_id = self.task.kwargs['country_id']
hid = self.task.kwargs['hid']
headers = {}
other_info = {'source_id': source_id, 'city_id': city_id}
if source in ['starwood', 'hyatt', 'gha', 'shangrila', 'fourseasons']:
error_code, res, page_store_key_list = hotel_detail_database(
url, source)
if error_code == 0:
result = parse_hotel_info(res)
else:
raise ServiceStandardError(error_code=error_code)
else:
with MySession(need_cache=True) as session:
# booking start
if source == 'booking':
headers['Referer'] = 'http://www.booking.com'
# booking end
session.headers.update(headers)
start = time.time()
if source not in ('hilton', 'ihg', 'holiday', 'accor',
'marriott'):
page = session.get(url, timeout=240)
page.encoding = 'utf8'
content = page.text
elif source == 'ihg':
url1, url2 = url.split('#####')
page1 = session.get(url1, timeout=240)
page1.encoding = 'utf8'
content1 = page1.text
page2 = session.get(url2, timeout=240)
page2.encoding = 'utf8'
content2 = page2.text
content = [content1, content2]
elif source == 'holiday':
url2, url1 = url.split('#####')
page1 = requests.get(
url1,
headers={
'x-ihg-api-key':
'se9ym5iAzaW8pxfBjkmgbuGjJcr3Pj6Y',
'ihg-language': 'zh-CN'
},
timeout=240)
page1.encoding = 'utf8'
content1 = page1.text
page2 = requests.get(
url2,
timeout=240,
headers={
'accept': 'application/json, text/plain, */*',
'Content-Type': 'application/json; charset=UTF-8',
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
'ihg-language': 'zh-CN',
})
page2.encoding = 'utf8'
content2 = page2.text
page3 = requests.get(url1,
headers={
'x-ihg-api-key':
'se9ym5iAzaW8pxfBjkmgbuGjJcr3Pj6Y'
},
timeout=240)
page3.encoding = 'utf8'
content3 = page3.text
content = (content1, content2, content3)
elif source == 'accor':
proxy_url = "http://10.10.239.46:8087/proxy?source=pricelineFlight&user=crawler&passwd=spidermiaoji2014"
r = requests.get(proxy_url)
proxies = {'https': "socks5://" + str(r.text)}
headers = {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"
}
page = requests.get(url,
headers=headers,
verify=False,
proxies=proxies)
page.encoding = 'utf8'
content = page.text
elif source == 'marriott':
url_list = url.split('#####')
url = url_list[0]
for i in url_list:
if len(i.split('=')) > 1:
key, value = i.split('=')[0], i.split('=')[1]
if key == 'longtitude':
other_info['longtitude'] = value
if key == 'latitude':
other_info['latitude'] = value
else:
if url_list.index(i) == 1:
other_info['hotel_name_en'] = i
url2 = url.replace("travel", "hotel-photos")
url3 = url.replace("travel/", "maps/travel/")
url4 = url.replace("hotels/", "hotels/fact-sheet/")
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/47.0'
}
if "https://www.marriott.com" in url:
page1 = requests.get(url, headers=headers, timeout=240)
page2 = requests.get(url2,
headers=headers,
timeout=240)
page3 = requests.get(url3,
headers=headers,
timeout=240)
page4 = requests.get(url4,
headers=headers,
timeout=240)
page1.encoding = 'utf8'
page2.encoding = 'utf8'
page3.encoding = 'utf8'
page4.encoding = 'utf8'
content1 = page1.text
content2 = page2.text
content3 = page3.text
content4 = page4.text
content = (content1, content2, content3, content4)
else:
url2 = url + "/hotel-overview"
page1 = requests.get(url, headers=headers, timeout=240)
page2 = requests.get(url2,
headers=headers,
timeout=240)
page1.encoding = 'utf8'
page2.encoding = 'utf8'
content1 = page1.text
content2 = page2.text
content = (content1, content2)
else:
session.auto_update_host = False
hilton_index = url.find('index.html')
if hilton_index > -1:
url = url[:hilton_index]
split_args = url.split('/')
detail_url = 'http://www3.hilton.com/zh_CN/hotels/{0}/{1}/popup/hotelDetails.html'.format(
split_args[-3], split_args[-2])
map_info_url = url + 'maps-directions.html'
desc_url = url + 'about.html'
page = session.get(url)
map_info_page = session.get(map_info_url)
desc_page = session.get(desc_url)
detail_page = session.get(detail_url, )
page.encoding = 'utf8'
detail_page.encoding = 'utf8'
map_info_page.encoding = 'utf8'
desc_page.encoding = 'utf8'
__content = page.text
logger.info(detail_url)
__detail_content = detail_page.text
__map_info_content = map_info_page.text
__desc_content = desc_page.text
content = [
__content, __detail_content, __map_info_content,
__desc_content
]
logger.debug("[crawl_data][Takes: {}]".format(time.time() -
start))
start = time.time()
result = parse_hotel(content=content,
url=url,
other_info=other_info,
source=source,
part=self.task.task_name,
retry_count=self.task.used_times)
logger.debug("[parse_hotel][func: {}][Takes: {}]".format(
parse_hotel.func_name,
time.time() - start))
try:
data_collections = mongo_data_client['ServicePlatform'][
self.task.task_name]
data_collections.create_index([('source', 1), ('source_id', 1)],
unique=True,
background=True)
data_collections.create_index([('location', '2dsphere')],
background=True)
tmp_result = deepcopy(result.values(backdict=True))
lon, lat = str(result.map_info).split(',')
lon, lat = float(lon), float(lat)
tmp_result.update(
{'location': {
'type': "Point",
'coordinates': [lon, lat]
}})
data_collections.save(tmp_result)
except pymongo.errors.DuplicateKeyError:
# logger.exception("[result already in db]", exc_info=e)
logger.warning("[result already in db]")
except Exception as exc:
raise ServiceStandardError(
error_code=ServiceStandardError.MONGO_ERROR,
wrapped_exception=exc)
start = time.time()
try:
service_platform_conn = service_platform_pool.connection()
cursor = service_platform_conn.cursor()
others_info = json.loads(result.others_info)
others_info['hid'] = hid
result.others_info = json.dumps(others_info)
sql = result.generation_sql()
sql = sql.format(table_name=self.task.task_name)
values = result.values()
self.logger.info(result.__dict__)
cursor.execute(sql, values)
service_platform_conn.commit()
cursor.close()
service_platform_conn.close()
except Exception as e:
logger.exception(e)
raise ServiceStandardError(
error_code=ServiceStandardError.MYSQL_ERROR,
wrapped_exception=e)
logger.debug("[Insert DB][Takes: {}]".format(time.time() - start))
self.task.error_code = 0
return self.task.error_code
def _execute(self, **kwargs):
with MySession(need_cache=True,need_proxies=True) as session:
try:
keyword = self.task.kwargs['keyword']
source = self.task.kwargs['source']
map_info = self.task.kwargs['map_info']
country_id = self.task.kwargs['country_id']
city_id = self.task.kwargs['city_id']
database_name = self.task.kwargs['database_name']
local_time = urllib.unquote(datetime.datetime.now(pytz.timezone(pytz.country_timezones('cn')[0])).strftime(
'%a %b %d %Y %H:%M:%S GMT+0800 (%Z)'))
if source in 'agoda':
url = source_interface[source].format(keyword,local_time)
header = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'zh-CN,zh;q=0.9',
'accept': 'application/json, text/javascript, */*; q=0.01',
'referer': 'https://www.agoda.com/zh-cn/',
'authority': 'www.agoda.com',
'x-requested-with': 'XMLHttpRequest'
}
response = session.get(url=url,headers=header)
get_suggest = getattr(sys.modules[__name__], 'get_{0}_suggest'.format(source))
elif source in 'daodao':
headers = {
'referer': 'https://www.tripadvisor.cn/',
'x-requested-with': 'XMLHttpRequest',
'accept-encoding': 'gzip, deflate, br',
'accept': 'text/javascript, text/html, application/xml, text/xml, */*',
'accept-language': 'zh-CN,zh;q=0.9',
'Origin': 'https://www.tripadvisor.cn',
'Host': 'www.tripadvisor.cn'
}
url = source_interface[source]
response = session.post(
url=url,
headers=headers,
data={
'action': 'API',
'uiOrigin': 'PTPT-dest',
'types': 'geo,dest',
'hglt': True,
'global': True,
'legacy_format': True,
'_ignoreMinCount': True,
'query': keyword
}
)
get_suggest = getattr(sys.modules[__name__], 'get_{0}_suggest'.format(source))
elif source in 'qyer':
headers = {
"Referer": "http://www.qyer.com/",
"Host": "www.qyer.com",
}
url = source_interface[source].format(keyword)
response = session.get(url,headers=headers)
get_suggest = getattr(sys.modules[__name__], 'get_{0}_suggest'.format(source))
elif source in 'ctrip':
headers = {
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Referer': 'http://hotels.ctrip.com/international/',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
'Connection': 'keep-alive'
}
url = source_interface[source].format(keyword)
response = session.get(url, headers=headers)
get_suggest = getattr(sys.modules[__name__], 'get_{0}_suggest'.format(source))
else:
url = source_interface[source].format(keyword)
response = session.get(url=url,)
get_suggest = getattr(sys.modules[__name__],'get_{0}_suggest'.format(source))
count = get_suggest(response.content,map_info,country_id,city_id,database_name,keyword)
if count >= 0:
self.task.error_code = 0
except Exception as e:
print(e)
raise ServiceStandardError(ServiceStandardError.REQ_ERROR,wrapped_exception=e)
return count