def __init__(self, message = None, status = None, http_status_code = None):
super().__init__(message, status)
self.message = message
self.status = status
self.http_status_code = http_status_code
log.error("an raise happen with message: {}, status: {}, http_status_code: {}".format(message, str(status), str(http_status_code)))
def remove_session(cls):
try:
cls.get_session().close()
except Exception as e:
log.error(str(e))
finally:
cls.get_instance()._session_factory.remove()
def load_or_create(cls, poiId, frontImg, title, avgScore, allCommentNum,
address, avgPrice, dealList):
meishi_obj = cls.by_id(poiId)
if meishi_obj:
return meishi_obj
meishi_obj = cls(
poiId=poiId,
frontImg=frontImg,
title=title,
avgScore=avgScore,
allCommentNum=allCommentNum,
address=address,
avgPrice=avgPrice,
dealList=dealList,
)
try:
Database.add(meishi_obj)
Database.commit()
return meishi_obj
#TODO(weiming liu) cache sqlalchemy for not cache base exception
except Exception as e:
log.error(str(e))
Database.rollback()
# TODO(weiming liu) raise error for not return None
return None
def handle_one_page(cls, one_page):
try:
url = one_page.xpath('article/div/div/a/@href')[0]
title = one_page.xpath('article/div/div/a/div/h4/text()')[0]
hash_id = HashUtil.get_hash_by_string(url)
Programming.load_or_create(hash_id, title, format_url(url))
except Exception as e:
log.error(str(e))
def load_or_create(cls, dianying_list, is_onshow):
try:
if is_onshow:
DianYingCache.set_onshow_dianying(dianying_list)
else:
DianYingCache.set_upcoming_dianying(dianying_list)
except Exception as e:
log.error(str(e))
raise
def load_or_create(cls, hash_id, title, url):
programming_obj = cls.by_hash_id(hash_id)
if programming_obj:
return programming_obj
programming_obj = cls(
hash_id=hash_id,
title=title,
url=url,
)
try:
Database.add(programming_obj)
Database.commit()
return programming_obj
#TODO(weiming liu) cache sqlalchemy for not cache base exception
except Exception as e:
Database.rollback()
log.error(str(e))
# TODO(weiming liu) raise error for not return None
return None
def load_or_create(cls, username, password):
user_obj = cls.by_name(username)
if user_obj:
if user_obj.password == password:
return user_obj
else:
raise ArgsError('unmatch password')
user_obj = cls(
username=username,
password=password,
)
try:
Database.add(user_obj)
Database.commit()
return user_obj
#TODO(weiming liu) cache sqlalchemy error for not cache base exception
except Exception as e:
log.error(str(e))
Database.rollback()
raise DatabaseError('can not create user {username}'.format(username=username))
def handle_upcoming_one_item(cls, one_item):
# print(etree.tostring(one_item).decode('utf-8'))
try:
url = one_item.xpath('a/@href')[0]
img = one_item.xpath('a/img/@src')[0]
title = one_item.xpath('div/h3/a/text()')[0]
detail_list = one_item.xpath('div/ul/li')
onshow_time, region, score = detail_list[0].xpath('text()')[0], detail_list[2].xpath('text()')[0], detail_list[3].xpath('span/text()')[0]
buy_ticket = one_item.xpath('div/ul/a/@href')
dianying_obj = {
'fileId': ReUtil.get_all_number_to_list(url)[0],
'img': img,
'title': title,
'score': score[:-3],
'region': region,
'url': url,
'buyTicket': buy_ticket,
'onShowTime': onshow_time,
}
return dianying_obj
except Exception as e:
log.error("crawler upcoming file list error " + repr(e))
def handle_onshow_one_item(cls, one_item):
# print(etree.tostring(one_item).decode('utf-8'))
# html = etree.HTML(etree.tostring(one_item).decode('utf-8'))
try:
file_id = one_item.xpath('@id')[0]
title = one_item.xpath('@data-title')[0]
score = one_item.xpath('@data-score')[0]
region = one_item.xpath('@data-region')[0]
img = one_item.xpath('ul/li/a/img/@src')[0]
url, buy_ticket = one_item.xpath('ul/li/a/@href')[0], one_item.xpath('ul/li/a/@href')[2]
dianying_obj = {
'fileId': file_id,
'img': img,
'title': title,
'score': score,
'region': region,
'url': url,
'buyTicket': buy_ticket,
'onShowTime': None,
}
return dianying_obj
except Exception as e:
log.error("crawler onshow file list error " + repr(e))