def parse(self, task):
#解析test，提取出字段
content = task.content
source = task.source
#数据表名，每天日期
today = str(datetime.datetime.now()).split(' ')[0].replace('-', '')
#table_name = today
table_name = ''
#任务开始时间
stime = time.time()
update_time = time.strftime('%Y-%m-%dT%H:%M:%S',
time.localtime(time.time()))
logger.info('feiquanqiuFlight: start a new task @ %s' %
str(update_time))
#初始化参数
para = []
error = ERR_CODE
#如果失败，重复抓取的次数
for i in range(2):
#返回值是一个dict，{'para':[(),()],'error':0}
result = feiquanqiu_task_parser(content)
try:
para = result['para']
flights = para['flight']
tickets = para['ticket']
error = result['error']
except Exception, e:
logger.error('feiquanqiuFlight error: Wrong Result Format %s' %
str(e))
return error
if tickets == None or tickets == []:
logger.info(
'feiquanqiuFlight: task failed with %s for %sth time' %
(content, i))
time.sleep(random.randint(1, 2))
continue
else:
try:
InsertFlight(tickets)
etime = time.time() #任务完成时间
dur = int((etime - stime) * 1000)
logger.info('task finish with %s using %d ms' %
(content, dur))
return error
except Exception, e:
logger.error('feiquanqiuFlight: Insertation Error: %s' %
str(e))
error = DB_ERR_CODE
return error
def parse(self, task):
#解析test，提取出字段
content = task.content
source = task.source
#任务开始时间
stime = time.time()
update_time = time.strftime('%Y-%m-%dT%H:%M:%S',
time.localtime(time.time()))
logger.info('expediaFlightFlight: start a new task @ %s' %
str(update_time))
#初始化参数
para = None
error = ERR_CODE
#如果失败，重复抓取的次数
for i in range(2):
#返回值是一个dict，{'para':['ticket':[], 'flight':{ }],'error':0}
result = expedia_task_parser(content)
try:
para = result['para']
error = result['error']
tickets = para['ticket']
flights = para['flight']
except Exception, e:
logger.error('expediaFlight error: Wrong Result Format %s' %
str(e))
return error
if tickets == []:
time.sleep(random.randint(1, 2))
continue
else:
try:
InsertFlight(tickets)
etime = time.time() #任务完成时间
dur = int((etime - stime) * 1000)
logger.info(
'expediaFlight: task finish with %s using %d ms' %
(content, dur))
return error
except Exception, e:
logger.error('expediaFlight: Insertation Error: %s' %
str(e))
error = DB_ERR_CODE
return error