def parse(self, task):
newTasks = []
ret, status = self.parseContent(task['__data'])
if status == 'OK':
#dump list
keyOutputPath = iPapa.iTsOutputPath
outputJsonLoc = os.path.join(keyOutputPath,
'hk_gem_stock_list_cn.json')
outputTxtLoc = os.path.join(keyOutputPath,
'hk_gem_stock_list_cn.txt')
if util.dump2JsonFile(ret['stockList'], outputJsonLoc) != True:
task.status = 'failed'
if util.dump2TxtFile(ret['stockList'], outputTxtLoc) != True:
task.status = 'failed'
for k in ret['stockInfoPage']:
if k not in self.historyKeys:
newT = Task(-1,
url=ret['stockInfoPage'][k],
handler='HkStockInfoCnPageHandler',
ref=task.url)
newT['key'] = "hk_stock_info_" + k
newTasks.append(newT)
else:
task.status = 'failed'
if newTasks != []:
return {'newTasks': newTasks}
return {}
def parse(self, task):
print "HkStockInfoCnPageHandler parse", task.url, task['key']
ret, status = self.parseContent(task['__data'])
if status == 'OK':
key = task['key']
keyOutputPath = iPapa.iTsOutputPath
outputJsonLoc = os.path.join(keyOutputPath, 'hk_stock_info_cn_'+key+'.json')
outputTxtLoc = os.path.join(keyOutputPath, 'hk_stock_info_cn_'+key+'.txt')
if util.dump2JsonFile(ret, outputJsonLoc) != True:
task.status = 'failed'
if util.dump2TxtFile(zip(ret.keys(), ret.values()), outputTxtLoc) != True:
task.status = 'failed'
else:
task.status = 'failed'
if task.status == 'ignore':
return {}
return {}
def parse(self, task):
print "HkStockInfoCnPageHandler parse", task.url, task['key']
ret, status = self.parseContent(task['__data'])
if status == 'OK':
key = task['key']
keyOutputPath = iPapa.iTsOutputPath
outputJsonLoc = os.path.join(keyOutputPath,
'hk_stock_info_cn_' + key + '.json')
outputTxtLoc = os.path.join(keyOutputPath,
'hk_stock_info_cn_' + key + '.txt')
if util.dump2JsonFile(ret, outputJsonLoc) != True:
task.status = 'failed'
if util.dump2TxtFile(zip(ret.keys(), ret.values()),
outputTxtLoc) != True:
task.status = 'failed'
else:
task.status = 'failed'
if task.status == 'ignore':
return {}
return {}
def parse(self, task):
newTasks = []
ret, status = self.parseContent(task['__data'])
if status == 'OK':
#dump list
keyOutputPath = iPapa.iTsOutputPath
outputJsonLoc = os.path.join(keyOutputPath, 'hk_main_stock_list_cn.json')
outputTxtLoc = os.path.join(keyOutputPath, 'hk_main_stock_list_cn.txt')
if util.dump2JsonFile(ret['stockList'], outputJsonLoc) != True:
task.status = 'failed'
if util.dump2TxtFile(ret['stockList'], outputTxtLoc) != True:
task.status = 'failed'
for k in ret['stockInfoPage']:
if k not in self.historyKeys:
newT = Task(-1, url=ret['stockInfoPage'][k], handler='HkStockInfoCnPageHandler', ref=task.url)
newT['key'] = "hk_stock_info_" + k
newTasks.append(newT)
else:
task.status = 'failed'
if newTasks != []:
return {'newTasks': newTasks}
return {}