예제 #1
0
    def parse(self, task):
        newTasks = []
        ret, status = self.parseContent(task['__data'])
        if status == 'OK':
            #dump list
            keyOutputPath = iPapa.iTsOutputPath
            outputJsonLoc = os.path.join(keyOutputPath,
                                         'hk_gem_stock_list_cn.json')
            outputTxtLoc = os.path.join(keyOutputPath,
                                        'hk_gem_stock_list_cn.txt')
            if util.dump2JsonFile(ret['stockList'], outputJsonLoc) != True:
                task.status = 'failed'

            if util.dump2TxtFile(ret['stockList'], outputTxtLoc) != True:
                task.status = 'failed'

            for k in ret['stockInfoPage']:
                if k not in self.historyKeys:
                    newT = Task(-1,
                                url=ret['stockInfoPage'][k],
                                handler='HkStockInfoCnPageHandler',
                                ref=task.url)
                    newT['key'] = "hk_stock_info_" + k
                    newTasks.append(newT)
        else:
            task.status = 'failed'
        if newTasks != []:
            return {'newTasks': newTasks}
        return {}
    def parse(self, task):
        print "HkStockInfoCnPageHandler parse", task.url, task['key']
        ret, status = self.parseContent(task['__data'])
        if status == 'OK':
            key = task['key']
            keyOutputPath = iPapa.iTsOutputPath

            outputJsonLoc = os.path.join(keyOutputPath, 'hk_stock_info_cn_'+key+'.json')
            outputTxtLoc = os.path.join(keyOutputPath, 'hk_stock_info_cn_'+key+'.txt')

            if util.dump2JsonFile(ret, outputJsonLoc) != True:
                task.status = 'failed'

            if util.dump2TxtFile(zip(ret.keys(), ret.values()), outputTxtLoc) != True:
                task.status = 'failed'
        
        else:
            task.status = 'failed'
        if task.status == 'ignore': 
            return {}
        return {}
예제 #3
0
    def parse(self, task):
        print "HkStockInfoCnPageHandler parse", task.url, task['key']
        ret, status = self.parseContent(task['__data'])
        if status == 'OK':
            key = task['key']
            keyOutputPath = iPapa.iTsOutputPath

            outputJsonLoc = os.path.join(keyOutputPath,
                                         'hk_stock_info_cn_' + key + '.json')
            outputTxtLoc = os.path.join(keyOutputPath,
                                        'hk_stock_info_cn_' + key + '.txt')

            if util.dump2JsonFile(ret, outputJsonLoc) != True:
                task.status = 'failed'

            if util.dump2TxtFile(zip(ret.keys(), ret.values()),
                                 outputTxtLoc) != True:
                task.status = 'failed'

        else:
            task.status = 'failed'
        if task.status == 'ignore':
            return {}
        return {}
예제 #4
0
    def parse(self, task):
        newTasks = []
        ret, status = self.parseContent(task['__data'])
        if status == 'OK':
            #dump list
            keyOutputPath = iPapa.iTsOutputPath
            outputJsonLoc = os.path.join(keyOutputPath, 'hk_main_stock_list_cn.json')
            outputTxtLoc = os.path.join(keyOutputPath, 'hk_main_stock_list_cn.txt')
            if util.dump2JsonFile(ret['stockList'], outputJsonLoc) != True:
                task.status = 'failed'

            if util.dump2TxtFile(ret['stockList'], outputTxtLoc) != True:
                task.status = 'failed'

            for k in ret['stockInfoPage']:
                if k not in self.historyKeys:
                    newT = Task(-1, url=ret['stockInfoPage'][k], handler='HkStockInfoCnPageHandler', ref=task.url)  
                    newT['key'] = "hk_stock_info_" + k
                    newTasks.append(newT)
        else:
            task.status = 'failed'
        if newTasks != []:
            return {'newTasks': newTasks}
        return {}