def save_result(data): try: del data['item_id'] except Exception as e: # print e pass try: result_file = get_result_name(plantform_e='toutiao', plantform_c='今日头条', date_time=data['publish_time'], urlOruid=data['url'], newsidOrtid=data['id'], datatype='news', full_data=data) if not result_file: return print datetime.datetime.now(), '--------', result_file save_data_to_mongodb(data={'data': data}, item_id=result_file, platform_e='toutiao', platform_c='今日头条', cache_data_list=self.cache_data_list) # producer.send(topic='1101_STREAM_SPIDER', value={'data': data}, key=result_file, updatetime=data['spider_time']) pass # host = '192.168.6.187:9092,192.168.6.188:9092,192.168.6.229:9092,192.168.6.230:9092' # producer = Producer(hosts=host) # result_file = get_result_name(plantform_c='今日头条',plantform_e='JinRiTouTiao', date_time=data['publish_time'], urlOruid=data['url'], # newsidOrtid=data['id'], # datatype='news', full_data=data) # # producer.send(topic='topic', value={'data': data}, key=result_file, updatetime=data['spider_time']) # comsumer = Consumer('topic', host, 'll') # what = comsumer.poll() # # for i in comsumer.poll(): # # print i.topic # for i in what: # topic = i.topic # partition = i.partition # offset = i.offset # key = i.key # value = i.value # Save_result(plantform='toutiao', date_time=data['publish_time'], urlOruid=data['url'], # newsidOrtid=data['id'], datatype='news', full_data=data) except Exception as e: # print e pass
def save_result(data): result_file = get_result_name(plantform_e='chengshiluntan', plantform_c='城市论坛', date_time=data['publish_time'], urlOruid=data['url'], newsidOrtid=data['id'], datatype='forum', full_data=data) if not result_file: return print datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '--------', result_file data['spider_time']=datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') save_data_to_mongodb(data={'data': data}, platform_c='城市论坛', platform_e='chengshiluntan', item_id=result_file,cache_data_list=self.cache_data_list)
def save_result(data): try: del data['is_movie'] except Exception as e: print e result_file = get_result_name(plantform_e='PengPai', plantform_c='澎湃新闻', date_time=data['publish_time'], urlOruid=data['url'], newsidOrtid=data['id'], datatype='news', full_data=data) if not result_file: return print datetime.now(),'---',result_file save_data_to_mongodb(data={'data':data},item_id=result_file,platform_c='澎湃新闻',platform_e='PengPai',cache_data_list=self.cache_data_list)
def save_result(data): result_file = get_result_name(plantform_e='xjbtssbtszhdj', plantform_c='第十师北屯市智慧党建', date_time=data['publish_time'], urlOruid=data['url'], newsidOrtid=data['id'], datatype='news', full_data=data) if not result_file: return print datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S'), '--------', result_file data['spider_time'] = datetime.datetime.now().strftime( '%Y-%m-%d %H:%M:%S') save_data_to_mongodb(data={'data': data}, item_id=result_file, platform_e='xjbtssbtszhdj', platform_c='第十师北屯市智慧党建', cache_data_list=self.cache_data_Queue)
def saveData(content, filename, platform_e, platform_c, news_type): #数据保存 host = '182.150.63.40' port = '12308' username = '******' password = '******' content = json.loads(content) # producer = Producer(hosts=host) producer = RemoteProducer(host=host, port=port, username=username, password=password) result_file = get_result_name(plantform_e=platform_e, plantform_c=platform_c, date_time=content['data']['publish_time'], urlOruid=content['data']['url'], newsidOrtid=content['data']['id'], datatype=news_type, full_data=content) # pass save_data_to_mongodb(data=content, item_id=result_file, platform_e=platform_e, platform_c=platform_c)