class CommitResult(threading.Thread): def __init__(self,queue,flag = 0,logfile = 'crawler_database.log'): threading.Thread.__init__(self) self.queue = queue self.flag = flag self.database = DatabaseOperator(logger = set_log(logfile)) self.start() def set_flag(self): self.flag = 1 def commit(self): while True: if self.flag: break if self.queue.empty(): time.sleep(5) result = self.queue.get() try: self.database.insert_url(result[0],result[1]) except: self.database.logger.error(result[0] + 'commit failed.') self.queue.task_done() def run(self): self.commit()
def submit_result(self): dbOp = DatabaseOperator(logger = self.logger, dbfile = self.dbfile) while True: if self.task_queue.empty() and self.result_queue.empty(): break elif not self.result_queue.empty(): #print 'result: ',self.result_queue.qsize() res = self.result_queue.get() dbOp.insert_item(res[0],res[1]) self.result_queue.task_done() elif not self.task_queue.empty(): time.sleep(10) dbOp.logger.info('************submit thread is existing.*****************')
# recommendation, flag = currentMeeting.recommend() # if recommendation == {}: # print("No recommendation available, please try some other time.") # elif flag == 0: # print("The recommended rooms are:") # for site_id in recommendation: # print(site_id+':') # for room_key in recommendation[str(site_id)]: # print(room_key, end=' ') # print('') # elif flag == 1: # print("We have lowered the capacity to schedule the meeting.\n") # print("The recommended rooms are:") # for site_id in recommendation: # print(site_id+':') # for room_key in recommendation[str(site_id)]: # print(room_key, end=' ') # print('') # DatabaseOperator.DatabaseOperator().selection_list(3) # DatabaseOperator.DatabaseOperator().selection_list_history('meeting_21_15324409580000') print(DatabaseOperator.DatabaseOperator().fetch_meeting_history(1)) # print(DatabaseOperator.convert_back(35)) # db_test() # new_room = MeetingRoom.MeetingRoom("room_1") # print(new_room.set_schedule('meeting_158_1532440958', 3, 8)) # db_test()
user = "******" password = "******" esip = "es-cn-zz11nl9y20001gczg.public.elasticsearch.aliyuncs.com" esport = 9200 #esip = "43.247.184.94" #esport = 9200 index_name = "bigdata_ic_gsb_change_1" index_type = "_doc" esuser = "******" espassword = "******" es = Elasticsearch([esip], http_auth=(esuser, espassword), port=esport) # es = Elasticsearch([esip], port=esport) sql = "select * from company_change_info_all_copy2" result = do.query(ip, user, password, database, port, sql) # print(result) if result != "error": bulkList = [] for re in result: try: isTrue, action = DataOper(re) if isTrue: bulkList.append(action) else: f1.write("获取" + str(re) + "的数据出错!\r\n") if len(bulkList) == 1000: try: bulk(es, bulkList, index=index_name,
def __init__(self,queue,flag = 0,logfile = 'crawler_database.log'): threading.Thread.__init__(self) self.queue = queue self.flag = flag self.database = DatabaseOperator(logger = set_log(logfile)) self.start()