class CommitResult(threading.Thread):
def __init__(self,queue,flag = 0,logfile = 'crawler_database.log'):
threading.Thread.__init__(self)
self.queue = queue
self.flag = flag
self.database = DatabaseOperator(logger = set_log(logfile))
self.start()
def set_flag(self):
self.flag = 1
def commit(self):
while True:
if self.flag:
break
if self.queue.empty():
time.sleep(5)
result = self.queue.get()
try:
self.database.insert_url(result[0],result[1])
except:
self.database.logger.error(result[0] + 'commit failed.')
self.queue.task_done()
def run(self):
self.commit()
def submit_result(self):
dbOp = DatabaseOperator(logger = self.logger, dbfile = self.dbfile)
while True:
if self.task_queue.empty() and self.result_queue.empty():
break
elif not self.result_queue.empty():
#print 'result: ',self.result_queue.qsize()
res = self.result_queue.get()
dbOp.insert_item(res[0],res[1])
self.result_queue.task_done()
elif not self.task_queue.empty():
time.sleep(10)
dbOp.logger.info('************submit thread is existing.*****************')
# recommendation, flag = currentMeeting.recommend()
# if recommendation == {}:
# print("No recommendation available, please try some other time.")
# elif flag == 0:
# print("The recommended rooms are:")
# for site_id in recommendation:
# print(site_id+':')
# for room_key in recommendation[str(site_id)]:
# print(room_key, end=' ')
# print('')
# elif flag == 1:
# print("We have lowered the capacity to schedule the meeting.\n")
# print("The recommended rooms are:")
# for site_id in recommendation:
# print(site_id+':')
# for room_key in recommendation[str(site_id)]:
# print(room_key, end=' ')
# print('')
# DatabaseOperator.DatabaseOperator().selection_list(3)
# DatabaseOperator.DatabaseOperator().selection_list_history('meeting_21_15324409580000')
print(DatabaseOperator.DatabaseOperator().fetch_meeting_history(1))
# print(DatabaseOperator.convert_back(35))
# db_test()
# new_room = MeetingRoom.MeetingRoom("room_1")
# print(new_room.set_schedule('meeting_158_1532440958', 3, 8))
# db_test()
user = "******"
password = "******"
esip = "es-cn-zz11nl9y20001gczg.public.elasticsearch.aliyuncs.com"
esport = 9200
#esip = "43.247.184.94"
#esport = 9200
index_name = "bigdata_ic_gsb_change_1"
index_type = "_doc"
esuser = "******"
espassword = "******"
es = Elasticsearch([esip], http_auth=(esuser, espassword), port=esport)
# es = Elasticsearch([esip], port=esport)
sql = "select * from company_change_info_all_copy2"
result = do.query(ip, user, password, database, port, sql)
# print(result)
if result != "error":
bulkList = []
for re in result:
try:
isTrue, action = DataOper(re)
if isTrue:
bulkList.append(action)
else:
f1.write("获取" + str(re) + "的数据出错！\r\n")
if len(bulkList) == 1000:
try:
bulk(es,
bulkList,
index=index_name,
def __init__(self,queue,flag = 0,logfile = 'crawler_database.log'):
threading.Thread.__init__(self)
self.queue = queue
self.flag = flag
self.database = DatabaseOperator(logger = set_log(logfile))
self.start()