def main(): log_fd = open('log/log', 'a') loginfo = logUtils.logutil(0) task_list = {} while True: db = pymysql.connect(database_host,database_user,database_pass,database_data) cursor = db.cursor() time.sleep(2) running_id = get_running_id(cursor,loginfo); if (running_id != -1): continue for (k, v) in task_list.items(): if (v.poll() != None): del task_list[k] mission_id = get_my_id(cursor,loginfo) loginfo.log_info('mission_id'+str(mission_id)) if mission_id is not -1: loginfo.log_info("task start") child = subprocess.Popen(['/usr/bin/python', 'fydiff.py','%d' % mission_id], shell = False, stdout = log_fd, stderr = log_fd, cwd=autoqps_path) task_list[mission_id] = child cancel_id = get_cancel_id(cursor,loginfo) if cancel_id is -1: continue if cancel_id in task_list: task_list[cancel_id].send_signal(10) loginfo.log_info('cancel:'+str(task_list)) else: sql = "UPDATE {_table} set status = 5 WHERE id = {_cancel_id}".format(_table=database_table,_cancel_id=cancel_id) cursor.execute(sql) try: db.commit() except: db.rollback()
def update_errorlog(log): logstr = logUtils.logutil(mission_id) # print(log.replace('\n', '')) log = log.replace("'", "\\'") sql = "UPDATE %s set errorlog=CONCAT(errorlog, '%s') where id=%d;" % (database_table, log, mission_id) cursor.execute(sql) data = cursor.fetchone() logstr.log_info(str(mission_id) + "\t" + log) try: db.commit() except: logstr.log_debug("error") return data
def main(): log_fd = open('log/log', 'a') loginfo = logUtils.logutil(0) while True: db = pymysql.connect(database_host, database_user, database_pass, database_data) cursor = db.cursor() time.sleep(2) running_id = get_running_id(cursor, loginfo); if (running_id != -1): continue mission_id = get_my_id(cursor, loginfo) loginfo.log_info('mission_id' + str(mission_id)) if mission_id is not -1: loginfo.log_info("task start") child = subprocess.Popen(['/search/odin/daemon/.venv/bin/python', 'run.py', '%d' % mission_id], shell=False,stdout=log_fd, stderr=log_fd, cwd=auto_path)
def main(): log_info = logUtils.logutil('qo') while True: db = pymysql.connect(database_host, database_user, database_pass, database_data) cursor = db.cursor() mission_id = check_new_task(cursor) if mission_id is -1: time.sleep(1) log_info.log_info('There is no task') continue ip = get_node(cursor) log_info.log_info('getnodeip' + ip) if ip is "": time.sleep(30) log_info.log_info("new task %d, but all servers are busy" % mission_id) continue log_info.log_info("task %d will run on %s" % (mission_id, ip)) ret = do_mission(mission_id, ip, db) log_info.log_info("return:%d" % ret)
def main(): loginfo = logUtils.logutil(mission_id) test_path = root_path + test_path_1 base_path = root_path + base_path_1 ol_data_path = ol_data_path_1 ol_conf_path = root_path + ol_conf_path_1 loginfo.log_info("test_path:" + test_path) loginfo.log_info("base_path:" + base_path) loginfo.log_info("ol_data_path:" + ol_data_path) loginfo.log_info("ol_conf_path:" + ol_conf_path) loginfo.log_info("mission_id:" + str(mission_id)) (testsvn, basesvn, testitem, newconfip, newconfuser, newconfpassw, newconfpath, newdataip, newdatauser, newdatapassw, newdatapath, newdata_topath, press_qps, press_time, press_expid, press_rate) = get_material() loginfo.log_info("testsvn:" + testsvn) loginfo.log_info("basesvn:" + basesvn) loginfo.log_info("testitem:" + str(testitem)) loginfo.log_info("newconfip:" + newconfip) loginfo.log_info("newconfuser:"******"newconfpassw:" + newconfpassw) loginfo.log_info("newconfpath:" + newconfpath) loginfo.log_info("newdataip:" + newdataip) loginfo.log_info("newdatauser:"******"newdatapassw:" + newdatapassw) loginfo.log_info("newdatapath:" + newdatapath) loginfo.log_info("newdata_topath:" + newdata_topath) loginfo.log_info("press_qps:" + str(press_qps)) loginfo.log_info("press_time:" + str(press_time)) loginfo.log_info("press_expid:" + str(press_expid)) loginfo.log_info("press_rate:" + str(press_rate)) ####configure sggp/ACE_Pressure_CACHE.ini # ret_configure_sggp = configure_sggp(sggp_conf,press_qps,press_time) # if ret_configure_sggp != 0: # update_errorlog("[%s] %s\n" % (get_now_time(), "configure sggp_conf has some error, pls check")) # set_status(3) # return -1 ret_configure_sggp_test = configure_sggp_test(sggp_path, press_qps, press_time, press_expid, press_rate) if ret_configure_sggp_test != 0: update_errorlog( "[%s] %s\n" % (get_now_time(), "configure sggp_conf has some error, pls check")) set_status(3) return -1 # ret_sync_ol_data = sync_ol_data_to_local(ol_data_path+"/data") # if ret_sync_ol_data != 0: # update_errorlog("[%s] %s\n" % (get_now_time(), "sync_ol_data_to_local has some error, pls check")) # set_status(3) # return -1 ret_sync_olbl_data = sync_olbl_data_to_local(black_data_path) if ret_sync_olbl_data != 0: update_errorlog("[%s] %s\n" % (get_now_time(), "sync_olbl_data_to_local has some error, pls check")) set_status(3) return -1 ret_sync_ol_conf = sync_ol_conf_to_local(ol_conf_path) if ret_sync_ol_conf != 0: update_errorlog("[%s] %s\n" % (get_now_time(), "sync_ol_conf_to_local has some error, pls check")) set_status(3) return -1 ##### just run test if testsvn.strip() != "": update_errorlog("[%s] %s\n" % (get_now_time(), "try start test")) update_errorlog("[%s] %s\n" % (get_now_time(), "start try build test enviroment")) ### check code try: update_errorlog("[%s] %s\n" % (get_now_time(), "test start try check code")) ret = checkcode_env(test_path, testsvn) except Exception as e: update_errorlog("[%s] %s\n" % (get_now_time(), e)) set_status(3) return -1 if (ret != 0): set_status(3) return 4 update_errorlog("[%s] %s\n" % (get_now_time(), "test check code ok")) ### make try: update_errorlog("[%s] %s\n" % (get_now_time(), "test start try make")) ret = make_env(test_path) except Exception as e: update_errorlog("[%s] %s\n" % (get_now_time(), e)) set_status(3) return -1 if (ret != 0): set_status(3) return 4 update_errorlog("[%s] %s\n" % (get_now_time(), "test make ok")) ### make test data link and scp new data to test env try: update_errorlog( "[%s] %s\n" % (get_now_time(), "test start try to make link with ol_data on test")) if newdatapath == '': if (os.path.exists(test_path + '/QueryOptimizer/data')): loginfo.log_info('test_path_data_dir is exist') os.popen('rm -rf %s' % (test_path + '/QueryOptimizer/data')) os.symlink(ol_data_path + '/data', test_path + '/QueryOptimizer/data') else: res = maketestlink(ol_data_path, test_path + '/QueryOptimizer', newdatapath) if (res != 0): set_status(3) return 4 if ";" in newdatapath and newdataip != '' and newdatauser != '' and newdatapassw != '': scpres = scpnewdata(test_path + '/QueryOptimizer', newdataip, newdatauser, newdatapassw, newdatapath) else: scpres = 1 update_errorlog( "[%s] %s\n" % (get_now_time(), "test new data configure is wrong")) if (scpres != 0): set_status(3) return 4 except Exception as e: update_errorlog("[%s] %s\n" % (get_now_time(), e)) set_status(3) return -1 update_errorlog( "[%s] %s\n" % (get_now_time(), "test start try to make link with ol_data on test ok")) ### scp new conf to test env try: update_errorlog("[%s] %s\n" % (get_now_time(), "test start to ctrl cfg ")) if newconfpath == '': loginfo.log_info('use cfg online') ret = cp_new_conf(ol_conf_path, test_path) elif newconfip != '' and newconfuser != '' and newconfpassw != '': loginfo.log_info('use rd cfg') ret = scp_new_conf(test_path, newconfip, newconfuser, newconfpassw, newconfpath) else: ret = 1 update_errorlog( "[%s] %s\n" % (get_now_time(), "test new conf configure is wrong")) except Exception, e: update_errorlog("[%s] %s\n" % (get_now_time(), e)) set_status(3) return -1 if (ret != 0): set_status(3) return 4 update_errorlog( "[%s] %s\n" % (get_now_time(), "test start to cp ol_data and ol_dev_conf ok")) ### cp start.sh to env try: update_errorlog( "[%s] %s\n" % (get_now_time(), "test start to cp start.sh to test env")) ret = cp_start_sc(test_path) except Exception, e: update_errorlog("[%s] %s\n" % (get_now_time(), e)) set_status(3) return -1