def exit_isr(a, b):
global running
pool = singletons[DownloaderWorkerPool.__name__]
log.debug("main proc : waiting for sub processes to die...")
pool.running = False
pool.stop_all()
log.debug("main proc : all sub processes gracefully stopped")
def run_post_upload(self, file="file.txt"):
"""
上传文件
:param file: 上传的文件名
"""
s = self.add_session()
for i in base.get_data(self):
url = self.get_url()
data = base.set_dict(self, i) # 调用函数，清除值为空的键值对
filePate = "./uploads/" + file
file = {'file': open(filePate, 'rb')}
response = s.post(url=url, data=data, files=file)
result = self.get_assert(self.nrow) # 获取实际结果的命令
expect = self.get_assert(self.nrow, 1) # 预期结果
log.info("调用第%s行，命令是：%s，预期结果是：%s" % (self.nrow, result, expect))
self.nrow = self.nrow + 1
ast = self.nrow - 1
if response.text == expect: # 判断返回的类型，根据不同的类型做出相应的判断
log.debug("返回值类型是text：%s，text断言成功" % response.text)
else:
response_json = response.json()
cmd = eval(result)
log.info("断言成功第%s页的%s行" % (self.assert_sheet + 1, ast))
self.ast.assertEqual(cmd, expect)
log.info("post请求成功")
def main(conf):
global running
signal.signal(signal.SIGTERM, exit_isr)
signal.signal(signal.SIGINT, exit_isr)
try:
pid_file = make_pid_file(sys.path[0])
try:
datasaver = DataSaver(conf)
singletons[DataSaver.__name__] = datasaver
worker_pool = DatasaverWorkerPool(
init_count=conf['server']['server_process_num'], conf=conf)
singletons[DatasaverWorkerPool.__name__] = worker_pool
worker_pool.start_all()
running = True
heartbeat_thread = HeartbeatThread("datasaver", conf)
heartbeat_thread.start()
handler = DataSaverHandler()
processor = DataSaverService.Processor(handler)
transport = TSocket.TServerSocket(port=conf['server']['port'])
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = InterruptableThreadPoolServer(processor, transport,
tfactory, pfactory)
server.setNumThreads(conf['server']['server_thread_num'])
log.debug("start main thrift")
server.serve()
except Exception, e:
log.debug(traceback.format_exc())
if running:
worker_pool.stop_all()
def run(self):
log.debug("starting input thread")
job_num = 0
while self.running and self.input_tube:
try:
job = self.beanstalk.reserve(self.input_tube, 3)
if job:
job_num += 1
body = job.body
resp = None
job.delete()
if self.processor is not None:
topic_id = None
try:
if type(self.processor).__name__ in (
'ExtractorProccessor',
'SingleSrcMergerProccessor'):
resp, topic_id = self.processor.do_task(body)
else:
resp = self.processor.do_task(body)
except Exception, e:
log.error("Process failed. " +
traceback.format_exc())
if resp is not None:
self.output_msg(resp, topic_id)
else:
self.log.debug(current_process().name +
" : no msg from : %s" % (self.input_tube))
def update_current_task_status():
with scheduler.app.app_context():
log.debug("Start maintain running task.")
print(f"Running task: {current_task_list}")
log.info(f"Running task: {current_task_list}")
task_list = copy.copy(current_task_list)
for task_id, task in task_list.items():
task_status = task.is_alive()
if task_status is not True:
exit_code = task.exitcode
if exit_code == 0:
log.info(
f"Task[{task_id}] normally finished. exit code:[{exit_code}]"
)
update_the_task_status(task_id, TASK_STATUS_FINISHED)
let_the_main_node_export_result_to_file(task_id)
else:
log.info(
f"Task[{task_id}] not normally finished. exit code:[{exit_code}]"
)
update_the_task_status(task_id, TASK_STATUS_INTERRUPT)
let_the_main_node_export_result_to_file(task_id)
update_time_info_for_the_task(task_id, "end_time")
current_task_list.pop(task_id)
update_node_info("cur_ps", len(current_task_list))
node_running_info.update({"cur_ps": len(current_task_list)})
def stop_handler(self, packet, from_main_queue, to_main_queue):
log.debug("Invoking stop handler for process " +
str(self.proc.process.name))
# self.input_thread.stop()
# self.input_thread.join()
# print "input thread in proc " + str(self.proc.process.name) + " joined"
self.proc.running = False
to_main_queue.put(ResponsePacket(packet, True))
def join(self):
resp = self.to_main_queue.get(block=True)
assert type(resp) == ResponsePacket
if resp.is_succ():
self.process.join()
log.debug("main proc : %s joined" % str(self.process.name))
else:
log.error("Failed to stop subprocess " + str(self.process.name))
def reload(self, parser_id=-1):
self.from_main_queue.put(ReloadPacket(parser_id=parser_id), block=True)
resp = self.to_main_queue.get(block=True)
assert type(resp) == ReloadRespPacket
if resp.is_succ():
log.debug(str(self.process.name) + " : reload success")
else:
log.debug(str(self.process.name) + " : reload failed, " + str(resp.msg))
return resp.is_succ(), resp.get_data()
def __put_deploy_conf():
log.debug("{}-generate supervisor deploy conf...".format(self.node_mark))
supervisor_tmp_file = os.path.join(self.local_node_tmp, "{}.conf".format(self.node_name))
self.__gen_deploy_conf(supervisor_tmp_file)
log.debug("{}-upload supervisor deploy conf...".format(self.node_mark))
self.run_ssh("rm -rf {}".format(self.remote_supervisor_node_file))
self.run_ssh("mkdir -p {}".format(self.cfg.remote_supervisor_tmp))
self.sftp.put(supervisor_tmp_file, self.remote_supervisor_node_file)
self.run_ssh("sudo -S -p '' cp {} /etc/supervisor/conf.d".format(self.remote_supervisor_node_file), True)
def __init():
cmd = '{} --datadir {} init {}'.format(self.remote_bin_file, self.remote_data_dir, self.remote_genesis_file)
result = self.run_ssh(cmd)
# todo ：fix init complete
# Adding a query here can only alleviate the problem of starting deployment without initialization.
self.run_ssh("ls {}".format(self.remote_data_dir))
if len(result) > 0:
log.error(failed_msg.format(self.node_mark, "init", result[0]))
raise Exception("Init failed:{}".format(result[0]))
log.debug("node-{} init success".format(self.node_mark))
def stop_handler(self, packet, from_main_queue, to_main_queue):
log.debug("%s : invoking stop handler for process " %
str(self.proc.process.name))
self.proc.running = False
if self.input_thread:
self.input_thread.stop()
self.input_thread.join()
log.debug("%s : input thread in proc joined" %
str(self.proc.process.name))
to_main_queue.put(ResponsePacket(packet, True))
def get_data(self):
"""
读取Excel里配置好的接口参数，以字典的方式存放
"""
data = xlrd.open_workbook(self.dir_case)
table = data.sheets()[self.param]
log.debug("参数文件路径是:%s,打开文件第:%s页" % (self.dir_case, self.param))
nor = table.nrows # 行
nol = table.ncols # 列
dict = {}
for i in range(1, nor):
for j in range(nol):
title = table.cell_value(0, j)
value = table.cell_value(i, j)
value_type = table.cell(i, j).ctype
if value_type == 2 and value % 1 == 0.0: # 如果ctype为2且取余于1等于0.0，转换成整型
value = int(value)
log.debug("获取到的参数是小数：%s,已转换成整型" % value)
if value == 0.0:
value = int(value)
log.debug("获取到的参数是：%s，已转换成整型" % value)
else:
pass
dict[title] = value
log.debug("获取到的参数名是:%s,参数值是:%s" % (title, value))
yield dict
log.info("参数读取成功")
def stop(self):
"""
close node
:return:
"""
log.debug("Stop node:{}".format(self.node_mark))
def __stop():
self.__is_connected = False
self.__is_ws_connected = False
if not self.running:
return True, "{}-node is not running".format(self.node_mark)
self.run_ssh("sudo -S -p '' supervisorctl stop {}".format(self.node_name), True)
return self.try_do_resturn(__stop)
def get_url(self, nrows=0, ncols=0):
"""
获取url地址
:param nrows: 行
:param ncols: 列
:return:
"""
data = xlrd.open_workbook(self.dir_case)
table = data.sheets()[self.assert_sheet]
url = table.cell_value(nrows, ncols)
log.debug("打开路径为： %s 的文件第：%s页，获取到的数据是：%s" %
(self.dir_case, self.assert_sheet, url))
log.info("获取url路径成功，地址第：%s" % url)
return url
def __pre_env():
ls = self.run_ssh("cd {};ls".format(
self.cfg.remote_compression_tmp_path))
if self.cfg.env_id and (self.cfg.env_id + ".tar.gz\n") in ls:
log.debug("{}-copy bin...".format(self.remote_node_path))
cmd = "cp -r {}/{}/* {}".format(
self.cfg.remote_compression_tmp_path, self.cfg.env_id,
self.remote_node_path)
self.run_ssh(cmd)
self.run_ssh("chmod +x {};mkdir {}".format(
self.remote_bin_file, self.remote_log_dir))
else:
self.put_bin()
self.put_config()
# self.put_static()
self.create_keystore()
if self.cfg.init_chain:
log.debug("{}-upload genesis...".format(self.node_mark))
self.put_genesis(genesis_file)
if self.cfg.is_need_static:
self.put_static()
log.debug("{}-upload blskey...".format(self.node_mark))
self.put_blskey()
log.debug("{}-upload nodekey...".format(self.node_mark))
self.put_nodekey()
self.put_deploy_conf()
self.run_ssh(
"sudo -S -p '' supervisorctl update " + self.node_name, True)
def start_handler(self, packet, from_main_queue, to_main_queue):
try:
self.on_start()
self.input_thread = InputThread(conf=self.conf,
proc_name=self.proc.process.name,
processor=self.processor)
self.input_thread.start()
self.proc.running = True
to_main_queue.put(ResponsePacket(packet, True))
except Exception as e:
log.error(str(traceback.format_exc()))
to_main_queue.put(ResponsePacket(packet, False))
raise e
log.debug("%s : invoking start handler for process " %
str(self.proc.process.name))
def clean(self):
"""
clear node data
:return:
"""
log.debug("Clean node:{}".format(self.node_mark))
def __clean():
is_success = self.stop()
if not is_success:
raise Exception("Stop failed")
self.run_ssh("sudo -S -p '' rm -rf {};mkdir -p {}".format(self.remote_node_path, self.remote_node_path),
True)
self.run_ssh("ls {}".format(self.remote_node_path))
return self.try_do_resturn(__clean)
def run(self):
self.running = True
while self.running and self.check_state({PoolState.RUNNING}):
log.debug("check worker status")
for idx, sub in enumerate(self.subs):
# log.debug("Worker %d is alive %s" % (idx, sub.process.is_ssssalive()))
if not sub.process.is_alive():
self.subs[idx] = self.work_class(conf = self.conf,
name=("%s-worker-%d" %(str(self.tag) \
if self.tag is not None else None,idx))
)
log.info("process %d is not active" % (idx))
self.subs[idx].start()
log.info("restart process %d success" % (idx))
time.sleep(1)
def process_backlog(self, resp, count=10):
lock_mgr = singletons[RecordLockManager.__name__]
for i in xrange(count):
backlog_task = lock_mgr.choose_backlog_and_lock_it()
if backlog_task is not None:
log.debug("Process backlog " + backlog_task.pickle_name)
# restore context
base_info = backlog_task.saved_context.base_info
extract_info = backlog_task.saved_context.extract_info
topic_id = extract_info.topic_id
download_time = backlog_task.saved_context.crawl_info.download_time
# extractor = self.route.get_extractor(topic_id)
self.process_single_entity_data(backlog_task.saved_context,
base_info, download_time,
backlog_task.data, topic_id,
resp, i)
lock_mgr.commit_task(backlog_task)
def start(self):
self.process.start()
self.from_main_queue.put(StartPacket(), block=True)
resp = self.to_main_queue.get(block=True)
assert type(resp) == ResponsePacket
if resp.is_succ():
log.debug("main proc : %s started successfully" %
str(self.process.name))
self.running = True
return True
else:
log.error("main proc : Failed to start subprocess " +
str(self.process.name))
self.running = False
#保证子进程启动不成功的is_alive状态为false
self.process.join()
return False
def run_get(self):
s = self.add_session()
for i in base.get_data(self):
url = self.get_url()
data = base.set_dict(self, i)
response = s.get(url, params=data, headers=self.header)
result = self.get_assert(self.nrow) # 实际结果命令
expect = self.get_assert(self.nrow, 1) # 预期结果
self.nrow = self.nrow + 1
ast = self.nrow - 1 # 现在调用的行数
if response.text == expect: # 判断返回的类型，根据不同的类型做出相应的判断
log.debug("返回值类型是text：%s，text断言成功" % response.text)
else:
response_json = response.json()
cmd = eval(result)
log.info("断言成功第%s页的%s行" % (self.assert_sheet, ast))
self.ast.assertEqual(cmd, expect)
log.info("get请求成功")
def add_session(self):
log.debug("获取到登录url为：%s" % (self.s_url))
log.debug("获取到登录的参数为：%s" % (self.s_data))
s = requests.session()
log.debug("登录请求成功")
response = s.get(url=self.s_url, params=self.s_data)
response_json = response.json()
try:
token = response_json['data']['token']
self.header['token'] = token
log.debug("获取到的token是：%s" % token)
log.info("token已添加到heder")
except:
log.debug("返回值中没有token")
pass
return s
def run_post(self):
s = self.add_session()
for i in base.get_data(self):
url = self.get_url()
data = base.set_dict(self, i) # 调用函数，清除值为空的键值对
response = s.post(url, data=data)
result = self.get_assert(self.nrow) # 获取实际结果的命令
expect = self.get_assert(self.nrow, 1) # 预期结果
log.info("调用第%s行，命令是：%s，预期结果是：%s" % (self.nrow, result, expect))
self.nrow = self.nrow + 1
ast = self.nrow - 1
if response.text == expect: # 判断返回的类型，根据不同的类型做出相应的判断
log.debug("返回值类型是text：%s，text断言成功" % response.text)
else:
response_json = response.json()
cmd = eval(result)
log.info("断言成功第%s页的%s行" % (self.assert_sheet, ast))
self.ast.assertEqual(cmd, expect)
log.info("post请求成功")
def deploy_me(self, genesis_file) -> tuple:
"""
deploy this node
1. Empty environmental data
2. According to the node server to determine whether it is necessary to upload files
3. Determine whether to initialize, choose to upload genesis
4. Upload the node key file
5. Upload the inter-node supervisor configuration
6. Start the node
:param genesis_file:
:return:
"""
log.debug("{}-clean node path...".format(self.node_mark))
is_success, msg = self.clean()
if not is_success:
return is_success, msg
self.clean_log()
is_success, msg = self.put_all_file(genesis_file)
if not is_success:
return is_success, msg
return self.start(self.cfg.init_chain)
def start(self, is_init=False) -> tuple:
"""
boot node
:param is_init:
:return:
"""
log.debug("Start node:{}".format(self.node_mark))
def __start():
is_success = self.stop()
if not is_success:
raise Exception("Stop failed")
if is_init:
self.init()
self.append_log_file()
result = self.run_ssh("sudo -S -p '' supervisorctl start " + self.node_name, True)
for r in result:
if "ERROR" in r or "Command 'supervisorctl' not found" in r:
raise Exception("Start failed:{}".format(r.strip("\n")))
return self.try_do_resturn(__start)
def worker_proc_main(self, from_main_queue, to_main_queue):
try:
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
set_proc_name(self.process.name)
try:
log.init_log(self.conf,
level=self.conf['logger']['level'],
console_out=self.conf['logger']['console'],
name=self.process.name)
except Exception as e:
packet = from_main_queue.get(block=True)
to_main_queue.put(ResponsePacket(packet, succ=False),
block=True)
raise e
# with daemon.DaemonContext:
log.debug(str(self.process.name) + " : started")
self.running = True
while self.running:
pack = from_main_queue.get(block=True)
self.controller.process_packet(pack, from_main_queue,
to_main_queue)
log.debug(str(self.process.name) + " : gracefully stopped")
finally:
log.debug(str(self.process.name) + " : exited")
def load_module(self, extractor_info):
if not extractor_info:
return True
topic_id = int(extractor_info.get('f_topic_id', -1))
module_path = extractor_info.get('f_module_path', '')
module_name = extractor_info.get('f_module_name', '')
log.debug("Initialize extractor %d %s %s" % (
topic_id, module_path, module_name))
topic = self.all_topics.get(topic_id, None)
if not topic:
return False
module_dir = module_path[0:module_path.find('/')]
module_path = module_path.replace("/", '.').replace(".py", '')
load_module = __import__(module_path, fromlist=[module_dir])
ConcreteExtractor = getattr(load_module, module_name)
concreteExtractor = ConcreteExtractor(topic, log)
self.extractors[topic_id] = concreteExtractor
log.debug("Initialize extractor %d %s %s success type %s" % (
topic_id, module_path, module_name, str(type(concreteExtractor))))
return True
def run_post_json(self):
"""
post请求，请求参数为json，参数以字典的方式存在每行的第一列
:return:
"""
self.header['Content-Type'] = "application/json;charset=UTF-8"
s = self.add_session()
data = xlrd.open_workbook(self.dir_case)
table = data.sheets()[self.param]
nrows = table.nrows
url = self.get_url()
for i in range(1, nrows):
data_list = table.row_values(i)
data_str = str(data_list)
data_str = data_str.strip("[]")
data_str = data_str.strip("'")
data_dict = json.loads(data_str)
response = s.post(url=url, json=data_dict, headers=self.header)
result = self.get_assert(self.nrow) # 获取实际结果的命令
expect = self.get_assert(self.nrow, 1) # 预期结果
log.info("调用第%s行，命令是：%s，预期结果是：%s" % (self.nrow, result, expect))
self.nrow = self.nrow + 1
ast = self.nrow - 1
log.info(response.json())
log.info(response.text)
if response.text == expect: # 判断返回的类型，根据不同的类型做出相应的判断
log.debug("返回值类型是text：%s，text断言成功" % response.text)
else:
response_json = response.json()
cmd = eval(result)
log.info("断言成功第%s页的%s行" % (self.assert_sheet, ast))
self.ast.assertEqual(cmd, expect)
log.info("post请求成功")
def exit_isr(a, b):
global running
pool = singletons[EEWorkerPool.__name__]
log.debug("main proc : waiting for sub processes to die...")
res = pool.stop_all()
if res:
log.debug("main proc : all sub processes gracefully stopped")
running = False
else:
log.debug("main proc : thread pool is not yet in running state")
def start_read():
with scheduler.app.app_context():
log.debug("Start to read redis to get task.")
print("start to read redis.")
if len(current_task_list) >= node_running_info.get("max_ps", 5):
log.debug("There is too many task now, wait to get new..")
else:
log.debug(f"Start to read redis in {redis_pool.connection_kwargs}")
task_info_redis = get_task_info_from_redis(redis_pool)
print(f"get task: {task_info_redis}")
log.debug(f"get task: {task_info_redis}")
if task_info_redis is not None and task_info_redis[
"status"] == "start":
task_info = get_task_info_with_id(task_info_redis["id"])
task_location_info = get_the_task_info_from_register_list(
task_info["template"], task_list)
task_info.update(task_location_info)
print(task_info)
start_the_task_with_task_info(task_info)
