def loop_check_distribute_crawler_info():
_programs_crawler_info = maintain_plans.GetValue(
).get_programs_crawler_info()
programs_info = copy.deepcopy(_programs_crawler_info)
while True:
try:
_programs_crawler_info = maintain_plans.GetValue(
).get_programs_crawler_info()
programs_info_new = copy.deepcopy(_programs_crawler_info)
hash_old = hashlib.md5(
json.dumps(programs_info).encode('utf-8')).hexdigest()
# 转换成dict
# 与执行计划比对
hash_new = hashlib.md5(
json.dumps(programs_info_new).encode('utf-8')).hexdigest()
except Exception as e:
print(e)
write_log(str(e))
else:
if hash_new != hash_old:
programs_info = copy.deepcopy(programs_info_new)
distribute = DistributePlans(_programs_crawler_info.keys())
distribute.distribute()
finally:
time_waite = 20
time.sleep(time_waite)
def loop_check_calculate_programs(self):
while True:
calculate_programs = copy.deepcopy(
maintain_plans.GetValue().get_programs_calculate_info_dic())
try:
threads = []
for version in calculate_programs:
programs_list = calculate_programs[version]
for program_info in programs_list:
flag = self.check_parents(program_info, version)
print(
"loop_check_calculate_programs[ program_info:%s, flag:%s]"
% (program_info, flag))
if flag:
ip = program_info['deploy_server']
port = program_info['port']
sid = program_info['sid']
program_info['version'] = version
maintain_plans.SetProgramsCalculateInfo(
sid, version).program_start_running()
send_program = SendProgram(ip, port, program_info)
threads.append(
threading.Thread(target=send_program.send))
for thread in threads:
thread.start()
except Exception as e:
err_str = "loop_check_calculate_programs Err:", str(e)
print(err_str)
write_log(err_str)
finally:
time.sleep(10)
def _monitor_stop():
log_str = '<slave服务状态监控启动>'
print(log_str)
write_log(log_str, name=slave_log)
t = threading.Thread(target=_stop)
t.setDaemon(True)
t.start()
def _url_exec_program(self):
time.sleep(5)
url_path = self._join_url_parameter()
print(url_path)
result = 1
extra_info = "url调用未执行"
try:
request.urlopen(url_path)
result = 0
except error.HTTPError:
result = 2
extra_info = "Http 访问错误"
except error.URLError:
result = 3
extra_info = "Url 路径错误"
except Exception as e:
result = 4
extra_info = str(e)
finally:
if result == 0:
string = 'sid:%s program:%s is accessed !result: %s' % (
self.sid, url_path, result)
extra_info = ''
else:
string = '[ err ] sid:%s program:%s is !result: %s, error info: %s' \
% (self.sid, self.program, result, extra_info)
extra_info = string
print(string)
write_log(string, name=slave_log)
self._pre_exec(extra_info)
def loop_flush_calculate_info():
while True:
global _programs_calculate_info_last
programs_info = list(copy.deepcopy(_programs_calculate_info_last))
try:
global _programs_calculate_info_dic
version = last_version()
sql_code_calculate = sql_codes.get_calculate_sql_codes()
programs_info_new = diff_programs_info(sql_code_calculate,
programs_info, 'calculate')
# ("programs_info_new:", programs_info_new)
if programs_info_new:
if version not in _programs_calculate_info_dic:
# 如果队列有更新 且 没有最新的版本则将最新版本赋值为 最新的队列
_programs_calculate_info_dic[version] = copy.deepcopy(
programs_info_new)
else:
# 更新最新版本 队列的 计算程序计划任务
add_programs_list = list()
remove_programs_list = list()
for program in programs_info_new:
if program not in programs_info:
add_programs_list.append(program)
for program in programs_info:
if program not in programs_info_new:
remove_programs_list.append(program)
_programs_calculate_info_last = copy.deepcopy(
programs_info_new)
# 将最新队列写入缓存
r.set('maintain_plans_programs_calculate_info_last',
_programs_calculate_info_last)
for program in add_programs_list:
if program not in _programs_calculate_info_dic[
version]:
_programs_calculate_info_dic[version].append(
program)
for program in remove_programs_list:
if program in _programs_calculate_info_dic[version]:
_programs_calculate_info_dic[version].remove(
program)
# 写入缓存数据库
r.set('maintain_plans_programs_calculate_info_dic',
_programs_calculate_info_dic)
else:
# 如果队列没有更新 且 最新版本不存在则用目前的队列生成最新版本
if version not in _programs_calculate_info_dic and len(
programs_info) != 0:
_programs_calculate_info_dic[version] = copy.deepcopy(
programs_info)
# 写入缓存数据库
r.set('maintain_plans_programs_calculate_info_dic',
_programs_calculate_info_dic)
except Exception as e:
print(e)
write_log(str(e))
finally:
# print("_programs_calculate_info_last:", _programs_calculate_info_last)
# print("_programs_calculate_info_dic:", _programs_calculate_info_dic)
time.sleep(30)
def _run_hello():
log_str = "<slave状态>监测线程启动!"
write_log(log_str)
t = threading.Thread(target=maintain_slaves_status.SendingHello().run)
t.setName("check-slave-stats")
print(log_str)
t.setDaemon(True)
t.start()
def _master_server():
global master_ip
global master_port
s_server = socketserver.ThreadingTCPServer((master_ip, master_port), MasterServer)
log_str = ('master_server start! bind_ip: %s bind_port:%s ' % (master_ip, master_port))
write_log(log_str)
with s_server:
s_server.serve_forever()
def _distribute_crawler():
log_str = "<Master执行计划监控程序>线程启动!"
print(log_str)
write_log(log_str)
t = threading.Thread(target=maintain_plans.run)
t.setName("distribute-plan")
t.setDaemon(True)
t.start()
def stop():
while True:
status = ModifyInfo.master_status
time.sleep(3)
if status == 1:
log_str = "master server shutdown!"
write_log(log_file, log_str)
os._exit(0)
def __init__(self, slave_ip, slave_port, cmd_list):
try:
self.data = {"type": "0", "list": cmd_list}
except Exception as e:
write_log('SendCmdClient Err:', str(e))
print('SendCmdClient Err:', str(e))
finally:
super(SendCmdClient, self).__init__(slave_ip, slave_port, self.data)
def _stop():
while True:
status = _get_slave_status()
time.sleep(3)
if status == 1:
log_str = 'slave服务器 关闭'
write_log(log_str, name=slave_log)
sys.exit(0)
def __init__(self, sid):
if isinstance(sid, str):
self.sid = int(sid)
elif isinstance(sid, int):
self.sid = sid
else:
err_str = "ParentsSons Err: program sid type should be Int!"
write_log(err_str)
raise ValueError(err_str)
def _server_start():
global slave_port
global slave_host
host = str(slave_host)
port = int(slave_port)
s_server = socketserver.ThreadingTCPServer((host, port), MyServer)
with s_server:
log_str = 'slave 服务器启动! bind_ip: %s bind_port: %s' % (host, port)
print(log_str)
write_log(log_str, name=slave_log)
s_server.serve_forever()
def start(self):
log_str = "Master Server 主程序启动"
write_log(log_str)
self._maintain_plans()
self._distribute_crawler()
self._send_mail()
time.sleep(10) # 等待数据库读取完毕
self._run_hello()
time.sleep(5)
self._loop_check_program_result()
self._sending_calculate_program()
self._monitor_stop()
self._master_server()
def _distribute(self, msg_type):
try:
if msg_type in self.task_list:
func_name = self.task_list[msg_type]
if hasattr(self, func_name):
func = getattr(self, func_name)
func()
else:
self.request.close()
else:
self.request.close()
except Exception as err:
write_log(err, name=slave_log)
self.request.close()
def _pre_exec(self, extra_info=''):
ti = time.strftime('%Y-%m-%d %H:%M:%S')
msg_type = 1
script_map = {
'sid': self.sid,
'time': ti,
'program': self.program,
'version': self.version,
'extra_info': extra_info,
'subversion': self.subversion
}
string = 'sid:%s program:%s is running start !' % (self.sid,
self.program)
print(string)
write_log(string, name=slave_log)
send_master = SendToMaster(msg_type, script_map)
send_master.sending()
def _after_exec(self, result, extra_info=""):
ti = time.strftime('%Y-%m-%d %H:%M:%S')
msg_type = 4
script_map = {
'sid': self.sid,
'time': ti,
'program': self.program,
"result": result,
"extra_info": extra_info,
'version': self.version,
'subversion': self.subversion
}
string = 'sid:%s program:%s is running ended !result: %s' % (
self.sid, self.program, result)
write_log(string, name=slave_log)
send_master = SendToMaster(msg_type, data=script_map)
send_master.sending()
def sons(self):
programs_list = list()
try:
sql = Mysql()
df = sql.mysql_read(get_sons.format(sid=self.sid))
programs_list = list()
if len(df) != 0:
for idx in df.index:
program_info = df.loc[idx, [
'sid', 'version', 'path', 'deploy_server', 'port',
'run_type'
]].to_dict()
programs_list.append(program_info)
except Exception as e:
err_str = "ParentsSons.sons Err :" + str(e)
write_log(err_str)
print(err_str)
return programs_list
def loop_flush_crawler_info():
while True:
global _programs_crawler_info
programs_info = copy.deepcopy(_programs_crawler_info)
try:
sql_code_crawler = sql_codes.get_crawler_sql_codes()
programs_info_new = diff_programs_info(sql_code_crawler,
programs_info, 'crawler')
if programs_info_new:
_programs_crawler_info = copy.deepcopy(programs_info_new)
distribute = DistributePlans()
distribute.distribute()
r.set('maintain_plans_programs_crawler_info',
_programs_crawler_info)
except Exception as e:
print(e)
write_log(str(e))
finally:
time.sleep(30)
def _program_start(self):
try:
sid = int(self.maps['data']["sid"])
event_time = self.maps['data']["time"]
name = self.maps['data']["program"]
version = self.maps['data']['version']
subversion = self.maps['data']["subversion"]
event_type = 100
sql_str = program_result_log.format(version=version, event_time=event_time, sid=sid,
event_type=event_type, extra_info="", subversion=subversion)
print(sql_str)
sql = mysql.Mysql()
sql.mysql_write(sql_str)
log_string = '程序编号: %s 程序名称: %s 程序执行开始时间: %s' % (str(sid), str(name), str(event_time))
write_log(log_string)
response = "ok"
self.request.sendall(response.encode('utf-8'))
except Exception as e:
print(str(e))
def run(self):
send_to_slave.SlaveStatus().load_slave_status()
while True:
waite_t = 5 - time.time() % 5
# a = send_to_slave.SlaveStatus().get_slave_status()
time.sleep(waite_t)
slaves_info = self._get_all_slaves()
# print(slaves_info)
threads = list()
try:
for slave in slaves_info:
ip = slave['ip']
port = slave['port']
slave_id = slave['id']
send_hello = SendHello(ip, int(port), slave_id)
threads.append(threading.Thread(target=send_hello.send))
for thread in threads:
thread.start()
except Exception as e:
write_log("SendingHello Err:", str(e))
print("SendingHello Err:", str(e))
def distribute(self):
try:
msg_type = int(self.maps['type'])
if msg_type in self.task_list:
func_name = self.task_list[msg_type]
print(func_name)
if hasattr(self, func_name):
func = getattr(self, func_name)
func()
else:
print(0)
self.request.close()
else:
log_string = "数据类型type: %s 没有定义！" % msg_type
print(log_string)
write_log(log_string)
self.request.close()
except Exception as err:
print("distribute Err", str(err))
write_log(str(err))
self.request.close()
def _connect_err(self, e):
global _slave_status
if self.ip in _slave_status:
if _slave_status[self.ip]["count"] < 3:
_slave_status[self.ip]["count"] += 1
elif _slave_status[self.ip]["count"] == 3:
del(_slave_status[self.ip])
log_string = "服务器: %s 下线 !" % self.ip
# 需要更新数据库状态
try:
update_time = time.strftime("%Y-%m-%d %H:%M:%S")
sql_code = self.sql_code.format(status=0, uptime=update_time, id=self.slave_id)
# print(sql_code)
sql = Mysql()
sql.mysql_write(sql_code)
except Exception as e:
print("SendHello._connect_err Err:", str(e))
write_log("SendHello._connect_err Err:", str(e))
finally:
write_log(log_string)
else:
del(_slave_status[self.ip])
def distribute(self):
try:
programs_info = copy.deepcopy(_programs_crawler_info)
self._get_valid_ip_list()
# print("self.valid_slaves_ip_list:", self.valid_slaves_ip_list)
if len(self.valid_slaves_ip_list) == 0:
raise ValueError(
'DistributePlan get a ip_list with no valid slave_ip!')
else:
for slave_info in self.valid_slaves_ip_list:
ip = slave_info['ip']
port = slave_info['port']
print(ip, port)
if ip in programs_info:
cmd_list = programs_info[ip]
send_cmd = SendCmdClient(ip, port, cmd_list)
send_cmd.send()
log_string = "更新服务器: %s 的执行计划成功" % ip
write_log(log_string)
break
except Exception as e:
write_log('DistributePlan get err:', str(e))
print('DistributePlans.distribute Err:', str(e))
def _response(self, receive, conn):
global _slave_status
if receive == "hello_ack":
if self.ip in _slave_status:
_slave_status[self.ip]["count"] = 0
else:
_slave_status[self.ip] = {"port": self.port, "count": 0}
log_string = "服务器: %s 上线 " % self.ip
# 需要更新数据库服务器状态
try:
update_time = time.strftime("%Y-%m-%d %H:%M:%S")
sql_code = self.sql_code.format(status=1, uptime=update_time, id=self.slave_id)
sql = Mysql()
sql.mysql_write(sql_code)
# print(sql_code)
except Exception as e:
print("SendHello._connect_err Err:", str(e))
write_log("SendHello._connect_err Err:", str(e))
finally:
write_log(log_string)
time.sleep(10)
send_plan = DistributePlans(ip_list=(self.ip, ))
send_plan.distribute()
conn.close()
def _program_end(self):
event_time = self.maps['data']['time']
sid = self.maps['data']['sid']
result_code = self.maps['data']['result']
version = self.maps['data']['version']
subversion = self.maps['data']["subversion"]
if result_code == 0:
result = "succeed"
event_type = 2
extra_info = ""
else:
result = "failed"
event_type = 3
extra_info = self.maps['data']['extra_info']
if extra_info == "":
extra_info = "Err_code:", str(result_code)
log_string = ('脚本编号: %s 脚本执行结束时间: %s result:%s' % (sid, event_time, result))
write_log(log_string)
sql_str = program_result_log.format(version=version, event_time=event_time, sid=sid,
event_type=event_type, extra_info=extra_info, subversion=subversion)
print("sql_str:", sql_str)
self.sql_write(sql_str)
response = "ok"
self.request.sendall(response.encode('utf-8'))
def _connect_success(self):
file_str = "更新服务器: %s 执行计划" % self.ip
write_log(file_str)
def _monitor_stop():
log_str = "<Master服务状态监测>线程启动!"
write_log(log_file, log_str)
t = threading.Thread(target=stop)
t.setDaemon(True)
t.start()
def _connect_err(self, e):
err_str = "连接服务器 :%s 异常,错误信息:%s" % (self.ip, str(e))
write_log(err_str)
