def loop_check_distribute_crawler_info(): _programs_crawler_info = maintain_plans.GetValue( ).get_programs_crawler_info() programs_info = copy.deepcopy(_programs_crawler_info) while True: try: _programs_crawler_info = maintain_plans.GetValue( ).get_programs_crawler_info() programs_info_new = copy.deepcopy(_programs_crawler_info) hash_old = hashlib.md5( json.dumps(programs_info).encode('utf-8')).hexdigest() # 转换成dict # 与执行计划比对 hash_new = hashlib.md5( json.dumps(programs_info_new).encode('utf-8')).hexdigest() except Exception as e: print(e) write_log(str(e)) else: if hash_new != hash_old: programs_info = copy.deepcopy(programs_info_new) distribute = DistributePlans(_programs_crawler_info.keys()) distribute.distribute() finally: time_waite = 20 time.sleep(time_waite)
def loop_check_calculate_programs(self): while True: calculate_programs = copy.deepcopy( maintain_plans.GetValue().get_programs_calculate_info_dic()) try: threads = [] for version in calculate_programs: programs_list = calculate_programs[version] for program_info in programs_list: flag = self.check_parents(program_info, version) print( "loop_check_calculate_programs[ program_info:%s, flag:%s]" % (program_info, flag)) if flag: ip = program_info['deploy_server'] port = program_info['port'] sid = program_info['sid'] program_info['version'] = version maintain_plans.SetProgramsCalculateInfo( sid, version).program_start_running() send_program = SendProgram(ip, port, program_info) threads.append( threading.Thread(target=send_program.send)) for thread in threads: thread.start() except Exception as e: err_str = "loop_check_calculate_programs Err:", str(e) print(err_str) write_log(err_str) finally: time.sleep(10)
def _monitor_stop(): log_str = '<slave服务状态监控启动>' print(log_str) write_log(log_str, name=slave_log) t = threading.Thread(target=_stop) t.setDaemon(True) t.start()
def _url_exec_program(self): time.sleep(5) url_path = self._join_url_parameter() print(url_path) result = 1 extra_info = "url调用未执行" try: request.urlopen(url_path) result = 0 except error.HTTPError: result = 2 extra_info = "Http 访问错误" except error.URLError: result = 3 extra_info = "Url 路径错误" except Exception as e: result = 4 extra_info = str(e) finally: if result == 0: string = 'sid:%s program:%s is accessed !result: %s' % ( self.sid, url_path, result) extra_info = '' else: string = '[ err ] sid:%s program:%s is !result: %s, error info: %s' \ % (self.sid, self.program, result, extra_info) extra_info = string print(string) write_log(string, name=slave_log) self._pre_exec(extra_info)
def loop_flush_calculate_info(): while True: global _programs_calculate_info_last programs_info = list(copy.deepcopy(_programs_calculate_info_last)) try: global _programs_calculate_info_dic version = last_version() sql_code_calculate = sql_codes.get_calculate_sql_codes() programs_info_new = diff_programs_info(sql_code_calculate, programs_info, 'calculate') # ("programs_info_new:", programs_info_new) if programs_info_new: if version not in _programs_calculate_info_dic: # 如果队列有更新 且 没有最新的版本则将最新版本赋值为 最新的队列 _programs_calculate_info_dic[version] = copy.deepcopy( programs_info_new) else: # 更新最新版本 队列的 计算程序计划任务 add_programs_list = list() remove_programs_list = list() for program in programs_info_new: if program not in programs_info: add_programs_list.append(program) for program in programs_info: if program not in programs_info_new: remove_programs_list.append(program) _programs_calculate_info_last = copy.deepcopy( programs_info_new) # 将最新队列写入缓存 r.set('maintain_plans_programs_calculate_info_last', _programs_calculate_info_last) for program in add_programs_list: if program not in _programs_calculate_info_dic[ version]: _programs_calculate_info_dic[version].append( program) for program in remove_programs_list: if program in _programs_calculate_info_dic[version]: _programs_calculate_info_dic[version].remove( program) # 写入缓存数据库 r.set('maintain_plans_programs_calculate_info_dic', _programs_calculate_info_dic) else: # 如果队列没有更新 且 最新版本不存在则用目前的队列生成最新版本 if version not in _programs_calculate_info_dic and len( programs_info) != 0: _programs_calculate_info_dic[version] = copy.deepcopy( programs_info) # 写入缓存数据库 r.set('maintain_plans_programs_calculate_info_dic', _programs_calculate_info_dic) except Exception as e: print(e) write_log(str(e)) finally: # print("_programs_calculate_info_last:", _programs_calculate_info_last) # print("_programs_calculate_info_dic:", _programs_calculate_info_dic) time.sleep(30)
def _run_hello(): log_str = "<slave状态>监测线程启动!" write_log(log_str) t = threading.Thread(target=maintain_slaves_status.SendingHello().run) t.setName("check-slave-stats") print(log_str) t.setDaemon(True) t.start()
def _master_server(): global master_ip global master_port s_server = socketserver.ThreadingTCPServer((master_ip, master_port), MasterServer) log_str = ('master_server start! bind_ip: %s bind_port:%s ' % (master_ip, master_port)) write_log(log_str) with s_server: s_server.serve_forever()
def _distribute_crawler(): log_str = "<Master执行计划监控程序>线程启动!" print(log_str) write_log(log_str) t = threading.Thread(target=maintain_plans.run) t.setName("distribute-plan") t.setDaemon(True) t.start()
def stop(): while True: status = ModifyInfo.master_status time.sleep(3) if status == 1: log_str = "master server shutdown!" write_log(log_file, log_str) os._exit(0)
def __init__(self, slave_ip, slave_port, cmd_list): try: self.data = {"type": "0", "list": cmd_list} except Exception as e: write_log('SendCmdClient Err:', str(e)) print('SendCmdClient Err:', str(e)) finally: super(SendCmdClient, self).__init__(slave_ip, slave_port, self.data)
def _stop(): while True: status = _get_slave_status() time.sleep(3) if status == 1: log_str = 'slave服务器 关闭' write_log(log_str, name=slave_log) sys.exit(0)
def __init__(self, sid): if isinstance(sid, str): self.sid = int(sid) elif isinstance(sid, int): self.sid = sid else: err_str = "ParentsSons Err: program sid type should be Int!" write_log(err_str) raise ValueError(err_str)
def _server_start(): global slave_port global slave_host host = str(slave_host) port = int(slave_port) s_server = socketserver.ThreadingTCPServer((host, port), MyServer) with s_server: log_str = 'slave 服务器启动! bind_ip: %s bind_port: %s' % (host, port) print(log_str) write_log(log_str, name=slave_log) s_server.serve_forever()
def start(self): log_str = "Master Server 主程序启动" write_log(log_str) self._maintain_plans() self._distribute_crawler() self._send_mail() time.sleep(10) # 等待数据库读取完毕 self._run_hello() time.sleep(5) self._loop_check_program_result() self._sending_calculate_program() self._monitor_stop() self._master_server()
def _distribute(self, msg_type): try: if msg_type in self.task_list: func_name = self.task_list[msg_type] if hasattr(self, func_name): func = getattr(self, func_name) func() else: self.request.close() else: self.request.close() except Exception as err: write_log(err, name=slave_log) self.request.close()
def _pre_exec(self, extra_info=''): ti = time.strftime('%Y-%m-%d %H:%M:%S') msg_type = 1 script_map = { 'sid': self.sid, 'time': ti, 'program': self.program, 'version': self.version, 'extra_info': extra_info, 'subversion': self.subversion } string = 'sid:%s program:%s is running start !' % (self.sid, self.program) print(string) write_log(string, name=slave_log) send_master = SendToMaster(msg_type, script_map) send_master.sending()
def _after_exec(self, result, extra_info=""): ti = time.strftime('%Y-%m-%d %H:%M:%S') msg_type = 4 script_map = { 'sid': self.sid, 'time': ti, 'program': self.program, "result": result, "extra_info": extra_info, 'version': self.version, 'subversion': self.subversion } string = 'sid:%s program:%s is running ended !result: %s' % ( self.sid, self.program, result) write_log(string, name=slave_log) send_master = SendToMaster(msg_type, data=script_map) send_master.sending()
def sons(self): programs_list = list() try: sql = Mysql() df = sql.mysql_read(get_sons.format(sid=self.sid)) programs_list = list() if len(df) != 0: for idx in df.index: program_info = df.loc[idx, [ 'sid', 'version', 'path', 'deploy_server', 'port', 'run_type' ]].to_dict() programs_list.append(program_info) except Exception as e: err_str = "ParentsSons.sons Err :" + str(e) write_log(err_str) print(err_str) return programs_list
def loop_flush_crawler_info(): while True: global _programs_crawler_info programs_info = copy.deepcopy(_programs_crawler_info) try: sql_code_crawler = sql_codes.get_crawler_sql_codes() programs_info_new = diff_programs_info(sql_code_crawler, programs_info, 'crawler') if programs_info_new: _programs_crawler_info = copy.deepcopy(programs_info_new) distribute = DistributePlans() distribute.distribute() r.set('maintain_plans_programs_crawler_info', _programs_crawler_info) except Exception as e: print(e) write_log(str(e)) finally: time.sleep(30)
def _program_start(self): try: sid = int(self.maps['data']["sid"]) event_time = self.maps['data']["time"] name = self.maps['data']["program"] version = self.maps['data']['version'] subversion = self.maps['data']["subversion"] event_type = 100 sql_str = program_result_log.format(version=version, event_time=event_time, sid=sid, event_type=event_type, extra_info="", subversion=subversion) print(sql_str) sql = mysql.Mysql() sql.mysql_write(sql_str) log_string = '程序编号: %s 程序名称: %s 程序执行开始时间: %s' % (str(sid), str(name), str(event_time)) write_log(log_string) response = "ok" self.request.sendall(response.encode('utf-8')) except Exception as e: print(str(e))
def run(self): send_to_slave.SlaveStatus().load_slave_status() while True: waite_t = 5 - time.time() % 5 # a = send_to_slave.SlaveStatus().get_slave_status() time.sleep(waite_t) slaves_info = self._get_all_slaves() # print(slaves_info) threads = list() try: for slave in slaves_info: ip = slave['ip'] port = slave['port'] slave_id = slave['id'] send_hello = SendHello(ip, int(port), slave_id) threads.append(threading.Thread(target=send_hello.send)) for thread in threads: thread.start() except Exception as e: write_log("SendingHello Err:", str(e)) print("SendingHello Err:", str(e))
def distribute(self): try: msg_type = int(self.maps['type']) if msg_type in self.task_list: func_name = self.task_list[msg_type] print(func_name) if hasattr(self, func_name): func = getattr(self, func_name) func() else: print(0) self.request.close() else: log_string = "数据类型type: %s 没有定义!" % msg_type print(log_string) write_log(log_string) self.request.close() except Exception as err: print("distribute Err", str(err)) write_log(str(err)) self.request.close()
def _connect_err(self, e): global _slave_status if self.ip in _slave_status: if _slave_status[self.ip]["count"] < 3: _slave_status[self.ip]["count"] += 1 elif _slave_status[self.ip]["count"] == 3: del(_slave_status[self.ip]) log_string = "服务器: %s 下线 !" % self.ip # 需要更新数据库状态 try: update_time = time.strftime("%Y-%m-%d %H:%M:%S") sql_code = self.sql_code.format(status=0, uptime=update_time, id=self.slave_id) # print(sql_code) sql = Mysql() sql.mysql_write(sql_code) except Exception as e: print("SendHello._connect_err Err:", str(e)) write_log("SendHello._connect_err Err:", str(e)) finally: write_log(log_string) else: del(_slave_status[self.ip])
def distribute(self): try: programs_info = copy.deepcopy(_programs_crawler_info) self._get_valid_ip_list() # print("self.valid_slaves_ip_list:", self.valid_slaves_ip_list) if len(self.valid_slaves_ip_list) == 0: raise ValueError( 'DistributePlan get a ip_list with no valid slave_ip!') else: for slave_info in self.valid_slaves_ip_list: ip = slave_info['ip'] port = slave_info['port'] print(ip, port) if ip in programs_info: cmd_list = programs_info[ip] send_cmd = SendCmdClient(ip, port, cmd_list) send_cmd.send() log_string = "更新服务器: %s 的执行计划成功" % ip write_log(log_string) break except Exception as e: write_log('DistributePlan get err:', str(e)) print('DistributePlans.distribute Err:', str(e))
def _response(self, receive, conn): global _slave_status if receive == "hello_ack": if self.ip in _slave_status: _slave_status[self.ip]["count"] = 0 else: _slave_status[self.ip] = {"port": self.port, "count": 0} log_string = "服务器: %s 上线 " % self.ip # 需要更新数据库服务器状态 try: update_time = time.strftime("%Y-%m-%d %H:%M:%S") sql_code = self.sql_code.format(status=1, uptime=update_time, id=self.slave_id) sql = Mysql() sql.mysql_write(sql_code) # print(sql_code) except Exception as e: print("SendHello._connect_err Err:", str(e)) write_log("SendHello._connect_err Err:", str(e)) finally: write_log(log_string) time.sleep(10) send_plan = DistributePlans(ip_list=(self.ip, )) send_plan.distribute() conn.close()
def _program_end(self): event_time = self.maps['data']['time'] sid = self.maps['data']['sid'] result_code = self.maps['data']['result'] version = self.maps['data']['version'] subversion = self.maps['data']["subversion"] if result_code == 0: result = "succeed" event_type = 2 extra_info = "" else: result = "failed" event_type = 3 extra_info = self.maps['data']['extra_info'] if extra_info == "": extra_info = "Err_code:", str(result_code) log_string = ('脚本编号: %s 脚本执行结束时间: %s result:%s' % (sid, event_time, result)) write_log(log_string) sql_str = program_result_log.format(version=version, event_time=event_time, sid=sid, event_type=event_type, extra_info=extra_info, subversion=subversion) print("sql_str:", sql_str) self.sql_write(sql_str) response = "ok" self.request.sendall(response.encode('utf-8'))
def _connect_success(self): file_str = "更新服务器: %s 执行计划" % self.ip write_log(file_str)
def _monitor_stop(): log_str = "<Master服务状态监测>线程启动!" write_log(log_file, log_str) t = threading.Thread(target=stop) t.setDaemon(True) t.start()
def _connect_err(self, e): err_str = "连接服务器 :%s 异常,错误信息:%s" % (self.ip, str(e)) write_log(err_str)