def updates(): import urllib2 import os distro_uri = conf["system"]["distro_server_uri"] platform = conf["system"]["platform"] build_info_path = os.path.join(os.getcwd(), "configs", "build_info.json") current_info = json.load(file(build_info_path)) develop_info = json.load( urllib2.urlopen(distro_uri + "/develop/build_info.json", timeout=15)) master_info = json.load( urllib2.urlopen(distro_uri + "/build_info.json", timeout=15)) log.debug(develop_info) log.debug(master_info) status = "" if request.method == "POST": tools = Tools() action = request.form["action"] if action == "update_to_master": tools.run_update_procedure(distro_uri, platform) if action == "update_to_develop": tools.run_update_procedure(distro_uri + "/develop", platform) status = "Update procedure is in progres..... please check /var/log/blackfly_upgrade.log for details " return render_template('updates.html', global_context=global_context, current_info=current_info, develop_info=develop_info, master_info=master_info, autoescape=False, status=status)
def request(exec_id, interface_id, job_id, status): """封装requests请求""" url = 'http://%(host)s:%(port)s/execute/callback/' % { 'host': config.master.host, 'port': config.master.port } params = { 'exec_id': exec_id, 'interface_id': interface_id, 'job_id': job_id, 'status': status } for _ in range(5): try: response = requests.get(url=url, params=params) log.debug('回调web服务响应结果:[status_code: %d][text: %s]' % (response.status_code, response.text)) if response.status_code == 200: return True else: return False except Exception as e: log.error('回调web服务失败: [params: %s][error: %s]' % (str(params), e), exc_info=True) return False
def __exit__(self, exc_type, exc_value, traceback): """释放锁""" try: sql = "SELECT RELEASE_LOCK('%s') AS 'release_lock'" % self.lock_name cursor = self.execute(sql) result = cursor.fetchone() if result['release_lock'] == 0: raise BaseException('锁[%s]未被释放, 该锁并非由此数据库连接创建.' % self.lock_name) elif result['release_lock'] == 1: log.debug("锁[%s]释放成功." % self.lock_name) else: raise BaseException("锁[%s]不存在." % self.lock_name) except Exception as e: log.error(e, exc_info=True) raise e finally: # 一定要释放连接, 不然会死锁 self.conn and self.conn.close()
def __enter__(self): """获取锁""" try: # 获取锁 sql = "SELECT GET_LOCK('%s', %s) AS 'lock'" % (self.lock_name, self.timeout) cursor = self.execute(sql) if cursor.rowcount != 1: raise BaseException('锁[%s]在lock函数中返回多行.') result = cursor.fetchone() if result['lock'] == 0: raise BaseException('锁[%s]已被其他客户端锁定.' % self.lock_name) elif result['lock'] == 1: log.debug('获取锁[%s]成功.' % self.lock_name) return else: raise BaseException('mysql lock函数其他异常.') except Exception as e: log.error(e, exc_info=True) raise e
def exec_job(exec_id, interface_id, job_id, server_dir, server_script, return_code, params, retry=0): """执行任务""" # 配置参数 params_str = ' '.join(params) if ' '.join(params).startswith( ' ') else ' ' + ' '.join(params) server_script_run = server_script + params_str # 文本日志 now_time = time.strftime('%Y-%m-%d_%H%M%S', time.localtime()) file_name = './logs/%s_%s_%s_%s.log' % (exec_id, interface_id, job_id, now_time) fw = open(file_name, 'w') # 添加执行任务开始日志 SchedulerModel.add_exec_detail_job(db.etl_db, exec_id, interface_id, job_id, 'INFO', server_dir, server_script_run, '任务开始', 1) # 子进程 p = subprocess.Popen(server_script_run, cwd=server_dir, stdout=fw, stderr=fw, shell=True, bufsize=0) # 执行任务开始 SchedulerModel.exec_job_start(db.etl_db, exec_id, interface_id, job_id, p.pid) # 获取返回码 ret_code = p.wait() for message in open(file_name, 'r'): message = message.rstrip() if message: log.debug('任务详情日志: [%s]' % message) # 添加执行任务详情日志 SchedulerModel.add_exec_detail_job(db.etl_db, exec_id, interface_id, job_id, 'INFO', server_dir, server_script_run, message, 2) # 结束 if ret_code is not None: # 添加执行任务结束日志 SchedulerModel.add_exec_detail_job(db.etl_db, exec_id, interface_id, job_id, 'INFO', server_dir, server_script_run, '任务结束', 3) # 异常 if ret_code != return_code: if retry >= 3: raise Exception('任务异常') # 重试三次 else: time.sleep(600) exec_job(exec_id, interface_id, job_id, server_dir, server_script, return_code, params, retry=retry + 1) return
def init_app_components(): # uri root prefix global root_uri, http_server_port global app, global_context global mqtt, sync_async_client, conf, conf_path, bf_containers logging.config.dictConfig(configs.log.config) log.info("Checking firewall configuration") log.info(Tools.open_port_in_firewall()) # Injecting root uri prefix mod_auth.url_prefix = root_uri login_manager.login_view = globals.get_full_url("/ui/login") log.debug("Login view url = %s" % login_manager.login_view) blackflow_ex.blackflow_bp.url_prefix = root_uri # Check and init application/service ID (sid) if not conf["system"]["sid"]: # the id is base on MAC address , which means it may not be unique sid = gen_sid() conf["system"]["sid"] = sid utils.save_config(conf_path, conf) else: sid = conf["system"]["sid"] # Flask init app = Flask(__name__) secret_key = "" try: secret_file = open("secret.db", "r") secret_key = secret_file.read() secret_file.close() except IOError: import os secret_file = open("secret.db", "w") secret_key = str(os.urandom(24)) secret_file.write(secret_key) secret_file.close() # app.secret_key = '\xb5\xd4\xa1\xa5_\xc9\x07"\xaa\xb5\x1d1\xea\xd0\x08\\\xe9\x0b\x056\xf9J\x8f\xd0' app.secret_key = secret_key app.config['SESSION_TYPE'] = 'filesystem' app.config['SESSION_FILE_DIR'] = '/tmp/' app.config['SESSION_COOKIE_NAME'] = 'bflowctr' app.config['LOGIN_DISABLED'] = conf["system"]["ui_security_disabled"] # app.config['APPLICATION_ROOT'] = root_uri app.register_blueprint(mod_auth) app.register_blueprint(blackflow_ex.blackflow_bp) login_manager.init_app(app) http_server_port = conf["system"]["http_server_port"] # Influx DB # Mqtt Adapter mqtt = MqttAdapter(conf["mqtt"]["client_id"]) mqtt.set_mqtt_params(conf["mqtt"]["client_id"], conf["mqtt"]["username"], conf["mqtt"]["password"], conf["mqtt"]["global_topic_prefix"], conf["mqtt"]["enable_sys"]) mqtt.set_global_context(global_context) try: if mqtt.connect(conf["mqtt"]["host"], int(conf["mqtt"]["port"])): mqtt.start() else: global_context['mqtt_conn_status'] = "offline" log.error("application can't connect to message broker.") except Exception as ex: global_context['mqtt_conn_status'] = "offline" log.error("application can't connect to message broker.") log.error(ex) # Sync async which implement sync service invocation over async sync_async_client = SyncToAsyncMsgConverter(mqtt) mqtt.set_message_handler(sync_async_client.on_message) auth_ex.global_context = global_context if global_context["auth_type"] == "local": auth_ex.um.load_from_storage() bf_containers = ServiceDiscovery(sync_async_client) blackflow_ex.global_context = global_context blackflow_ex.sync_async_client = sync_async_client blackflow_ex.svc_discovery = bf_containers