def updates():
import urllib2
import os
distro_uri = conf["system"]["distro_server_uri"]
platform = conf["system"]["platform"]
build_info_path = os.path.join(os.getcwd(), "configs",
"build_info.json")
current_info = json.load(file(build_info_path))
develop_info = json.load(
urllib2.urlopen(distro_uri + "/develop/build_info.json",
timeout=15))
master_info = json.load(
urllib2.urlopen(distro_uri + "/build_info.json", timeout=15))
log.debug(develop_info)
log.debug(master_info)
status = ""
if request.method == "POST":
tools = Tools()
action = request.form["action"]
if action == "update_to_master":
tools.run_update_procedure(distro_uri, platform)
if action == "update_to_develop":
tools.run_update_procedure(distro_uri + "/develop", platform)
status = "Update procedure is in progres..... please check /var/log/blackfly_upgrade.log for details "
return render_template('updates.html',
global_context=global_context,
current_info=current_info,
develop_info=develop_info,
master_info=master_info,
autoescape=False,
status=status)
def request(exec_id, interface_id, job_id, status):
"""封装requests请求"""
url = 'http://%(host)s:%(port)s/execute/callback/' % {
'host': config.master.host,
'port': config.master.port
}
params = {
'exec_id': exec_id,
'interface_id': interface_id,
'job_id': job_id,
'status': status
}
for _ in range(5):
try:
response = requests.get(url=url, params=params)
log.debug('回调web服务响应结果：[status_code: %d][text: %s]' %
(response.status_code, response.text))
if response.status_code == 200:
return True
else:
return False
except Exception as e:
log.error('回调web服务失败: [params: %s][error: %s]' % (str(params), e),
exc_info=True)
return False
def __exit__(self, exc_type, exc_value, traceback):
"""释放锁"""
try:
sql = "SELECT RELEASE_LOCK('%s') AS 'release_lock'" % self.lock_name
cursor = self.execute(sql)
result = cursor.fetchone()
if result['release_lock'] == 0:
raise BaseException('锁[%s]未被释放, 该锁并非由此数据库连接创建.' %
self.lock_name)
elif result['release_lock'] == 1:
log.debug("锁[%s]释放成功." % self.lock_name)
else:
raise BaseException("锁[%s]不存在." % self.lock_name)
except Exception as e:
log.error(e, exc_info=True)
raise e
finally:
# 一定要释放连接, 不然会死锁
self.conn and self.conn.close()
def __enter__(self):
"""获取锁"""
try:
# 获取锁
sql = "SELECT GET_LOCK('%s', %s) AS 'lock'" % (self.lock_name,
self.timeout)
cursor = self.execute(sql)
if cursor.rowcount != 1:
raise BaseException('锁[%s]在lock函数中返回多行.')
result = cursor.fetchone()
if result['lock'] == 0:
raise BaseException('锁[%s]已被其他客户端锁定.' % self.lock_name)
elif result['lock'] == 1:
log.debug('获取锁[%s]成功.' % self.lock_name)
return
else:
raise BaseException('mysql lock函数其他异常.')
except Exception as e:
log.error(e, exc_info=True)
raise e
def exec_job(exec_id,
interface_id,
job_id,
server_dir,
server_script,
return_code,
params,
retry=0):
"""执行任务"""
# 配置参数
params_str = ' '.join(params) if ' '.join(params).startswith(
' ') else ' ' + ' '.join(params)
server_script_run = server_script + params_str
# 文本日志
now_time = time.strftime('%Y-%m-%d_%H%M%S', time.localtime())
file_name = './logs/%s_%s_%s_%s.log' % (exec_id, interface_id, job_id,
now_time)
fw = open(file_name, 'w')
# 添加执行任务开始日志
SchedulerModel.add_exec_detail_job(db.etl_db, exec_id, interface_id,
job_id, 'INFO', server_dir,
server_script_run, '任务开始', 1)
# 子进程
p = subprocess.Popen(server_script_run,
cwd=server_dir,
stdout=fw,
stderr=fw,
shell=True,
bufsize=0)
# 执行任务开始
SchedulerModel.exec_job_start(db.etl_db, exec_id, interface_id, job_id,
p.pid)
# 获取返回码
ret_code = p.wait()
for message in open(file_name, 'r'):
message = message.rstrip()
if message:
log.debug('任务详情日志: [%s]' % message)
# 添加执行任务详情日志
SchedulerModel.add_exec_detail_job(db.etl_db, exec_id,
interface_id, job_id, 'INFO',
server_dir, server_script_run,
message, 2)
# 结束
if ret_code is not None:
# 添加执行任务结束日志
SchedulerModel.add_exec_detail_job(db.etl_db, exec_id, interface_id,
job_id, 'INFO', server_dir,
server_script_run, '任务结束', 3)
# 异常
if ret_code != return_code:
if retry >= 3:
raise Exception('任务异常')
# 重试三次
else:
time.sleep(600)
exec_job(exec_id,
interface_id,
job_id,
server_dir,
server_script,
return_code,
params,
retry=retry + 1)
return
def init_app_components():
# uri root prefix
global root_uri, http_server_port
global app, global_context
global mqtt, sync_async_client, conf, conf_path, bf_containers
logging.config.dictConfig(configs.log.config)
log.info("Checking firewall configuration")
log.info(Tools.open_port_in_firewall())
# Injecting root uri prefix
mod_auth.url_prefix = root_uri
login_manager.login_view = globals.get_full_url("/ui/login")
log.debug("Login view url = %s" % login_manager.login_view)
blackflow_ex.blackflow_bp.url_prefix = root_uri
# Check and init application/service ID (sid)
if not conf["system"]["sid"]:
# the id is base on MAC address , which means it may not be unique
sid = gen_sid()
conf["system"]["sid"] = sid
utils.save_config(conf_path, conf)
else:
sid = conf["system"]["sid"]
# Flask init
app = Flask(__name__)
secret_key = ""
try:
secret_file = open("secret.db", "r")
secret_key = secret_file.read()
secret_file.close()
except IOError:
import os
secret_file = open("secret.db", "w")
secret_key = str(os.urandom(24))
secret_file.write(secret_key)
secret_file.close()
# app.secret_key = '\xb5\xd4\xa1\xa5_\xc9\x07"\xaa\xb5\x1d1\xea\xd0\x08\\\xe9\x0b\x056\xf9J\x8f\xd0'
app.secret_key = secret_key
app.config['SESSION_TYPE'] = 'filesystem'
app.config['SESSION_FILE_DIR'] = '/tmp/'
app.config['SESSION_COOKIE_NAME'] = 'bflowctr'
app.config['LOGIN_DISABLED'] = conf["system"]["ui_security_disabled"]
# app.config['APPLICATION_ROOT'] = root_uri
app.register_blueprint(mod_auth)
app.register_blueprint(blackflow_ex.blackflow_bp)
login_manager.init_app(app)
http_server_port = conf["system"]["http_server_port"]
# Influx DB
# Mqtt Adapter
mqtt = MqttAdapter(conf["mqtt"]["client_id"])
mqtt.set_mqtt_params(conf["mqtt"]["client_id"], conf["mqtt"]["username"],
conf["mqtt"]["password"],
conf["mqtt"]["global_topic_prefix"],
conf["mqtt"]["enable_sys"])
mqtt.set_global_context(global_context)
try:
if mqtt.connect(conf["mqtt"]["host"], int(conf["mqtt"]["port"])):
mqtt.start()
else:
global_context['mqtt_conn_status'] = "offline"
log.error("application can't connect to message broker.")
except Exception as ex:
global_context['mqtt_conn_status'] = "offline"
log.error("application can't connect to message broker.")
log.error(ex)
# Sync async which implement sync service invocation over async
sync_async_client = SyncToAsyncMsgConverter(mqtt)
mqtt.set_message_handler(sync_async_client.on_message)
auth_ex.global_context = global_context
if global_context["auth_type"] == "local":
auth_ex.um.load_from_storage()
bf_containers = ServiceDiscovery(sync_async_client)
blackflow_ex.global_context = global_context
blackflow_ex.sync_async_client = sync_async_client
blackflow_ex.svc_discovery = bf_containers