def remove_some_container(task, number):
"""
删除若干个容器
:param task:
:param number:
:return:
"""
containers = AppContainModel.select().where(AppContainModel.app_id == task.app_id).limit(number)
for obj in containers:
api_url = obj.api_url + "remove"
data = {
"containerId": obj.container_id
}
r = request_api(api_url, data)
if r.status_code == 200:
d = json.loads(r.text)
if d['code'] == 0:
# 删除容器成功
dao = AppContainModel.delete().where(AppContainModel.id == obj.id)
dao.execute()
login_log("common", "删除容器成功，触发应用id：%d，容器id：%s，容器服务器域名：%s" % (task.app_id, obj.container_id, obj.host))
continue
# 删除出错处理
login_log("error", "删除容器失败，触发应用id：%d，容器id：%s，容器服务器域名：%s" % (task.app_id, obj.container_id, obj.host))
return None
def get_app_avg_message(app_id):
"""
获取应用平均状态
:return: cpu，内存
"""
cpu = 0
memory = 0
count = AppContainModel.select().where(AppContainModel.app_id == app_id).count()
objs = AppContainModel.select().where(AppContainModel.app_id == app_id)
for obj in objs:
r = request_api(obj.api_url + "stats", {"containerId": obj.container_id})
if r.status_code != 200:
cpu += 100
memory += 100
continue
d = json.loads(r.text)
if d["code"] != 0:
cpu += 100
memory += 100
continue
cpu += d["result"]["cpu"]
memory += d["result"]["memory"]
if count == 0:
return 100, 100
cpu = cpu / count
cpu = float("%.2f" % cpu)
memory = memory / count
memory = float("%.2f" % memory)
return cpu, memory
def remove_some_container(task, number):
"""
删除若干个容器
:param task:
:param number:
:return:
"""
containers = AppContainModel.select().where(
AppContainModel.app_id == task.app_id).limit(number)
for obj in containers:
api_url = obj.api_url + "remove"
data = {"containerId": obj.container_id}
r = request_api(api_url, data)
if r.status_code == 200:
d = json.loads(r.text)
if d['code'] == 0:
# 删除容器成功
dao = AppContainModel.delete().where(
AppContainModel.id == obj.id)
dao.execute()
login_log(
"common", "删除容器成功，触发应用id：%d，容器id：%s，容器服务器域名：%s" %
(task.app_id, obj.container_id, obj.host))
continue
# 删除出错处理
login_log(
"error", "删除容器失败，触发应用id：%d，容器id：%s，容器服务器域名：%s" %
(task.app_id, obj.container_id, obj.host))
return None
def build_nginx_config(app_id):
"""
生成应用nginx配置文件
:param app_id:
:return:
"""
nginx_str = """
upstream site%d {
%s
}
server{
listen %s:%d;
index index.html index.htm;
location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_pass http://site%d;
}
}
"""
try:
app = AppModel.select().where(AppModel.id == app_id).get()
except:
return False
app_lists = ""
containers = AppContainModel.select().where(
AppContainModel.app_id == app_id)
for obj in containers:
app_lists = app_lists + ("server %s:%d;\n" % (obj.host, obj.port))
config_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
config_path = config_path + "/data/nginx_config/" + str(app_id) + ".conf"
if os.path.exists(config_path):
os.remove(config_path)
if app_lists == "":
return False
config_str = nginx_str % (app_id, app_lists, app.app_host, app.app_port,
app_id)
fp = open(config_path, "w")
fp.write(config_str)
fp.close()
subprocess.getstatusoutput("nginx -s reload")
def get_app_avg_message(app_id):
"""
获取应用平均状态
:return: cpu，内存
"""
cpu = 0
memory = 0
count = AppContainModel.select().where(
AppContainModel.app_id == app_id).count()
objs = AppContainModel.select().where(AppContainModel.app_id == app_id)
for obj in objs:
r = request_api(obj.api_url + "stats",
{"containerId": obj.container_id})
if r.status_code != 200:
cpu += 100
memory += 100
continue
d = json.loads(r.text)
if d['code'] != 0:
cpu += 100
memory += 100
continue
cpu += d['result']['cpu']
memory += d['result']['memory']
if count == 0:
return 100, 100
cpu = cpu / count
cpu = float("%.2f" % cpu)
memory = memory / count
memory = float("%.2f" % memory)
return cpu, memory
def get_dict_from_model_obj(cls, model_obj):
r = dict()
r['id'] = model_obj.id
r['title'] = model_obj.title
r['description'] = model_obj.description
r['memory'] = model_obj.memory
r['env'] = model_obj.env
r['code_address'] = model_obj.code_address
r['app_host'] = model_obj.app_host
r['app_port'] = model_obj.app_port
r['min_container_number'] = model_obj.min_container_number
r['max_container_number'] = model_obj.max_container_number
r['create_time'] = model_obj.create_time.strftime("%Y-%m-%d %H:%M:%S")
r['now_container_number'] = AppContainModel.select().where(AppContainModel.app_id == model_obj.id).count()
return r
def build_nginx_config(app_id):
"""
生成应用nginx配置文件
:param app_id:
:return:
"""
nginx_str = """
upstream site%d {
%s
}
server{
listen %s:%d;
index index.html index.htm;
location / {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_pass http://site%d;
}
}
"""
try:
app = AppModel.select().where(AppModel.id == app_id).get()
except:
return False
app_lists = ""
containers = AppContainModel.select().where(AppContainModel.app_id == app_id)
for obj in containers:
app_lists = app_lists + ("server %s:%d;\n" % (obj.host, obj.port))
config_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
config_path = config_path + "/data/nginx_config/" + str(app_id) + ".conf"
if os.path.exists(config_path):
os.remove(config_path)
if app_lists == "":
return False
config_str = nginx_str % (app_id, app_lists, app.app_host, app.app_port, app_id)
fp = open(config_path, "w")
fp.write(config_str)
fp.close()
subprocess.getstatusoutput("nginx -s reload")
def handle_task(task):
"""
处理每一个任务
:param task: 任务对象
:return:
"""
if task.command_code == 1:
# 添加一个容器
r, obj = add_one_container(task)
if r:
# 保存容器服务器申请到的容器
dao = AppContainModel(
app_id=task.app_id,
host=obj['host'],
port=obj['r']['result']['port'],
container_id=obj['r']['result']['containerId'],
create_time=datetime.datetime.now(),
api_url=obj['api_url'])
dao.save()
login_log(
"common", "申请容器成功，触发应用id：%d，申请到的容器域名：%s，端口：%d，id：%s" %
(task.app_id, obj['host'], obj['r']['result']['port'],
obj['r']['result']['containerId']))
else:
login_log("error", "申请容器失败，触发应用id：%d" % task.app_id)
elif task.command_code == 2:
# 移除一个容器
remove_some_container(task, 1)
elif task.command_code == 3:
# 移除所有容器
count = AppContainModel.select().where(
AppContainModel.app_id == task.app_id).count()
remove_some_container(task, count)
else:
# 未定义任务码
login_log("error",
"未定义任务码：%d，触发应用id：%d" % (task.command_code, task.app_id))
build_nginx_config(task.app_id)
# 删除任务队列
dao = TaskQueueModel.delete().where(TaskQueueModel.id == task.id)
dao.execute()
def handle_task(task):
"""
处理每一个任务
:param task: 任务对象
:return:
"""
if task.command_code == 1:
# 添加一个容器
r, obj = add_one_container(task)
if r:
# 保存容器服务器申请到的容器
dao = AppContainModel(app_id=task.app_id, host=obj['host'], port=obj['r']['result']['port'],
container_id=obj['r']['result']['containerId'], create_time=datetime.datetime.now(),
api_url=obj['api_url'])
dao.save()
login_log("common", "申请容器成功，触发应用id：%d，申请到的容器域名：%s，端口：%d，id：%s" % (task.app_id,
obj['host'], obj['r']['result']['port'],
obj['r']['result']['containerId']))
else:
login_log("error", "申请容器失败，触发应用id：%d"%task.app_id)
elif task.command_code == 2:
# 移除一个容器
remove_some_container(task, 1)
elif task.command_code == 3:
# 移除所有容器
count = AppContainModel.select().where(AppContainModel.app_id == task.app_id).count()
remove_some_container(task, count)
else:
# 未定义任务码
login_log("error", "未定义任务码：%d，触发应用id：%d" % (task.command_code, task.app_id))
build_nginx_config(task.app_id)
# 删除任务队列
dao = TaskQueueModel.delete().where(TaskQueueModel.id == task.id)
dao.execute()
def handle_app(app):
"""
处理每一个APP
:param app: app对象
:return:
"""
container_nums = AppContainModel.select().where(
AppContainModel.app_id == app.id).count()
cpu, memory = get_app_avg_message(app.id)
# 判断是否需要增加容器
if container_nums < app.min_container_number:
add_sign = 1
else:
if cpu > 5.0 and memory > 95.5 and container_nums < app.max_container_number:
add_sign = 1
else:
add_sign = 0
# 判断是否需要减少容器
if container_nums > app.max_container_number:
add_sign = 0
reduce_sign = -1
else:
if cpu < 1.5 and memory < 90 and container_nums > app.min_container_number:
reduce_sign = -1
else:
reduce_sign = 0
if container_nums >= app.max_container_number:
add_sign = 0
if container_nums <= app.min_container_number:
reduce_sign = 0
sign = add_sign + reduce_sign
if sign == 0:
return None
obj = TaskQueueModel(user_id=0,
app_id=app.id,
create_time=datetime.datetime.now())
obj.command_content = '{}'
if sign == 1:
obj.command_code = 1 # 增加容器
elif sign == -1:
obj.command_code = 2 # 减少容器
# 判断命令有没有重复
if TaskQueueModel.select().where(
TaskQueueModel.user_id == obj.user_id,
TaskQueueModel.app_id == obj.app_id,
TaskQueueModel.command_code == obj.command_code).count():
return None
# 删除同应用不同操作
dao = TaskQueueModel.delete().where(TaskQueueModel.user_id == obj.user_id,
TaskQueueModel.app_id == obj.app_id)
dao.execute()
obj.save()
def handle_app(app):
"""
处理每一个APP
:param app: app对象
:return:
"""
container_nums = AppContainModel.select().where(AppContainModel.app_id == app.id).count()
cpu, memory = get_app_avg_message(app.id)
# 判断是否需要增加容器
if container_nums < app.min_container_number:
add_sign = 1
else:
if cpu > 5.0 and memory > 95.5 and container_nums < app.max_container_number:
add_sign = 1
else:
add_sign = 0
# 判断是否需要减少容器
if container_nums > app.max_container_number:
add_sign = 0
reduce_sign = -1
else:
if cpu < 1.5 and memory < 90 and container_nums > app.min_container_number:
reduce_sign = -1
else:
reduce_sign = 0
if container_nums >= app.max_container_number:
add_sign = 0
if container_nums <= app.min_container_number:
reduce_sign = 0
sign = add_sign + reduce_sign
if sign == 0:
return None
obj = TaskQueueModel(user_id=0, app_id=app.id, create_time=datetime.datetime.now())
obj.command_content = "{}"
if sign == 1:
obj.command_code = 1 # 增加容器
elif sign == -1:
obj.command_code = 2 # 减少容器
# 判断命令有没有重复
if (
TaskQueueModel.select()
.where(
TaskQueueModel.user_id == obj.user_id,
TaskQueueModel.app_id == obj.app_id,
TaskQueueModel.command_code == obj.command_code,
)
.count()
):
return None
# 删除同应用不同操作
dao = TaskQueueModel.delete().where(TaskQueueModel.user_id == obj.user_id, TaskQueueModel.app_id == obj.app_id)
dao.execute()
obj.save()
for t in objs:
obj['id'] = t[0]
obj['api_url'] = t[1]
obj['container_id'] = t[2]
obj['app_id'] = t[3]
obj['host'] = t[4]
obj['port'] = t[5]
login_log("maintain", "发现孤儿容器，所属应用id：%d，容器id：%s，容器域名：%s，容器端口：%d" % (obj['app_id'],
obj['container_id'], obj['host'], obj['port']))
api_url = obj['api_url'] + "remove"
data = {
"containerId": obj['container_id']
}
r = request_api(api_url, data)
if r.status_code == 200:
d = json.loads(r.text)
if d['code'] == 0:
# 删除容器成功
dao = AppContainModel.delete().where(AppContainModel.id == obj['id'])
dao.execute()
login_log("maintain", "删除孤儿容器成功，所属应用id：%d，容器id：%s，容器域名：%s，容器端口：%d" % (obj['app_id'],
obj['container_id'], obj['host'],
obj['port']))
continue
login_log("maintain_error", "删除孤儿容器失败，所属应用id：%d，容器id：%s，容器域名：%s，容器端口：%d" % (obj['app_id'],
obj['container_id'], obj['host'],
obj['port']))
close_connect(sqlite_db_use=False)
print("清除孤儿容器结束")
