def sse_worker(product):
# job_pattern = re.compile('salt/job/\d+/ret/')
mine_pattern = re.compile(r'"fun": "mine.update"')
saltutil_pattern = re.compile(r'"fun": "saltutil.find_job"')
running_pattern = re.compile(r'"fun": "saltutil.running"')
lookup_pattern = re.compile(r'"fun": "runner.jobs.lookup_jid"')
event_pattern = re.compile(r'"tag": "salt/event/new_client"')
salt_api = salt_api_for_product(product)
event_response = salt_api.events()
client = sseclient.SSEClient(event_response)
for event in client.events():
if mine_pattern.search(event.data):
pass
elif saltutil_pattern.search(event.data):
pass
elif running_pattern.search(event.data):
pass
elif lookup_pattern.search(event.data):
pass
elif event_pattern.search(event.data):
pass
else:
print(event.data)
event_dict = ast.literal_eval(event.data.replace('true', 'True').replace('false', 'False').
replace('null', '""'))
event_dict['data']['_stamp'] = utc_to_local(event_dict['data']['_stamp'] + "Z")
event_dict['data']['product_id'] = product
db = DB()
db.insert("event", json.dumps(event_dict, ensure_ascii=False))
db.close_mysql()
def get(self):
db = DB()
status, result = db.select("product", "")
db.close_mysql()
product_list = []
if status is True:
if result:
for i in result:
try:
product_list.append(eval(i[0]))
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {
"status": False,
"message": "Product does not exist"
}, 404
else:
return {"status": False, "message": result}, 500
return {
"products": {
"product": product_list
},
"status": True,
"message": ""
}, 200
def get(self):
user = g.user_info["username"]
args = parser.parse_args()
salt_api = salt_api_for_product(args["product_id"])
minions = []
minions_mysql = []
if isinstance(salt_api, dict):
return salt_api, 500
else:
result = salt_api.list_all_key()
if result:
if result.get("status") is False:
return result, 500
for minion in result.get("minions"):
minions.append(minion)
# 同步产品线下的Grains
Grains.create_grains(minions, args["product_id"], user)
db = DB()
status, result = db.select("grains", "where data -> '$.product_id'='%s'" % args["product_id"])
db.close_mysql()
if status is True and result:
for i in result:
minions_mysql.append(i.get("id"))
# 对比数据库中的minion与已经同意的minion的不同，删掉数据库中多余的minion
diff = list(set(minions_mysql).difference(minions))
Grains.delete_grains(diff, args["product_id"], user)
return {"status": True, "message": ""}, 200
def post(self):
user = g.user_info["username"]
args = parser.parse_args()
args["id"] = uuid_prefix("u")
db = DB()
status, result = db.select(
"user", "where data -> '$.username'='%s'" % args["username"])
if status is True:
if len(result) == 0:
password_hash = custom_app_context.encrypt(args["password"])
args["password"] = password_hash
users = args
# 默认新添加的用户都是默认用户
role_id = get_common_user()
if isinstance(role_id, dict):
return role_id
users["role"].append(role_id)
insert_status, insert_result = db.insert(
"user", json.dumps(users, ensure_ascii=False))
db.close_mysql()
if insert_status is not True:
logger.error("Add user error: %s" % insert_result)
return {"status": False, "message": insert_result}, 500
audit_log(user, args["id"], "", "user", "add")
return {"status": True, "message": ""}, 201
else:
return {
"status": False,
"message": "The user name already exists"
}, 200
else:
logger.error("Select user error: %s" % result)
return {"status": False, "message": result}, 500
def delete(self, groups_id):
db = DB()
# 首先获得所需项目
status, result = db.select_by_id("groups", groups_id)
if status:
group = result
else:
return {"status": False, "message": str(result)}, 500
# 执行删除
status, result = db.delete_by_id("groups", groups_id)
if status is not True:
return {"status": False, "message": result}, 500
if result is 0:
return {
"status": False,
"message": "%s does not exist" % groups_id
}, 404
# 完成数据的统一，将project中的组类别删除
project_list = group['projects']
for project_name in project_list:
status, result = db.select(
'projects', "where data -> '$.name'='%s'" % project_name)
project_origion = dict(result[0])
group_list = list(project_origion['groups'])
group_list.remove(groups_id)
project_origion['groups'] = group_list
db.update_by_id("projects",
json.dumps(project_origion, ensure_ascii=False),
result[0]['id'])
db.close_mysql()
return {"status": True, "message": ""}, 200
def put(self, role_id):
user = g.user_info["username"]
args = parser.parse_args()
args["id"] = role_id
role = args
db = DB()
status, result = db.select(
"role", "where data -> '$.name'='%s'" % args["name"])
if status is True:
if len(result) != 0:
info = eval(result[0][0])
if role_id != info.get("id"):
return {
"status": False,
"message": "The role name already exists"
}, 200
status, result = db.update_by_id("role",
json.dumps(role, ensure_ascii=False),
role_id)
db.close_mysql()
if status is not True:
logger.error("Modify role error: %s" % result)
return {"status": False, "message": result}, 500
audit_log(user, role_id, "", "role", "edit")
return {"status": True, "message": ""}, 200
def get(self):
args = parser.parse_args()
action = args["action"]
if action == "start":
db = DB()
status, result = db.select("product", "")
db.close_mysql()
if status is True:
if result:
for product in result:
event_to_mysql.delay(product['id'])
return {"data": "", "status": True, "message": ""}, 200
else:
return {
"data": "",
"status": False,
"message": "Product does not exist"
}, 404
else:
return {"data": "", "status": False, "message": result}, 500
if action == "stop":
echo = os.popen("celery -A app.celery inspect active --json")
info = echo.readline()
try:
result = eval(
info.replace("true", "'true'").replace("false", "'false'"))
for k, v in result.items():
for w in v:
os.popen("kill -9 %s" % w.get("worker_pid"))
return {"data": "", "status": True, "message": ""}, 200
except Exception as e:
logger.error("Stop celery error: %s" % e)
return {"data": "", "status": False, "message": "%s" % e}, 200
def put(self, target_id):
args = parser.parse_args()
logger.info(args['host_id'])
args["id"] = target_id
logger.info('id:' + target_id)
del args['path'], args['key_word'], args['file_name'], args[
'target_id'], args['cipher']
target = args
db = DB()
status, result = db.select_by_id('target', target_id)
origion_IP = result['IP']
host_id = result['host_id']
args['host_id'] = host_id
if origion_IP != args['IP']:
status, message = judge_target_IP_exist(args['IP'],
args['host_id'])
if status is not True:
return {"status": False, "message": message}, 500
status, result = db.update_by_id(
"target", json.dumps(target, ensure_ascii=False), target_id)
db.close_mysql()
if status is not True:
logger.error("Modify target: %s" % result)
return {"status": False, "message": result}, 500
return {"status": True, "message": result}, 200
def get(self):
db = DB()
user_info = g.user_info
sql_list = []
groups_list = []
if user_info["groups"]:
for group in user_info["groups"]:
sql_list.append("data -> '$.id'='%s'" % group)
sql = " or ".join(sql_list)
status, result = db.select("groups", "where %s" % sql)
db.close_mysql()
if status is True:
if result:
for i in result:
try:
groups_list.append(eval(i[0]))
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {
"status": False,
"message": "Group does not exist"
}, 404
else:
return {"status": False, "message": result}, 500
return {"groups": {"groups": groups_list}}, 200
else:
return {"groups": {"groups": groups_list}}, 200
def delete(self):
args = parser.parse_args()
db = DB()
status, result = db.select_by_id("product", args["product_id"])
db.close_mysql()
if status is True:
if result:
product = eval(result[0][0])
else:
return {"status": False, "message": "%s does not exist" % args["product_id"]}
else:
return {"status": False, "message": result}
salt_api = salt_api_for_product(args["product_id"])
user = g.user_info["username"]
if isinstance(salt_api, dict):
return salt_api, 500
else:
if args["action"] == "kill" and args["jid"]:
kill = "salt %s saltutil.kill_job %s" % (product.get("salt_master_id"), args["jid"])
try:
result = salt_api.shell_remote_execution(product.get("salt_master_id"), kill)
audit_log(user, args["jid"], args["product_id"], "job id", "kill")
return {"status": True, "message": result}, 200
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {"status": False, "message": "The specified job id does not exist or arguments error"}, 400
def get_gitlab(product_id):
db = DB()
status, result = db.select_by_id("product", product_id)
db.close_mysql()
if status is True:
if result:
product = result
else:
return {"status": False, "message": "%s does not exist" % product_id}
else:
return {"status": False, "message": result}
try:
if product.get("file_server") == "gitfs":
gl = gitlab.Gitlab(url=product.get("gitlab_url"),
private_token=None if product.get("private_token") is "" else product.get(
"private_token"),
oauth_token=None if product.get("oauth_token") is "" else product.get("oauth_token"),
email=None if product.get("email") is "" else product.get("email"),
password=None if product.get("password") is "" else product.get("password"),
ssl_verify=True,
http_username=None if product.get("http_username") is "" else product.get(
"http_username"),
http_password=None if product.get("http_password") is "" else product.get(
"http_password"),
timeout=120,
api_version=None if product.get("api_version") is "" else product.get("api_version")
)
return gl
else:
return {"status": False, "message": "File server is not gitfs"}, ""
except Exception as e:
return {"status": False, "message": str(e)}
def get(self):
product_id = request.args.get("product_id")
scheduler_type = request.args.get("scheduler_type")
db = DB()
task = []
if scheduler_type:
sql = "where data -> '$.product_id'='%s' and data -> '$.scheduler'!='%s' " \
"order by data -> '$.timestamp' desc" % (product_id, scheduler_type)
else:
sql = "where data -> '$.product_id'='%s' order by data -> '$.timestamp' desc" % product_id
status, result = db.select("period_task", sql)
if status is True:
for period in result:
target = []
for group_id in period.get("target"):
group_status, group_result = db.select_by_id(
"groups", group_id)
if group_status is True:
target.append({
"id": group_id,
"name": group_result.get("name")
})
period["target"] = target
period_audit_status, period_audit_result = db.select(
"period_audit",
"where data -> '$.id'='%s' order by data -> '$.result.timestamp' desc limit 1"
% period["id"])
period["audit"].extend(period_audit_result)
task.append(period)
db.close_mysql()
return {"data": task, "status": True, "message": ""}, 200
else:
return {"status": False, "message": task}, 500
def create_git_project(product_id, project_name):
logger.info("create_git_project1")
db = DB()
status, result = db.select(
'projects', "where data -> '$.gitlab_name'='%s'" % project_name)
if status is True:
if len(result) == 0:
logger.info('project_name:' + project_name)
gl = get_gitlab(product_id)
try:
gl.projects.create({'name': project_name})
except Exception as e:
raise Exception('该gitlab项目名已被占用')
projects = gl.projects.list(all=True)
for pr in projects:
if str(pr.__dict__.get('_attrs').get(
'path_with_namespace')).replace('root/',
'') == project_name:
project = gl.projects.get(
pr.__dict__.get('_attrs').get('id'))
commit_init_file(project)
return True
else:
db.close_mysql()
return False
else:
db.close_mysql()
raise Exception('mysql select from projects error')
def get(self, user_id):
user = g.user_info["username"]
db = DB()
status, result = db.select_by_id("user", user_id)
if status is True and result:
# 生成12位随机密码
password = ''.join(
random.sample(string.ascii_letters + string.digits, 12))
send_mail(result["mail"], "Saltshaker 重置密码", "新密码：" + password)
# 离散hash
password_hash = custom_app_context.encrypt(password)
result["password"] = password_hash
insert_status, insert_result = db.update_by_id(
"user", json.dumps(result, ensure_ascii=False), user_id)
db.close_mysql()
if insert_status is not True:
logger.error("Reset %s password error: %s" %
(user_id, insert_result))
return {"status": False, "message": insert_result}, 500
audit_log(user, user_id, "", "user", "reset")
return {"status": True, "message": ""}, 201
else:
db.close_mysql()
logger.error("Select user error: %s" % result)
return {"status": False, "message": result}, 500
def post(self):
args = parser.parse_args()
sls = args["sls"]
if not sls:
return {
"status": False,
"message": "The specified sls parameter does not exist"
}, 400
# 去掉后缀
sls = sls.replace(".sls", "")
minion_id = args["minion_id"]
salt_api = salt_api_for_product(args["product_id"])
user_info = g.user_info
audit_log(user_info["username"], minion_id, args["product_id"],
"minion", "sls")
if isinstance(salt_api, dict):
return salt_api, 500
result = salt_api.target_deploy(minion_id, sls)
db = DB()
cmd_history = {
"user_id": user_info["id"],
"product_id": args["product_id"],
"command": args["sls"],
"type": "sls",
"minion_id": minion_id,
"result": result,
"time": time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
}
db.insert("cmd_history", json.dumps(cmd_history, ensure_ascii=False))
db.close_mysql()
audit_log(user_info["username"], minion_id, args["product_id"],
"minion", "sls")
minion_count = str(len(minion_id))
cmd_succeed = str(len(result))
cmd_failure = str(len(minion_id) - len(result))
succeed_minion = []
for i in result:
succeed_minion.append(i)
failure_minion = ','.join(
list(set(minion_id).difference(set(succeed_minion))))
if result.get("status") is False:
status = False
message = result.get("message")
else:
status = True
message = ""
return {
"data": {
"result": result,
"command": args["sls"],
"total": minion_count,
"succeed": cmd_succeed,
"failure": cmd_failure,
"failure_minion": failure_minion
},
"status": status,
"message": message
}, 200
def insert_period_audit(period_id, period_audits):
db = DB()
results = {"id": period_id, "result": period_audits}
status, result = db.insert("period_audit",
json.dumps(results, ensure_ascii=False))
if status is False:
db.close_mysql()
logger.error("Insert period audit error: %s" % result)
def get(self):
db = DB()
status, result = db.select("role", "")
db.close_mysql()
if status is True:
return {"data": result, "status": True, "message": ""}, 200
else:
return {"status": False, "message": result}, 500