def get(self):
db = DB()
user_info = g.user_info
sql_list = []
groups_list = []
if user_info["groups"]:
for group in user_info["groups"]:
sql_list.append("data -> '$.id'='%s'" % group)
sql = " or ".join(sql_list)
status, result = db.select("groups", "where %s" % sql)
db.close_mysql()
if status is True:
if result:
for i in result:
try:
groups_list.append(eval(i[0]))
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {
"status": False,
"message": "Group does not exist"
}, 404
else:
return {"status": False, "message": result}, 500
return {"groups": {"groups": groups_list}}, 200
else:
return {"groups": {"groups": groups_list}}, 200
def updatePriceLimit(self, p):
DB.update(self.collection, {'_id': self._id}, {
'name': self.name,
"item_id": self.item_id,
'priceLimit': p,
'_id': self._id
})
def main():
load_dotenv(find_dotenv())
HOST = getenv("MYSQL_HOST")
PORT = getenv("MYSQL_PORT")
USER = getenv("MYSQL_USER")
PASS = getenv("MYSQL_PASS")
DATABASE = getenv("MYSQL_DB")
db = DB(HOST, PORT, USER, PASS, DATABASE)
store = Store(getenv("S3_BUCKET"))
files = store.getFiles(PREFIX)
file_names = list(
filter(lambda x: ".csv" in x, map(lambda x: x.key, files)))
latest_date = max(
list(map(lambda x: extractDate(x, PREFIX, ".csv"), file_names)))
latest_file = "{}/{}.{}".format(PREFIX, latest_date, "csv")
# Get File
body = store.getFile(latest_file)
csv = pd.read_csv(StringIO(body), low_memory=False)
csv = csv[["track_id", "played_at"]]
csv_tuple = [tuple(x) for x in csv.to_numpy()]
# Load data to sql
db.insertRecentPlays(csv_tuple)
def put(self, host_id):
logger.info("HOSTPUT")
user = g.user_info["username"]
args = parser.parse_args()
args["id"] = host_id
db = DB()
# 判断是否存在
select_status, select_result = db.select_by_id("host", host_id)
if select_status is False:
db.close_mysql()
logger.error("Modify host error: %s" % select_result)
return {"status": False, "message": select_result}, 500
if select_result:
try:
host = select_result
if args['tag'] != []:
host["tag"] = args["tag"]
if args['rename'] != '':
host["rename"] = args["rename"]
status, result = db.update_by_id(
"host", json.dumps(host, ensure_ascii=False), host_id)
db.close_mysql()
if status is not True:
logger.error("Modify host error: %s" % result)
return {"status": False, "message": result}, 500
except Exception as e:
db.close_mysql()
logger.error("Modify %s host error: %s" % (host_id, e))
return {"status": False, "message": str(e)}, 500
audit_log(user, args["id"], args["product_id"], "host", "edit")
return {"status": True, "message": ""}, 200
def get(self):
user = g.user_info["username"]
args = parser.parse_args()
salt_api = salt_api_for_product(args["product_id"])
minions = []
minions_mysql = []
if isinstance(salt_api, dict):
return salt_api, 500
else:
result = salt_api.list_all_key()
if result:
if result.get("status") is False:
return result, 500
for minion in result.get("minions"):
minions.append(minion)
# 同步产品线下的Grains
Grains.create_grains(minions, args["product_id"], user)
db = DB()
status, result = db.select("grains", "where data -> '$.product_id'='%s'" % args["product_id"])
db.close_mysql()
if status is True and result:
for i in result:
minions_mysql.append(i.get("id"))
# 对比数据库中的minion与已经同意的minion的不同，删掉数据库中多余的minion
diff = list(set(minions_mysql).difference(minions))
Grains.delete_grains(diff, args["product_id"], user)
return {"status": True, "message": ""}, 200
def get(self):
args = parser.parse_args()
action = args["action"]
if action == "start":
db = DB()
status, result = db.select("product", "")
db.close_mysql()
if status is True:
if result:
for product in result:
event_to_mysql.delay(product['id'])
return {"data": "", "status": True, "message": ""}, 200
else:
return {
"data": "",
"status": False,
"message": "Product does not exist"
}, 404
else:
return {"data": "", "status": False, "message": result}, 500
if action == "stop":
echo = os.popen("celery -A app.celery inspect active --json")
info = echo.readline()
try:
result = eval(
info.replace("true", "'true'").replace("false", "'false'"))
for k, v in result.items():
for w in v:
os.popen("kill -9 %s" % w.get("worker_pid"))
return {"data": "", "status": True, "message": ""}, 200
except Exception as e:
logger.error("Stop celery error: %s" % e)
return {"data": "", "status": False, "message": "%s" % e}, 200
def post(self):
logger.info("SinglePing")
args = parser.parse_args()
target_id = args['target_id']
cipher = args['cipher']
# 获得所需参数minion_id、product_id、target_ip
db = DB()
state, result = db.select_by_id('target', target_id)
target_ip = result['IP']
host_id = result['host_id']
state, result = db.select_by_id('host', host_id)
minion_id = result['minion_id']
product_id = result['product_id']
salt_api = salt_api_for_product(product_id)
command = 'snmpwalk -v 2c -t 0.5 -c \'' + cipher + '\' ' + target_ip + ' 1.3.6.1.2.1.1.1'
logger.info('command:' + command)
sysDescr = salt_api.shell_remote_execution([minion_id], command)
response_data = {}
if str(sysDescr[minion_id]).__contains__("Timeout") | str(
sysDescr[minion_id]).__contains__("Unknown"):
response_data['status'] = '设备网络不通'
else:
response_data['status'] = "设备正常"
response_data['sysDescr'] = str(sysDescr[minion_id])
return {"status": True, "message": '成功', "data": response_data}, 200
def post(self):
logger.info("PingList")
args = parser.parse_args()
db = DB()
host_id = args['host_id']
cipher = args['cipher']
state, result = db.select('host',
"where data -> '$.id'='%s'" % host_id)
minion_id = result[0]['minion_id']
logger.info('minion_id:' + minion_id)
product_id = result[0]['product_id']
salt_api = salt_api_for_product(product_id)
state, targets = db.select('target',
"where data -> '$.host_id'='%s'" % host_id)
targets_not = []
thread_pool = ThreadPoolExecutor(max_workers=10,
thread_name_prefix="target_")
futures = []
for target in targets:
future = thread_pool.submit(pingTarget, target, minion_id,
salt_api, cipher)
futures.append(future)
thread_pool.shutdown(wait=True)
for future in futures:
result = future.result()
logger.info(str(result['status']))
if str(result['status']).__contains__("Timeout") | str(
result['status']).__contains__("Unknown"):
targets_not.append(result["target"])
return {"status": True, "message": '配置发送成功', "data": targets_not}, 200
def get_gitlab(product_id):
db = DB()
status, result = db.select_by_id("product", product_id)
db.close_mysql()
if status is True:
if result:
product = result
else:
return {"status": False, "message": "%s does not exist" % product_id}
else:
return {"status": False, "message": result}
try:
if product.get("file_server") == "gitfs":
gl = gitlab.Gitlab(url=product.get("gitlab_url"),
private_token=None if product.get("private_token") is "" else product.get(
"private_token"),
oauth_token=None if product.get("oauth_token") is "" else product.get("oauth_token"),
email=None if product.get("email") is "" else product.get("email"),
password=None if product.get("password") is "" else product.get("password"),
ssl_verify=True,
http_username=None if product.get("http_username") is "" else product.get(
"http_username"),
http_password=None if product.get("http_password") is "" else product.get(
"http_password"),
timeout=120,
api_version=None if product.get("api_version") is "" else product.get("api_version")
)
return gl
else:
return {"status": False, "message": "File server is not gitfs"}, ""
except Exception as e:
return {"status": False, "message": str(e)}
def main():
load_dotenv(find_dotenv())
HOST = getenv("MYSQL_HOST")
PORT = getenv("MYSQL_PORT")
USER = getenv("MYSQL_USER")
PASS = getenv("MYSQL_PASS")
DATABASE = getenv("MYSQL_DB")
db = DB(HOST, PORT, USER, PASS, DATABASE)
store = Store(getenv("S3_BUCKET"))
files = store.getFiles(PREFIX)
file_names = list(
filter(lambda x: ".csv" in x, map(lambda x: x.key, files)))
latest_date = max(
list(map(lambda x: extractDate(x, PREFIX, ".csv"), file_names)))
latest_file = "{}/{}.{}".format(PREFIX, latest_date, "csv")
# Get File
body = store.getFile(latest_file)
csv = pd.read_csv(StringIO(body), low_memory=False)
csv = csv[[
"artist", "album", "track", "track_id", "danceability", "energy",
"key", "loudness", "mode", "speechiness", "acousticness",
"instrumentalness", "liveness", "valence", "tempo", "duration_ms",
"lyrics", "popularity", "explicit"
]]
csv = csv.dropna()
csv_tuple = [tuple(x) for x in csv.to_numpy()]
# Load data to sql
db.insertSongs(csv_tuple)
def post(self):
args = parser.parse_args()
args["id"] = uuid_prefix("project")
db = DB()
project = transfer_args_to_project(args)
# 如果是创建，group们必然是增加此项目名
status, result = db.select(
"projects",
"where data -> '$.name'='%s' and data -> '$.product_id'='%s'" %
(args["name"], args["product_id"]))
if status is True:
if len(result) == 0:
try:
create_git_project(args['product_id'], args['gitlab_name'])
git_clone(args['product_id'], args['gitlab_name'])
except Exception as e:
return {"status": False, "message": str(e)}, 500
insert_status, insert_result = db.insert(
"projects", json.dumps(project, ensure_ascii=False))
update_group_for_create_project(project['name'],
project['groups'])
db.close_mysql()
if insert_status is not True:
return {"status": False, "message": insert_result}, 500
return {"status": True, "message": ""}, 200
else:
db.close_mysql()
return {
"status": False,
"message": "The projects name already exists"
}, 500
else:
db.close_mysql()
return {"status": False, "message": result}, 500
def get(self):
product_id = request.args.get("product_id")
db = DB()
user_info = g.user_info
sql_list = []
groups_list = []
if user_info["groups"]:
for group in user_info["groups"]:
sql_list.append(
"data -> '$.id'='%s' and data -> '$.product_id'='%s'" %
(group, product_id))
sql = " or ".join(sql_list)
status, result = db.select("groups", "where %s" % sql)
db.close_mysql()
if status is True:
if result:
groups_list = result
return {
"data": groups_list,
"status": True,
"message": ""
}, 200
else:
return {
"status": False,
"message": "Group does not exist"
}, 404
else:
return {"status": False, "message": result}, 500
else:
return {"data": groups_list, "status": True, "message": ""}, 200
def post(self):
args = parser.parse_args()
args["id"] = uuid_prefix("a")
user = g.user_info["username"]
acl = args
db = DB()
status, result = db.select(
"acl", "where data -> '$.name'='%s'" % args["name"])
if status is True:
if len(result) == 0:
insert_status, insert_result = db.insert(
"acl", json.dumps(acl, ensure_ascii=False))
db.close_mysql()
if insert_status is not True:
logger.error("Add acl error: %s" % insert_result)
return {"status": False, "message": insert_result}, 500
audit_log(user, args["id"], "", "acl", "add")
return {"status": True, "message": ""}, 201
else:
db.close_mysql()
return {
"status": False,
"message": "The acl name already exists"
}, 200
else:
db.close_mysql()
logger.error("Select acl name error: %s" % result)
return {"status": False, "message": result}, 500
def create_grains(minion_list, product_id, user):
salt_api = salt_api_for_product(product_id)
if isinstance(salt_api, dict):
return salt_api, 500
db = DB()
for minion in minion_list:
select_status, select_result = db.select(
"grains", "where data -> '$.id'='%s' and data -> "
"'$.product_id'='%s'" % (minion, product_id))
grains = salt_api.grains(minion)
grains[minion].update({"product_id": product_id})
if select_status is True:
if len(select_result) > 1:
for m in select_result:
db.delete_by_id("grains", m["id"])
insert_status, insert_result = db.insert(
"grains", json.dumps(grains[minion],
ensure_ascii=False))
if insert_status is not True:
logger.error("Add Grains error: %s" % insert_result)
elif len(select_result) == 1:
update_status, update_result = db.update_by_id(
"grains", json.dumps(grains[minion],
ensure_ascii=False),
select_result[0]["id"])
if update_status is not True:
logger.error("Update Grains error: %s" % update_result)
else:
insert_status, insert_result = db.insert(
"grains", json.dumps(grains[minion],
ensure_ascii=False))
if insert_status is not True:
logger.error("Add Grains error: %s" % insert_result)
db.close_mysql()
def get(self):
product_id = request.args.get("product_id")
db = DB()
status, result = db.select(
"groups", "where data -> '$.product_id'='%s'" % product_id)
db.close_mysql()
groups_list = []
if status is True:
if result:
for i in result:
try:
groups_list.append(eval(i[0]))
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {
"groups": {
"group": groups_list
},
"status": True,
"message": ""
}, 200
else:
return {"status": False, "message": result}, 500
return {
"groups": {
"group": groups_list
},
"status": True,
"message": ""
}, 200
def get(self):
db = DB()
status, result = db.select("product", "")
db.close_mysql()
product_list = []
if status is True:
if result:
for i in result:
try:
product_list.append(eval(i[0]))
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {
"status": False,
"message": "Product does not exist"
}, 404
else:
return {"status": False, "message": result}, 500
return {
"products": {
"product": product_list
},
"status": True,
"message": ""
}, 200
def get(self):
logger.info("HOSTLIST")
product_id = request.args.get("product_id")
db = DB()
status, result = db.select(
"host", "where data -> '$.product_id'='%s'" % product_id)
if status is True:
host_list = result
else:
db.close_mysql()
return {"status": False, "message": result}, 500
status, result = db.select(
"groups", "where data -> '$.product_id'='%s'" % product_id)
if status is True:
groups_list = result
else:
db.close_mysql()
return {"status": False, "message": result}, 500
for host in host_list:
for group in groups_list:
for minion in group["minion"]:
if host["minion_id"] == minion:
host["groups"].append(group["name"])
db.close_mysql()
return {"data": host_list, "status": True, "message": ""}, 200
def post(self):
logger.info("HOSTLIST")
args = parser.parse_args()
args["id"] = uuid_prefix("h")
user = g.user_info["username"]
host = args
db = DB()
status, result = db.select(
"host", "where data -> '$.minion_id'='%s'" % args["minion_id"])
if status is True:
if len(result) == 0:
insert_status, insert_result = db.insert(
"host", json.dumps(host, ensure_ascii=False))
db.close_mysql()
if insert_status is not True:
logger.error("Add host error: %s" % insert_result)
return {"status": False, "message": insert_result}, 500
audit_log(user, args["id"], args["product_id"], "host", "add")
return {"status": True, "message": ""}, 201
else:
db.close_mysql()
return {
"status": False,
"message": "The host name already exists"
}, 200
else:
db.close_mysql()
logger.error("Select host name error: %s" % result)
return {"status": False, "message": result}, 500
def post(self):
user = g.user_info["username"]
args = parser.parse_args()
args["id"] = uuid_prefix("u")
db = DB()
status, result = db.select(
"user", "where data -> '$.username'='%s'" % args["username"])
if status is True:
if len(result) == 0:
# 默认新添加的用户都是默认用户
role_id = get_common_user()
if isinstance(role_id, dict):
return role_id
args["role"].append(role_id)
insert_status, insert_result = db.insert(
"user", json.dumps(args, ensure_ascii=False))
db.close_mysql()
if insert_status is not True:
logger.error("Add user error: %s" % insert_result)
return {"status": False, "message": insert_result}, 500
audit_log(user, args["id"], "", "user", "add")
return {"status": True, "message": ""}, 201
else:
db.close_mysql()
return {
"status": False,
"message": "The user name already exists"
}, 200
else:
db.close_mysql()
logger.error("Select user error: %s" % result)
return {"status": False, "message": result}, 500
def db(self):
if not self._db:
app = import_module(self.__module__.split('.')[0])
conf = import_module('config')
connection = conf.DATABASES[app.database][conf.TEST]
self._db = DB(**connection).session
return self._db
def put(self, host_id):
user = g.user_info["username"]
args = parser.parse_args()
args["id"] = host_id
host = args
db = DB()
# 判断是否存在
select_status, select_result = db.select_by_id("host", host_id)
if select_status is False:
db.close_mysql()
logger.error("Modify host error: %s" % select_result)
return {"status": False, "message": select_result}, 500
if select_result:
for i in select_result:
try:
host = eval(i[0])
host["group_id"] = args["group_id"]
host["tag"] = args["tag"]
except Exception as e:
db.close_mysql()
logger.error("Modify %s host error: %s" % (host_id, e))
return {"status": False, "message": str(e)}, 500
status, result = db.update_by_id("host", json.dumps(host, ensure_ascii=False), host_id)
db.close_mysql()
if status is not True:
logger.error("Modify host error: %s" % result)
return {"status": False, "message": result}, 500
audit_log(user, args["id"], args["product_id"], "host", "edit")
return {"status": True, "message": ""}, 200
def delete(self):
args = parser.parse_args()
db = DB()
status, result = db.select_by_id("product", args["product_id"])
db.close_mysql()
if status is True:
if result:
product = eval(result[0][0])
else:
return {"status": False, "message": "%s does not exist" % args["product_id"]}
else:
return {"status": False, "message": result}
salt_api = salt_api_for_product(args["product_id"])
user = g.user_info["username"]
if isinstance(salt_api, dict):
return salt_api, 500
else:
if args["action"] == "kill" and args["jid"]:
kill = "salt %s saltutil.kill_job %s" % (product.get("salt_master_id"), args["jid"])
try:
result = salt_api.shell_remote_execution(product.get("salt_master_id"), kill)
audit_log(user, args["jid"], args["product_id"], "job id", "kill")
return {"status": True, "message": result}, 200
except Exception as e:
return {"status": False, "message": str(e)}, 500
else:
return {"status": False, "message": "The specified job id does not exist or arguments error"}, 400
def post(self):
args = parser.parse_args()
args["id"] = uuid_prefix("p")
user = g.user_info["username"]
product = args
db = DB()
status, result = db.select(
"product", "where data -> '$.name'='%s'" % args["name"])
if status is True:
if len(result) == 0:
insert_status, insert_result = db.insert(
"product", json.dumps(product, ensure_ascii=False))
db.close_mysql()
if insert_status is not True:
logger.error("Add product error: %s" % insert_result)
return {"status": False, "message": insert_result}, 500
audit_log(user, args["id"], "", "product", "add")
# 更新Rsync配置
if args["file_server"] == "rsync":
rsync_config()
return {"status": True, "message": ""}, 201
else:
db.close_mysql()
return {
"status": False,
"message": "The product name already exists"
}, 200
else:
db.close_mysql()
logger.error("Select product name error: %s" % result)
return {"status": False, "message": result}, 500
def verify_acl(acl_list, command):
if acl_list:
db = DB()
sql_list = []
for acl_id in acl_list:
sql_list.append("data -> '$.id'='%s'" % acl_id)
sql = " or ".join(sql_list)
status, result = db.select("acl", "where %s" % sql)
db.close_mysql()
if status is True:
if result:
for i in result:
try:
acl = eval(i[0])
for deny in acl["deny"]:
deny_pattern = re.compile(deny)
if deny_pattern.search(command):
return {
"status":
False,
"message":
"Deny Warning : You don't have permission run [ %s ]"
% command
}
except Exception as e:
return {"status": False, "message": str(e)}
return {"status": True, "message": ""}
else:
return {"status": False, "message": "acl does not exist"}
else:
return {"status": False, "message": result}
else:
return {"status": True, "message": ""}
def get(self):
db = DB()
status, result = db.select("role", "")
db.close_mysql()
if status is True:
return {"data": result, "status": True, "message": ""}, 200
else:
return {"status": False, "message": result}, 500
def insert_period_audit(period_id, period_audits):
db = DB()
results = {"id": period_id, "result": period_audits}
status, result = db.insert("period_audit",
json.dumps(results, ensure_ascii=False))
if status is False:
db.close_mysql()
logger.error("Insert period audit error: %s" % result)
def get_gitlab_project(project_id, product_id):
db = DB()
status, result = db.select_by_id('projects', project_id)
project_gitlab_name = result['gitlab_name']
db.close_mysql()
project, product_name = gitlab_project_name(product_id,
project_gitlab_name)
return project, product_name
def setUp(self) -> None:
#实例化DB
self.db = DB()
#登录
url = 'http://ecshop.itsoso.cn/ECMobile/?url=/user/signin'
data = {"name": "lzj", "password": "******"}
response = Interface_login.login(method='post', url=url, data=data)
self.sid = get_result_one(response, 'sid')
self.uid = get_result_one(response, 'uid')
def get(self):
args = parser.parse_args()
db = DB()
status, result = db.select("audit_log", "where data -> '$.product_id'='%s' order by data -> '$.time' desc"
% args["product_id"])
db.close_mysql()
if status is True:
return {"data": result, "status": True, "message": ""}, 200
else:
return {"status": False, "message": result}, 500
def get(self):
product_id = request.args.get("product_id")
db = DB()
status, result = db.select(
"grains", "where data -> '$.product_id'='%s'" % product_id)
db.close_mysql()
if status is True:
return {"data": result, "status": True, "message": ""}, 200
else:
return {"status": False, "message": result}, 500
