def get(self): db = DB() user_info = g.user_info sql_list = [] groups_list = [] if user_info["groups"]: for group in user_info["groups"]: sql_list.append("data -> '$.id'='%s'" % group) sql = " or ".join(sql_list) status, result = db.select("groups", "where %s" % sql) db.close_mysql() if status is True: if result: for i in result: try: groups_list.append(eval(i[0])) except Exception as e: return {"status": False, "message": str(e)}, 500 else: return { "status": False, "message": "Group does not exist" }, 404 else: return {"status": False, "message": result}, 500 return {"groups": {"groups": groups_list}}, 200 else: return {"groups": {"groups": groups_list}}, 200
def updatePriceLimit(self, p): DB.update(self.collection, {'_id': self._id}, { 'name': self.name, "item_id": self.item_id, 'priceLimit': p, '_id': self._id })
def main(): load_dotenv(find_dotenv()) HOST = getenv("MYSQL_HOST") PORT = getenv("MYSQL_PORT") USER = getenv("MYSQL_USER") PASS = getenv("MYSQL_PASS") DATABASE = getenv("MYSQL_DB") db = DB(HOST, PORT, USER, PASS, DATABASE) store = Store(getenv("S3_BUCKET")) files = store.getFiles(PREFIX) file_names = list( filter(lambda x: ".csv" in x, map(lambda x: x.key, files))) latest_date = max( list(map(lambda x: extractDate(x, PREFIX, ".csv"), file_names))) latest_file = "{}/{}.{}".format(PREFIX, latest_date, "csv") # Get File body = store.getFile(latest_file) csv = pd.read_csv(StringIO(body), low_memory=False) csv = csv[["track_id", "played_at"]] csv_tuple = [tuple(x) for x in csv.to_numpy()] # Load data to sql db.insertRecentPlays(csv_tuple)
def put(self, host_id): logger.info("HOSTPUT") user = g.user_info["username"] args = parser.parse_args() args["id"] = host_id db = DB() # 判断是否存在 select_status, select_result = db.select_by_id("host", host_id) if select_status is False: db.close_mysql() logger.error("Modify host error: %s" % select_result) return {"status": False, "message": select_result}, 500 if select_result: try: host = select_result if args['tag'] != []: host["tag"] = args["tag"] if args['rename'] != '': host["rename"] = args["rename"] status, result = db.update_by_id( "host", json.dumps(host, ensure_ascii=False), host_id) db.close_mysql() if status is not True: logger.error("Modify host error: %s" % result) return {"status": False, "message": result}, 500 except Exception as e: db.close_mysql() logger.error("Modify %s host error: %s" % (host_id, e)) return {"status": False, "message": str(e)}, 500 audit_log(user, args["id"], args["product_id"], "host", "edit") return {"status": True, "message": ""}, 200
def get(self): user = g.user_info["username"] args = parser.parse_args() salt_api = salt_api_for_product(args["product_id"]) minions = [] minions_mysql = [] if isinstance(salt_api, dict): return salt_api, 500 else: result = salt_api.list_all_key() if result: if result.get("status") is False: return result, 500 for minion in result.get("minions"): minions.append(minion) # 同步产品线下的Grains Grains.create_grains(minions, args["product_id"], user) db = DB() status, result = db.select("grains", "where data -> '$.product_id'='%s'" % args["product_id"]) db.close_mysql() if status is True and result: for i in result: minions_mysql.append(i.get("id")) # 对比数据库中的minion与已经同意的minion的不同,删掉数据库中多余的minion diff = list(set(minions_mysql).difference(minions)) Grains.delete_grains(diff, args["product_id"], user) return {"status": True, "message": ""}, 200
def get(self): args = parser.parse_args() action = args["action"] if action == "start": db = DB() status, result = db.select("product", "") db.close_mysql() if status is True: if result: for product in result: event_to_mysql.delay(product['id']) return {"data": "", "status": True, "message": ""}, 200 else: return { "data": "", "status": False, "message": "Product does not exist" }, 404 else: return {"data": "", "status": False, "message": result}, 500 if action == "stop": echo = os.popen("celery -A app.celery inspect active --json") info = echo.readline() try: result = eval( info.replace("true", "'true'").replace("false", "'false'")) for k, v in result.items(): for w in v: os.popen("kill -9 %s" % w.get("worker_pid")) return {"data": "", "status": True, "message": ""}, 200 except Exception as e: logger.error("Stop celery error: %s" % e) return {"data": "", "status": False, "message": "%s" % e}, 200
def post(self): logger.info("SinglePing") args = parser.parse_args() target_id = args['target_id'] cipher = args['cipher'] # 获得所需参数minion_id、product_id、target_ip db = DB() state, result = db.select_by_id('target', target_id) target_ip = result['IP'] host_id = result['host_id'] state, result = db.select_by_id('host', host_id) minion_id = result['minion_id'] product_id = result['product_id'] salt_api = salt_api_for_product(product_id) command = 'snmpwalk -v 2c -t 0.5 -c \'' + cipher + '\' ' + target_ip + ' 1.3.6.1.2.1.1.1' logger.info('command:' + command) sysDescr = salt_api.shell_remote_execution([minion_id], command) response_data = {} if str(sysDescr[minion_id]).__contains__("Timeout") | str( sysDescr[minion_id]).__contains__("Unknown"): response_data['status'] = '设备网络不通' else: response_data['status'] = "设备正常" response_data['sysDescr'] = str(sysDescr[minion_id]) return {"status": True, "message": '成功', "data": response_data}, 200
def post(self): logger.info("PingList") args = parser.parse_args() db = DB() host_id = args['host_id'] cipher = args['cipher'] state, result = db.select('host', "where data -> '$.id'='%s'" % host_id) minion_id = result[0]['minion_id'] logger.info('minion_id:' + minion_id) product_id = result[0]['product_id'] salt_api = salt_api_for_product(product_id) state, targets = db.select('target', "where data -> '$.host_id'='%s'" % host_id) targets_not = [] thread_pool = ThreadPoolExecutor(max_workers=10, thread_name_prefix="target_") futures = [] for target in targets: future = thread_pool.submit(pingTarget, target, minion_id, salt_api, cipher) futures.append(future) thread_pool.shutdown(wait=True) for future in futures: result = future.result() logger.info(str(result['status'])) if str(result['status']).__contains__("Timeout") | str( result['status']).__contains__("Unknown"): targets_not.append(result["target"]) return {"status": True, "message": '配置发送成功', "data": targets_not}, 200
def get_gitlab(product_id): db = DB() status, result = db.select_by_id("product", product_id) db.close_mysql() if status is True: if result: product = result else: return {"status": False, "message": "%s does not exist" % product_id} else: return {"status": False, "message": result} try: if product.get("file_server") == "gitfs": gl = gitlab.Gitlab(url=product.get("gitlab_url"), private_token=None if product.get("private_token") is "" else product.get( "private_token"), oauth_token=None if product.get("oauth_token") is "" else product.get("oauth_token"), email=None if product.get("email") is "" else product.get("email"), password=None if product.get("password") is "" else product.get("password"), ssl_verify=True, http_username=None if product.get("http_username") is "" else product.get( "http_username"), http_password=None if product.get("http_password") is "" else product.get( "http_password"), timeout=120, api_version=None if product.get("api_version") is "" else product.get("api_version") ) return gl else: return {"status": False, "message": "File server is not gitfs"}, "" except Exception as e: return {"status": False, "message": str(e)}
def main(): load_dotenv(find_dotenv()) HOST = getenv("MYSQL_HOST") PORT = getenv("MYSQL_PORT") USER = getenv("MYSQL_USER") PASS = getenv("MYSQL_PASS") DATABASE = getenv("MYSQL_DB") db = DB(HOST, PORT, USER, PASS, DATABASE) store = Store(getenv("S3_BUCKET")) files = store.getFiles(PREFIX) file_names = list( filter(lambda x: ".csv" in x, map(lambda x: x.key, files))) latest_date = max( list(map(lambda x: extractDate(x, PREFIX, ".csv"), file_names))) latest_file = "{}/{}.{}".format(PREFIX, latest_date, "csv") # Get File body = store.getFile(latest_file) csv = pd.read_csv(StringIO(body), low_memory=False) csv = csv[[ "artist", "album", "track", "track_id", "danceability", "energy", "key", "loudness", "mode", "speechiness", "acousticness", "instrumentalness", "liveness", "valence", "tempo", "duration_ms", "lyrics", "popularity", "explicit" ]] csv = csv.dropna() csv_tuple = [tuple(x) for x in csv.to_numpy()] # Load data to sql db.insertSongs(csv_tuple)
def post(self): args = parser.parse_args() args["id"] = uuid_prefix("project") db = DB() project = transfer_args_to_project(args) # 如果是创建,group们必然是增加此项目名 status, result = db.select( "projects", "where data -> '$.name'='%s' and data -> '$.product_id'='%s'" % (args["name"], args["product_id"])) if status is True: if len(result) == 0: try: create_git_project(args['product_id'], args['gitlab_name']) git_clone(args['product_id'], args['gitlab_name']) except Exception as e: return {"status": False, "message": str(e)}, 500 insert_status, insert_result = db.insert( "projects", json.dumps(project, ensure_ascii=False)) update_group_for_create_project(project['name'], project['groups']) db.close_mysql() if insert_status is not True: return {"status": False, "message": insert_result}, 500 return {"status": True, "message": ""}, 200 else: db.close_mysql() return { "status": False, "message": "The projects name already exists" }, 500 else: db.close_mysql() return {"status": False, "message": result}, 500
def get(self): product_id = request.args.get("product_id") db = DB() user_info = g.user_info sql_list = [] groups_list = [] if user_info["groups"]: for group in user_info["groups"]: sql_list.append( "data -> '$.id'='%s' and data -> '$.product_id'='%s'" % (group, product_id)) sql = " or ".join(sql_list) status, result = db.select("groups", "where %s" % sql) db.close_mysql() if status is True: if result: groups_list = result return { "data": groups_list, "status": True, "message": "" }, 200 else: return { "status": False, "message": "Group does not exist" }, 404 else: return {"status": False, "message": result}, 500 else: return {"data": groups_list, "status": True, "message": ""}, 200
def post(self): args = parser.parse_args() args["id"] = uuid_prefix("a") user = g.user_info["username"] acl = args db = DB() status, result = db.select( "acl", "where data -> '$.name'='%s'" % args["name"]) if status is True: if len(result) == 0: insert_status, insert_result = db.insert( "acl", json.dumps(acl, ensure_ascii=False)) db.close_mysql() if insert_status is not True: logger.error("Add acl error: %s" % insert_result) return {"status": False, "message": insert_result}, 500 audit_log(user, args["id"], "", "acl", "add") return {"status": True, "message": ""}, 201 else: db.close_mysql() return { "status": False, "message": "The acl name already exists" }, 200 else: db.close_mysql() logger.error("Select acl name error: %s" % result) return {"status": False, "message": result}, 500
def create_grains(minion_list, product_id, user): salt_api = salt_api_for_product(product_id) if isinstance(salt_api, dict): return salt_api, 500 db = DB() for minion in minion_list: select_status, select_result = db.select( "grains", "where data -> '$.id'='%s' and data -> " "'$.product_id'='%s'" % (minion, product_id)) grains = salt_api.grains(minion) grains[minion].update({"product_id": product_id}) if select_status is True: if len(select_result) > 1: for m in select_result: db.delete_by_id("grains", m["id"]) insert_status, insert_result = db.insert( "grains", json.dumps(grains[minion], ensure_ascii=False)) if insert_status is not True: logger.error("Add Grains error: %s" % insert_result) elif len(select_result) == 1: update_status, update_result = db.update_by_id( "grains", json.dumps(grains[minion], ensure_ascii=False), select_result[0]["id"]) if update_status is not True: logger.error("Update Grains error: %s" % update_result) else: insert_status, insert_result = db.insert( "grains", json.dumps(grains[minion], ensure_ascii=False)) if insert_status is not True: logger.error("Add Grains error: %s" % insert_result) db.close_mysql()
def get(self): product_id = request.args.get("product_id") db = DB() status, result = db.select( "groups", "where data -> '$.product_id'='%s'" % product_id) db.close_mysql() groups_list = [] if status is True: if result: for i in result: try: groups_list.append(eval(i[0])) except Exception as e: return {"status": False, "message": str(e)}, 500 else: return { "groups": { "group": groups_list }, "status": True, "message": "" }, 200 else: return {"status": False, "message": result}, 500 return { "groups": { "group": groups_list }, "status": True, "message": "" }, 200
def get(self): db = DB() status, result = db.select("product", "") db.close_mysql() product_list = [] if status is True: if result: for i in result: try: product_list.append(eval(i[0])) except Exception as e: return {"status": False, "message": str(e)}, 500 else: return { "status": False, "message": "Product does not exist" }, 404 else: return {"status": False, "message": result}, 500 return { "products": { "product": product_list }, "status": True, "message": "" }, 200
def get(self): logger.info("HOSTLIST") product_id = request.args.get("product_id") db = DB() status, result = db.select( "host", "where data -> '$.product_id'='%s'" % product_id) if status is True: host_list = result else: db.close_mysql() return {"status": False, "message": result}, 500 status, result = db.select( "groups", "where data -> '$.product_id'='%s'" % product_id) if status is True: groups_list = result else: db.close_mysql() return {"status": False, "message": result}, 500 for host in host_list: for group in groups_list: for minion in group["minion"]: if host["minion_id"] == minion: host["groups"].append(group["name"]) db.close_mysql() return {"data": host_list, "status": True, "message": ""}, 200
def post(self): logger.info("HOSTLIST") args = parser.parse_args() args["id"] = uuid_prefix("h") user = g.user_info["username"] host = args db = DB() status, result = db.select( "host", "where data -> '$.minion_id'='%s'" % args["minion_id"]) if status is True: if len(result) == 0: insert_status, insert_result = db.insert( "host", json.dumps(host, ensure_ascii=False)) db.close_mysql() if insert_status is not True: logger.error("Add host error: %s" % insert_result) return {"status": False, "message": insert_result}, 500 audit_log(user, args["id"], args["product_id"], "host", "add") return {"status": True, "message": ""}, 201 else: db.close_mysql() return { "status": False, "message": "The host name already exists" }, 200 else: db.close_mysql() logger.error("Select host name error: %s" % result) return {"status": False, "message": result}, 500
def post(self): user = g.user_info["username"] args = parser.parse_args() args["id"] = uuid_prefix("u") db = DB() status, result = db.select( "user", "where data -> '$.username'='%s'" % args["username"]) if status is True: if len(result) == 0: # 默认新添加的用户都是默认用户 role_id = get_common_user() if isinstance(role_id, dict): return role_id args["role"].append(role_id) insert_status, insert_result = db.insert( "user", json.dumps(args, ensure_ascii=False)) db.close_mysql() if insert_status is not True: logger.error("Add user error: %s" % insert_result) return {"status": False, "message": insert_result}, 500 audit_log(user, args["id"], "", "user", "add") return {"status": True, "message": ""}, 201 else: db.close_mysql() return { "status": False, "message": "The user name already exists" }, 200 else: db.close_mysql() logger.error("Select user error: %s" % result) return {"status": False, "message": result}, 500
def db(self): if not self._db: app = import_module(self.__module__.split('.')[0]) conf = import_module('config') connection = conf.DATABASES[app.database][conf.TEST] self._db = DB(**connection).session return self._db
def put(self, host_id): user = g.user_info["username"] args = parser.parse_args() args["id"] = host_id host = args db = DB() # 判断是否存在 select_status, select_result = db.select_by_id("host", host_id) if select_status is False: db.close_mysql() logger.error("Modify host error: %s" % select_result) return {"status": False, "message": select_result}, 500 if select_result: for i in select_result: try: host = eval(i[0]) host["group_id"] = args["group_id"] host["tag"] = args["tag"] except Exception as e: db.close_mysql() logger.error("Modify %s host error: %s" % (host_id, e)) return {"status": False, "message": str(e)}, 500 status, result = db.update_by_id("host", json.dumps(host, ensure_ascii=False), host_id) db.close_mysql() if status is not True: logger.error("Modify host error: %s" % result) return {"status": False, "message": result}, 500 audit_log(user, args["id"], args["product_id"], "host", "edit") return {"status": True, "message": ""}, 200
def delete(self): args = parser.parse_args() db = DB() status, result = db.select_by_id("product", args["product_id"]) db.close_mysql() if status is True: if result: product = eval(result[0][0]) else: return {"status": False, "message": "%s does not exist" % args["product_id"]} else: return {"status": False, "message": result} salt_api = salt_api_for_product(args["product_id"]) user = g.user_info["username"] if isinstance(salt_api, dict): return salt_api, 500 else: if args["action"] == "kill" and args["jid"]: kill = "salt %s saltutil.kill_job %s" % (product.get("salt_master_id"), args["jid"]) try: result = salt_api.shell_remote_execution(product.get("salt_master_id"), kill) audit_log(user, args["jid"], args["product_id"], "job id", "kill") return {"status": True, "message": result}, 200 except Exception as e: return {"status": False, "message": str(e)}, 500 else: return {"status": False, "message": "The specified job id does not exist or arguments error"}, 400
def post(self): args = parser.parse_args() args["id"] = uuid_prefix("p") user = g.user_info["username"] product = args db = DB() status, result = db.select( "product", "where data -> '$.name'='%s'" % args["name"]) if status is True: if len(result) == 0: insert_status, insert_result = db.insert( "product", json.dumps(product, ensure_ascii=False)) db.close_mysql() if insert_status is not True: logger.error("Add product error: %s" % insert_result) return {"status": False, "message": insert_result}, 500 audit_log(user, args["id"], "", "product", "add") # 更新Rsync配置 if args["file_server"] == "rsync": rsync_config() return {"status": True, "message": ""}, 201 else: db.close_mysql() return { "status": False, "message": "The product name already exists" }, 200 else: db.close_mysql() logger.error("Select product name error: %s" % result) return {"status": False, "message": result}, 500
def verify_acl(acl_list, command): if acl_list: db = DB() sql_list = [] for acl_id in acl_list: sql_list.append("data -> '$.id'='%s'" % acl_id) sql = " or ".join(sql_list) status, result = db.select("acl", "where %s" % sql) db.close_mysql() if status is True: if result: for i in result: try: acl = eval(i[0]) for deny in acl["deny"]: deny_pattern = re.compile(deny) if deny_pattern.search(command): return { "status": False, "message": "Deny Warning : You don't have permission run [ %s ]" % command } except Exception as e: return {"status": False, "message": str(e)} return {"status": True, "message": ""} else: return {"status": False, "message": "acl does not exist"} else: return {"status": False, "message": result} else: return {"status": True, "message": ""}
def get(self): db = DB() status, result = db.select("role", "") db.close_mysql() if status is True: return {"data": result, "status": True, "message": ""}, 200 else: return {"status": False, "message": result}, 500
def insert_period_audit(period_id, period_audits): db = DB() results = {"id": period_id, "result": period_audits} status, result = db.insert("period_audit", json.dumps(results, ensure_ascii=False)) if status is False: db.close_mysql() logger.error("Insert period audit error: %s" % result)
def get_gitlab_project(project_id, product_id): db = DB() status, result = db.select_by_id('projects', project_id) project_gitlab_name = result['gitlab_name'] db.close_mysql() project, product_name = gitlab_project_name(product_id, project_gitlab_name) return project, product_name
def setUp(self) -> None: #实例化DB self.db = DB() #登录 url = 'http://ecshop.itsoso.cn/ECMobile/?url=/user/signin' data = {"name": "lzj", "password": "******"} response = Interface_login.login(method='post', url=url, data=data) self.sid = get_result_one(response, 'sid') self.uid = get_result_one(response, 'uid')
def get(self): args = parser.parse_args() db = DB() status, result = db.select("audit_log", "where data -> '$.product_id'='%s' order by data -> '$.time' desc" % args["product_id"]) db.close_mysql() if status is True: return {"data": result, "status": True, "message": ""}, 200 else: return {"status": False, "message": result}, 500
def get(self): product_id = request.args.get("product_id") db = DB() status, result = db.select( "grains", "where data -> '$.product_id'='%s'" % product_id) db.close_mysql() if status is True: return {"data": result, "status": True, "message": ""}, 200 else: return {"status": False, "message": result}, 500