def jira_integration(args, project): try: if args["test"]: url = f'{args["config"]["jira_url"]}/rest/api/2/project' res = get(url, auth=(args["config"]["jira_login"], args["config"]["jira_password"])) if res.status_code == 200: message = "Successfully connected to Jira" else: message = "Connection failed" if "failed" not in message and args["config"]["jira_project"] not in res.text: message = "Connection succeed but project not found" return message else: jira_perf_api_config = { "assignee": args["config"]["jira_login"], "check_functional_errors": "True", "check_performance_degradation": "True", "check_missed_thresholds": "True", "performance_degradation_rate": 20, "missed_thresholds_rate": 50, "jira_labels": "performance, api", "jira_watchers": "", "jira_epic_key": "" } secrets = get_project_secrets(project.id) hidden_secrets = get_project_hidden_secrets(project.id) hidden_secrets["jira"] = dumps(args["config"]) secrets["jira_perf_api"] = dumps(jira_perf_api_config) set_project_secrets(project.id, secrets) set_project_hidden_secrets(project.id, hidden_secrets) return "Jira settings saved" except Exception as e: return f"Failed. Jira settings not saved. {str(e)}"
def get(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=R0201,C0111 # Check project_id for validity project = Project.get_or_404(project_id) # Get secret secrets = get_project_secrets(project.id) _secret = secrets.get(secret) if secrets.get(secret) else get_project_hidden_secrets(project.id).get(secret) return {"secret": _secret}, 200
def delete(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=C0111 project = Project.get_or_404(project_id) secrets = get_project_secrets(project.id) if secret in secrets: del secrets[secret] set_project_secrets(project.id, secrets) return {"message": "deleted"}, 200
def smtp_integration(args, project): try: if args["test"]: try: s = smtplib.SMTP_SSL(host=args['config']['smtp_host'], port=int(args['config']['smtp_port'])) s.ehlo() s.login(args['config']['smtp_user'], args['config']['smtp_password']) return "SMTP server connected" except smtplib.SMTPException as e: return f"SMTP server not connected. {str(e)}" else: secrets = get_project_secrets(project.id) hidden_secrets = get_project_hidden_secrets(project.id) hidden_secrets["smtp"] = dumps(args["config"]) env_vars = args["config"] env_vars["error_rate"] = 10 env_vars["performance_degradation_rate"] = 20 env_vars["missed_thresholds"] = 50 if "email_notification_id" in secrets: update_task(secrets["email_notification_id"], dumps(env_vars)) elif "email_notification_id" in hidden_secrets: update_task(hidden_secrets["email_notification_id"], dumps(env_vars)) else: email_notification_args = { "funcname": "email_notification", "invoke_func": "lambda_function.lambda_handler", "runtime": "Python 3.7", "env_vars": dumps(env_vars) } email_notification = create_task(project, File(EMAIL_NOTIFICATION_PATH), email_notification_args) hidden_secrets["email_notification_id"] = email_notification.task_id set_project_hidden_secrets(project.id, hidden_secrets) return "SMTP setting saved" except Exception as e: return f"Failed. SMTP server not connected. {str(e)}"
def rp_integration(args, project): if args["test"]: url = f'{args["config"]["rp_host"]}/api/v1/project/{args["config"]["rp_project"]}' headers = { 'content-type': 'application/json', 'Authorization': f'bearer {args["config"]["rp_token"]}' } res = get(url, headers=headers) if res.status_code == 200: message = "Successfully connected to RP" else: message = "Connection failed" return message else: rp_perf_api_config = { "rp_launch_name": "carrier", "check_functional_errors": "True", "check_performance_degradation": "True", "check_missed_thresholds": "True", "performance_degradation_rate": 20, "missed_thresholds_rate": 50 } secrets = get_project_secrets(project.id) hidden_secrets = get_project_hidden_secrets(project.id) hidden_secrets["rp"] = dumps(args["config"]) secrets["rp_perf_api"] = dumps(rp_perf_api_config) set_project_secrets(project.id, secrets) set_project_hidden_secrets(project.id, hidden_secrets) return "RP settings saved"
def compile_tests(project_id, file_name, runner): client = docker.from_env() container_name = JOB_CONTAINER_MAPPING.get(runner)["container"] secrets = get_project_secrets(project_id) env_vars = {"artifact": file_name, "bucket": "tests", "galloper_url": secrets["galloper_url"], "token": secrets["auth_token"], "project_id": project_id, "compile": "true"} client.containers.run(container_name, stderr=True, remove=True, environment=env_vars, tty=True, user='******')
def get(self, project_id: int) -> Tuple[dict, int]: # pylint: disable=R0201,C0111 # Check project_id for validity project = Project.get_or_404(project_id) # Get secrets secrets_dict = get_project_secrets(project.id) resp = [] for key in secrets_dict.keys(): resp.append({"name": key, "secret": "******"}) return resp
def post(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=C0111 data = self._parser_post.parse_args() # Check project_id for validity project = Project.get_or_404(project_id) # Set secret secrets = get_project_secrets(project.id) secrets[secret] = data["secret"] set_project_secrets(project.id, secrets) return {"message": f"Project secret was saved"}, 200
def put(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=C0111 # Check project_id for validity project = Project.get_or_404(project_id) # Set secret secrets = get_project_secrets(project.id) hidden_secrets = get_project_hidden_secrets(project.id) hidden_secrets[secret] = secrets[secret] secrets.pop(secret, None) set_project_secrets(project.id, secrets) set_project_hidden_secrets(project.id, hidden_secrets) return {"message": f"Project secret was moved to hidden secrets"}, 200
def get_client(project_id, db_name=None): secrets = get_project_secrets(project_id) hidden_secrets = get_project_hidden_secrets(project_id) influx_host = secrets.get( "influx_ip") if "influx_ip" in secrets else hidden_secrets.get( "influx_ip", "") influx_user = secrets.get( "influx_user") if "influx_user" in secrets else hidden_secrets.get( "influx_user", "") influx_password = secrets.get("influx_password") if "influx_password" in secrets else \ hidden_secrets.get("influx_password", "") return InfluxDBClient(influx_host, 8086, influx_user, influx_password, db_name)
def run_task(project_id, event, task_id=None): secrets = get_project_secrets(project_id) if "control_tower_id" not in secrets: secrets = get_project_hidden_secrets(project_id) task_id = task_id if task_id else secrets["control_tower_id"] task = Task.query.filter(and_(Task.task_id == task_id)).first().to_json() app = run.connect_to_celery(1) celery_task = app.signature("tasks.execute", kwargs={ "task": unsecret(task, project_id=project_id), "event": unsecret(event, project_id=project_id) }) celery_task.apply_async() return {"message": "Accepted", "code": 200, "task_id": task_id}
def run_task(project_id, event, task_id=None): secrets = get_project_secrets(project_id) if "control_tower_id" not in secrets: secrets = get_project_hidden_secrets(project_id) task_id = task_id if task_id else secrets["control_tower_id"] task = Task.query.filter(and_(Task.task_id == task_id)).first().to_json() check_tasks_quota(task) statistic = Statistic.query.filter(Statistic.project_id == task['project_id']).first() setattr(statistic, 'tasks_executions', Statistic.tasks_executions + 1) statistic.commit() arbiter = get_arbiter() task_kwargs = {"task": unsecret(task, project_id=project_id), "event": unsecret(event, project_id=project_id), "galloper_url": unsecret("{{secret.galloper_url}}", project_id=task['project_id']), "token": unsecret("{{secret.auth_token}}", project_id=task['project_id'])} arbiter.apply("execute_lambda", queue=RABBIT_QUEUE_NAME, task_kwargs=task_kwargs) arbiter.close() return {"message": "Accepted", "code": 200, "task_id": task_id}
def get_project_queues(project_id): secrets = get_project_secrets(project_id) hidden_secrets = get_project_hidden_secrets(project_id) root_user = secrets[ "rabbit_user"] if "rabbit_user" in secrets else hidden_secrets[ "rabbit_user"] root_password = secrets[ "rabbit_password"] if "rabbit_password" in secrets else hidden_secrets[ "rabbit_password"] user = secrets[ "rabbit_project_user"] if "rabbit_project_user" in secrets else hidden_secrets[ "rabbit_project_user"] password = secrets["rabbit_project_password"] if "rabbit_project_password" in secrets \ else hidden_secrets["rabbit_project_password"] vhost = secrets["rabbit_project_vhost"] if "rabbit_project_vhost" in secrets \ else hidden_secrets["rabbit_project_vhost"] queues = {"public": [], "project": [], "clouds": []} # Check public on demand queues arbiter = get_arbiter(user=root_user, password=root_password, vhost="carrier") try: queues["public"] = list(arbiter.workers().keys()) except: queues["public"] = [] arbiter.close() # Check project on demand queues arbiter = get_arbiter(user=user, password=password, vhost=vhost) try: queues["project"] = list(arbiter.workers().keys()) except: queues["project"] = [] arbiter.close() # Check project Cloud integrations for each in ["aws", "azure_cloud", "gcp", "kubernetes"]: if each in hidden_secrets: queues["clouds"].append(each) return queues
def post(self, project_id: int): args = self._parser_post.parse_args(strict=False) project = Project.get_or_404(project_id) report = APIReport.query.filter_by( project_id=project_id, id=args.get("report_id")).first().to_json() event = { "galloper_url": "{{secret.galloper_url}}", "project_id": project.id, "token": "{{secret.auth_token}}", "report_id": args["report_id"], "influx_host": "{{secret.influx_ip}}", "config_file": "{}", "bucket": str(report["name"]).lower().replace(" ", "").replace("_", "").replace( "-", ""), "prefix": f'test_results_{uuid4()}_', } task = PerformanceTests.query.filter( and_(PerformanceTests.project_id == project.id, PerformanceTests.test_uid == report["test_uid"])).first() event["email_recipients"] = task.emails integration = [] for each in ["jira", "report_portal", "email", "azure_devops"]: if each in task.reporting: integration.append(each) junit = True if "junit" in task.reporting else False event["integration"] = integration event["junit"] = junit secrets = get_project_secrets(project_id) if "post_processor_id" not in secrets: secrets = get_project_hidden_secrets(project_id) return run_task(project.id, event, secrets["post_processor_id"])
def compile_tests(project_id, file_name, runner): client = docker.from_env() container_mapper = { "gatling2": "getcarrier/perfgun:2.3", "gatling3": "getcarrier/perfgun:latest" } container_name = container_mapper.get(runner) secrets = get_project_secrets(project_id) env_vars = { "artifact": file_name, "bucket": "tests", "galloper_url": secrets["galloper_url"], "token": secrets["auth_token"], "project_id": project_id, "compile": "true" } client.containers.run(container_name, stderr=True, remove=True, environment=env_vars, tty=True, user='******')
def get(self, project_id): project = Project.get_or_404(project_id) args = self.get_parser.parse_args(strict=False) if args['name'] not in ['post_processor', 'control_tower', 'all']: return {"message": "go away", "code": 400}, 400 secrets = get_project_hidden_secrets(project.id) project_secrets = get_project_secrets(project.id) if args['name'] == 'post_processor': self.create_pp_task(project) elif args['name'] == 'control_tower': self.create_cc_task(project) elif args['name'] == 'all': self.create_pp_task(project) self.create_cc_task(project) project_secrets["galloper_url"] = APP_HOST project_secrets["project_id"] = project.id secrets[ "post_processor"] = f"{APP_HOST}/task/{secrets['post_processor_id']}" secrets["redis_host"] = APP_IP secrets["loki_host"] = EXTERNAL_LOKI_HOST.replace( "https://", "http://") secrets["influx_ip"] = APP_IP secrets["influx_port"] = INFLUX_PORT secrets["influx_user"] = INFLUX_USER secrets["influx_password"] = INFLUX_PASSWORD secrets["loki_port"] = LOKI_PORT secrets["redis_password"] = REDIS_PASSWORD secrets["rabbit_host"] = APP_IP secrets["rabbit_user"] = RABBIT_USER secrets["rabbit_password"] = RABBIT_PASSWORD set_project_secrets(project.id, project_secrets) else: return {"message": "go away", "code": 400}, 400 set_project_hidden_secrets(project.id, secrets) if "rabbit_project_user" not in project_secrets and "rabbit_project_user" not in secrets: create_project_user_and_vhost(project_id) return {"message": "Done", "code": 200}
def create_project_user_and_vhost(project_id): secrets = get_project_secrets(project_id) hidden_secrets = get_project_hidden_secrets(project_id) # connect to RabbitMQ management api rabbit_api = AdminAPI(url=f'http://carrier-rabbit:15672', auth=(hidden_secrets["rabbit_user"], hidden_secrets["rabbit_password"])) # prepare user credentials user = f"rabbit_user_{project_id}" password = password_generator() vhost = f"project_{project_id}_vhost" # create project user and vhost rabbit_api.create_vhost(vhost) rabbit_api.create_user(user, password) rabbit_api.create_user_permission(user, vhost) # set project secrets secrets["rabbit_project_user"] = user secrets["rabbit_project_password"] = password secrets["rabbit_project_vhost"] = vhost set_project_secrets(project_id, secrets)
def set_grafana_datasources(project_id): secrets = get_project_secrets(project_id) hidden_secrets = get_project_hidden_secrets(project_id) influx_user = secrets.get( "influx_user") if "influx_user" in secrets else hidden_secrets.get( "influx_user", "") influx_password = secrets.get("influx_password") if "influx_password" in secrets else \ hidden_secrets.get("influx_password", "") grafana_api_key = secrets.get( "gf_api_key") if "gf_api_key" in secrets else hidden_secrets.get( "gf_api_key", "") url = f"http://carrier-grafana:3000/grafana/api/datasources" headers = { "Authorization": f"Bearer {grafana_api_key}", "Content-Type": "application/json" } for each in ["jmeter", "gatling", "telegraf"]: data = deepcopy(DATASOURCE) data["name"] = f"{each}_{project_id}" data["database"] = f"{each}_{project_id}" data["user"] = influx_user data["password"] = influx_password post(url, json=data, headers=headers)