def jira_integration(args, project):
try:
if args["test"]:
url = f'{args["config"]["jira_url"]}/rest/api/2/project'
res = get(url, auth=(args["config"]["jira_login"], args["config"]["jira_password"]))
if res.status_code == 200:
message = "Successfully connected to Jira"
else:
message = "Connection failed"
if "failed" not in message and args["config"]["jira_project"] not in res.text:
message = "Connection succeed but project not found"
return message
else:
jira_perf_api_config = {
"assignee": args["config"]["jira_login"],
"check_functional_errors": "True",
"check_performance_degradation": "True",
"check_missed_thresholds": "True",
"performance_degradation_rate": 20,
"missed_thresholds_rate": 50,
"jira_labels": "performance, api",
"jira_watchers": "",
"jira_epic_key": ""
}
secrets = get_project_secrets(project.id)
hidden_secrets = get_project_hidden_secrets(project.id)
hidden_secrets["jira"] = dumps(args["config"])
secrets["jira_perf_api"] = dumps(jira_perf_api_config)
set_project_secrets(project.id, secrets)
set_project_hidden_secrets(project.id, hidden_secrets)
return "Jira settings saved"
except Exception as e:
return f"Failed. Jira settings not saved. {str(e)}"
def get(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=R0201,C0111
# Check project_id for validity
project = Project.get_or_404(project_id)
# Get secret
secrets = get_project_secrets(project.id)
_secret = secrets.get(secret) if secrets.get(secret) else get_project_hidden_secrets(project.id).get(secret)
return {"secret": _secret}, 200
def delete(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=C0111
project = Project.get_or_404(project_id)
secrets = get_project_secrets(project.id)
if secret in secrets:
del secrets[secret]
set_project_secrets(project.id, secrets)
return {"message": "deleted"}, 200
def smtp_integration(args, project):
try:
if args["test"]:
try:
s = smtplib.SMTP_SSL(host=args['config']['smtp_host'], port=int(args['config']['smtp_port']))
s.ehlo()
s.login(args['config']['smtp_user'], args['config']['smtp_password'])
return "SMTP server connected"
except smtplib.SMTPException as e:
return f"SMTP server not connected. {str(e)}"
else:
secrets = get_project_secrets(project.id)
hidden_secrets = get_project_hidden_secrets(project.id)
hidden_secrets["smtp"] = dumps(args["config"])
env_vars = args["config"]
env_vars["error_rate"] = 10
env_vars["performance_degradation_rate"] = 20
env_vars["missed_thresholds"] = 50
if "email_notification_id" in secrets:
update_task(secrets["email_notification_id"], dumps(env_vars))
elif "email_notification_id" in hidden_secrets:
update_task(hidden_secrets["email_notification_id"], dumps(env_vars))
else:
email_notification_args = {
"funcname": "email_notification",
"invoke_func": "lambda_function.lambda_handler",
"runtime": "Python 3.7",
"env_vars": dumps(env_vars)
}
email_notification = create_task(project, File(EMAIL_NOTIFICATION_PATH), email_notification_args)
hidden_secrets["email_notification_id"] = email_notification.task_id
set_project_hidden_secrets(project.id, hidden_secrets)
return "SMTP setting saved"
except Exception as e:
return f"Failed. SMTP server not connected. {str(e)}"
def rp_integration(args, project):
if args["test"]:
url = f'{args["config"]["rp_host"]}/api/v1/project/{args["config"]["rp_project"]}'
headers = {
'content-type': 'application/json',
'Authorization': f'bearer {args["config"]["rp_token"]}'
}
res = get(url, headers=headers)
if res.status_code == 200:
message = "Successfully connected to RP"
else:
message = "Connection failed"
return message
else:
rp_perf_api_config = {
"rp_launch_name": "carrier",
"check_functional_errors": "True",
"check_performance_degradation": "True",
"check_missed_thresholds": "True",
"performance_degradation_rate": 20,
"missed_thresholds_rate": 50
}
secrets = get_project_secrets(project.id)
hidden_secrets = get_project_hidden_secrets(project.id)
hidden_secrets["rp"] = dumps(args["config"])
secrets["rp_perf_api"] = dumps(rp_perf_api_config)
set_project_secrets(project.id, secrets)
set_project_hidden_secrets(project.id, hidden_secrets)
return "RP settings saved"
def compile_tests(project_id, file_name, runner):
client = docker.from_env()
container_name = JOB_CONTAINER_MAPPING.get(runner)["container"]
secrets = get_project_secrets(project_id)
env_vars = {"artifact": file_name, "bucket": "tests", "galloper_url": secrets["galloper_url"],
"token": secrets["auth_token"], "project_id": project_id, "compile": "true"}
client.containers.run(container_name, stderr=True, remove=True, environment=env_vars, tty=True, user='******')
def get(self, project_id: int) -> Tuple[dict, int]: # pylint: disable=R0201,C0111
# Check project_id for validity
project = Project.get_or_404(project_id)
# Get secrets
secrets_dict = get_project_secrets(project.id)
resp = []
for key in secrets_dict.keys():
resp.append({"name": key, "secret": "******"})
return resp
def post(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=C0111
data = self._parser_post.parse_args()
# Check project_id for validity
project = Project.get_or_404(project_id)
# Set secret
secrets = get_project_secrets(project.id)
secrets[secret] = data["secret"]
set_project_secrets(project.id, secrets)
return {"message": f"Project secret was saved"}, 200
def put(self, project_id: int, secret: str) -> Tuple[dict, int]: # pylint: disable=C0111
# Check project_id for validity
project = Project.get_or_404(project_id)
# Set secret
secrets = get_project_secrets(project.id)
hidden_secrets = get_project_hidden_secrets(project.id)
hidden_secrets[secret] = secrets[secret]
secrets.pop(secret, None)
set_project_secrets(project.id, secrets)
set_project_hidden_secrets(project.id, hidden_secrets)
return {"message": f"Project secret was moved to hidden secrets"}, 200
def get_client(project_id, db_name=None):
secrets = get_project_secrets(project_id)
hidden_secrets = get_project_hidden_secrets(project_id)
influx_host = secrets.get(
"influx_ip") if "influx_ip" in secrets else hidden_secrets.get(
"influx_ip", "")
influx_user = secrets.get(
"influx_user") if "influx_user" in secrets else hidden_secrets.get(
"influx_user", "")
influx_password = secrets.get("influx_password") if "influx_password" in secrets else \
hidden_secrets.get("influx_password", "")
return InfluxDBClient(influx_host, 8086, influx_user, influx_password,
db_name)
def run_task(project_id, event, task_id=None):
secrets = get_project_secrets(project_id)
if "control_tower_id" not in secrets:
secrets = get_project_hidden_secrets(project_id)
task_id = task_id if task_id else secrets["control_tower_id"]
task = Task.query.filter(and_(Task.task_id == task_id)).first().to_json()
app = run.connect_to_celery(1)
celery_task = app.signature("tasks.execute",
kwargs={
"task": unsecret(task,
project_id=project_id),
"event": unsecret(event,
project_id=project_id)
})
celery_task.apply_async()
return {"message": "Accepted", "code": 200, "task_id": task_id}
def run_task(project_id, event, task_id=None):
secrets = get_project_secrets(project_id)
if "control_tower_id" not in secrets:
secrets = get_project_hidden_secrets(project_id)
task_id = task_id if task_id else secrets["control_tower_id"]
task = Task.query.filter(and_(Task.task_id == task_id)).first().to_json()
check_tasks_quota(task)
statistic = Statistic.query.filter(Statistic.project_id == task['project_id']).first()
setattr(statistic, 'tasks_executions', Statistic.tasks_executions + 1)
statistic.commit()
arbiter = get_arbiter()
task_kwargs = {"task": unsecret(task, project_id=project_id),
"event": unsecret(event, project_id=project_id),
"galloper_url": unsecret("{{secret.galloper_url}}", project_id=task['project_id']),
"token": unsecret("{{secret.auth_token}}", project_id=task['project_id'])}
arbiter.apply("execute_lambda", queue=RABBIT_QUEUE_NAME, task_kwargs=task_kwargs)
arbiter.close()
return {"message": "Accepted", "code": 200, "task_id": task_id}
def get_project_queues(project_id):
secrets = get_project_secrets(project_id)
hidden_secrets = get_project_hidden_secrets(project_id)
root_user = secrets[
"rabbit_user"] if "rabbit_user" in secrets else hidden_secrets[
"rabbit_user"]
root_password = secrets[
"rabbit_password"] if "rabbit_password" in secrets else hidden_secrets[
"rabbit_password"]
user = secrets[
"rabbit_project_user"] if "rabbit_project_user" in secrets else hidden_secrets[
"rabbit_project_user"]
password = secrets["rabbit_project_password"] if "rabbit_project_password" in secrets \
else hidden_secrets["rabbit_project_password"]
vhost = secrets["rabbit_project_vhost"] if "rabbit_project_vhost" in secrets \
else hidden_secrets["rabbit_project_vhost"]
queues = {"public": [], "project": [], "clouds": []}
# Check public on demand queues
arbiter = get_arbiter(user=root_user,
password=root_password,
vhost="carrier")
try:
queues["public"] = list(arbiter.workers().keys())
except:
queues["public"] = []
arbiter.close()
# Check project on demand queues
arbiter = get_arbiter(user=user, password=password, vhost=vhost)
try:
queues["project"] = list(arbiter.workers().keys())
except:
queues["project"] = []
arbiter.close()
# Check project Cloud integrations
for each in ["aws", "azure_cloud", "gcp", "kubernetes"]:
if each in hidden_secrets:
queues["clouds"].append(each)
return queues
def post(self, project_id: int):
args = self._parser_post.parse_args(strict=False)
project = Project.get_or_404(project_id)
report = APIReport.query.filter_by(
project_id=project_id, id=args.get("report_id")).first().to_json()
event = {
"galloper_url":
"{{secret.galloper_url}}",
"project_id":
project.id,
"token":
"{{secret.auth_token}}",
"report_id":
args["report_id"],
"influx_host":
"{{secret.influx_ip}}",
"config_file":
"{}",
"bucket":
str(report["name"]).lower().replace(" ", "").replace("_",
"").replace(
"-", ""),
"prefix":
f'test_results_{uuid4()}_',
}
task = PerformanceTests.query.filter(
and_(PerformanceTests.project_id == project.id,
PerformanceTests.test_uid == report["test_uid"])).first()
event["email_recipients"] = task.emails
integration = []
for each in ["jira", "report_portal", "email", "azure_devops"]:
if each in task.reporting:
integration.append(each)
junit = True if "junit" in task.reporting else False
event["integration"] = integration
event["junit"] = junit
secrets = get_project_secrets(project_id)
if "post_processor_id" not in secrets:
secrets = get_project_hidden_secrets(project_id)
return run_task(project.id, event, secrets["post_processor_id"])
def compile_tests(project_id, file_name, runner):
client = docker.from_env()
container_mapper = {
"gatling2": "getcarrier/perfgun:2.3",
"gatling3": "getcarrier/perfgun:latest"
}
container_name = container_mapper.get(runner)
secrets = get_project_secrets(project_id)
env_vars = {
"artifact": file_name,
"bucket": "tests",
"galloper_url": secrets["galloper_url"],
"token": secrets["auth_token"],
"project_id": project_id,
"compile": "true"
}
client.containers.run(container_name,
stderr=True,
remove=True,
environment=env_vars,
tty=True,
user='******')
def get(self, project_id):
project = Project.get_or_404(project_id)
args = self.get_parser.parse_args(strict=False)
if args['name'] not in ['post_processor', 'control_tower', 'all']:
return {"message": "go away", "code": 400}, 400
secrets = get_project_hidden_secrets(project.id)
project_secrets = get_project_secrets(project.id)
if args['name'] == 'post_processor':
self.create_pp_task(project)
elif args['name'] == 'control_tower':
self.create_cc_task(project)
elif args['name'] == 'all':
self.create_pp_task(project)
self.create_cc_task(project)
project_secrets["galloper_url"] = APP_HOST
project_secrets["project_id"] = project.id
secrets[
"post_processor"] = f"{APP_HOST}/task/{secrets['post_processor_id']}"
secrets["redis_host"] = APP_IP
secrets["loki_host"] = EXTERNAL_LOKI_HOST.replace(
"https://", "http://")
secrets["influx_ip"] = APP_IP
secrets["influx_port"] = INFLUX_PORT
secrets["influx_user"] = INFLUX_USER
secrets["influx_password"] = INFLUX_PASSWORD
secrets["loki_port"] = LOKI_PORT
secrets["redis_password"] = REDIS_PASSWORD
secrets["rabbit_host"] = APP_IP
secrets["rabbit_user"] = RABBIT_USER
secrets["rabbit_password"] = RABBIT_PASSWORD
set_project_secrets(project.id, project_secrets)
else:
return {"message": "go away", "code": 400}, 400
set_project_hidden_secrets(project.id, secrets)
if "rabbit_project_user" not in project_secrets and "rabbit_project_user" not in secrets:
create_project_user_and_vhost(project_id)
return {"message": "Done", "code": 200}
def create_project_user_and_vhost(project_id):
secrets = get_project_secrets(project_id)
hidden_secrets = get_project_hidden_secrets(project_id)
# connect to RabbitMQ management api
rabbit_api = AdminAPI(url=f'http://carrier-rabbit:15672',
auth=(hidden_secrets["rabbit_user"],
hidden_secrets["rabbit_password"]))
# prepare user credentials
user = f"rabbit_user_{project_id}"
password = password_generator()
vhost = f"project_{project_id}_vhost"
# create project user and vhost
rabbit_api.create_vhost(vhost)
rabbit_api.create_user(user, password)
rabbit_api.create_user_permission(user, vhost)
# set project secrets
secrets["rabbit_project_user"] = user
secrets["rabbit_project_password"] = password
secrets["rabbit_project_vhost"] = vhost
set_project_secrets(project_id, secrets)
def set_grafana_datasources(project_id):
secrets = get_project_secrets(project_id)
hidden_secrets = get_project_hidden_secrets(project_id)
influx_user = secrets.get(
"influx_user") if "influx_user" in secrets else hidden_secrets.get(
"influx_user", "")
influx_password = secrets.get("influx_password") if "influx_password" in secrets else \
hidden_secrets.get("influx_password", "")
grafana_api_key = secrets.get(
"gf_api_key") if "gf_api_key" in secrets else hidden_secrets.get(
"gf_api_key", "")
url = f"http://carrier-grafana:3000/grafana/api/datasources"
headers = {
"Authorization": f"Bearer {grafana_api_key}",
"Content-Type": "application/json"
}
for each in ["jmeter", "gatling", "telegraf"]:
data = deepcopy(DATASOURCE)
data["name"] = f"{each}_{project_id}"
data["database"] = f"{each}_{project_id}"
data["user"] = influx_user
data["password"] = influx_password
post(url, json=data, headers=headers)
