def add_widget_config(widget_config=None, token_info=None, user=None):
"""Create a new widget config
:param widget_config: The widget_config to save
:type widget_config: dict | bytes
:rtype: WidgetConfig
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
data = connexion.request.json
if data["widget"] not in WIDGET_TYPES.keys():
return "Bad request, widget type does not exist", 400
# add default weight of 10
if not data.get("weight"):
data["weight"] = 10
# Look up the project id
if data.get("project"):
project = get_project(data.pop("project"))
if not project_has_user(project, user):
return "Forbidden", 403
data["project_id"] = project.id
# default to make views navigable
if data.get("navigable") and isinstance(data["navigable"], str):
data["navigable"] = data["navigable"][0] in ALLOWED_TRUE_BOOLEANS
if data.get("type") == "view" and data.get("navigable") is None:
data["navigable"] = True
widget_config = WidgetConfig.from_dict(**data)
session.add(widget_config)
session.commit()
return widget_config.to_dict(), 201
def update_dashboard(id_, dashboard=None, token_info=None, user=None):
"""Update a dashboard
:param id: ID of test dashboard
:type id: str
:param body: Dashboard
:type body: dict | bytes
:rtype: Dashboard
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
dashboard_dict = connexion.request.get_json()
if dashboard_dict.get("metadata", {}).get("project") and not project_has_user(
dashboard_dict["metadata"]["project"], user
):
return "Forbidden", 403
dashboard = Dashboard.query.get(id_)
if not dashboard:
return "Dashboard not found", 404
if project_has_user(dashboard.project, user):
return "Forbidden", 403
dashboard.update(connexion.request.get_json())
session.add(dashboard)
session.commit()
return dashboard.to_dict()
def auth(provider):
"""Auth redirect URL"""
if not connexion.request.args.get("code"):
return "Bad request", 400
code = connexion.request.args["code"]
frontend_url = build_url(
current_app.config.get("FRONTEND_URL", "http://localhost:3000"),
"login")
provider_config = _get_provider_config(provider)
user = _get_user_from_provider(provider, provider_config, code)
if not user:
return "Unauthorized", 401
jwt_token = generate_token(user.id)
token = _find_or_create_token("login-token", user)
token.token = jwt_token
session.add(token)
session.commit()
if provider == "keycloak":
query_params = urlencode({
"email": user.email,
"name": user.name,
"token": jwt_token
})
return redirect(f"{frontend_url}?{query_params}")
elif provider == "google":
return {"email": user.email, "name": user.name, "token": jwt_token}
else:
return make_response(
AUTH_WINDOW.format(data=json.dumps({
"email": user.email,
"name": user.name,
"token": jwt_token
})))
def add_run(run=None, token_info=None, user=None):
"""Create a new run
:param body: Run object
:type body: dict | bytes
:rtype: Run
"""
if not connexion.request.is_json:
return "Bad request, JSON is required", 400
run = Run.from_dict(**connexion.request.get_json())
if run.data and not (run.data.get("project") or run.project_id):
return "Bad request, project or project_id is required", 400
project = get_project(run.data["project"])
if not project_has_user(project, user):
return "Forbidden", 403
run.project = project
run.env = run.data.get("env") if run.data else None
run.component = run.data.get("component") if run.data else None
# allow start_time to be set by update_run task if no start_time present
run.start_time = run.start_time if run.start_time else datetime.utcnow()
# if not present, created is the time at which the run is added to the DB
run.created = run.created if run.created else datetime.utcnow()
session.add(run)
session.commit()
update_run_task.apply_async((run.id, ), countdown=5)
return run.to_dict(), 201
def update_run(id_, run=None, token_info=None, user=None):
"""Updates a single run
:param id: ID of run to update
:type id: int
:param body: Run
:type body: dict
:rtype: Run
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
run_dict = connexion.request.get_json()
if run_dict.get("metadata", {}).get("project"):
run_dict["project_id"] = get_project_id(
run_dict["metadata"]["project"])
if not project_has_user(run_dict["project_id"], user):
return "Forbidden", 403
run = Run.query.get(id_)
if run and not project_has_user(run.project, user):
return "Forbidden", 403
if not run:
return "Run not found", 404
run.update(run_dict)
session.add(run)
session.commit()
update_run_task.apply_async((id_, ), countdown=5)
return run.to_dict()
def update_widget_config(id_, token_info=None, user=None):
"""Updates a single widget config
:param id: ID of widget to update
:type id: int
:param body: Result
:type body: dict
:rtype: Result
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
data = connexion.request.get_json()
if data.get("widget") and data["widget"] not in WIDGET_TYPES.keys():
return "Bad request, widget type does not exist", 400
# Look up the project id
if data.get("project"):
project = get_project(data.pop("project"))
if not project_has_user(project, user):
return "Forbidden", 403
data["project_id"] = project.id
widget_config = WidgetConfig.query.get(id_)
# add default weight of 10
if not widget_config.weight:
widget_config.weight = 10
# default to make views navigable
if data.get("navigable") and isinstance(data["navigable"], str):
data["navigable"] = data["navigable"][0] in ALLOWED_TRUE_BOOLEANS
if data.get("type") and data["type"] == "view" and data.get("navigable") is None:
data["navigable"] = True
widget_config.update(data)
session.add(widget_config)
session.commit()
return widget_config.to_dict()
def generate_csv_report(report):
"""Generate a CSV report"""
_update_report(report)
results = _get_results(report)
if not results:
_set_report_empty(report)
return
# First, loop through ALL the results and collect the names of the columns
field_names = set()
for result in results:
row = _make_row(result)
field_names |= set(row.keys())
# Now rewind the cursor and write the results to the CSV
csv_file = StringIO()
csv_writer = DictWriter(csv_file,
fieldnames=list(field_names),
extrasaction="ignore")
csv_writer.writeheader()
for result in results:
csv_writer.writerow(_make_row(result))
# Write the report to the database
csv_file.seek(0)
report_file = ReportFile(
filename=report["filename"],
data={"contentType": "application/csv"},
report_id=report["id"],
content=csv_file.read().encode("utf8"),
)
session.add(report_file)
session.commit()
_set_report_done(report)
def admin_add_user(new_user=None, token_info=None, user=None):
"""Create a new user in the system"""
check_user_is_admin(user)
if not connexion.request.is_json:
return "Bad request, JSON required", 400
new_user = User.from_dict(**connexion.request.get_json())
session.add(new_user)
session.commit()
return _hide_sensitive_fields(new_user.to_dict()), 201
def add_import(
import_file: Optional[FileStorage] = None,
project: Optional[str] = None,
metadata: Optional[str] = None,
source: Optional[str] = None,
token_info: Optional[str] = None,
user: Optional[str] = None,
):
"""Imports a JUnit XML file and creates a test run and results from it.
:param import_file: file to upload
:type import_file: werkzeug.datastructures.FileStorage
:param project: the project to add this test run to
:type project: str
:param metadata: extra metadata to add to the run and the results, in a JSON string
:type metadata: str
:param source: the source of the test run
:type source: str
:rtype: Import
"""
if "importFile" in connexion.request.files:
import_file = connexion.request.files["importFile"]
if not import_file:
return "Bad request, no file uploaded", 400
data = {}
if connexion.request.form.get("project"):
project = connexion.request.form["project"]
if project:
project = get_project(project)
if not project_has_user(project, user):
return "Forbidden", 403
data["project_id"] = project.id
if connexion.request.form.get("metadata"):
metadata = json.loads(connexion.request.form.get("metadata"))
data["metadata"] = metadata
if connexion.request.form.get("source"):
data["source"] = connexion.request.form["source"]
new_import = Import.from_dict(
**{
"status": "pending",
"filename": import_file.filename,
"format": "",
"data": data
})
session.add(new_import)
session.commit()
new_file = ImportFile(import_id=new_import.id, content=import_file.read())
session.add(new_file)
session.commit()
if import_file.filename.endswith(".xml"):
run_junit_import.delay(new_import.to_dict())
elif import_file.filename.endswith(".tar.gz"):
run_archive_import.delay(new_import.to_dict())
else:
return "Unsupported Media Type", 415
return new_import.to_dict(), 202
def login(email=None, password=None):
"""login
:param email: The e-mail address of the user
:type email: str
:param password: The password for the user
:type password: str
:rtype: LoginToken
"""
if not connexion.request.is_json:
return "Bad request, JSON is required", 400
login = connexion.request.get_json()
if not login.get("email") or not login.get("password"):
return {
"code": "EMPTY",
"message": "Username and/or password are empty"
}, 401
user = User.query.filter_by(email=login["email"]).first()
# superadmins can login even if local login is disabled
if user and not user.is_superadmin and not current_app.config.get(
"USER_LOGIN_ENABLED", True):
return {
"code":
"INVALID",
"message":
"Username/password auth is disabled. "
"Please login via one of the links below.",
}, 401
if user and user.check_password(login["password"]):
login_token = generate_token(user.id)
token = Token.query.filter(Token.name == "login-token",
Token.user_id == user.id).first()
if not token:
token = Token(name="login-token", user_id=user.id)
token.token = login_token
session.add(token)
session.commit()
return {"name": user.name, "email": user.email, "token": login_token}
else:
if not current_app.config.get("USER_LOGIN_ENABLED", True):
return {
"code":
"INVALID",
"message":
"Username/password auth is disabled. "
"Please login via one of the links below.",
}, 401
else:
return {
"code": "INVALID",
"message": "Username and/or password are invalid"
}, 401
def update_current_user(token_info=None, user=None):
"""Return the current user"""
user = User.query.get(user)
if not user:
return "Not authorized", 401
user_dict = connexion.request.get_json()
user_dict.pop("is_superadmin", None)
user.update(user_dict)
session.add(user)
session.commit()
return _hide_sensitive_fields(user.to_dict())
def add_group(group=None):
"""Create a new group
:param body: Group
:type body: dict | bytes
:rtype: Group
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
group = Group.from_dict(**connexion.request.get_json())
session.add(group)
return group.to_dict(), 201
def update_run(run_id):
"""Update the run summary from the results, this task will retry 1000 times"""
with lock(f"update-run-lock-{run_id}"):
run = Run.query.get(run_id)
if not run:
return
# initialize some necessary variables
summary = {
"errors": 0,
"failures": 0,
"skips": 0,
"tests": 0,
"xpasses": 0,
"xfailures": 0,
"collected": run.summary.get("collected", 0),
}
run.duration = 0.0
metadata = run.data or {}
# Fetch all the results for the runs and calculate the summary
results = (Result.query.filter(Result.run_id == run_id).order_by(
Result.start_time.asc()).all())
for i, result in enumerate(results):
if i == 0:
# on the first result, copy over some metadata
for column in COLUMNS_TO_COPY:
_copy_column(result, run, column)
for key in METADATA_TO_COPY:
_copy_result_metadata(result, metadata, key)
key = _status_to_summary(result.result)
if key in summary:
summary[key] = summary.get(key, 0) + 1
# update the number of tests that actually ran
summary["tests"] += 1
if result.duration:
run.duration += result.duration
# determine the number of passes
summary["passes"] = summary["tests"] - (
summary["errors"] + summary["xpasses"] + summary["xfailures"] +
summary["failures"] + summary["skips"])
# determine the number of tests that didn't run
summary["not_run"] = max(summary["collected"] - summary["tests"], 0)
run.update({"summary": summary, "data": metadata})
session.add(run)
session.commit()
def get_user_from_provider(provider, auth_data):
"""Get a user object from the ``provider``, using the ``auth_data``"""
provider_config = get_provider_config(provider, is_private=True)
if provider == "google":
user_dict = {
"id": auth_data["iat"],
"email": auth_data["email"],
"name": auth_data["name"]
}
else:
access_token = auth_data.get("accessToken",
auth_data.get("access_token"))
response = requests.get(
provider_config["user_url"],
headers={"Authorization": f"Bearer {access_token}"},
)
if response.status_code == 200:
user_dict = response.json()
else:
return None
if not user_dict.get("email"):
if provider_config.get("email_url"):
# GitHub only returns the publically visible e-mail address with the user, so we need
# to make another request to get the e-mail address, see the this answer for more info:
# https://stackoverflow.com/a/35387123
response = requests.get(
provider_config["email_url"],
headers={"Authorization": f"Bearer {access_token}"})
if response.status_code == 200:
emails = response.json()
primary_email = [email for email in emails if email["primary"]]
user_dict["email"] = (primary_email[0]["email"]
if primary_email else emails[0]["email"])
else:
return None
else:
return None
user = User.query.filter(User.email == user_dict["email"]).first()
if not user:
user = User(
email=user_dict["email"],
name=user_dict["name"],
_password=user_dict["id"],
is_active=True,
is_superadmin=False,
)
session.add(user)
session.commit()
return user
def add_dashboard(dashboard=None, token_info=None, user=None):
"""Create a dashboard
:param body: Dashboard
:type body: dict | bytes
:rtype: Dashboard
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
dashboard = Dashboard.from_dict(**connexion.request.get_json())
if dashboard.project_id and not project_has_user(dashboard.project_id, user):
return "Forbidden", 403
session.add(dashboard)
session.commit()
return dashboard.to_dict(), 201
def bulk_update(filter_=None, page_size=1, token_info=None, user=None):
"""Updates multiple runs with common metadata
Note: can only be used to update metadata on runs, limited to 25 runs
:param filter_: A list of filters to apply
:param page_size: Limit the number of runs updated, defaults to 1
:rtype: List[Run]
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
run_dict = connexion.request.get_json()
if not run_dict.get("metadata"):
return "Bad request, can only update metadata", 401
# ensure only metadata is updated
run_dict = {"metadata": run_dict.pop("metadata")}
if page_size > 25:
return "Bad request, cannot update more than 25 runs at a time", 405
if run_dict.get("metadata", {}).get("project"):
project = get_project(run_dict["metadata"]["project"])
if not project_has_user(project, user):
return "Forbidden", 403
run_dict["project_id"] = project.id
runs = get_run_list(filter_=filter_, page_size=page_size,
estimate=True).get("runs")
if not runs:
return f"No runs found with {filter_}", 404
model_runs = []
for run_json in runs:
run = Run.query.get(run_json.get("id"))
# update the json dict of the run with the new metadata
merge_dicts(run_dict, run_json)
run.update(run_json)
session.add(run)
model_runs.append(run)
session.commit()
return [run.to_dict() for run in model_runs]
def register(email=None, password=None):
"""Register a user
:param email: The e-mail address of the user
:type email: str
:param password: The password for the user
:type password: str
"""
if not connexion.request.is_json:
return "Bad request, JSON is required", 400
details = connexion.request.get_json()
if not details.get("email") or not details.get("password"):
return {
"code": "EMPTY",
"message": "Username and/or password are empty"
}, 401
# Create a random activation code. Base64 just for funsies
activation_code = urlsafe_b64encode(str(
uuid4()).encode("utf8")).strip(b"=").decode()
# Create a user
user = User(email=details["email"],
password=details["password"],
activation_code=activation_code)
session.add(user)
session.commit()
# Send an activation e-mail
activation_url = build_url(
current_app.config.get("BACKEND_URL", "http://localhost:8080/"),
"api",
"login",
"activate",
activation_code,
)
mail = current_app.extensions.get("mail")
if mail and hasattr(mail, "state") and mail.state is not None:
mail.send_message(
"[Ibutsu] Registration Confirmation",
recipients=[email],
body=ACTIVATION_EMAIL.format(activation_url=activation_url),
)
else:
print(
f"No e-mail configuration. Email: {email} - activation URL: {activation_url}"
)
return {}, 201
def admin_update_user(id_, user_info=None, token_info=None, user=None):
"""Update a single user in the system"""
check_user_is_admin(user)
if not connexion.request.is_json:
return "Bad request, JSON required", 400
user_dict = connexion.request.get_json()
projects = user_dict.pop("projects", [])
requested_user = User.query.get(id_)
if not requested_user:
abort(404)
requested_user.update(user_dict)
requested_user.projects = [
Project.query.get(project["id"]) for project in projects
]
session.add(requested_user)
session.commit()
return _hide_sensitive_fields(requested_user.to_dict())
def generate_text_report(report):
"""Generate a text report"""
_update_report(report)
results = _get_results(report)
if not results:
_set_report_empty(report)
return
# Create file with header
text_file = StringIO()
text_file.write("Test Report\n")
text_file.write("\n")
text_file.write("Filter: {}\n".format(report["params"].get("filter", "")))
text_file.write("Source: {}\n".format(report["params"]["source"]))
text_file.write("\n")
# Now loop through the results and summarise them
summary = {
"passed": 0,
"failed": 0,
"skipped": 0,
"error": 0,
"xpassed": 0,
"xfailed": 0,
"other": 0,
}
for result in results:
if result["result"] in summary:
summary[result["result"]] += 1
else:
summary["other"] += 1
text_file.writelines(
["{}: {}\n".format(key, value) for key, value in summary.items()])
text_file.write("\n")
for result in results:
result_path = _make_result_path(result)
text_file.write("{}: {}\n".format(result_path, result["result"]))
# Write the report to the database
text_file.seek(0)
report_file = ReportFile(
filename=report["filename"],
data={"contentType": "text/plain"},
report_id=report["id"],
content=text_file.read().encode("utf8"),
)
session.add(report_file)
session.commit()
_set_report_done(report)
def generate_json_report(report):
"""Generate a JSON report"""
_update_report(report)
results = _get_results(report)
if not results:
_set_report_empty(report)
return
report_dict = _make_dict(results)
# Write the report to the database
report_file = ReportFile(
filename=report["filename"],
data={"contentType": "application/json"},
report_id=report["id"],
content=json.dumps(report_dict, indent=2).encode("utf8"),
)
session.add(report_file)
session.commit()
_set_report_done(report)
def add_project(project=None, token_info=None, user=None):
"""Create a project
:param body: Project
:type body: dict | bytes
:rtype: Project
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
project = Project.from_dict(**connexion.request.get_json())
user = User.query.get(user)
if user:
project.owner = user
project.users.append(user)
session.add(project)
session.commit()
return project.to_dict(), 201
def _create_result(tar,
run_id,
result,
artifacts,
project_id=None,
metadata=None):
"""Create a result with artifacts, used in the archive importer"""
old_id = None
result_id = result.get("id")
if is_uuid(result_id):
result_record = session.query(Result).get(result_id)
else:
result_record = None
if result_record:
result_record.run_id = run_id
else:
old_id = result["id"]
if "id" in result:
result.pop("id")
result["run_id"] = run_id
if project_id:
result["project_id"] = project_id
if metadata:
result["metadata"] = result.get("metadata", {})
result["metadata"].update(metadata)
result["env"] = result.get("metadata", {}).get("env")
result["component"] = result.get("metadata", {}).get("component")
result_record = Result.from_dict(**result)
session.add(result_record)
session.commit()
result = result_record.to_dict()
for artifact in artifacts:
session.add(
Artifact(
filename=artifact.name.split("/")[-1],
result_id=result["id"],
data={
"contentType": "text/plain",
"resultId": result["id"]
},
content=tar.extractfile(artifact).read(),
))
session.commit()
return old_id
def admin_update_project(id_, project=None, token_info=None, user=None):
"""Update a project
:param id: ID of test project
:type id: str
:param body: Project
:type body: dict | bytes
:rtype: Project
"""
check_user_is_admin(user)
if not connexion.request.is_json:
return "Bad request, JSON required", 400
if not is_uuid(id_):
id_ = convert_objectid_to_uuid(id_)
project = Project.query.get(id_)
if not project:
abort(404)
# Grab the fields from the request
project_dict = connexion.request.get_json()
# If the "owner" field is set, ignore it
project_dict.pop("owner", None)
# handle updating users separately
for username in project_dict.pop("users", []):
user_to_add = User.query.filter_by(email=username).first()
if user_to_add and user_to_add not in project.users:
project.users.append(user_to_add)
# Make sure the project owner is in the list of users
if project_dict.get("owner_id"):
owner = User.query.get(project_dict["owner_id"])
if owner and owner not in project.users:
project.users.append(owner)
# update the rest of the project info
project.update(project_dict)
session.add(project)
session.commit()
return project.to_dict()
def recover(email=None):
"""Recover a user account
:param email: The e-mail address of the user
"""
if not connexion.request.is_json:
return "Bad request, JSON is required", 400
login = connexion.request.get_json()
if not login.get("email"):
return "Bad request", 400
user = User.query.filter(User.email == login["email"]).first()
if not user:
return "Bad request", 400
# Create a random activation code. Base64 just for funsies
user.activation_code = urlsafe_b64encode(str(
uuid4()).encode("utf8")).strip(b"=")
session.add(user)
session.commit()
return {}, 201
def generate_html_report(report):
"""Generate an HTML report"""
_update_report(report)
results = _get_results(report)
if not results:
_set_report_empty(report)
return
report_dict = _make_dict(results)
tree = deepcopy(TREE_ROOT)
counts = {
"passed": 0,
"failed": 0,
"skipped": 0,
"error": 0,
"xpassed": 0,
"xfailed": 0,
"other": 0,
}
for _, result in report_dict.items():
_build_tree(result["name"], tree, result)
try:
counts[result["statuses"]["overall"]] += 1
except Exception:
counts["other"] += 1
html_report = render_template(
"reports/html-report.html",
report_name=report["name"],
tree=tree,
results=report_dict,
report=report,
counts=counts,
current_counts=counts,
)
# Write the report to the database
report_file = ReportFile(
filename=report["filename"],
data={"contentType": "text/hmtl"},
report_id=report["id"],
content=html_report.encode("utf8"),
)
session.add(report_file)
session.commit()
_set_report_done(report)
def update_group(id_, group=None):
"""Update a group
:param id: The ID of the group
:type id: str
:param body: The updated group
:type body: dict | bytes
:rtype: Group
"""
if not connexion.request.is_json:
return "Bad request, JSON required", 400
group = Group.query.get(id_)
if not group:
return "Group not found", 404
group.update(connexion.request.get_json())
session.add(group)
session.commit()
return group.to_dict()
def reset_password(activation_code=None, password=None):
"""Reset the password from the recover page
:param e-mail: The e-mail address of the user
:param activation_code: The activation_code supplied to the reset page
:param password: The new password for the user
"""
if not connexion.request.is_json:
return "Bad request, JSON is required", 400
login = connexion.request.get_json()
if not login.get("activation_code") or not login.get("password"):
return "Bad request", 400
user = User.query.filter(
User.activation_code == login["activation_code"]).first()
if not user:
return "Invalid activation code", 400
user.password = login["password"]
user.activation_code = None
session.add(user)
session.commit()
return {}, 201
def activate(activation_code=None):
"""Activate a user's account
:param activation_code: The activation code
"""
if not activation_code:
return "Not Found", 404
user = User.query.filter(User.activation_code == activation_code).first()
login_url = build_url(
current_app.config.get("FRONTEND_URL", "http://localhost:3000"),
"login")
if user:
user.is_active = True
user.activation_code = None
session.add(user)
session.commit()
return redirect(
f"{login_url}?st=success&msg=Account+activated,+please+log+in.")
else:
return redirect(
f"{login_url}?st=error&msg=Invalid+activation+code,+please+check+the+link"
"+in+your+email.")
def add_token(token=None, token_info=None, user=None):
"""Create a new token
:param body: Token object
:type body: dict | bytes
:rtype: Token
"""
if not connexion.request.is_json:
return "Bad request, JSON is required", 400
user = User.query.get(user)
if not user:
return "Not authorized", 401
token = Token.from_dict(**connexion.request.get_json())
token.user = user
token.expires = datetime.fromisoformat(token.expires.replace(
"Z", "+00:00"))
token.token = generate_token(user.id, token.expires.timestamp())
session.add(token)
session.commit()
return token.to_dict(), 201
def get_user_from_keycloak(auth_data):
"""Get a user object from the keycloak server"""
config = get_keycloak_config(is_private=True)
response = requests.get(
config["user_url"],
headers={"Authorization": "Bearer " + auth_data["access_token"]})
if response.status_code == 200:
user_json = response.json()
user = User.query.filter(User.email == user_json["email"]).first()
if not user:
user = User(
email=user_json["email"],
name=user_json["name"],
_password=user_json["sub"],
is_active=True,
is_superadmin=False,
)
session.add(user)
session.commit()
return user
else:
print("Error getting user, response:", response.text)
return None
