def import_datamodel(database: Database) -> None:
"""Read the data model and store it in the database."""
data_model_path = pathlib.Path(__file__).resolve().parent.parent / "data" / "datamodel.json"
with data_model_path.open() as json_data_model:
data_model = json.load(json_data_model)
if latest := latest_datamodel(database):
del latest["timestamp"]
del latest["_id"]
if data_model == latest: # pragma: no cover-behave
logging.info("Skipping loading the data model; it is unchanged")
return
def import_datamodel(database: Database) -> None:
"""Read the data model and store it in the database."""
# The coverage measurement of the behave feature tests is unstable. Most of the time it reports the last two lines
# as covered, sometimes not. It's unclear why. To prevent needless checking of the coverage report coverage
# measurement of the last two lines and the if-statement has been turned off.
data_model_path = pathlib.Path(
__file__).resolve().parent.parent / "data" / "datamodel.json"
with data_model_path.open() as json_data_model:
data_model = json.load(json_data_model)
if latest := latest_datamodel(database): # pragma: no-cover behave
del latest["timestamp"]
del latest["_id"]
if data_model == latest: # pragma: no cover-behave
logging.info("Skipping loading the data model; it is unchanged")
return
def post_move_source(source_uuid: SourceId, target_metric_uuid: MetricId, database: Database):
"""Move the source to another metric."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
source = SourceData(data_model, reports, source_uuid)
target = MetricData(data_model, reports, target_metric_uuid)
delta_description = (
f"{{user}} moved the source '{source.source_name}' from metric '{source.metric_name}' of subject "
f"'{source.subject_name}' in report '{source.report_name}' to metric '{target.metric_name}' of subject "
f"'{target.subject_name}' in report '{target.report_name}'."
)
target.metric["sources"][source_uuid] = source.source
target_uuids: List[Union[Optional[ReportId], Optional[SubjectId], Optional[MetricId], Optional[SourceId]]] = [
target.report_uuid
]
reports_to_insert = [(target.report, target_uuids)]
if target.report_uuid == source.report_uuid:
# Source is moved within the same report
del target.report["subjects"][source.subject_uuid]["metrics"][source.metric_uuid]["sources"][source_uuid]
if target.subject_uuid != source.subject_uuid:
# Source is moved from one subject to another subject, include both subject uuids in the delta
target_uuids.append(source.subject_uuid)
target_uuids.extend([target.subject_uuid, source.metric_uuid])
else:
# Source is moved from one report to another, update both
del source.metric["sources"][source_uuid]
source_uuids = [source.report_uuid, source.subject_uuid, source.metric_uuid, source_uuid]
reports_to_insert.append((source.report, source_uuids))
target_uuids.append(target.subject_uuid)
target_uuids.extend([target_metric_uuid, source_uuid])
return insert_new_report(database, delta_description, *reports_to_insert)
def post_source_copy(source_uuid: SourceId, metric_uuid: MetricId,
database: Database):
"""Add a copy of the source to the metric (new in v3)."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
source = SourceData(data_model, reports, source_uuid)
target = MetricData(data_model, reports, metric_uuid)
target.metric["sources"][(source_copy_uuid :=
uuid())] = copy_source(source.source,
source.datamodel)
user = sessions.user(database)
target.report["delta"] = dict(
uuids=[
target.report_uuid, target.subject_uuid, target.metric_uuid,
source_copy_uuid
],
email=user["email"],
description=
f"{user['user']} copied the source '{source.source_name}' of metric "
f"'{source.metric_name}' of subject '{source.subject_name}' from report '{source.report_name}' to "
f"metric '{target.metric_name}' of subject '{target.subject_name}' in report "
f"'{target.report_name}'.")
result = insert_new_report(database, target.report)
result["new_source_uuid"] = source_copy_uuid
return result
def post_move_metric(metric_uuid: MetricId, target_subject_uuid: SubjectId,
database: Database):
"""Move the metric to another subject."""
data_model, reports = latest_datamodel(database), latest_reports(database)
source = MetricData(data_model, reports, metric_uuid)
target = SubjectData(data_model, reports, target_subject_uuid)
delta_description = (
f"{{user}} moved the metric '{source.metric_name}' from subject '{source.subject_name}' in report "
f"'{source.report_name}' to subject '{target.subject_name}' in report '{target.report_name}'."
)
target.subject["metrics"][metric_uuid] = source.metric
if target.report_uuid == source.report_uuid:
# Metric is moved within the same report
del target.report["subjects"][
source.subject_uuid]["metrics"][metric_uuid]
target_uuids = [
target.report_uuid, source.subject_uuid, target_subject_uuid,
metric_uuid
]
reports_to_insert = [(target.report, target_uuids)]
else:
# Metric is moved from one report to another, update both
del source.subject["metrics"][metric_uuid]
source_uuids = [source.report_uuid, source.subject_uuid, metric_uuid]
target_uuids = [target.report_uuid, target_subject_uuid, metric_uuid]
reports_to_insert = [(target.report, target_uuids),
(source.report, source_uuids)]
return insert_new_report(database, delta_description, *reports_to_insert)
def post_metric_attribute(metric_uuid: MetricId, metric_attribute: str,
database: Database):
"""Set the metric attribute."""
new_value = dict(bottle.request.json)[metric_attribute]
data = MetricData(latest_datamodel(database), latest_reports(database),
metric_uuid)
if metric_attribute == "comment" and new_value:
new_value = sanitize_html(new_value)
old_value: Any
if metric_attribute == "position":
old_value, new_value = move_item(data, new_value, "metric")
else:
old_value = data.metric.get(metric_attribute) or ""
if old_value == new_value:
return dict(ok=True) # Nothing to do
data.metric[metric_attribute] = new_value
if metric_attribute == "type":
data.metric.update(default_metric_attributes(database, new_value))
description = (
f"{{user}} changed the {metric_attribute} of metric '{data.metric_name}' of subject "
f"'{data.subject_name}' in report '{data.report_name}' from '{old_value}' to '{new_value}'."
)
uuids = [data.report_uuid, data.subject_uuid, metric_uuid]
insert_new_report(database, description, (data.report, uuids))
if metric_attribute in ATTRIBUTES_IMPACTING_STATUS and (
latest := latest_measurement(database, metric_uuid)):
return insert_new_measurement(database, data.datamodel, data.metric,
latest.copy(), latest)
def post_metric_attribute(metric_uuid: MetricId, metric_attribute: str,
database: Database):
"""Set the metric attribute."""
value = dict(bottle.request.json)[metric_attribute]
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = MetricData(data_model, reports, metric_uuid)
if metric_attribute == "comment" and value:
value = sanitize_html(value)
old_value: Any
if metric_attribute == "position":
old_value, value = move_item(data, value, "metric")
else:
old_value = data.metric.get(metric_attribute) or ""
if old_value == value:
return dict(ok=True) # Nothing to do
data.metric[metric_attribute] = value
if metric_attribute == "type":
data.metric.update(default_metric_attributes(database, value))
user = sessions.user(database)
data.report["delta"] = dict(
uuids=[data.report_uuid, data.subject_uuid, metric_uuid],
email=user["email"],
description=
f"{user['user']} changed the {metric_attribute} of metric '{data.metric_name}' "
f"of subject '{data.subject_name}' in report '{data.report_name}' from '{old_value}' to '{value}'."
)
insert_new_report(database, data.report)
attributes_impacting_status = ("accept_debt", "debt_target",
"debt_end_date", "direction", "near_target",
"target")
if metric_attribute in attributes_impacting_status and (
latest := latest_measurement(database, metric_uuid)):
return insert_new_measurement(database, data.datamodel, data.metric,
latest)
def post_move_metric(metric_uuid: MetricId, target_subject_uuid: SubjectId,
database: Database):
"""Move the metric to another subject."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
source = MetricData(data_model, reports, metric_uuid)
target = SubjectData(data_model, reports, target_subject_uuid)
user = sessions.user(database)
delta_description = f"{user['user']} moved the metric '{source.metric_name}' from subject " \
f"'{source.subject_name}' in report '{source.report_name}' to subject " \
f"'{target.subject_name}' in report '{target.report_name}'."
target.subject["metrics"][metric_uuid] = source.metric
reports_to_insert = [target.report]
if target.report_uuid == source.report_uuid:
# Metric is moved within the same report
del target.report["subjects"][
source.subject_uuid]["metrics"][metric_uuid]
target_uuids = [
target.report_uuid, source.subject_uuid, target_subject_uuid,
metric_uuid
]
else:
# Metric is moved from one report to another, update both
reports_to_insert.append(source.report)
del source.subject["metrics"][metric_uuid]
source.report["delta"] = dict(
uuids=[source.report_uuid, source.subject_uuid, metric_uuid],
email=user["email"],
description=delta_description)
target_uuids = [target.report_uuid, target_subject_uuid, metric_uuid]
target.report["delta"] = dict(uuids=target_uuids,
email=user["email"],
description=delta_description)
return insert_new_report(database, *reports_to_insert)
def post_source_parameter(source_uuid: SourceId, parameter_key: str,
database: Database):
"""Set the source parameter."""
data = SourceData(latest_datamodel(database), latest_reports(database),
source_uuid)
new_value = new_parameter_value(data, parameter_key)
old_value = data.source["parameters"].get(parameter_key) or ""
if old_value == new_value:
return dict(ok=True) # Nothing to do
edit_scope = cast(EditScope,
dict(bottle.request.json).get("edit_scope", "source"))
changed_ids = change_source_parameter(data, parameter_key, old_value,
new_value, edit_scope)
if is_password_parameter(data.datamodel, data.source["type"],
parameter_key):
new_value, old_value = "*" * len(new_value), "*" * len(old_value)
source_description = _source_description(data, edit_scope, parameter_key,
old_value)
delta_description = (
f"{{user}} changed the {parameter_key} of {source_description} from '{old_value}' to '{new_value}'."
)
reports_to_insert = [(report, changed_ids) for report in data.reports
if report["report_uuid"] in changed_ids]
result = insert_new_report(database, delta_description, *reports_to_insert)
if availability_checks := _availability_checks(data, parameter_key):
result["availability"] = availability_checks
def post_source_attribute(source_uuid: SourceId, source_attribute: str,
database: Database):
"""Set a source attribute."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = SourceData(data_model, reports, source_uuid)
value = dict(bottle.request.json)[source_attribute]
old_value: Any
if source_attribute == "position":
old_value, value = move_item(data, value, "source")
else:
old_value = data.source.get(source_attribute) or ""
data.source[source_attribute] = value
if old_value == value:
return dict(ok=True) # Nothing to do
delta_description = (
f"{{user}} changed the {source_attribute} of source '{data.source_name}' of metric '{data.metric_name}' of "
f"subject '{data.subject_name}' in report '{data.report_name}' from '{old_value}' to '{value}'."
)
uuids = [
data.report_uuid, data.subject_uuid, data.metric_uuid, source_uuid
]
if source_attribute == "type":
data.source["parameters"] = default_source_parameters(
database, data.metric["type"], value)
return insert_new_report(database, delta_description, (data.report, uuids))
def post_notification_destination_attributes(
report_uuid: ReportId,
notification_destination_uuid: NotificationDestinationId,
database: Database):
"""Set specified notification destination attributes."""
data = ReportData(latest_datamodel(database), latest_reports(database),
report_uuid)
notification_destination_name = data.report["notification_destinations"][
notification_destination_uuid]["name"]
attributes = dict(bottle.request.json)
old_values = []
for key in attributes:
old_values.append(data.report["notification_destinations"]
[notification_destination_uuid].get(key) or "")
data.report["notification_destinations"][
notification_destination_uuid][key] = attributes[key]
if set(old_values) == set(attributes.values()):
return dict(ok=True) # Nothing to do
separator = "' and '"
delta_description = (
f"{{user}} changed the '{separator.join(attributes.keys())}' of notification destination "
f"'{notification_destination_name}' in report '{data.report_name}' "
f"from '{separator.join(old_values)}' to '{separator.join(attributes.values())}'."
)
uuids = [data.report_uuid, notification_destination_uuid]
return insert_new_report(database, delta_description, (data.report, uuids))
def set_entity_attribute(metric_uuid: MetricId, source_uuid: SourceId,
entity_key: str, attribute: str,
database: Database) -> dict:
"""Set an entity attribute."""
data = SourceData(latest_datamodel(database), latest_reports(database),
source_uuid)
old_measurement = latest_measurement(database, metric_uuid)
new_measurement = old_measurement.copy()
source = [
s for s in new_measurement["sources"]
if s["source_uuid"] == source_uuid
][0]
entity = [e for e in source["entities"] if e["key"] == entity_key][0]
entity_description = "/".join(
[entity[key] for key in entity.keys() if key not in ("key", "url")])
old_value = source.get("entity_user_data", {}).get(entity_key,
{}).get(attribute) or ""
new_value = dict(bottle.request.json)[attribute]
source.setdefault("entity_user_data",
{}).setdefault(entity_key, {})[attribute] = new_value
user = sessions.user(database)
new_measurement["delta"] = dict(
uuids=[data.report_uuid, data.subject_uuid, metric_uuid, source_uuid],
description=
f"{user['user']} changed the {attribute} of '{entity_description}' from '{old_value}' to "
f"'{new_value}'.",
email=user["email"],
)
return insert_new_measurement(database, data.datamodel, data.metric,
new_measurement, old_measurement)
def post_metric_new(subject_uuid: SubjectId, database: Database):
"""Add a new metric."""
data = SubjectData(latest_datamodel(database), latest_reports(database), subject_uuid)
data.subject["metrics"][(metric_uuid := uuid())] = default_metric_attributes(database)
description = f"{{user}} added a new metric to subject '{data.subject_name}' in report '{data.report_name}'."
uuids = [data.report_uuid, data.subject_uuid, metric_uuid]
result = insert_new_report(database, description, (data.report, uuids))
result["new_metric_uuid"] = metric_uuid
return result
def delete_report(report_uuid: ReportId, database: Database):
"""Delete a report."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = ReportData(data_model, reports, report_uuid)
data.report["deleted"] = "true"
delta_description = f"{{user}} deleted the report '{data.report_name}'."
return insert_new_report(database, delta_description,
(data.report, [report_uuid]))
def delete_subject(subject_uuid: SubjectId, database: Database):
"""Delete the subject."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = SubjectData(data_model, reports, subject_uuid)
del data.report["subjects"][subject_uuid]
delta_description = f"{{user}} deleted the subject '{data.subject_name}' from report '{data.report_name}'."
uuids = [data.report_uuid, subject_uuid]
return insert_new_report(database, delta_description, (data.report, uuids))
def get_data_model(database: Database):
"""Return the data model."""
data_model = latest_datamodel(database, report_date_time())
if data_model:
md5 = md5_hash(data_model['timestamp'])
if "W/" + md5 == bottle.request.headers.get("If-None-Match"): # pylint: disable=no-member
bottle.abort(304) # Data model unchanged
bottle.response.set_header("ETag", md5)
return data_model
def delete_metric(metric_uuid: MetricId, database: Database):
"""Delete a metric."""
data = MetricData(latest_datamodel(database), latest_reports(database), metric_uuid)
description = (
f"{{user}} deleted metric '{data.metric_name}' from subject '{data.subject_name}' in report "
f"'{data.report_name}'."
)
uuids = [data.report_uuid, data.subject_uuid, metric_uuid]
del data.subject["metrics"][metric_uuid]
return insert_new_report(database, description, (data.report, uuids))
def delete_subject(subject_uuid: SubjectId, database: Database):
"""Delete the subject."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = SubjectData(data_model, reports, subject_uuid)
del data.report["subjects"][subject_uuid]
user = sessions.user(database)
data.report["delta"] = dict(
uuids=[data.report_uuid, subject_uuid], email=user["email"],
description=f"{user['user']} deleted the subject '{data.subject_name}' from report '{data.report_name}'.")
return insert_new_report(database, data.report)
def delete_report(report_uuid: ReportId, database: Database):
"""Delete a report."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = ReportData(data_model, reports, report_uuid)
data.report["deleted"] = "true"
user = sessions.user(database)
data.report["delta"] = dict(
uuids=[report_uuid],
email=user["email"],
description=f"{user['user']} deleted the report '{data.report_name}'.")
return insert_new_report(database, data.report)
def post_report_copy(report_uuid: ReportId, database: Database):
"""Copy a report."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = ReportData(data_model, reports, report_uuid)
report_copy = copy_report(data.report, data.datamodel)
delta_description = f"{{user}} copied the report '{data.report_name}'."
uuids = [report_uuid, report_copy["report_uuid"]]
result = insert_new_report(database, delta_description,
(report_copy, uuids))
result["new_report_uuid"] = report_copy["report_uuid"]
return result
def delete_source(source_uuid: SourceId, database: Database):
"""Delete a source."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = SourceData(data_model, reports, source_uuid)
delta_description = (
f"{{user}} deleted the source '{data.source_name}' from metric "
f"'{data.metric_name}' of subject '{data.subject_name}' in report '{data.report_name}'."
)
uuids = [data.report_uuid, data.subject_uuid, data.metric_uuid, source_uuid]
del data.metric["sources"][source_uuid]
return insert_new_report(database, delta_description, (data.report, uuids))
def get_report(database: Database, report_uuid: ReportId):
"""Return the quality report, including information about other reports needed for move/copy actions."""
date_time = report_date_time()
data_model = latest_datamodel(database, date_time)
reports = latest_reports(database, date_time)
for report in reports:
if report["report_uuid"] == report_uuid:
recent_measurements = recent_measurements_by_metric_uuid(
database, date_time)
summarize_report(report, recent_measurements, data_model)
break
hide_credentials(data_model, *reports)
return dict(reports=reports)
def delete_notification_destination(
report_uuid: ReportId,
notification_destination_uuid: NotificationDestinationId,
database: Database):
"""Delete a destination from a report."""
data = ReportData(latest_datamodel(database), latest_reports(database),
report_uuid)
destination_name = data.report["notification_destinations"][
notification_destination_uuid]["name"]
del data.report["notification_destinations"][notification_destination_uuid]
delta_description = f"{{user}} deleted destination {destination_name} from report '{data.report_name}'."
uuids = [report_uuid, notification_destination_uuid]
return insert_new_report(database, delta_description, (data.report, uuids))
def get_reports(database: Database):
"""Return the quality reports."""
date_time = report_date_time()
data_model = latest_datamodel(database, date_time)
overview = latest_reports_overview(database, date_time)
overview["reports"] = []
recent_measurements = recent_measurements_by_metric_uuid(
database, date_time)
for report in latest_reports(database, date_time):
summarize_report(report, recent_measurements, data_model)
overview["reports"].append(report)
hide_credentials(data_model, *overview["reports"])
return overview
def post_new_subject(report_uuid: ReportId, database: Database):
"""Create a new subject."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = ReportData(data_model, reports, report_uuid)
data.report["subjects"][(subject_uuid :=
uuid())] = default_subject_attributes(database)
delta_description = f"{{user}} created a new subject in report '{data.report_name}'."
uuids = [report_uuid, subject_uuid]
result = insert_new_report(database, delta_description,
(data.report, uuids))
result["new_subject_uuid"] = subject_uuid
return result
def post_new_subject(report_uuid: ReportId, database: Database):
"""Create a new subject."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = ReportData(data_model, reports, report_uuid)
data.report["subjects"][(subject_uuid := uuid())] = default_subject_attributes(database)
user = sessions.user(database)
data.report["delta"] = dict(
uuids=[report_uuid, subject_uuid], email=user["email"],
description=f"{user['user']} created a new subject in report '{data.report_name}'.")
result = insert_new_report(database, data.report)
result["new_subject_uuid"] = subject_uuid
return result
def post_report_copy(report_uuid: ReportId, database: Database):
"""Copy a report."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = ReportData(data_model, reports, report_uuid)
report_copy = copy_report(data.report, data.datamodel)
user = sessions.user(database)
report_copy["delta"] = dict(
uuids=[report_uuid, report_copy["report_uuid"]],
email=user["email"],
description=f"{user['user']} copied the report '{data.report_name}'.")
result = insert_new_report(database, report_copy)
result["new_report_uuid"] = report_copy["report_uuid"]
return result
def delete_metric(metric_uuid: MetricId, database: Database):
"""Delete a metric."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
data = MetricData(data_model, reports, metric_uuid)
user = sessions.user(database)
data.report["delta"] = dict(
uuids=[data.report_uuid, data.subject_uuid, metric_uuid],
email=user["email"],
description=
f"{user['user']} deleted metric '{data.metric_name}' from subject "
f"'{data.subject_name}' in report '{data.report_name}'.")
del data.subject["metrics"][metric_uuid]
return insert_new_report(database, data.report)
def post_metric_copy(metric_uuid: MetricId, subject_uuid: SubjectId, database: Database):
"""Add a copy of the metric to the subject (new in v3)."""
data_model, reports = latest_datamodel(database), latest_reports(database)
source = MetricData(data_model, reports, metric_uuid)
target = SubjectData(data_model, reports, subject_uuid)
target.subject["metrics"][(metric_copy_uuid := uuid())] = copy_metric(source.metric, source.datamodel)
description = (
f"{{user}} copied the metric '{source.metric_name}' of subject '{source.subject_name}' from report "
f"'{source.report_name}' to subject '{target.subject_name}' in report '{target.report_name}'."
)
uuids = [target.report_uuid, target.subject_uuid, metric_copy_uuid]
result = insert_new_report(database, description, (target.report, uuids))
result["new_metric_uuid"] = metric_copy_uuid
return result
def post_subject_copy(subject_uuid: SubjectId, report_uuid: ReportId, database: Database):
"""Add a copy of the subject to the report (new in v3)."""
data_model = latest_datamodel(database)
reports = latest_reports(database)
source = SubjectData(data_model, reports, subject_uuid)
target = ReportData(data_model, reports, report_uuid)
target.report["subjects"][(subject_copy_uuid := uuid())] = copy_subject(source.subject, source.datamodel)
user = sessions.user(database)
target.report["delta"] = dict(
uuids=[target.report_uuid, subject_copy_uuid], email=user["email"],
description=f"{user['user']} copied the subject '{source.subject_name}' from report " \
f"'{source.report_name}' to report '{target.report_name}'.")
result = insert_new_report(database, target.report)
result["new_subject_uuid"] = subject_copy_uuid
return result