def recent_measurements_by_metric_uuid(database: Database,
max_iso_timestamp: str = "",
days=7):
"""Return all recent measurements."""
max_iso_timestamp = max_iso_timestamp or iso_timestamp()
min_iso_timestamp = (datetime.fromisoformat(max_iso_timestamp) -
timedelta(days=days)).isoformat()
recent_measurements = database.measurements.find(
filter={
"end": {
"$gte": min_iso_timestamp
},
"start": {
"$lte": max_iso_timestamp
}
},
sort=[("start", pymongo.ASCENDING)],
projection={
"_id": False,
"sources.entities": False
},
)
measurements_by_metric_uuid: dict[MetricId, list] = {}
for measurement in recent_measurements:
measurements_by_metric_uuid.setdefault(measurement["metric_uuid"],
[]).append(measurement)
return measurements_by_metric_uuid
def latest_reports_overview(database: Database, max_iso_timestamp: str = "") -> Dict:
"""Return the latest reports overview."""
timestamp_filter = dict(timestamp={"$lt": max_iso_timestamp or iso_timestamp()})
overview = database.reports_overviews.find_one(timestamp_filter, sort=TIMESTAMP_DESCENDING)
if overview: # pragma: no cover-behave
overview["_id"] = str(overview["_id"])
return overview or {}
def update_measurement_end(database: Database, measurement_id: MeasurementId):
"""Set the end date and time of the measurement to the current date and time."""
return database.measurements.update_one(
filter={"_id": measurement_id},
update={"$set": {
"end": iso_timestamp()
}})
def insert_new_measurement(database: Database, data_model, metric_data: dict,
measurement: dict,
previous_measurement: dict) -> dict:
"""Insert a new measurement."""
if "_id" in measurement:
del measurement["_id"]
metric = Metric(data_model, metric_data)
metric_type = data_model["metrics"][metric.type()]
measurement["start"] = measurement["end"] = now = iso_timestamp()
for scale in metric_type["scales"]:
value = calculate_measurement_value(data_model, metric,
measurement["sources"], scale)
status = metric.status(value)
measurement[scale] = dict(value=value,
status=status,
direction=metric.direction())
# We can't cover determine_status_start() returning False in the feature tests because all new measurements have
# a status start timestamp, hence the pragma: no cover-behave:
if status_start := determine_status_start(
status, previous_measurement, scale,
now): # pragma: no cover-behave
measurement[scale]["status_start"] = status_start
for target in ("target", "near_target", "debt_target"):
target_type = cast(TargetType, target)
measurement[scale][target] = determine_target_value(
metric, measurement, scale, target_type)
def copy_source_entity_user_data(old_source, new_source) -> None:
"""Copy the user entity data of the source."""
new_entity_keys = {
entity["key"]
for entity in new_source.get("entities", [])
}
# Sometimes the key Quality-time generates for entities needs to change, e.g. when it turns out not to be
# unique. Create a mapping of old keys to new keys so we can move the entity user data to the new keys
changed_entity_keys = {
entity["old_key"]: entity["key"]
for entity in new_source.get("entities", []) if "old_key" in entity
}
# Copy the user data of entities, keeping 'orphaned' entity user data around for a while in case the entity
# returns in a later measurement:
max_days_to_keep_orphaned_entity_user_data = 21
for entity_key, attributes in old_source.get("entity_user_data",
{}).items():
entity_key = changed_entity_keys.get(entity_key, entity_key)
if entity_key in new_entity_keys:
if "orphaned_since" in attributes:
del attributes[
"orphaned_since"] # The entity returned, remove the orphaned since date/time
else:
if "orphaned_since" in attributes:
days_since_orphaned = days_ago(
datetime.fromisoformat(attributes["orphaned_since"]))
if days_since_orphaned > max_days_to_keep_orphaned_entity_user_data: # pragma: no cover-behave
continue # Don't copy this user data, it has been orphaned too long
else:
# The entity user data refers to a disappeared entity. Keep it around in case the entity
# returns, but also set the current date/time so we can eventually remove the user data.
attributes["orphaned_since"] = iso_timestamp()
new_source.setdefault("entity_user_data", {})[entity_key] = attributes
def latest_datamodel(database: Database, max_iso_timestamp: str = ""):
"""Return the latest data model."""
timestamp_filter = dict(
timestamp={"$lte": max_iso_timestamp or iso_timestamp()})
if data_model := database.datamodels.find_one(timestamp_filter,
sort=[("timestamp",
pymongo.DESCENDING)]):
data_model["_id"] = str(data_model["_id"])
def _prepare_documents_for_insertion(*documents, **extra_attributes) -> None:
"""Prepare the documents for insertion in the database by removing any ids and setting the extra attributes."""
now = iso_timestamp()
for document in documents:
if "_id" in document:
del document["_id"]
document["timestamp"] = now
for key, value in extra_attributes.items():
document[key] = value
def test_iso_timestamp(self):
"""Test that the iso timestamp has the correct format."""
expected_time_stamp = "2020-03-03T10:04:05+00:00"
with patch("server_utilities.functions.datetime") as date_time:
date_time.now.return_value = datetime(2020,
3,
3,
10,
4,
5,
567,
tzinfo=timezone.utc)
self.assertEqual(expected_time_stamp, iso_timestamp())
def latest_reports(database: Database, max_iso_timestamp: str = ""):
"""Return the latest, undeleted, reports in the reports collection."""
if max_iso_timestamp and max_iso_timestamp < iso_timestamp():
report_filter = dict(deleted=DOES_NOT_EXIST, timestamp={"$lt": max_iso_timestamp})
report_uuids = database.reports.distinct("report_uuid", report_filter)
reports = []
for report_uuid in report_uuids:
report_filter["report_uuid"] = report_uuid
reports.append(database.reports.find_one(report_filter, sort=TIMESTAMP_DESCENDING))
else:
reports = list(database.reports.find({"last": True, "deleted": DOES_NOT_EXIST}))
for report in reports:
report["_id"] = str(report["_id"])
return reports
def insert_new_measurement(database: Database, data_model, metric: Dict, measurement: Dict) -> Dict:
"""Insert a new measurement."""
if "_id" in measurement:
del measurement["_id"]
metric_type = data_model["metrics"][metric["type"]]
direction = metric.get("direction") or metric_type["direction"]
for scale in metric_type["scales"]:
value = calculate_measurement_value(data_model, metric, measurement["sources"], scale)
status = determine_measurement_status(metric, direction, value)
measurement[scale] = dict(value=value, status=status, direction=direction)
for target in ("target", "near_target", "debt_target"):
measurement[scale][target] = determine_target(
metric, measurement, metric_type, scale, cast(TargetType, target))
measurement["start"] = measurement["end"] = iso_timestamp()
database.measurements.insert_one(measurement)
del measurement["_id"]
return measurement
def get_tag_report(tag: str, database: Database):
"""Get a report with all metrics that have the specified tag."""
date_time = report_date_time()
reports = latest_reports(database, date_time)
data_model = latest_datamodel(database, date_time)
subjects = _get_subjects_and_metrics_by_tag(data_model, reports, tag)
tag_report = dict(
title=f'Report for tag "{tag}"',
subtitle="Note: tag reports are read-only",
report_uuid=f"tag-{tag}",
timestamp=iso_timestamp(),
subjects=subjects,
)
hide_credentials(data_model, tag_report)
summarize_report(tag_report,
recent_measurements_by_metric_uuid(database, date_time),
data_model)
return tag_report
def _prepare_documents_for_insertion(database: Database,
delta_description: str, *documents,
**extra_attributes) -> None:
"""Prepare the documents for insertion in the database by removing any ids and setting the extra attributes."""
now = iso_timestamp()
user = sessions.user(database) or {}
email = user.get("email", "")
username = user.get("user", "An operator")
# Don't use str.format because there may be curly braces in the delta description, e.g. due to regular expressions:
description = delta_description.replace("{user}", username, 1)
for document, uuids in documents:
if "_id" in document:
del document["_id"]
document["timestamp"] = now
document["delta"] = dict(description=description, email=email)
if uuids:
document["delta"]["uuids"] = uuids
for key, value in extra_attributes.items():
document[key] = value
def test_iso_timestamp(self):
"""Test that the iso timestamp has the correct format."""
with patch("server_utilities.functions.datetime") as date_time:
date_time.now.return_value = self.now
self.assertEqual(self.expected_time_stamp, iso_timestamp())
def insert_new_datamodel(database: Database, data_model):
"""Insert a new data model in the data models collection."""
if "_id" in data_model: # pragma: no cover-behave
del data_model["_id"]
data_model["timestamp"] = iso_timestamp()
return database.datamodels.insert_one(data_model)
