def recent_measurements_by_metric_uuid(database: Database, max_iso_timestamp: str = "", days=7): """Return all recent measurements.""" max_iso_timestamp = max_iso_timestamp or iso_timestamp() min_iso_timestamp = (datetime.fromisoformat(max_iso_timestamp) - timedelta(days=days)).isoformat() recent_measurements = database.measurements.find( filter={ "end": { "$gte": min_iso_timestamp }, "start": { "$lte": max_iso_timestamp } }, sort=[("start", pymongo.ASCENDING)], projection={ "_id": False, "sources.entities": False }, ) measurements_by_metric_uuid: dict[MetricId, list] = {} for measurement in recent_measurements: measurements_by_metric_uuid.setdefault(measurement["metric_uuid"], []).append(measurement) return measurements_by_metric_uuid
def latest_reports_overview(database: Database, max_iso_timestamp: str = "") -> Dict: """Return the latest reports overview.""" timestamp_filter = dict(timestamp={"$lt": max_iso_timestamp or iso_timestamp()}) overview = database.reports_overviews.find_one(timestamp_filter, sort=TIMESTAMP_DESCENDING) if overview: # pragma: no cover-behave overview["_id"] = str(overview["_id"]) return overview or {}
def update_measurement_end(database: Database, measurement_id: MeasurementId): """Set the end date and time of the measurement to the current date and time.""" return database.measurements.update_one( filter={"_id": measurement_id}, update={"$set": { "end": iso_timestamp() }})
def insert_new_measurement(database: Database, data_model, metric_data: dict, measurement: dict, previous_measurement: dict) -> dict: """Insert a new measurement.""" if "_id" in measurement: del measurement["_id"] metric = Metric(data_model, metric_data) metric_type = data_model["metrics"][metric.type()] measurement["start"] = measurement["end"] = now = iso_timestamp() for scale in metric_type["scales"]: value = calculate_measurement_value(data_model, metric, measurement["sources"], scale) status = metric.status(value) measurement[scale] = dict(value=value, status=status, direction=metric.direction()) # We can't cover determine_status_start() returning False in the feature tests because all new measurements have # a status start timestamp, hence the pragma: no cover-behave: if status_start := determine_status_start( status, previous_measurement, scale, now): # pragma: no cover-behave measurement[scale]["status_start"] = status_start for target in ("target", "near_target", "debt_target"): target_type = cast(TargetType, target) measurement[scale][target] = determine_target_value( metric, measurement, scale, target_type)
def copy_source_entity_user_data(old_source, new_source) -> None: """Copy the user entity data of the source.""" new_entity_keys = { entity["key"] for entity in new_source.get("entities", []) } # Sometimes the key Quality-time generates for entities needs to change, e.g. when it turns out not to be # unique. Create a mapping of old keys to new keys so we can move the entity user data to the new keys changed_entity_keys = { entity["old_key"]: entity["key"] for entity in new_source.get("entities", []) if "old_key" in entity } # Copy the user data of entities, keeping 'orphaned' entity user data around for a while in case the entity # returns in a later measurement: max_days_to_keep_orphaned_entity_user_data = 21 for entity_key, attributes in old_source.get("entity_user_data", {}).items(): entity_key = changed_entity_keys.get(entity_key, entity_key) if entity_key in new_entity_keys: if "orphaned_since" in attributes: del attributes[ "orphaned_since"] # The entity returned, remove the orphaned since date/time else: if "orphaned_since" in attributes: days_since_orphaned = days_ago( datetime.fromisoformat(attributes["orphaned_since"])) if days_since_orphaned > max_days_to_keep_orphaned_entity_user_data: # pragma: no cover-behave continue # Don't copy this user data, it has been orphaned too long else: # The entity user data refers to a disappeared entity. Keep it around in case the entity # returns, but also set the current date/time so we can eventually remove the user data. attributes["orphaned_since"] = iso_timestamp() new_source.setdefault("entity_user_data", {})[entity_key] = attributes
def latest_datamodel(database: Database, max_iso_timestamp: str = ""): """Return the latest data model.""" timestamp_filter = dict( timestamp={"$lte": max_iso_timestamp or iso_timestamp()}) if data_model := database.datamodels.find_one(timestamp_filter, sort=[("timestamp", pymongo.DESCENDING)]): data_model["_id"] = str(data_model["_id"])
def _prepare_documents_for_insertion(*documents, **extra_attributes) -> None: """Prepare the documents for insertion in the database by removing any ids and setting the extra attributes.""" now = iso_timestamp() for document in documents: if "_id" in document: del document["_id"] document["timestamp"] = now for key, value in extra_attributes.items(): document[key] = value
def test_iso_timestamp(self): """Test that the iso timestamp has the correct format.""" expected_time_stamp = "2020-03-03T10:04:05+00:00" with patch("server_utilities.functions.datetime") as date_time: date_time.now.return_value = datetime(2020, 3, 3, 10, 4, 5, 567, tzinfo=timezone.utc) self.assertEqual(expected_time_stamp, iso_timestamp())
def latest_reports(database: Database, max_iso_timestamp: str = ""): """Return the latest, undeleted, reports in the reports collection.""" if max_iso_timestamp and max_iso_timestamp < iso_timestamp(): report_filter = dict(deleted=DOES_NOT_EXIST, timestamp={"$lt": max_iso_timestamp}) report_uuids = database.reports.distinct("report_uuid", report_filter) reports = [] for report_uuid in report_uuids: report_filter["report_uuid"] = report_uuid reports.append(database.reports.find_one(report_filter, sort=TIMESTAMP_DESCENDING)) else: reports = list(database.reports.find({"last": True, "deleted": DOES_NOT_EXIST})) for report in reports: report["_id"] = str(report["_id"]) return reports
def insert_new_measurement(database: Database, data_model, metric: Dict, measurement: Dict) -> Dict: """Insert a new measurement.""" if "_id" in measurement: del measurement["_id"] metric_type = data_model["metrics"][metric["type"]] direction = metric.get("direction") or metric_type["direction"] for scale in metric_type["scales"]: value = calculate_measurement_value(data_model, metric, measurement["sources"], scale) status = determine_measurement_status(metric, direction, value) measurement[scale] = dict(value=value, status=status, direction=direction) for target in ("target", "near_target", "debt_target"): measurement[scale][target] = determine_target( metric, measurement, metric_type, scale, cast(TargetType, target)) measurement["start"] = measurement["end"] = iso_timestamp() database.measurements.insert_one(measurement) del measurement["_id"] return measurement
def get_tag_report(tag: str, database: Database): """Get a report with all metrics that have the specified tag.""" date_time = report_date_time() reports = latest_reports(database, date_time) data_model = latest_datamodel(database, date_time) subjects = _get_subjects_and_metrics_by_tag(data_model, reports, tag) tag_report = dict( title=f'Report for tag "{tag}"', subtitle="Note: tag reports are read-only", report_uuid=f"tag-{tag}", timestamp=iso_timestamp(), subjects=subjects, ) hide_credentials(data_model, tag_report) summarize_report(tag_report, recent_measurements_by_metric_uuid(database, date_time), data_model) return tag_report
def _prepare_documents_for_insertion(database: Database, delta_description: str, *documents, **extra_attributes) -> None: """Prepare the documents for insertion in the database by removing any ids and setting the extra attributes.""" now = iso_timestamp() user = sessions.user(database) or {} email = user.get("email", "") username = user.get("user", "An operator") # Don't use str.format because there may be curly braces in the delta description, e.g. due to regular expressions: description = delta_description.replace("{user}", username, 1) for document, uuids in documents: if "_id" in document: del document["_id"] document["timestamp"] = now document["delta"] = dict(description=description, email=email) if uuids: document["delta"]["uuids"] = uuids for key, value in extra_attributes.items(): document[key] = value
def test_iso_timestamp(self): """Test that the iso timestamp has the correct format.""" with patch("server_utilities.functions.datetime") as date_time: date_time.now.return_value = self.now self.assertEqual(self.expected_time_stamp, iso_timestamp())
def insert_new_datamodel(database: Database, data_model): """Insert a new data model in the data models collection.""" if "_id" in data_model: # pragma: no cover-behave del data_model["_id"] data_model["timestamp"] = iso_timestamp() return database.datamodels.insert_one(data_model)