def test_insert_intersection_no_value(self, tag_type):
timestamp = parse_timestamp("2020-09-30T12:34:21.9855")
intxn_mappings = INTXN_TABLE_MAPPINGS[tag_type]
try:
self.adapter.insert_intersections(
1,
timestamp,
[
{
"tag_id": 1,
"value_id": None
}
],
tag_type,
3,
producer="test_producer"
)
except Exception as e:
traceback.print_exc()
self.fail("insert_intersections raised exception '%s'" % type(e))
cursor = self.adapter.db_connection.cursor(
cursor_factory=psycopg2.extras.DictCursor
)
cursor.execute(
"""
SELECT
%(start_ts)s AS start_ts,
%(start_date)s AS start_date,
%(measured_at)s AS measured_at,
%(producer)s AS producer
FROM %(table_name)s
WHERE
%(id)s = 3
AND (%(taxonomy_id)s = 1)
AND (%(tag_id)s = 1)
AND (%(value_id)s IS NULL)
AND (%(end_date)s IS NULL)
AND (%(end_ts)s IS NULL)
""" % intxn_mappings
)
_row = cursor.fetchone()
self.assertIsNotNone(_row)
# only a single row should be returned
self.assertIsNone(cursor.fetchone())
self.assertEqual(_row["start_ts"], timestamp)
self.assertEqual(_row["start_date"], int(timestamp.strftime("%Y%m%d")))
self.assertEqual(_row["measured_at"], timestamp)
self.assertEqual(_row["producer"], "test_producer")
def test_end_intersection_with_value(self, tag_type):
timestamp = parse_timestamp("2020-09-30T12:34:21.9855")
intxn_mappings = INTXN_TABLE_MAPPINGS[tag_type]
self.adapter.end_intersections(
1,
timestamp,
[
{
"tag_id": 1,
"value_id": 1
}
],
tag_type,
2
)
cursor = self.adapter.db_connection.cursor(
cursor_factory=psycopg2.extras.DictCursor
)
cursor.execute(
"""
SELECT
%(start_ts)s AS start_ts,
%(start_date)s AS start_date,
%(end_date)s AS end_date,
%(measured_at)s AS measured_at,
%(producer)s AS producer
FROM %(table_name)s
WHERE
%(id)s = 2
AND (%(taxonomy_id)s = 1)
AND (%(tag_id)s = 1)
AND (%(value_id)s = 1)
AND (%(end_ts)s = %%s)
""" % intxn_mappings,
(timestamp,)
)
_row = cursor.fetchone()
self.assertIsNotNone(_row)
# only a single row should be returned
self.assertIsNone(cursor.fetchone())
self.assertLess(_row["start_ts"], timestamp)
self.assertLess(_row["start_date"], int(timestamp.strftime("%Y%m%d")))
self.assertEqual(_row["end_date"], int(timestamp.strftime("%Y%m%d")))
self.assertEqual(_row["measured_at"], timestamp)
self.assertIsNone(_row["producer"])
def handle_measurement(self, msm, skip_validation=False):
"""
Handles a single measurement, from validating the measurement to
writing the resulting tags to the database
The return looks like this:
{
"tag_type": str, # 'delegation' or 'domain'
"tagged_id": int, # ID in domain or delegation table
"taxonomy_id": int, # taxonomy ID
"measured_at": str, # timestamp of the input measurement
"tag_changes": {
"insert": Tuple[TagStateTuple], # new tags inserted
"prolong": Tuple[TagStateTuple], # tags that were extended
"end": Tuple[TagStateTuple] # tags that were ended
}
}
The tag TagStateTuple are named tuples defined in MeasurementToTags.
They have two attributes: .tag_id and .value_id corresponding to IDs
in the tags and taxonomy_tag_val table.
Parameters
----------
measurement - dict
dict that conforms to the measurement_schema
skip_validation - dict
skip the validation of msm against the measurement schema. Tis can
be useful when the validation has already been done.
Raises
------
InvalidMeasurementException
the measurement msm is not in the right format or there are keys
missing or the measurement is for an unknown tag type
StaleMeasurementException
the measurement msm is older than max_measurement_age
Return
------
dict - contains information about the tag changes triggered by the
measurement
"""
t_start_total = time.time()
self.logger.debug("received measurement")
if not skip_validation:
# throws InvalidMeasurementException if msm is invalid
self.validate_measurement(msm)
self.logger.debug(json.dumps(msm, indent=4))
msm_timestamp = parse_timestamp(msm["measured_at"])
if "measurement_id" in msm:
self.logger.info(
"received measurement %s:%s measured at %s" % (
msm["producer"],
msm["measurement_id"],
msm["measured_at"]
)
)
else:
self.logger.info(
"received measurement from producer %s measured at %s" % (
msm["producer"], msm["measured_at"]
)
)
# throw an InvalidMeasurmentException if the tag type is not known
if not self.db_adapter.is_valid_tag_type(msm["tag_type"]):
raise InvalidMeasurementException(
"unknown tag_type '%s'" % msm["tag_type"]
)
# throws StaleMeasurementException if msm is invalid
self.check_max_age(msm_timestamp)
# throws InvalidMeasurementException if msm does not fit into
# tag2domain tables
_t_start = time.time()
taxonomy_db_info = self.prepare_tag2domain_taxonomy(msm)
self.logger.debug(
"finished preparing taxonomy in %5.3f ms",
1000 * (time.time() - _t_start)
)
_t_start = time.time()
required_intersection_changes = self.calculate_changes(
msm["tagged_id"],
msm["tag_type"],
msm_timestamp,
msm["producer"],
taxonomy_db_info
)
self.logger.debug(
"finished calculating intersection changes in %5.3f ms",
1000 * (time.time() - _t_start)
)
_t_start = time.time()
self.write_intersection_changes(
taxonomy_db_info["taxonomy"]["id"],
msm_timestamp,
required_intersection_changes,
msm["tag_type"],
msm["tagged_id"],
msm["producer"]
)
self.logger.debug(
"finished writing intersection changes in %5.3f ms",
1000 * (time.time() - _t_start)
)
self.logger.debug("committing to DB")
_t_start = time.time()
self.db_adapter.commit()
self.logger.debug(
"finished committing DB changes in %5.3f ms",
1000 * (time.time() - _t_start)
)
self.logger.info(
"finished handling measurement in %5.3f ms",
1000 * (time.time() - t_start_total)
)
return {
"tag_type": msm["tag_type"],
"tagged_id": msm["tagged_id"],
"taxonomy_id": taxonomy_db_info["taxonomy"]["id"],
"measured_at": msm_timestamp,
"tag_changes": required_intersection_changes
}