def test_insert_intersection_no_value(self, tag_type): timestamp = parse_timestamp("2020-09-30T12:34:21.9855") intxn_mappings = INTXN_TABLE_MAPPINGS[tag_type] try: self.adapter.insert_intersections( 1, timestamp, [ { "tag_id": 1, "value_id": None } ], tag_type, 3, producer="test_producer" ) except Exception as e: traceback.print_exc() self.fail("insert_intersections raised exception '%s'" % type(e)) cursor = self.adapter.db_connection.cursor( cursor_factory=psycopg2.extras.DictCursor ) cursor.execute( """ SELECT %(start_ts)s AS start_ts, %(start_date)s AS start_date, %(measured_at)s AS measured_at, %(producer)s AS producer FROM %(table_name)s WHERE %(id)s = 3 AND (%(taxonomy_id)s = 1) AND (%(tag_id)s = 1) AND (%(value_id)s IS NULL) AND (%(end_date)s IS NULL) AND (%(end_ts)s IS NULL) """ % intxn_mappings ) _row = cursor.fetchone() self.assertIsNotNone(_row) # only a single row should be returned self.assertIsNone(cursor.fetchone()) self.assertEqual(_row["start_ts"], timestamp) self.assertEqual(_row["start_date"], int(timestamp.strftime("%Y%m%d"))) self.assertEqual(_row["measured_at"], timestamp) self.assertEqual(_row["producer"], "test_producer")
def test_end_intersection_with_value(self, tag_type): timestamp = parse_timestamp("2020-09-30T12:34:21.9855") intxn_mappings = INTXN_TABLE_MAPPINGS[tag_type] self.adapter.end_intersections( 1, timestamp, [ { "tag_id": 1, "value_id": 1 } ], tag_type, 2 ) cursor = self.adapter.db_connection.cursor( cursor_factory=psycopg2.extras.DictCursor ) cursor.execute( """ SELECT %(start_ts)s AS start_ts, %(start_date)s AS start_date, %(end_date)s AS end_date, %(measured_at)s AS measured_at, %(producer)s AS producer FROM %(table_name)s WHERE %(id)s = 2 AND (%(taxonomy_id)s = 1) AND (%(tag_id)s = 1) AND (%(value_id)s = 1) AND (%(end_ts)s = %%s) """ % intxn_mappings, (timestamp,) ) _row = cursor.fetchone() self.assertIsNotNone(_row) # only a single row should be returned self.assertIsNone(cursor.fetchone()) self.assertLess(_row["start_ts"], timestamp) self.assertLess(_row["start_date"], int(timestamp.strftime("%Y%m%d"))) self.assertEqual(_row["end_date"], int(timestamp.strftime("%Y%m%d"))) self.assertEqual(_row["measured_at"], timestamp) self.assertIsNone(_row["producer"])
def handle_measurement(self, msm, skip_validation=False): """ Handles a single measurement, from validating the measurement to writing the resulting tags to the database The return looks like this: { "tag_type": str, # 'delegation' or 'domain' "tagged_id": int, # ID in domain or delegation table "taxonomy_id": int, # taxonomy ID "measured_at": str, # timestamp of the input measurement "tag_changes": { "insert": Tuple[TagStateTuple], # new tags inserted "prolong": Tuple[TagStateTuple], # tags that were extended "end": Tuple[TagStateTuple] # tags that were ended } } The tag TagStateTuple are named tuples defined in MeasurementToTags. They have two attributes: .tag_id and .value_id corresponding to IDs in the tags and taxonomy_tag_val table. Parameters ---------- measurement - dict dict that conforms to the measurement_schema skip_validation - dict skip the validation of msm against the measurement schema. Tis can be useful when the validation has already been done. Raises ------ InvalidMeasurementException the measurement msm is not in the right format or there are keys missing or the measurement is for an unknown tag type StaleMeasurementException the measurement msm is older than max_measurement_age Return ------ dict - contains information about the tag changes triggered by the measurement """ t_start_total = time.time() self.logger.debug("received measurement") if not skip_validation: # throws InvalidMeasurementException if msm is invalid self.validate_measurement(msm) self.logger.debug(json.dumps(msm, indent=4)) msm_timestamp = parse_timestamp(msm["measured_at"]) if "measurement_id" in msm: self.logger.info( "received measurement %s:%s measured at %s" % ( msm["producer"], msm["measurement_id"], msm["measured_at"] ) ) else: self.logger.info( "received measurement from producer %s measured at %s" % ( msm["producer"], msm["measured_at"] ) ) # throw an InvalidMeasurmentException if the tag type is not known if not self.db_adapter.is_valid_tag_type(msm["tag_type"]): raise InvalidMeasurementException( "unknown tag_type '%s'" % msm["tag_type"] ) # throws StaleMeasurementException if msm is invalid self.check_max_age(msm_timestamp) # throws InvalidMeasurementException if msm does not fit into # tag2domain tables _t_start = time.time() taxonomy_db_info = self.prepare_tag2domain_taxonomy(msm) self.logger.debug( "finished preparing taxonomy in %5.3f ms", 1000 * (time.time() - _t_start) ) _t_start = time.time() required_intersection_changes = self.calculate_changes( msm["tagged_id"], msm["tag_type"], msm_timestamp, msm["producer"], taxonomy_db_info ) self.logger.debug( "finished calculating intersection changes in %5.3f ms", 1000 * (time.time() - _t_start) ) _t_start = time.time() self.write_intersection_changes( taxonomy_db_info["taxonomy"]["id"], msm_timestamp, required_intersection_changes, msm["tag_type"], msm["tagged_id"], msm["producer"] ) self.logger.debug( "finished writing intersection changes in %5.3f ms", 1000 * (time.time() - _t_start) ) self.logger.debug("committing to DB") _t_start = time.time() self.db_adapter.commit() self.logger.debug( "finished committing DB changes in %5.3f ms", 1000 * (time.time() - _t_start) ) self.logger.info( "finished handling measurement in %5.3f ms", 1000 * (time.time() - t_start_total) ) return { "tag_type": msm["tag_type"], "tagged_id": msm["tagged_id"], "taxonomy_id": taxonomy_db_info["taxonomy"]["id"], "measured_at": msm_timestamp, "tag_changes": required_intersection_changes }