def __trace_metrics(self, events):
return events \
.withColumn("counter",
custom_translate_like(
source_field=col("message"),
mappings_pair=[
(["HTTP request received", "Referer: cdvr-bs", "<Result>success</Result>"],
"vrm_success_recorded"),
(["HTTP request received", "Referer: cdvr-bs", "<Result>failed</Result>"],
"vrm_failed_recorded"),
(["HTTP request", ":8080/RE/", "learnAction"], "reng_success_action"),
(["HTTP request received", "IsAuthorized.traxis"], "irdeto_success_request"),
(["HTTP request received", "User-Agent", "vod-service"], "vod_service_success"),
(["HTTP request received", "x-application-name: purchase-service"],
"purchase_service_success"),
(["HTTP request received", "x-application-name: discovery-service"],
"discovery_service_success"),
(["HTTP request received", "x-application-name: epg-packager"], "epg_success"),
(["HTTP request received", "x-application-name: recording-service"],
"recording_service_success"),
(["HTTP request received", "x-application-name: session-service"],
"session_service_success")
],
default_value="unclassified")) \
.where("counter != 'unclassified'") \
.aggregate(Count(group_fields=["hostname", "counter"],
aggregation_name=self._component_name))
def __process_hi_res_events(self, read_stream):
"""
Aggregation for events with information about loading high_resolution images
:param read_stream: input stream with events from dag kafka topic
:return: list of aggregated metrics
"""
perform_high_res_images_events = read_stream \
.where("task == 'perform_high_resolution_images_qc'")
def __process_images_processed_status(column_name, regex_group_number,
component_suffix):
"""
Calculate aggregated metric for specific column
:param column_name: New column name for value of processed images
:param regex_group_number: index of group in regex pattern
:param component_suffix: name of suffix for metric
:return: aggregated metric for specific column
"""
return perform_high_res_images_events \
.where("subtask_message like 'Images processed:%'") \
.withColumn(column_name,
regexp_extract("subtask_message",
r"^Images processed: qc_success: (\d+), qc_retry: (\d+), qc_error: (\d+).*",
regex_group_number)) \
.aggregate(Sum(group_fields=["dag", "task"], aggregation_field=column_name,
aggregation_name=self._component_name + "." + component_suffix))
perform_high_res_images_processed_success_sum = \
__process_images_processed_status("images_success", 1, "hi_res_images_processed_success")
perform_high_res_images_processed_retry_sum = \
__process_images_processed_status("images_retry", 2, "hi_res_images_processed_retry")
perform_high_res_images_processed_error_sum = \
__process_images_processed_status("images_error", 3, "hi_res_images_processed_error")
__mapping_image_type = [
(["image_type='HighResPortrait'",
"status='qc_success'"], "hi_res_images_portrait"),
(["image_type='HighResLandscape'",
"status='qc_success'"], "hi_res_images_landscape")
]
perform_high_res_images_type_count = perform_high_res_images_events \
.withColumn("image_type", custom_translate_like(source_field=col("subtask_message"),
mappings_pair=__mapping_image_type,
default_value="unclassified")) \
.where("image_type != 'unclassified'") \
.aggregate(Count(group_fields=["dag", "task", "image_type"],
aggregation_name=self._component_name))
return [
perform_high_res_images_processed_success_sum,
perform_high_res_images_processed_retry_sum,
perform_high_res_images_processed_error_sum,
perform_high_res_images_type_count
]
def __info_metrics(self, events):
return events \
.withColumn("counter",
custom_translate_like(
source_field=col("message"),
mappings_pair=[
(["Loading tva version", "took"], "metadata_success")
],
default_value="unclassified")) \
.where("counter != 'unclassified'") \
.aggregate(Count(group_fields=["hostname", "counter"],
aggregation_name=self._component_name))
def __memory_flushing(self, events):
return events \
.where("message like '%Flushing%'") \
.withColumn("column_family", custom_translate_like(
source_field=col("message"),
mappings_pair=[(["Channels"], "channels"),
(["Titles"], "titles"),
(["Groups"], "groups")],
default_value="unclassified")) \
.where("column_family != 'unclassified'") \
.aggregate(Count(group_fields=["column_family"],
aggregation_name=self._component_name + ".memory_flushing"))
def __warn_metrics(self, events):
return events \
.withColumn("counter",
custom_translate_like(
source_field=col("message"),
mappings_pair=[
(["Error", ":8080/RE"], "reng_error_action"),
(["Genre", "is not known"], "metadata_warning"),
(["Invalid parameter"], "invalid_parameter_warning")
],
default_value="unclassified")) \
.where("counter != 'unclassified'") \
.aggregate(Count(group_fields=["hostname", "counter"],
aggregation_name=self._component_name))
def __error_metrics(self, events):
error_stream = events.where("level = 'ERROR'") \
.withColumn("counter",
custom_translate_like(
source_field=col("message"),
mappings_pair=[
(["Eventis.Traxis.Cassandra.CassandraException"], "traxis_cassandra_error"),
(["NetworkTimeCheckError"], "ntp_error")
],
default_value="unclassifed_errors"))
warn_and_fatal_stream = events.where("level in ('WARN', 'FATAL')") \
.withColumn("counter", lit("unclassifed_errors"))
return error_stream.union(warn_and_fatal_stream) \
.aggregate(Count(group_fields=["hostname", "counter"],
aggregation_name=self._component_name))
def _process_pipeline(self, uxp_stream):
"""
Returns list with streams for aggragated fields.
:param uxp_stream: input stream
:return: list of processed streams
"""
filtered_exp_stream = uxp_stream \
.where(uxp_stream.url.isin(self.__processing_urls)) \
.select(custom_translate_like(col("url"), self.__url_mapping, lit("undefined")).alias("action"),
col("status code").alias("statusCode"), col("responseTime"), col("@timestamp"))
uxp_count_stream = filtered_exp_stream \
.aggregate(Count(group_fields=["action", "statusCode"], aggregation_name=self._component_name))
uxp_avg_response_time_stream = filtered_exp_stream \
.aggregate(Avg(aggregation_field="responseTime", group_fields=["action"],
aggregation_name=self._component_name))
return [uxp_count_stream, uxp_avg_response_time_stream]
def _process_pipeline(self, source_stream):
filtered_stream = source_stream \
.where(col("level") != "DEBUG") \
.withColumn("level_updated",
when((col("level") == "INFO"), "SUCCESS")
.when((col("level") == "FATAL"), "ERROR")
.otherwise(col("level"))) \
.withColumn("level", col("level_updated"))
stream_with_mapped_groups = filtered_stream \
.withColumn("group", custom_translate_like(source_field=lower(col("thread_name")),
mappings_pair=self.__mapping_thread_name_to_group,
default_value="unclassified"))
count_by_level = stream_with_mapped_groups \
.where((col("level") != "SUCCESS") | lower(col("message")).like("%succe%")) \
.aggregate(
Count(group_fields=["hostname", "level"], aggregation_name=self._component_name + ".prodis_operations"))
total_count = stream_with_mapped_groups.aggregate(
Count(group_fields=["hostname"],
aggregation_name=self._component_name +
".prodis_operations"))
total_count_by_group = stream_with_mapped_groups.aggregate(
Count(group_fields=["hostname", "group"],
aggregation_name=self._component_name))
count_by_group_and_level = stream_with_mapped_groups.aggregate(
Count(group_fields=["hostname", "group", "level"],
aggregation_name=self._component_name))
return [
total_count, count_by_level, total_count_by_group,
count_by_group_and_level
]
def _process_pipeline(self, read_stream):
info_messages = [
InfoMessage('Nagra ELK Import DEX EPG',
'Imported a Nagra file successfully'),
InfoMessage('TVA listings Ingester',
"Commit succeeded for Model 'TVA listings Ingester'"),
InfoMessage(
"TVA_Eredivisie Ingester",
"Commit succeeded for Model 'TVA_Eredivisie Ingester'"),
InfoMessage('DefaultsProvider',
"Commit succeeded for Model 'DefaultsProvider'"),
InfoMessage('Online Prodis VOD ingest',
"Data has been successfully ingested"),
InfoMessage('PullBasedPublisher', 'Successfully published'),
InfoMessage('Nagra ELK Export DIM', 'Export to Nagra succeeded'),
InfoMessage('TVA Filepublisher', 'Successfully published'),
InfoMessage('Tva2Prodis', 'Successfully published')
]
where_column = col("level").isin("ERROR", "WARN")
for info_message in info_messages:
where_column = where_column | \
(col("level") == "INFO") \
& (col("instance_name") == info_message.instance_name) \
& (col("message").like("%" + info_message.message + "%"))
by_specific_info_messages = read_stream.where(where_column)\
.aggregate(Count(group_fields=["hostname", "instance_name", "level"],
aggregation_name=self._component_name))
count_by_classname = read_stream.aggregate(
Count(group_fields=["class_name"],
aggregation_name=self._component_name))
count_by_hostname = read_stream.aggregate(
Count(group_fields=["hostname"],
aggregation_name=self._component_name))
count_by_hostname_and_common_levels = read_stream\
.where((col("level") == "INFO") | (col("level") == "WARN") | (col("level") == "ERROR"))\
.aggregate(Count(group_fields=["hostname", "level"], aggregation_name=self._component_name))
count_by_hostname_and_other_levels = read_stream \
.where((col("level") != "INFO") & (col("level") != "WARN") & (col("level") != "ERROR")) \
.withColumn("level", lit("other"))\
.aggregate(Count(group_fields=["hostname", "level"], aggregation_name=self._component_name))
count_by_instance_and_level = read_stream.aggregate(
Count(group_fields=["instance_name", "level"],
aggregation_name=self._component_name))
count_by_instance = read_stream.aggregate(
Count(group_fields=["instance_name"],
aggregation_name=self._component_name))
other_metrics = read_stream \
.where((col("message").like("%saved%") & (col("level") == "INFO"))
| (col("message").like("%cannot download% TVA%") & (col("level") == "ERROR"))
| (col("message").like("Successfully published. 'Full'%"))
| (col("message").like("Successfully published. 'Delta'%"))
| (col("message").like("Updating%") & (col("level") == "INFO") & (lower(col("class_name")) == "p"))
| (col("message").like("Inserting%") & (col("level") == "INFO") & (lower(col("class_name")) == "p"))
| (col("message").like("'PRODIS%"))
| (col("message").like("%traxis% succeeded%"))
| (col("message").like("%Subscriber added%"))
| (col("message").like("%Subscriber removed%"))
) \
.withColumn("message_type",
custom_translate_like(
source_field=col("message"),
mappings_pair=[
(["saved"], "catalog_ingestion_success"),
(["cannot download", "TVA"], "catalog_ingestion_failure"),
(["Successfully published. 'Full'"], "full_feed_published"),
(["Successfully published. 'Delta'"], "delta_feeds_published"),
(["Updating"], "catalog_update"),
(["Inserting"], "catalog_insert"),
(["'PRODIS"], "prodis_delta_pull_requests"),
(["traxis", "succeeded"], "subscription_manager_traxis"),
(["Subscriber added"], "subscriber_addition"),
(["Subscriber removed"], "subscriber_removal")
],
default_value="unclassified")) \
.aggregate(Count(group_fields="message_type", aggregation_name=self._component_name))
return [
by_specific_info_messages, count_by_classname, count_by_hostname,
count_by_hostname_and_common_levels,
count_by_hostname_and_other_levels, count_by_instance_and_level,
count_by_instance, other_metrics
]