def create_event_creators(configuration):
"""
Method creates configured event_creator for logs from Mongo
:param configuration
:return: Composite event creator for Mongo
"""
timezone_name = configuration.property("timezone.name")
timezones_property = configuration.property("timezone.priority", "idc")
event_creator = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp",
"%Y-%m-%dT%H:%M:%S.%f",
timezone_name,
timezones_property,
"@timestamp",
include_timezone=True),
StringField("level"),
StringField("event_type"),
StringField("thread"),
StringField("message")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+.\d{4})\s+(?P<level>.*?)\s+"
r"(?P<event_type>.*?)\s+\[(?P<thread>.*?)\]\s(?P<message>.*)"))
return MatchField(
"source", {
"mongo.log":
SourceConfiguration(
event_creator,
Utils.get_output_topic(configuration, "mongo_parsed"))
})
def ee_interface_event_creator(timezone_name, timezones_priority):
return EventCreator(
Metadata([
ConfigurableTimestampField("@timestamp",
"%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("instance_name"),
StringField("thread_id"),
StringField("class_name"),
StringField("message")
]), SplitterParser("|", is_trim=True))
def general_worker_creator(timezone_name, timezones_property):
return EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("script"),
StringField("level"),
StringField("message")
]),
RegexpParser(
r"^\[(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] "
r"\{(?P<script>[^\}]+)\} (?P<level>\w+?) - (?P<message>(.|\s)*)"
))
def subtask_creator(timezone_name, timezones_property):
return EventCreator(
Metadata([
ConfigurableTimestampField("subtask_timestamp",
"%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_property),
StringField("subtask_script"),
StringField("subtask_level"),
StringField("subtask_message")
]),
RegexpParser(
r"^Subtask: \[(?P<subtask_timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]"
r" \{(?P<subtask_script>[^\}]+):\d+\} (?P<subtask_level>\w+?) - (?P<subtask_message>(?:.|\s)*)",
return_empty_dict=True),
matcher=SubstringMatcher("Subtask:"),
field_to_parse="message")
def ip_webui_manager_creator(timezone_name, timezones_property):
return EventCreator(
Metadata([
ConfigurableTimestampField("timestamp",
"%d/%b/%Y:%H:%M:%S",
timezone_name,
timezones_property,
"@timestamp",
include_timezone=True),
StringField("message"),
StringField("ip")
]),
RegexpParser(
r"^(?P<ip>.*?)\s+-\s+-\s\[(?P<timestamp>\d{2}\/\w+\/\d{4}:\d{2}:\d{2}:\d{2}\s.\d{4})\]\s+"
r"(?P<message>.*)",
return_empty_dict=True))
def script_webui_manager_creator(timezone_name, timezones_property):
return EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("thread_id"),
StringField("script"),
StringField("script_line"),
StringField("message"),
StringField("level")
]),
RegexpParser(
r"^\[(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\s+\[(?P<thread_id>.*?)\]\s+"
r"\{(?P<script>.*?):(?P<script_line>.*?)\}\s+(?P<level>\w+?)\s+-\s+(?P<message>.*)",
return_empty_dict=False))
def manager_scheduler_airflow_event_creator(timezone_name,
timezones_property):
return EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("script"),
StringField("script_line"),
StringField("level"),
StringField("message")
]),
RegexpParser(
r"^\[(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\s+"
r"\{(?P<script>.*?):(?P<script_line>.*?)\}\s+(?P<level>\w+?)\s+-\s+(?P<message>.*)"
))
def ee_wcf_event_creator(timezone_name, timezones_priority):
def concat(item1, item2):
""" method concatenates params """
return item1 + item2
def take_first(item1, item2):
""" method takes left parameter only """
return item1
return MultilineEventCreator(
Metadata([
ConfigurableTimestampField("@timestamp",
"%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("message")
]), SplitterParser("|", is_trim=True), [take_first, concat])
def webui_manager_creator(timezone_name, timezones_property):
return EventCreator(
Metadata([
ConfigurableTimestampField("timestamp",
"%Y-%m-%d %H:%M:%S",
timezone_name,
timezones_property,
"@timestamp",
include_timezone=True),
StringField("thread_id"),
StringField("message"),
StringField("level")
]),
RegexpParser(
r"^\[(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\s+?.\d*?)\]\s+?"
r"\[(?P<thread_id>.*?)\]\s+?\[(?P<level>\w+?)\]\s+?(?P<message>.*)",
return_empty_dict=True))
def create_event_creators(configuration=None):
"""
Tree of different parsers for all types of logs for poster server
:param configuration: YML config
:return: Tree of event_creators
"""
timezone_name = configuration.property("timezone.name")
timezones_property = configuration.property("timezone.priority", "dic")
poster_server_log = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("level"),
StringField("module"),
StringField("message")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\,\d{3})"
r"\s+(?P<level>\w+?)\s+(?P<module>\w+)\s+(?P<message>.*)"))
crid_creator = EventCreator(Metadata([StringField("crid")]),
RegexpParser(r".*(?P<crid>crid[^\\]*)",
return_empty_dict=True),
matcher=SubstringMatcher("crid"),
field_to_parse="message")
composite_event_creator = CompositeEventCreator() \
.add_source_parser(poster_server_log) \
.add_intermediate_result_parser(crid_creator)
return MatchField(
"source", {
"PosterServer.Error.log":
SourceConfiguration(
composite_event_creator,
Utils.get_output_topic(configuration,
"poster_server_error_log")),
"PosterServer.log":
SourceConfiguration(
composite_event_creator,
Utils.get_output_topic(configuration, "poster_server_log"))
})
def create_event_creators(configuration):
timezone_name = configuration.property("timezone.name")
timezones_priority = configuration.property("timezone.priority", "dic")
return MatchField(
"topic", {
"traxis_cassandra_log_gen":
SourceConfiguration(
MultipleEventCreator([
EventCreator(
Metadata([
StringField("level"),
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("message")
]),
RegexpParser(
r"^.*?\:\s+(\S+)\s\[[^\]]*\]\s+(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d+)\s+(\S[\s\S]+)"
)),
EventCreator(
Metadata([
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("level"),
StringField("message")
]),
RegexpParser(
r"^.*?\:\s(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d+)\s(\S+)\s+\[[^\]]*\]\s+(\S[\s\S]+)"
))
]), Utils.get_output_topic(configuration, "general")),
"traxis_cassandra_log_err":
SourceConfiguration(
MultipleEventCreator([
EventCreator(
Metadata([
StringField("level"),
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("message")
]),
RegexpParser(
r"^.*\:\s.*\:\s(\S+)\s\[[^\]]*\]\s+(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d+)\s+(\S[\s\S]+)"
)),
EventCreator(
Metadata([
StringField("level"),
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("message")
]),
RegexpParser(
r"^.*\:\s(\S+)\s\[[^\]]*\]\s+(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d+)\s+(\S[\s\S]+)"
)),
EventCreator(
Metadata([
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("level"),
StringField("message")
]),
RegexpParser(
r"^.*\:\s(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d+)\s(\S+)\s\[[^\]]*\]\s+(\S[\s\S]+)"
))
]), Utils.get_output_topic(configuration, "error"))
})
def create_event_creators(configuration=None):
"""
Tree of different parsers for all types of logs for Nokia VRM
:param configuration: YML config
:return: Tree of event_creators
"""
timezone_name = configuration.property("timezone.name")
timezones_property = configuration.property("timezone.priority", "dic")
nokia_vrm_scheduler_audit_csv = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%d-%b-%Y %H:%M:%S.%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("level"),
StringField("field1"),
StringField("event_id"),
StringField("domain"),
StringField("ip"),
StringField("method"),
StringField("params"),
StringField("description"),
StringField("message")
]), SplitterParser("|", is_trim=True))
nokia_vrm_audit_csv = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%d-%b-%Y %H:%M:%S.%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("level"),
StringField("event_id"),
StringField("domain"),
StringField("ip"),
StringField("method"),
StringField("params"),
StringField("description"),
StringField("message")
]), SplitterParser("|", is_trim=True))
nokia_vrm_ds_audit_csv = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%d-%b-%Y %H:%M:%S.%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("level"),
StringField("event_id_1"),
StringField("event_id_2"),
StringField("domain"),
StringField("ip"),
StringField("method"),
StringField("params"),
StringField("description"),
StringField("message")
]), SplitterParser("|", is_trim=True))
nokia_vrm_bs_lgi_lgienh_api_audit_csv = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S.%f",
timezone_name, timezones_property,
"@timestamp"),
StringField("level"),
StringField("endpoint"),
StringField("request")
]), SplitterParser("|", is_trim=True))
return MatchField(
"source", {
"scheduler_bs_audit.log":
SourceConfiguration(
nokia_vrm_scheduler_audit_csv,
Utils.get_output_topic(configuration, "scheduler_bs_audit")),
"console_bs_audit.log":
SourceConfiguration(
nokia_vrm_audit_csv,
Utils.get_output_topic(configuration, "console_bs_audit")),
"authentication_bs_audit.log":
SourceConfiguration(
nokia_vrm_audit_csv,
Utils.get_output_topic(configuration,
"authentication_bs_audit")),
"cdvr_bs_audit.log":
SourceConfiguration(
nokia_vrm_audit_csv,
Utils.get_output_topic(configuration, "cdvr_bs_audit")),
"epg_audit.log":
MatchField(
"topic", {
"nokiavrmds_epgaudit":
SourceConfiguration(
nokia_vrm_ds_audit_csv,
Utils.get_output_topic(configuration, "epg_ds_audit")),
"nokiavrmbs_epgaudit":
SourceConfiguration(
nokia_vrm_ds_audit_csv,
Utils.get_output_topic(configuration, "epg_bs_audit"))
}),
"cDVR_audit.log":
SourceConfiguration(
nokia_vrm_ds_audit_csv,
Utils.get_output_topic(configuration, "cdvr_ds_audit")),
"user_audit.log":
SourceConfiguration(
nokia_vrm_ds_audit_csv,
Utils.get_output_topic(configuration, "user_ds_audit")),
"lgienhapi_bs.log":
SourceConfiguration(
nokia_vrm_bs_lgi_lgienh_api_audit_csv,
Utils.get_output_topic(configuration, "lgienh_api_bs_audit")),
"schange_import_bs_audit.log":
SourceConfiguration(
nokia_vrm_audit_csv,
Utils.get_output_topic(configuration,
"schange_import_bs_audit")),
"vspp_adapter_bs_audit.log":
SourceConfiguration(
nokia_vrm_audit_csv,
Utils.get_output_topic(configuration, "vspp_adapter_bs_audit"))
})
def create_event_creators(configuration=None):
"""
Method creates configuration for CDN
:param configuration
:return: Composite event creator for CDN
"""
timezone_name = configuration.property("timezone.name")
timezones_property = configuration.property("timezone.priority", "dic")
cdn_log = EventCreator(
Metadata([
StringField("s_dns"),
StringField("date"),
StringField("time"),
StringField("x_duration"),
StringField("c_ip"),
StringField("c_port"),
StringField("c_vx_zone"),
StringField("c_vx_gloc"),
StringField("unknown_field1"),
StringField("unknown_field2"),
StringField("cs_method"),
StringField("cs_uri"),
StringField("cs_version"),
StringField("cs_user_agent"),
StringField("cs_refer"),
StringField("cs_cookie"),
StringField("cs_range"),
StringField("cs_status"),
StringField("s_cache_status"),
StringField("sc_bytes"),
StringField("sc_stream_bytes"),
StringField("sc_dscp"),
StringField("s_ip"),
StringField("s_vx_rate"),
StringField("s_vx_rate_status"),
StringField("s_vx_serial"),
StringField("rs_stream_bytes"),
StringField("rs_bytes"),
StringField("cs_vx_token"),
StringField("sc_vx_download_rate"),
StringField("x_protohash"),
StringField("additional_headers"),
StringField("unknown_field3"),
StringField("unknown_field4")
]), SplitterParser("\t", is_trim=True))
cdn_log_with_timestamp = MutateEventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S",
timezone_name, timezones_property,
"@timestamp")
]), [
FieldsMapping(["date", "time"], "timestamp",
lambda x, y: x + " " + y, True)
])
return SourceConfiguration(
CompositeEventCreator().add_source_parser(
cdn_log).add_intermediate_result_parser(cdn_log_with_timestamp),
Utils.get_output_topic(configuration, "cdn_log"))
def create_event_creators(config):
"""
Method creates configuration for UServices Component
:param config, configuration
:return: Composite event creator for UServices
"""
timezone_name = config.property("timezone.name")
timezones_priority = config.property("timezone.priority", "dic")
json_event_creator = SingleTypeEventCreator(
StringField(None),
JsonParser(keys_mapper=None,
values_mapper=None,
flatten=True,
delimiter='_',
fields_to_flat=["http", "header"]))
timestamp_event_creator = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%dT%H:%M:%S.%fZ",
timezone_name, timezones_priority,
"@timestamp")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)"),
field_to_parse="@timestamp")
http_url_query_event_creator = EventWithUrlCreator(
url_query_field="http_urlquery", delete_source_field=False)
api_methods_event_creator = PredicateEventCreator(
["app", "header_x-original-uri"],
[(["recording-service", "bookings"], {
"api_method": "bookings"
}), (["recording-service", "recordings"], {
"api_method": "recordings"
}), (["purchase-service", "history"], {
"api_method": "history"
}),
(["purchase-service", "entitlements"], {
"api_method": "entitlements"
}), (["vod-service", "contextualvod"], {
"api_method": "contextualvod"
}), (["vod-service", "detailscreen"], {
"api_method": "detailscreen"
}), (["vod-service", "gridscreen"], {
"api_method": "gridscreen"
}),
(["discovery-service", "learn-actions"], {
"api_method": "learn-actions"
}), (["discovery-service", "search"], {
"api_method": "search"
}),
(["discovery-service", "recommendations"], {
"api_method": "recommendations"
}), (["session-service", "channels"], {
"api_method": "channels"
}), (["session-service", "cpes"], {
"api_method": "cpes"
})])
return SourceConfiguration(
CompositeEventCreator().add_source_parser(json_event_creator).
add_intermediate_result_parser(
timestamp_event_creator).add_intermediate_result_parser(
http_url_query_event_creator).add_intermediate_result_parser(
api_methods_event_creator, final=True),
Utils.get_output_topic(config, "uservices_parsed_logs"))
def create_event_creators(configuration):
"""
Method creates configuration for Traxis Frontend Component
:param configuration
:return: MatchField configuration for Traxis Frontend
"""
timezone_name = configuration.property("timezone.name")
timezones_priority = configuration.property("timezone.priority", "dic")
event_creator = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority,
"@timestamp"),
StringField("level"),
StringField("thread_name"),
StringField("component"),
StringField("message")
]),
RegexpParser(
r"(?s)^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})"
r"\s*"
r"(?P<level>\w+)"
r"\s*"
r"\[(?P<thread_name>.*?)\]"
r"\s*"
r"(?P<component>\w+)"
r"\s*-\s*"
r"(?P<message>.*)$"))
ip_event_creator = EventCreator(
Metadata([StringField("ip")]),
RegexpParser(r"^\[(?P<ip>[0-9,\.: ]*?)\].*", return_empty_dict=True))
request_id_event_creator = EventCreator(
Metadata([StringField("request_id", "request-id")]),
RegexpParser(r"^.*\[RequestId = (?P<request_id>.*?)\].*",
return_empty_dict=True),
matcher=SubstringMatcher("RequestId ="))
obo_customer_id_event_creator = EventCreator(
Metadata([StringField("obo_customer_id", "obo-customer-id")]),
RegexpParser(r"^.*\[CustomerId = (?P<obo_customer_id>.*?)\].*",
return_empty_dict=True),
matcher=SubstringMatcher("CustomerId ="))
x_request_id_event_creator = EventCreator(
Metadata([StringField("x_request_id", "x-request-id")]),
RegexpParser(r"(\n|.)*x-request-id:\s(?P<x_request_id>[a-z0-9- ]*).*",
return_empty_dict=True),
matcher=SubstringMatcher("x-request-id:"))
method_duration_event_creator = EventCreator(
Metadata([StringField("method"),
StringField("duration")]),
RegexpParser(
r"^.*Executing method \'(?P<method>.*?)\' took \'(?P<duration>.*?)\'.*",
return_empty_dict=True),
matcher=SubstringMatcher("Executing method"))
method_invoked_event_creator = EventCreator(
Metadata([
StringField("method"),
StringField("identity"),
StringField("product_id", "productId")
]),
RegexpParser(
r"^.*Method \'(?P<method>.*?)\' invoked with parameters\: identity = (?P<identity>.*?)\, productId ="
r" (?P<product_id>.*?)(\,.*|$)",
return_empty_dict=True),
matcher=SubstringMatcher("invoked with parameters"))
cannot_purchase_product_event_creator = EventCreator(
Metadata([StringField("product_id", "productId")]),
RegexpParser(
r"^.*Cannot purchase products of type \'Subscription\'.*productId \'(?P<product_id>.*?)\'$",
return_empty_dict=True),
matcher=SubstringMatcher("Cannot purchase products of type"))
query_metrics_event_creator = EventCreator(
Metadata([StringField("query_metrics")]),
RegexpParser(r"^.*QueryMetrics:(?P<query_metrics>.*)",
return_empty_dict=True),
matcher=SubstringMatcher("QueryMetrics"))
key_value_event_creator = SingleTypeEventCreator(
IntField(None),
KeyValueParser(",", "="),
field_to_parse="query_metrics")
id_event_creator = CompositeEventCreator() \
.add_source_parser(event_creator) \
.add_intermediate_result_parser(ip_event_creator) \
.add_intermediate_result_parser(request_id_event_creator) \
.add_intermediate_result_parser(obo_customer_id_event_creator) \
.add_intermediate_result_parser(x_request_id_event_creator)
return MatchField(
"source", {
"TraxisService.log":
SourceConfiguration(
id_event_creator.add_intermediate_result_parser(
query_metrics_event_creator).
add_intermediate_result_parser(
key_value_event_creator,
final=True).add_intermediate_result_parser(
method_duration_event_creator,
final=True).add_intermediate_result_parser(
method_invoked_event_creator,
final=True).add_intermediate_result_parser(
cannot_purchase_product_event_creator,
final=True),
Utils.get_output_topic(configuration, "general")),
"TraxisServiceError.log":
SourceConfiguration(id_event_creator,
Utils.get_output_topic(configuration,
"error")),
"TraxisServiceDistributedScheduler.log":
SourceConfiguration(
id_event_creator,
Utils.get_output_topic(configuration, "scheduler")),
"TraxisServiceLogManagement.log":
SourceConfiguration(
id_event_creator,
Utils.get_output_topic(configuration, "management"))
})
def create_event_creators(configuration):
"""
Tree of different parsers for all types of logs for THINK ANALYTICS
:param configuration: YML config
:return: Tree of event_creators
"""
timezone_name = configuration.property("timezone.name")
timezones_priority = configuration.property("timezone.priority", "idc")
duration_event_creator = MutateEventCreator(None, [
FieldsMapping([
"started_script", "finished_script", "finished_time", "@timestamp"
], "duration", duration_update)
])
concat_httpaccess_timestamp_event_creator = MutateEventCreator(
Metadata([
ConfigurableTimestampField("timestamp",
"%d/%b/%Y:%H:%M:%S",
timezone_name,
timezones_priority,
"@timestamp",
include_timezone=True)
]), [
FieldsMapping(["date", "time"],
"timestamp",
agg_func=lambda x, y: (x + " " + y)[1:-1],
remove_intermediate_fields=True)
])
concat_central_timestamp_event_creator = MutateEventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%a %d/%m/%y %H:%M:%S",
timezone_name, timezones_priority,
"@timestamp")
]), [
FieldsMapping(
["date", "time"], "timestamp", remove_intermediate_fields=True)
])
traxis_profile_id_event_creator = MutateEventCreator(fields_mappings=[
FieldsMapping(["subscriberId"], "traxis-profile-id", lambda x: x, True)
])
content_item_id_event_creator = MutateEventCreator(fields_mappings=[
FieldsMapping(["contentItemId"], "crid",
lambda x: "crid{}".format(x.split('crid')[-1]), False)
])
int_request_id_event_creator = MutateEventCreator(fields_mappings=[
FieldsMapping(["intRequestId"], "request-id", lambda x: x, True)
])
return MatchField(
"source", {
"localhost_access_log":
SourceConfiguration(
CompositeEventCreator().add_source_parser(
EventCreator(
Metadata([
StringField("date"),
StringField("time"),
StringField("ip"),
StringField("thread"),
StringField("http_method"),
StringField("url"),
StringField("http_version"),
StringField("response_code"),
StringField("response_time")
]), SplitterParser(
delimiter=" ",
is_trim=True))).add_intermediate_result_parser(
concat_httpaccess_timestamp_event_creator
).add_intermediate_result_parser(
EventWithUrlCreator(delete_source_field=True,
keys_to_underscore=False)).
add_intermediate_result_parser(traxis_profile_id_event_creator)
.add_intermediate_result_parser(content_item_id_event_creator).
add_intermediate_result_parser(int_request_id_event_creator),
Utils.get_output_topic(configuration, "httpaccess")),
"RE_SystemOut.log":
SourceConfiguration(
EventCreator(
Metadata([
ConfigurableTimestampField("@timestamp",
"%d/%m/%y %H:%M:%S.%f",
timezone_name,
timezones_priority,
dayfirst=True,
use_smart_parsing=True),
StringField("level"),
StringField("script"),
StringField("message")
]),
RegexpParser(
r"^\[(?P<timestamp>\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}\.\d{3}\s\D+?)\] "
r"(?P<level>\w+?)\s+?-\s+?(?P<script>\S+?)\s+?:\s+?(?P<message>.*)"
)), Utils.get_output_topic(configuration, "resystemout")),
"REMON_SystemOut.log":
SourceConfiguration(
EventCreator(
Metadata([
ConfigurableTimestampField("@timestamp",
None,
timezone_name,
timezones_priority,
dayfirst=True,
use_smart_parsing=True),
StringField("level"),
StringField("script"),
StringField("type"),
StringField("message")
]),
RegexpParser(
r"^\[(?P<timestamp>\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}.\d{3}\s\D+?)\] "
r"(?P<level>\w+?)\s+?-\s+?(?P<script>\S+?)\s+?:\s+?\[(?P<type>\S+?)\]\s+?-\s+?(?P<message>.*)"
)), Utils.get_output_topic(configuration,
"remonsystemout")),
"Central.log":
SourceConfiguration(
CompositeEventCreator().add_source_parser(
EventCreator(
Metadata([
StringField("date"),
StringField("time"),
StringField("level"),
StringField("message"),
StringField("thread"),
StringField("c0"),
StringField("c1"),
StringField("c2"),
StringField("role")
]), CsvParser(
",", '"'))).add_intermediate_result_parser(
concat_central_timestamp_event_creator),
Utils.get_output_topic(configuration, "central")),
"thinkenterprise.log":
SourceConfiguration(
EventCreator(
Metadata([
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("level"),
StringField("message")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})"
r":\s+?(?P<level>\w+?)\s+?-\s+?(?P<message>.*)")),
Utils.get_output_topic(configuration, "thinkenterprise")),
"gcollector.log":
SourceConfiguration(
EventCreator(
Metadata([
ConfigurableTimestampField("@timestamp",
"%Y-%m-%dT%H:%M:%S.%f",
timezone_name,
timezones_priority,
include_timezone=True),
StringField("process_uptime"),
StringField("message")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}.\d{4})"
r":\s+?(?P<process_uptime>\d+?\.\d{3}):\s+?(?P<message>.*)"
)), Utils.get_output_topic(configuration, "gcollector")),
"server.log":
SourceConfiguration(
EventCreator(
Metadata([
ConfigurableTimestampField(
"@timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority),
StringField("level"),
StringField("class_name"),
StringField("thread"),
StringField("message")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+?"
r"(?P<level>\w+?)\s+?\[(?P<class_name>.+?)\]\s+?\((?P<thread>.+?)\)\s+?(?P<message>.*)"
)), Utils.get_output_topic(configuration, "server")),
"RE_Ingest.log":
SourceConfiguration(
CompositeEventCreator().add_source_parser(
DictEventCreator(
Metadata([
StringField("started_script"),
ConfigurableTimestampField("timestamp",
None,
timezone_name,
timezones_priority,
"@timestamp",
use_smart_parsing=True),
StringField("message"),
StringField("finished_script"),
ConfigurableTimestampField("finished_time",
None,
timezone_name,
timezones_priority,
use_smart_parsing=True)
]),
RegexpMatchesParser(
r"Started\s+?(?P<started_script>.*?\.sh)\s+?"
r"(?P<timestamp>\w+?\s+?\w+?\s+?\d{1,2}\s+?\d{2}:\d{2}:\d{2}\s+?\w+?\s+?\d{4})"
r"(?P<message>(?:.|\s)*)Finished\s+?(?P<finished_script>.*?\.sh)\s+?"
r"(?P<finished_time>\w+?\s+?\w+?\s+?\d{1,2}\s+?\d{2}:\d{2}:\d{2}\s+?\w+?\s+?\d{4}).*"
))).add_intermediate_result_parser(
duration_event_creator),
Utils.get_output_topic(configuration, "reingest"))
})
def create_event_creators(config):
"""
Method to create a list of event creators for parsing of Traxis Backend logs.
:param config: Job configuration.
:return: A list of event creators.
"""
timezone_name = config.property("timezone.name")
timezones_priority = config.property("timezone.priority", "dic")
general_event_creator = EventCreator(
Metadata([
ConfigurableTimestampField("timestamp", "%Y-%m-%d %H:%M:%S,%f",
timezone_name, timezones_priority,
"@timestamp"),
StringField("level"),
StringField("message")
]),
RegexpParser(
r"^(?P<timestamp>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}.\d+)\s+"
r"(?P<level>\S+)\s+\[[^\]]+\]\s+(?P<message>[\s\S]*)"))
tva_ingest_event_creator = EventCreator(
Metadata([StringField("activity"),
StringField("request_id")]),
RegexpParser(
r"^(?P<activity>OnlineTvaIngest).*\[RequestId\s=\s(?P<request_id>[^]]+)\][\s\S]*",
return_empty_dict=True),
matcher=SubstringMatcher("OnlineTvaIngest"))
tva_manager_event_creator = EventCreator(
Metadata([
StringField("activity"),
StringField("task"),
IntField("duration_ms")
]),
RegexpParser(
r"^(?P<activity>TvaManager).*\[Task\s=\s(?P<task>[^]]+)\].*took\s'(?P<duration_ms>\d+)'\sms[\s\S]*",
return_empty_dict=True),
matcher=SubstringMatcher("TvaManager"))
parsing_context_event_creator = EventCreator(
Metadata([
StringField("activity"),
StringField("task"),
IntField("duration_ms")
]),
RegexpParser(
r"^(?P<activity>ParsingContext).*\[Task\s=\s(?P<task>[^]]+)\]\s"
r"Tva\singest\scompleted,\sduration\s=\s(?P<duration_ms>\d+)\sms[\s\S]*",
return_empty_dict=True),
matcher=SubstringMatcher("Tva ingest completed, duration"))
write_actions_event_creator = EventCreator(
Metadata([
StringField("activity"),
StringField("task"),
IntField("duration_ms")
]),
RegexpParser(
r"^(?P<activity>ParsingContext).*\[Task\s=\s(?P<task>[^]]+)\]\s"
r"Number\sof\swrite\sactions\squeued.*took\s(?P<duration_ms>\d+)\sms[\s\S]*",
return_empty_dict=True),
matcher=SubstringMatcher("Number of write actions queued"))
return MatchField(
"topic", {
"traxis_backend_log_gen":
MatchField(
"source", {
"TraxisService.log":
SourceConfiguration(
CompositeEventCreator().add_source_parser(
general_event_creator).
add_intermediate_result_parser(
tva_ingest_event_creator,
final=True).add_intermediate_result_parser(
tva_manager_event_creator,
final=True).add_intermediate_result_parser(
parsing_context_event_creator,
final=True).add_intermediate_result_parser(
write_actions_event_creator,
final=True),
Utils.get_output_topic(config, "general")),
"TraxisServiceDistributedScheduler.log":
SourceConfiguration(
general_event_creator,
Utils.get_output_topic(config, "scheduler")),
"TraxisServiceLogManagement.log":
SourceConfiguration(
general_event_creator,
Utils.get_output_topic(config, "management"))
}),
"traxis_backend_log_err":
SourceConfiguration(general_event_creator,
Utils.get_output_topic(config, "error"))
})