def test_get_member_name(self):
report_manager = ReportManager(None, None, None, None, "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
member_code = "MemberCodeA"
subsystem_code = "SubsystemCodeA"
member_class = "MemberClassA"
x_road_instance = "CI-REPORTS"
member_name_dict = [{
"x_road_instance": "CI-REPORTS",
"subsystem_name": {
"et": "Subsystem Name ET",
"en": "Subsystem Name EN"
},
"member_class": "MemberClassA",
"email": [],
"subsystem_code": "SubsystemCodeA",
"member_code": "MemberCodeA",
"member_name": "Member Name"
}]
member_name = report_manager.get_member_name(member_code,
subsystem_code,
member_class,
x_road_instance,
member_name_dict)
self.assertEqual(member_name, "Member Name")
member_name_dict = None
member_name = report_manager.get_member_name(member_code,
subsystem_code,
member_class,
x_road_instance,
member_name_dict)
self.assertEqual(member_name, "")
def test_get_name_and_average(self):
report_manager = ReportManager(None, None, None, "ss", "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
key = "ee-dev"
report_row = ReportRow(None)
report_row.duration_avg = (5123.123, 15)
produced_service = True
service_name = "service"
self.assertEqual(
report_manager.get_name_and_average(key, report_row,
produced_service,
service_name),
("ss: ee-dev",
round(report_row.duration_avg[0] / (report_row.duration_avg[1]))))
produced_service = False
self.assertEqual(
report_manager.get_name_and_average(key, report_row,
produced_service,
service_name),
("ee-dev: service",
round(report_row.duration_avg[0] / (report_row.duration_avg[1]))))
report_row.duration_avg = (None, None)
self.assertEqual(
report_manager.get_name_and_average(key, report_row,
produced_service,
service_name),
("ee-dev: service", None))
def test_merge_document_fields(self):
report_manager = ReportManager(None, None, None, None, "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
subsystem_code = "subsystem_code"
member_code = "member_code"
member_class = None
x_road_instance = ""
service_version = "1.0"
inner_document = dict()
inner_document["serviceSubsystemCode"] = subsystem_code
inner_document["serviceMemberCode"] = member_code
inner_document["serviceMemberClass"] = member_class
inner_document["serviceXRoadInstance"] = x_road_instance
inner_document["serviceVersion"] = service_version
merged_field = report_manager.merge_document_fields(
inner_document, [
"serviceSubsystemCode", "serviceMemberCode",
"serviceMemberClass", "serviceXRoadInstance", "serviceVersion"
], "new_field", ".")
self.assertEqual(merged_field, "subsystem_code.member_code.1.0")
def test_get_duration_top(self):
report_manager = ReportManager(None, None, None, "ss", "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
report_row = ReportRow(None)
report_row.duration_avg = (100, 10)
report_row_2 = ReportRow(None)
report_row_2.duration_avg = (1000, 10)
report_row_3 = ReportRow(None)
report_row_3.duration_avg = (200, 10)
report_row_4 = ReportRow(None)
report_row_4.duration_avg = (109, 10)
report_row_5 = ReportRow(None)
report_row_5.duration_avg = (1100, 10)
report_row_6 = ReportRow(None)
report_row_6.duration_avg = (209, 10)
report_row_7 = ReportRow(None)
report_row_7.duration_avg = (90, 10)
report_row_8 = ReportRow(None)
report_row_8.duration_avg = (900, 10)
report_row_9 = ReportRow(None)
report_row_9.duration_avg = (190, 10)
test_data = {
"service_1": {
"ee-dev": report_row,
"ee-dev_2": report_row_2,
"ee-dev_3": report_row_3
},
"service_2": {
"ee-dev_4": report_row_7,
"ee-dev_5": report_row_8,
"ee-dev_6": report_row_9
},
"service_3": {
"ee-dev_7": report_row_4,
"ee-dev_8": report_row_5,
"ee-dev_9": report_row_6
},
}
self.assertEqual(report_manager.get_duration_top(test_data, True),
[("ss: service_2", 90), ("ss: service_1", 100),
("ss: service_3", 110)])
self.assertEqual(report_manager.get_duration_top(test_data, False),
[("service_1: ee-dev_3", 20),
("service_3: ee-dev_9", 21),
("service_2: ee-dev_5", 90),
("service_1: ee-dev_2", 100),
("service_3: ee-dev_8", 110)])
def test_get_succeeded_top(self):
report_manager = ReportManager(None, None, None, "ss", "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
report_row = ReportRow(None)
report_row.succeeded_queries = 10
report_row_2 = ReportRow(None)
report_row_2.succeeded_queries = 15
report_row_3 = ReportRow(None)
report_row_3.succeeded_queries = 7
report_row_4 = ReportRow(None)
report_row_4.succeeded_queries = 12
report_row_5 = ReportRow(None)
report_row_5.succeeded_queries = 13
report_row_6 = ReportRow(None)
report_row_6.succeeded_queries = 8
report_row_7 = ReportRow(None)
report_row_7.succeeded_queries = 11
report_row_8 = ReportRow(None)
report_row_8.succeeded_queries = 14
report_row_9 = ReportRow(None)
report_row_9.succeeded_queries = 9
test_data = {
"service_1": {
"ee-dev": report_row,
"ee-dev_2": report_row_2,
"ee-dev_3": report_row_3
},
"service_2": {
"ee-dev_4": report_row_7,
"ee-dev_5": report_row_8,
"ee-dev_6": report_row_9
},
"service_3": {
"ee-dev_7": report_row_4,
"ee-dev_8": report_row_5,
"ee-dev_9": report_row_6
},
}
self.assertEqual(report_manager.get_succeeded_top(test_data, True),
[("ss: service_3", 13), ("ss: service_2", 14),
("ss: service_1", 15)])
self.assertEqual(report_manager.get_succeeded_top(test_data, False),
[("service_2: ee-dev_4", 11),
("service_3: ee-dev_7", 12),
("service_3: ee-dev_8", 13),
("service_2: ee-dev_5", 14),
("service_1: ee-dev_2", 15)])
def test_is_producer_document(self):
report_manager = ReportManager(None, None, None, None, "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
document = dict()
subsystem_code = "subsystem_code"
member_code = "member_code"
member_class = None
x_road_instance = ""
document["serviceSubsystemCode"] = subsystem_code
document["serviceMemberCode"] = member_code
document["serviceMemberClass"] = member_class
document["serviceXRoadInstance"] = x_road_instance
self.assertTrue(
report_manager.is_producer_document(document, subsystem_code,
member_code, member_class,
x_road_instance))
self.assertFalse(
report_manager.is_producer_document(document, subsystem_code,
subsystem_code, member_class,
x_road_instance))
self.assertFalse(
report_manager.is_producer_document(document, subsystem_code,
member_code, member_code,
x_road_instance))
self.assertFalse(
report_manager.is_producer_document(document, subsystem_code,
member_code, member_class,
member_class))
self.assertFalse(
report_manager.is_producer_document(document, x_road_instance,
member_code, member_class,
x_road_instance))
def test_get_name_and_count(self):
report_manager = ReportManager(None, None, None, "ss", "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
key = "ee-dev"
report_row = ReportRow(None)
suc_queries = 10
report_row.succeeded_queries = suc_queries
produced_service = True
service_name = "service"
self.assertEqual(
report_manager.get_name_and_count(key, report_row,
produced_service, service_name),
("ss: ee-dev", suc_queries))
produced_service = False
self.assertEqual(
report_manager.get_name_and_count(key, report_row,
produced_service, service_name),
("ee-dev: service", suc_queries))
def test_get_min_mean_max(self):
report_manager = ReportManager(None, None, None, None, "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
self.assertEqual(report_manager.get_min_mean_max(5, (11, 2), 5),
"5 / 6 / 5")
self.assertEqual(report_manager.get_min_mean_max(-5, (-11, 2), -5),
"-5 / -6 / -5")
self.assertEqual(
report_manager.get_min_mean_max(5.44235, (11.12412, 3.1253),
5.415123), "5 / 4 / 5")
self.assertEqual(report_manager.get_min_mean_max(5.6, (11, 4), 5.6),
"6 / 3 / 6")
self.assertEqual(report_manager.get_min_mean_max(None, (11, 2), 5),
"None / 6 / 5")
self.assertEqual(
report_manager.get_min_mean_max(None, (None, None), None),
"None / None / None")
def test_get_service_type(self):
report_manager_producer = ReportManager("", None, "member_code",
"subsystem_code", "2017-01-01",
"2017-01-01", None, None, None,
None, [], None, None, None,
None, None, None, None)
report_manager_producer_meta = ReportManager(
"", None, "member_code", "subsystem_code", "2017-01-01",
"2017-01-01", None, None, None, None, ["meta"], None, None, None,
None, None, None, None)
subsystem_code = "subsystem_code"
member_code = "member_code"
member_class = None
x_road_instance = ""
inner_document = dict()
inner_document["serviceSubsystemCode"] = subsystem_code
inner_document["serviceMemberCode"] = member_code
inner_document["serviceMemberClass"] = member_class
inner_document["serviceXRoadInstance"] = x_road_instance
inner_document["clientSubsystemCode"] = "not_producer"
inner_document["clientMemberCode"] = member_code
inner_document["clientMemberClass"] = member_class
inner_document["clientXRoadInstance"] = x_road_instance
inner_document["serviceCode"] = "meta"
service_type = report_manager_producer.get_service_type(inner_document)
self.assertEqual(service_type, "ps")
service_type = report_manager_producer_meta.get_service_type(
inner_document)
self.assertEqual(service_type, "pms")
inner_document["serviceMemberCode"] = "not_client"
inner_document["clientSubsystemCode"] = subsystem_code
service_type = report_manager_producer.get_service_type(inner_document)
self.assertEqual(service_type, "cs")
service_type = report_manager_producer_meta.get_service_type(
inner_document)
self.assertEqual(service_type, "cms")
def test_reduce_to_plain_json(self):
report_manager = ReportManager(None, None, None, None, "2017-01-01",
"2017-01-01", None, None, None, None,
None, None, None, None, None, None,
None, None)
subsystem_code = "subsystem_code"
member_code = "member_code"
member_class = None
x_road_instance = ""
inner_document = dict()
inner_document["serviceSubsystemCode"] = subsystem_code
inner_document["serviceMemberCode"] = member_code
inner_document["serviceMemberClass"] = member_class
inner_document["serviceXRoadInstance"] = x_road_instance
inner_document["clientSubsystemCode"] = subsystem_code
inner_document["clientMemberCode"] = member_code
inner_document["clientMemberClass"] = member_class
inner_document["clientXRoadInstance"] = x_road_instance
document = dict()
document['client'] = inner_document.copy()
document['producer'] = inner_document.copy()
doc = report_manager.reduce_to_plain_json(document)
self.assertEqual(inner_document, doc)
document = dict()
document['client'] = None
document['producer'] = inner_document.copy()
doc = report_manager.reduce_to_plain_json(document)
for key in list(inner_document.keys()):
self.assertTrue(doc[key] == inner_document[key])
document = dict()
document['client'] = inner_document.copy()
document['producer'] = None
doc = report_manager.reduce_to_plain_json(document)
for key in list(inner_document.keys()):
self.assertTrue(doc[key] == inner_document[key])
document = dict()
document['client'] = None
document['producer'] = None
with self.assertRaises(TypeError):
report_manager.reduce_to_plain_json(document)
def main(logger_manager):
"""
The main method.
:param logger_manager: The LoggerManager object.
:return:
"""
# Start timer
start_processing_time = time.time()
# Parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--member_code',
dest='member_code',
help='MemberCode',
required=True)
parser.add_argument('--subsystem_code',
dest='subsystem_code',
help='SubsystemCode',
default="")
parser.add_argument('--member_class',
dest='member_class',
help='MemberClass',
required=True)
parser.add_argument('--x_road_instance',
dest='x_road_instance',
help='XRoadInstance',
required=True)
parser.add_argument('--start_date',
dest='start_date',
help='StartDate "YYYY-MM-DD"',
required=True)
parser.add_argument('--end_date',
dest='end_date',
help='EndDate "YYYY-MM-DD"',
required=True)
parser.add_argument('--language',
dest='language',
help='Language ("et"/"en")',
required=True)
args = parser.parse_args()
# Initialize variables
member_code = args.member_code
subsystem_code = args.subsystem_code
member_class = args.member_class
x_road_instance = args.x_road_instance
start_date = args.start_date
end_date = args.end_date
language = args.language
# Initialize handlers
mdb_suffix = settings.MONGODB_SUFFIX
mdb_user = settings.MONGODB_USER
mdb_pwd = settings.MONGODB_PWD
mdb_server = settings.MONGODB_SERVER
db_handler = MongoDBHandler(mdb_suffix, mdb_user, mdb_pwd, mdb_server)
db_m = DatabaseManager(db_handler, logger_manager)
# Initialize translator
with open(settings.TRANSLATION_FILES.format(LANGUAGE=language),
'rb') as language_file:
language_template = json.loads(language_file.read().decode("utf-8"))
translator = Translator(language_template)
# Initialize variables
riha_json = read_in_json(settings.SUBSYSTEM_INFO_PATH, logger_manager)
meta_services = settings.META_SERVICE_LIST
html_template = settings.HTML_TEMPLATE_PATH
css_files = settings.CSS_FILES
report_path = settings.REPORTS_PATH
ria_image_1 = settings.RIA_IMAGE_1.format(LANGUAGE=language)
ria_image_2 = settings.RIA_IMAGE_2.format(LANGUAGE=language)
ria_image_3 = settings.RIA_IMAGE_3.format(LANGUAGE=language)
# Generate a report manager
report_manager = ReportManager(x_road_instance, member_class, member_code,
subsystem_code, start_date, end_date,
riha_json, logger_manager, db_m, language,
meta_services, translator, html_template,
report_path, css_files, ria_image_1,
ria_image_2, ria_image_3)
# Log starting
logger_manager.log_heartbeat("report_worker start",
settings.HEARTBEAT_LOGGER_PATH,
settings.REPORT_HEARTBEAT_NAME, "SUCCEEDED")
logger_manager.log_info('report_worker', 'report_worker_start')
# Generate report
report_manager.generate_report()
# Log ending of the reports generation
end_processing_time = time.time()
total_time = time.strftime(
"%H:%M:%S", time.gmtime(end_processing_time - start_processing_time))
logger_manager.log_info('report_worker', 'report_worker_end')
logger_manager.log_info('report_worker',
'Total generation time: {0}'.format(total_time))
logger_manager.log_heartbeat("report_worker end",
settings.HEARTBEAT_LOGGER_PATH,
settings.REPORT_HEARTBEAT_NAME, "SUCCEEDED")
def test_generate_report(self):
self.maxDiff = None
settings = Settings()
mongodb_h = MongoDBHandler(settings.MONGODB_USER, settings.MONGODB_PWD,
settings.MONGODB_SERVER)
mongodb_h.remove_all()
mongodb_h.create_indexes()
log_manager = MagicMock()
db_manager = DatabaseManager(mongodb_h, log_manager)
# Clean database state
number_of_produced_services = 7
number_of_produced_meta_services = 11
number_of_consumed_services = 13
number_of_consumed_meta_services = 17
# Test machine code to be used as producer
test_machine = ("CI-REPORTS", "MemberClassA", "MemberCodeA",
"SubsystemCodeA")
service_code = "serviceCodeA"
service_version = "1.0"
docs = []
producer_request_duration_list = []
request_size = []
response_size = []
# Creating producers
for i in range(number_of_produced_services):
doc_producer = ci_helper.create_clean_document(
specify_producer_machine=test_machine)
doc_producer['client']['clientMemberClass'] = "MemberClass1"
doc_producer['client']['clientMemberCode'] = "MemberCode1"
doc_producer['client']['clientSubsystemCode'] = "SubsystemCode1"
doc_producer['client']['serviceCode'] = service_code
doc_producer['client']['serviceVersion'] = service_version
producer_request_duration_list.append(
doc_producer['producerDurationProducerView'])
request_size.append(doc_producer['clientRequestSize'])
response_size.append(doc_producer['clientResponseSize'])
docs.append(doc_producer)
# Creating 1 failed producer
docs[0]['client']['succeeded'] = False
meta_producer_request_duration_list = []
meta_request_size = []
meta_response_size = []
# Creating metaproducers
for i in range(number_of_produced_meta_services):
doc_producer = ci_helper.create_clean_document(
specify_producer_machine=test_machine)
doc_producer['producer']['serviceCode'] = "getWsdl"
doc_producer['client']['serviceCode'] = doc_producer['producer'][
'serviceCode']
doc_producer['client']['serviceCode'] = "getWsdl"
doc_producer['client']['serviceVersion'] = service_version
doc_producer['client']['clientMemberClass'] = "MemberClass1"
doc_producer['client']['clientMemberCode'] = "MemberCode1"
doc_producer['client']['clientSubsystemCode'] = "SubsystemCode1"
meta_producer_request_duration_list.append(
doc_producer['producerDurationProducerView'])
meta_request_size.append(doc_producer['clientRequestSize'])
meta_response_size.append(doc_producer['clientResponseSize'])
docs.append(doc_producer)
consumer_request_duration_list = []
cons_request_size = []
cons_response_size = []
# Creating consumers
for i in range(number_of_consumed_services):
doc_producer = ci_helper.create_clean_document(
specify_client_machine=test_machine)
doc_producer['client']['serviceCode'] = service_code
doc_producer['client']['serviceVersion'] = service_version
doc_producer['client']['serviceMemberClass'] = "MemberClass1"
doc_producer['client']['serviceMemberCode'] = "MemberCode1"
doc_producer['client']['serviceSubsystemCode'] = "SubsystemCode1"
consumer_request_duration_list.append(
doc_producer['totalDuration'])
cons_request_size.append(doc_producer['clientRequestSize'])
cons_response_size.append(doc_producer['clientResponseSize'])
docs.append(doc_producer)
# Creating 1 failed consumer
docs[20]['client']['succeeded'] = False
meta_consumer_request_duration_list = []
meta_cons_request_size = []
meta_cons_response_size = []
# Creating metaconsumers
for i in range(number_of_consumed_meta_services):
doc_producer = ci_helper.create_clean_document(
specify_client_machine=test_machine)
doc_producer['client']['serviceCode'] = "getWsdl"
doc_producer['producer']['serviceCode'] = doc_producer['client'][
'serviceCode']
doc_producer['client']['serviceMemberClass'] = "MemberClass1"
doc_producer['client']['serviceMemberCode'] = "MemberCode1"
doc_producer['client']['serviceSubsystemCode'] = "SubsystemCode1"
doc_producer['client']['serviceVersion'] = service_version
meta_consumer_request_duration_list.append(
doc_producer['totalDuration'])
meta_cons_request_size.append(doc_producer['clientRequestSize'])
meta_cons_response_size.append(doc_producer['clientResponseSize'])
docs.append(doc_producer)
# Add documents into the MongoDB
mongodb_h.add_clean_documents(docs)
# Produced service:
# producer_request_duration_list
# [-82400, 9810, -61287, -78865, -38082, 13780, 76099]
# request_size
# [49797, 13238, 28785, 97912, 89166, 99943, 69163]
# response_zise
# [45082, 59429, 7331, 76484, 89353, 69514, 27760]
# Produced metaservice:
# meta_producer_request_duration_list
# [-21810, -24887, -7648, 32715, -32418, -88489, 58120, 32870, -57522, 4230, -44214]
# meta_request_size
# [92778, 55519, 70855, 28534, 75880, 81416, 34664, 34335, 87416, 45830, 56959]
# meta_response_size
# [30007, 95561, 98771, 96434, 81183, 9602, 97310, 57912, 73385, 29219, 41114]
# Consumed service:
# consumer_request_duration_list
# [41114, 63779, -49667, -18066, -18385, -8924, 53571, 23562, -36997, -477, 21489, -66685, -11144]
# cons_request_size
# [66469, 85813, 62277, 17477, 10155, 76556, 52994, 98567, 28569, 39529, 85256, 27802, 90673]
# cons_response_size
# [53528, 57905, 59692, 69616, 30735, 75456, 61261, 10735, 33586, 15866, 70539, 44608, 87026]
# Consumed metaservice:
# meta_consumer_request_duration_list
# [73851, 63965, -54644, 44225, 24367, -28349, -22960, -19809, -62915, 52779, 6305,
# -28436, 23416, -33396, -81974, 15363, 80896]
# meta_cons_request_size
# [99901, 91411, 73772, 34687, 86662, 57377, 83202, 67449, 79806, 50277, 59682,
# 59890, 89705, 89364, 45293, 79482, 11813]
# meta_cons_response_size
# [25105, 98669, 32530, 20585, 15043, 5224, 61605, 71118, 20758, 66616, 91422,
# 18894, 97880, 22946, 83746, 37512, 97973]
# Generate the final report MemberCodeA", "MemberClassA", "CI-REPORTS
HTML_TEMPLATE_PATH = "/reports_module/pdf_files/pdf_report_template.html"
CSS_FILES = ["reports_module/pdf_files/pdf_report_style.css"]
# Initialize translator
with open('reports_module/lang/et_lang.json', 'rb') as language_file:
language_template = json.loads(
language_file.read().decode("utf-8"))
translator = Translator(language_template)
language = "et"
RIA_IMAGE_1 = "reports_module/pdf_files/header_images/ria_75_{LANGUAGE}.png".format(
LANGUAGE=language)
RIA_IMAGE_2 = "reports_module/pdf_files/header_images/eu_rdf_75_{LANGUAGE}.png".format(
LANGUAGE=language)
RIA_IMAGE_3 = "reports_module/pdf_files/header_images/xroad_75_{LANGUAGE}.png".format(
LANGUAGE=language)
report_manager = ReportManager(
"CI-REPORTS", "MemberClassA", "MemberCodeA", "SubsystemCodeA",
"1970-1-1", "1980-1-1", None, log_manager, db_manager, language, [
"getWsdl", "listMethods", "allowedMethods",
"getSecurityServerMetrics", "getSecurityServerOperationalData",
"getSecurityServerHealthData"
], translator, HTML_TEMPLATE_PATH, "integration_tests/ci_reports",
CSS_FILES, RIA_IMAGE_1, RIA_IMAGE_2, RIA_IMAGE_3)
report_manager.generate_report()
# Clean before exit
mongodb_h.remove_all()