def __init__(self, name=str, host=str, port=int, max_retries=10, timeout=None, logger=None, pb2_module=None) -> None: """ Class creates the protobuf payload with the marriage of Fuzzed data. :param name: name of the class :param host: target host :param port: target port :param max_retries: maximum retries of connection :param timeout: timeout :param logger: kitty logger """ super(ProtobufTarget, self).__init__(name, logger) self.host = host self.port = port if (host is None) or (port is None): raise ValueError('Host and port may not be None') self.timeout = timeout self.socket = None self.max_retries = max_retries self.pb2_module = pb2_module self.set_expect_response(False) self.config = ConfigParser() self.verbosity = self.config.get_generic_verbosity() self.logger.setLevel(self.verbosity) self.module_path = self.config.get_module_path() self._uuid = GenerateUUID.generate_uuid() self.frmwrk_utils = FrameworkUtils()
def __init__(self, pb2_api, name='ProtobufRunner', logger=None): super(ProtobufRunner, self).__init__(name, logger) self.pb2_api = pb2_api self.config = ConfigParser() self.target_host = self.config.get_target_host_name() self.target_port = self.config.get_target_port() self.frmu = FrameworkUtils()
def __init__(self, name='Runner', logger=None): super(Runner, self).__init__(name, logger) self.config = ConfigParser() self.framework_utils = FrameworkUtils() self.put_archive_to_s3 = PutArchiveToS3() self.proto_path = self.config.get_generic_proto_path() self.pb2_path = self.config.get_generic_pb2_path() self.verbosity = self.config.get_generic_verbosity() self.module_path = self.config.get_module_path() self.proto_modules = self.config.get_protobuf_modules() self.proto_classes = self.config.get_protobuf_classes_to_send() self.protocol_proto = self.config.get_protocol_protobuf() self.protocol_http = self.config.get_protocol_http() self.protocol_dns = self.config.get_protocol_dns() self.set_verbosity(self.verbosity) self.use_s3_to_archive = self.config.get_archive_to_s3()
def execute_api_parse(self) -> tuple: """ Will load the protobuf module and returns with the parsed protobuf api. :return: tuple key is the api dict value is the module """ # Cut '.py' from the end of file) pb2_module_obj: object = FrameworkUtils.load_pb2_module( self.pb2_api_file[:-3], self.moule_path) return self.explore_pb2_api(pb2_module_obj), pb2_module_obj
class ProtobufTarget(ServerTarget): """ ProtobufTarget will create files with the fuzzed payloads """ def __init__(self, name=str, host=str, port=int, max_retries=10, timeout=None, logger=None, pb2_module=None) -> None: """ Class creates the protobuf payload with the marriage of Fuzzed data. :param name: name of the class :param host: target host :param port: target port :param max_retries: maximum retries of connection :param timeout: timeout :param logger: kitty logger """ super(ProtobufTarget, self).__init__(name, logger) self.host = host self.port = port if (host is None) or (port is None): raise ValueError('Host and port may not be None') self.timeout = timeout self.socket = None self.max_retries = max_retries self.pb2_module = pb2_module self.set_expect_response(False) self.config = ConfigParser() self.verbosity = self.config.get_generic_verbosity() self.logger.setLevel(self.verbosity) self.module_path = self.config.get_module_path() self._uuid = GenerateUUID.generate_uuid() self.frmwrk_utils = FrameworkUtils() def pre_test(self, test_num=int) -> None: """ This is only checks whether the target is available or not. :param test_num: The number of the test case. """ super(ProtobufTarget, self).pre_test(test_num) retry_count = 0 while self.socket is None and retry_count < self.max_retries: sock = self._get_socket() if self.timeout is not None: sock.settimeout(self.timeout) try: retry_count += 1 sock.connect((self.host, self.port)) self.socket = sock except Exception: sock.close() self.logger.error( f"PBTARGET - TCP Error: {traceback.format_exc()}") self.logger.error( f"PBTARGET - TCP Failed to connect to target server, retrying..." ) time.sleep(1) if self.socket is None: raise (KittyException( 'PBTARGET - TCPTarget: (pre_test) cannot connect to server (retries = %d' % retry_count)) def _get_socket(self) -> socket: """ Get a socket object """ return socket.socket(socket.AF_INET, socket.SOCK_STREAM) def post_test(self, test_num=int): """ Called after a test is completed, perform cleanup etc. """ super(ProtobufTarget, self).post_test(test_num) if self.socket is not None: self.socket.close() self.socket = None if self.report.get('status') != Report.PASSED: if not os.path.exists(os.getcwd() + '/results/'): os.makedirs(os.getcwd() + '/results/') test_num = self.report.get('test_number') with open(os.getcwd() + '/results/{}-result-protobuf-{}.json'.format( test_num, str(self._uuid)), 'w', encoding='utf-8') as file: json.dump(self.report.to_dict(), file, ensure_ascii=False, indent=4) def _construct_message(self, data, pb2_module): msg = None to_dict = json.loads(data.decode()) counter = 0 for key, val in to_dict.items(): if counter == 0: module_to_put_into = getattr(pb2_module, key) msg = json_format.ParseDict(val, module_to_put_into()) else: # TODO Should implement a mechanism to send together # TODO top level messages. # TODO Currently sending only one top level for.ie.: Request # TODO and all its nested messages. raise NotImplementedError counter += 1 return msg def _send_to_target(self, data): """ HTTP POST with protobuf binary payload: POST / HTTP/1.1 Host: localhost:8000 Content-Type: : 'application/octet-stream' "b'|\x00\x00\x00\n\x05fdbcd\x10{\x1a\x05fdbcd"\t\n\x05fdbcd\x10\x01:_\n\x05fdbcd\x10{\x18\x01""\n\x05fdbcd\x10{\x1a\x05fdbcd"\x10\n\x05fdbcd\x10{\x1a\x05fdbcd*\x10\n\x05fdbcd\x10{\x1a\x05fdbcd2\x10\n\x05fdbcd\x10{\x1a\x05fdbcdB\n\n\x06%u0000\x10{'" https://stackoverflow.com/questions/28670835/python-socket-client-post-parameters :param data: :return: """ proto_payload = self._construct_message(data, self.pb2_module) host = None port = None headers = """\ POST / HTTP/1.1\r \r\n\ Content-Type: {content_type}\r Content-Length: {content_length}\r \r\n""" body_bytes = self.frmwrk_utils.encode_message(proto_payload) # Evaluation of the message. #self.frmwrk_utils.decode_message(body_bytes, lookup_pb2, 'Request') header_bytes = headers.format(content_type="application/octet-stream", content_length=len(body_bytes), host=str(host) + ":" + str(port)).encode('iso-8859-1') payload = header_bytes + body_bytes self.socket.send(payload) def _receive_from_target(self): return self.socket.recv(10000) def transmit(self, payload): """ This is the original transmit method from ServerTarget overwritten with special cases such as 40X or 50X according to the aim of the test. Accordin to https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500 500 Internal Server Error 501 Not Implemented 502 Bad Gateway 503 Service Unavailable 504 Gateway Timeout 505 HTTP Version Not Supported 506 Variant Also Negotiates 507 Insufficient Storage 508 Loop Detected 510 Not Extended 511 Network Authentication Required Original method docstring: Transmit single payload, and receive response, if expected. The actual implementation of the send/receive should be in ``_send_to_target`` and ``_receive_from_target``. :type payload: str :param payload: payload to send :rtype: str :return: the response (if received) """ SERVER_50x_CODES = [ '500 Internal Server Error', '501 Not Implemented', '502 Bad Gateway', '503 Service Unavailable', '504 Gateway Timeout', '505 HTTP Version Not Supported', '506 Variant Also Negotiates', '507 Insufficient Storage', '508 Loop Detected', '510 Not Extended', '511 Network Authentication Required' ] SERVER_40xCODES = [ '400 Bad Request', '401 Unauthorized', '402 Payment Required', '403 Forbidden', '404 Not Found', '405 Method Not Allowed', '406 Not Acceptable', '407 Proxy Authentication Required', '408 Request Timeout', '409 Conflict', '410 Gone', '411 Length Required', '412 Precondition Failed', '413 Payload Too Large', '414 URI Too Long', '415 Unsupported Media Type', '416 Range Not Satisfiable', '417 Expectation Failed', '422 Unprocessable Entity', '425 Too Early', '426 Upgrade Required', '428 Precondition Required', '429 Too Many Requests', '431 Request Header Fields Too Large', '451 Unavailable For Legal Reasons' ] response = None trans_report_name = 'transmission_0x%04x' % self.transmission_count trans_report = Report(trans_report_name) self.transmission_report = trans_report self.report.add(trans_report_name, trans_report) try: trans_report.add('request (hex)', hexlify(payload).decode()) trans_report.add('request (raw)', '%s' % payload) trans_report.add('request length', len(payload)) trans_report.add('request time', time.time()) request = hexlify(payload).decode() request = request if len(request) < 100 else (request[:100] + ' ...') self.logger.info(f"request({len(payload)}): {request}") self.logger.debug(f"payload {payload}") self._send_to_target(payload) trans_report.success() if self.expect_response: try: response = self._receive_from_target() trans_report.add('response time', time.time()) trans_report.add('response (hex)', hexlify(response).decode()) trans_report.add('response (raw)', '%s' % response) trans_report.add('response length', len(response)) trans_report.add('Session ID', str(self._uuid)) printed_response = hexlify(response).decode() printed_response = printed_response if len( printed_response) < 100 else (printed_response[:100] + ' ...') self.logger.info( f"response({len(response)}): {printed_response}") string_response = response.decode('utf-8') response_code_string = string_response.splitlines()[0] response_code = response_code_string.replace( 'HTTP/1.1 ', '') if response_code in SERVER_40xCODES or response_code in SERVER_50x_CODES: self.logger.info( f"response failure {response.decode('utf-8')}") trans_report.failed('Failure in HTTP-PROTO response.') trans_report.add('Response', response.decode('utf-8')) self.report.set_status('failed') self.receive_failure = True except Exception as ex2: trans_report.failed('failed to receive response: %s' % ex2) trans_report.add('traceback', traceback.format_exc()) self.logger.error( f"target.transmit - failure in receive (exception: {ex2})" ) self.logger.error(traceback.format_exc()) self.receive_failure = True else: response = '' except Exception as ex1: #trans_report.failed('failed to send payload: %s' % ex1) #trans_report.add('traceback', traceback.format_exc()) self.logger.error( f"target.transmit - failure in send (exception: {ex1})") self.logger.error(traceback.format_exc()) #self.send_failure = True self.transmission_count += 1 return response
def junit_xml_report(self, report_list=list) -> None: """ Creates a JUnit compatible XML output for CI systems. According to https://help.catchsoftware.com/display/ET/JUnit+Format :param report_list: list :return: None """ if len(report_list) != 0: get_values = FrameworkUtils() test_suite = ET.Element("testsuite", errors="", failures=str(len(report_list)), name=str(self._get_case_description()), skipped="0", tests="", time="0", timestamp=str( time.strftime("%Y%m%d-%H%M%S"))) for report in report_list: controller_start = report['controller']['start_time'] try: controller_stop = report['controller']['stop_time'] controller_duration = controller_stop - controller_start except KeyError as err: self.logger.error( f"Key Error {err}, controller running duration value will be false!" ) controller_duration = 0 self.logger.error( f"Duration has been set to {controller_duration}") reason = base64.b64decode( report['transmission_0x0000']['reason']) request = base64.b64decode( report['transmission_0x0000']['request (raw)']) controller_name = base64.b64decode( report['controller']['name']) test_case = ET.SubElement( test_suite, "testcase", classname=request.decode('utf-8'), name=controller_name.decode('utf-8'), time=str(timedelta(seconds=controller_duration)), timestamp=time.strftime("%Y%m%d-%H%M%S")) if len(get_values.extract_values(report, 'traceback')) != 0: failure_traceback = base64.b64decode( report['transmission_0x0000']['traceback']).decode( 'utf-8') ET.SubElement( test_case, "failure", message="Failure traceback", type=reason.decode( 'utf-8')).text = f"<![CDATA[{failure_traceback}]]>" else: ET.SubElement( test_case, "failure", message="Failure traceback", type=reason.decode('utf-8')).text = f"<![CDATA[]]>" ET.SubElement(test_case, "system-out").text = "<![CDATA[]]>" ET.SubElement(test_case, "system-err").text = "<![CDATA[]]>" tree = ET.ElementTree(test_suite) tree.write(open(os.getcwd() + '/results/' + 'results.xml', 'wb'), encoding='utf-8', xml_declaration=True) else: test_suite = ET.Element("testsuite", errors="0", failures="0", name=str(self._get_case_description()), skipped="0", tests="", time="0", timestamp=str( time.strftime("%Y%m%d-%H%M%S"))) test_case = ET.SubElement(test_suite, "testcase", classname="", name=str(self._get_case_description()), time="", timestamp=time.strftime("%Y%m%d-%H%M%S")) ET.SubElement(test_case, "system-out").text = "<![CDATA[]]>" ET.SubElement(test_case, "system-err").text = "<![CDATA[]]>" tree = ET.ElementTree(test_suite) tree.write(open(os.getcwd() + '/results/' + 'results.xml', 'wb'), encoding='utf-8', xml_declaration=True)
class Runner(FuzzObject): """ This is the main runner class of the Fuzzer Framework. It integrates the kitty/katnip components and the framework components. See documentation for the usage. """ def __init__(self, name='Runner', logger=None): super(Runner, self).__init__(name, logger) self.config = ConfigParser() self.framework_utils = FrameworkUtils() self.put_archive_to_s3 = PutArchiveToS3() self.proto_path = self.config.get_generic_proto_path() self.pb2_path = self.config.get_generic_pb2_path() self.verbosity = self.config.get_generic_verbosity() self.module_path = self.config.get_module_path() self.proto_modules = self.config.get_protobuf_modules() self.proto_classes = self.config.get_protobuf_classes_to_send() self.protocol_proto = self.config.get_protocol_protobuf() self.protocol_http = self.config.get_protocol_http() self.protocol_dns = self.config.get_protocol_dns() self.set_verbosity(self.verbosity) self.use_s3_to_archive = self.config.get_archive_to_s3() def search_files(self) -> tuple: """ Creates lists of raw and compiled proto files paired into tuple; :return: raw and compiled proto file lists in tuple; """ search_raw_proto = SearchFile(file_type='.proto', file_path=self.proto_path) raw_proto_files = search_raw_proto.search_file self.logger.debug(f'RAW proto files {raw_proto_files}') search_api = SearchFile(file_type='_pb2.py', file_path=self.pb2_path) compiled_proto_files = search_api.search_file self.logger.debug(f'Compiled proto files {compiled_proto_files}') return raw_proto_files, compiled_proto_files def checksum_files(self, raw_proto: list) -> list: # TODO for further usage, function not in use. checksum_files = self.framework_utils.md5_checksum_for_proto_files(raw_proto) self.logger.debug(f'Checksum {checksum_files}') return checksum_files @property def process_pb2(self) -> object: """ This function parses the compiled protobuf API and produces a Dictionary representation of it. :return: object """ compiled_proto_files = self.search_files()[1] for item in compiled_proto_files: if len(item.split('_')) > 2: raise FileNameError # raise if file name contains more then 2 underline. elif item.split('_')[0] in self.config.get_protobuf_modules(): parse_file = ParseProtoApi(pb2_api_file=item, module_path=self.module_path) return parse_file.execute_api_parse() else: self.logger.info(f'File {item} was not part of the inspection.') def tear_down(self): """ The common tear down sequence of a fuzzing session :return: None """ self.framework_utils.results_dir() self.framework_utils.rename_log_file() self.framework_utils.archive_locally() self.report() self.framework_utils.archive_xml() if self.use_s3_to_archive: self.put_archive_to_s3.do_upload(self.framework_utils.dir_files_to_archive()) def report(self): _report = CreateReport() _report.run_report() def run(self): if self.config.get_banner(): file1 = open(os.getcwd() + '/banner', 'r') Lines = file1.readlines() for line in Lines: print(f' {line.strip()}') for i in range(5, 0, -1): sys.stdout.write(str(i) + '.....') sys.stdout.flush() time.sleep(1) if self.protocol_proto: self.logger.info(f"[{time.strftime('%H:%M:%S')}] Starting PROTOBUF Fuzzer session") try: api = self.process_pb2 do_proto = ProtobufRunner(pb2_api=api) do_proto.run_proto() except SystemError as e: self.logger.error(f"Interpreter found an internal error: {e}") except EnvironmentError as e: self.logger(f"Error occur outside the Python environment: {e}") finally: self.logger.info(f"[{time.strftime('%H:%M:%S')}] Flush cache...") self.logger.info(f"[{time.strftime('%H:%M:%S')}] Rename kitty related log files...") self.tear_down() elif self.protocol_dns: self.logger.info(f"[{time.strftime('%H:%M:%S')}] Starting DNS Fuzzer session") try: # query record type do_dns = DnsRunner() do_dns.run_dns() self.logger.info(f"[{time.strftime('%H:%M:%S')}] DNS runner started") except SystemError as e: self.logger.error(f"Interpreter finds an internal problem {e}") except EnvironmentError as e: self.logger(f"Error occur outside the Python environment: {e}") finally: self.logger.info(f"[{time.strftime('%H:%M:%S')}] Rename kitty related log files...") self.tear_down() elif self.protocol_http: self.logger.info(f"[{time.strftime('%H:%M:%S')}] Starting HTTP Fuzzer session") try: do_http = HttpRunner() do_http.run_http() self.logger.info(f"[{time.strftime('%H:%M:%S')}] HTTP runner started") except SystemError as e: self.logger.error(f"Interpreter finds an internal problem {e}") except EnvironmentError as e: self.logger(f"Error occur outside the Python environment: {e}") finally: self.logger.info(f"[{time.strftime('%H:%M:%S')}] Rename kitty related log files...") self.tear_down()
def __init__(self, name='TestFramework'): super(TestFramework, self).__init__(name) self.framework_utils = FrameworkUtils() self.protobuf_target = ProtobufTarget()
class TestFramework(unittest.TestCase): def __init__(self, name='TestFramework'): super(TestFramework, self).__init__(name) self.framework_utils = FrameworkUtils() self.protobuf_target = ProtobufTarget() def test_01_convert_filepath_to_module_path(self): module_import_path = \ self.framework_utils.convert_filepath_to_module_path(file_path='/framework/models/pb2/api/') self.assertEqual(module_import_path, 'framework.models.pb2.api.') def test_02_encode_message(self): with open( os.path.abspath(os.path.join(os.path.dirname(__file__))) + '/msg_encode_assert') as file_data: data = file_data.read().encode() file_data.close() msg = self.protobuf_target._construct_message(data, query_pb2) msg_encoded = self.framework_utils.encode_message(msg) self.assertEqual(msg_encoded, msg_encoded) def test_03_decode_message(self): result = self.framework_utils.decode_message(msg_encoded, query_pb2, 'Request') msg = json_format.MessageToDict(result[0]) with open( os.path.join(os.path.dirname(__file__)) + '/msg_decoded_assert') as file_data: msg_decoded_assert = json.load(file_data) self.assertEqual(msg, msg_decoded_assert) def test_04_create_result_dir_if_not_exists(self): is_exits = False self.framework_utils.results_dir() if os.path.exists(os.getcwd() + '/results/'): is_exits = True self.assertTrue(is_exits, True) #shutil.rmtree(os.getcwd() + '/results/', ignore_errors=True) def test_05_rename_log_file(self): self._uuid = GenerateUUID.generate_uuid() self.framework_utils.rename_log_file() file_name = glob.glob(os.getcwd() + '/kittylogs/' + '*.log')[0].split('/')[-1:] expected_file_name = str( self._uuid) + '-kitty_' + time.strftime("%Y%m%d-%H%M%S") + '.log' shutil.rmtree(os.getcwd() + '/kittylogs/', ignore_errors=True) self.assertAlmostEqual(file_name[0], expected_file_name) def test_06_rename_log_file_file_not_found(self): shutil.rmtree(os.getcwd() + '/kittylogs/', ignore_errors=True) with self.assertRaises(FileNotFoundError): self.framework_utils.rename_log_file() def test_07_rename_log_file_could_not_read_from_file(self): os.makedirs(os.getcwd() + '/kittylogs/') with open(os.getcwd() + '/kittylogs/foo.gld', 'w') as file: file.write('dummy') file.close() with self.assertLogs('fuzzer', level='WARNING') as cm: self.framework_utils.rename_log_file() self.assertIn( 'WARNING:fuzzer:Could not read from file ' + os.getcwd() + '/kittylogs/', cm.output) shutil.rmtree(os.getcwd() + '/kittylogs/', ignore_errors=True) def test_08_archive_locally(self): dir_exists = False self.framework_utils.archive_locally() self._uuid = GenerateUUID.generate_uuid() dir_ = str(self._uuid) + '-log' if os.path.exists(os.getcwd() + '/' + dir_): dir_exists = True self.assertTrue(dir_exists, True) def test_09_extract_values(self): input_ = { "a": 1, "b": 2, "c": { "d": 3, "e": 4, "f": [{ "g": 5 }], "h": { "i": 6 } } } key_ = 'i' self.assertEqual(self.framework_utils.extract_values(input_, key_), [6]) def test_10_update_nested_object(self): context_ = [("a", 1), ("c", 2), ("h", 1), ("j", 1)] appnd_ = {"m": 9} key_ = "j" input_ = { "a": 1, "b": 2, "c": { "d": 3, "e": 4, "f": [{ "g": 5 }], "h": { "i": 6, "j": [{ "k": 7, "l": 8 }] } } } result = self.framework_utils.update_nested(input_, key_, appnd_, context_) expected = { 'a': 1, 'b': 2, 'c': { 'd': 3, 'e': 4, 'f': [{ 'g': 5 }], 'h': { 'i': 6, 'j': [{ 'k': 7, 'l': 8, 'm': 9 }] } } } self.assertEqual(result, expected) def test_11_update_nested_object_same_name(self): context_ = [("a", 1), ("c", 2), ("h", 1)] appnd_ = {"h": 9} key_ = "h" input_ = { "a": 1, "b": 2, "c": { "d": 3, "e": 4, "f": { "g": 5 }, "h": { "i": 6, "j": [{ "k": 7, "l": 8 }] } } } result = self.framework_utils.update_nested(input_, key_, appnd_, context_) expected = { 'a': 1, 'b': 2, 'c': { 'd': 3, 'e': 4, 'f': { 'g': 5 }, 'h': { 'i': 6, 'j': [{ 'k': 7, 'l': 8 }], 'h': 9 } } } self.assertEqual(result, expected) def test_12_update_nested_object_same_name_nested(self): context_ = [("a", 1), ("q", 2), ("q", 1)] appnd_ = {"q": 9} key_ = "q" input_ = { "a": 1, "b": 2, "q": { "q": { "x": 1 } }, "c": { "d": 3, "e": 4, "f": { "g": 5 }, "h": { "i": 6, "j": [{ "k": 7, "l": 8 }] } } } result = self.framework_utils.update_nested(input_, key_, appnd_, context_) expected = { 'a': 1, 'b': 2, 'q': { 'q': { 'x': 1, 'q': 9 } }, 'c': { 'd': 3, 'e': 4, 'f': { 'g': 5 }, 'h': { 'i': 6, 'j': [{ 'k': 7, 'l': 8 }] } } } self.assertEqual(result, expected) def test_13_field_randomizer_double(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(1, default_value=None, enum_list=None) if 1.0 <= result <= 1.9: is_close = True self.assertTrue(is_close, True) def test_13_field_randomizer_double_default_value(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer( 1, default_value=1.12345679, enum_list=None) if 1.0 <= result <= 1.9: is_close = True self.assertTrue(is_close, True) def test_14_field_randomizer_float(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(2, default_value=None, enum_list=None) if 1.0 <= result <= 1.9: is_close = True self.assertTrue(is_close, True) def test_15_field_randomizer_float_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer( 2, default_value=1.123456789, enum_list=None) if 1.0 <= result <= 1.9: is_close = True self.assertTrue(is_close, True) def test_14_field_randomizer_int64(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(3, default_value=None, enum_list=None) if -9223372036854775808 <= result <= 9223372036854775808: is_close = True self.assertTrue(is_close, True) def test_15_field_randomizer_int64_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer( 3, default_value=1234567899999999, enum_list=None) if -9223372036854775808 <= result <= 9223372036854775808: is_close = True self.assertTrue(is_close, True) def test_16_field_randomizer_uint64(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(4, default_value=None, enum_list=None) if -0 <= result <= 18446744073709551615: is_close = True self.assertTrue(is_close, True) def test_17_field_randomizer_uint64_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer( 4, default_value=1844674407370955161, enum_list=None) if -0 <= result <= 18446744073709551615: is_close = True self.assertTrue(is_close, True) def test_18_field_randomizer_int32(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(5, default_value=None, enum_list=None) if -2147483648 <= result <= 2147483648: is_close = True self.assertTrue(is_close, True) def test_19_field_randomizer_int32_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(5, default_value=1234567, enum_list=None) if -2147483648 <= result <= 2147483648: is_close = True self.assertTrue(is_close, True) def test_20_field_randomizer_fixed64(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(6, default_value=None, enum_list=None) if -9223372036854775808 <= result <= 9223372036854775808: is_close = True self.assertTrue(is_close, True) def test_21_field_randomizer_fixed64_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer( 6, default_value=92233720368547758, enum_list=None) if -9223372036854775808 <= result <= 9223372036854775808: is_close = True self.assertTrue(is_close, True) def test_22_field_randomizer_fixed32(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(7, default_value=None, enum_list=None) if -2147483648 <= result <= 2147483648: is_close = True self.assertTrue(is_close, True) def test_23_field_randomizer_fixed32_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(7, default_value=214748364, enum_list=None) if -2147483648 <= result <= 2147483648: is_close = True self.assertTrue(is_close, True) def test_22_field_randomizer_bool(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(8, default_value=None, enum_list=None) if result or not result: is_close = True self.assertTrue(is_close, True) def test_23_field_randomizer_bool_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(8, default_value=True, enum_list=None) if result or not result: is_close = True self.assertTrue(is_close, True) def test_24_field_randomizer_string(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False regex = re.compile('[a-zA-Z0-9]', re.I) result = self.framework_utils.field_randomizer(9, default_value=None, enum_list=None) res = bool(regex.match(str(result))) if isinstance(result, str) and res: is_close = True self.assertTrue(is_close, True) def test_25_field_randomizer_string_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False regex = re.compile('[a-zA-Z0-9]', re.I) result = self.framework_utils.field_randomizer( 9, default_value='abcdefghijk', enum_list=None) res = bool(regex.match(str(result))) if isinstance(result, str) and res: is_close = True self.assertTrue(is_close, True) def test_26_field_randomizer_group(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) with self.assertRaises(NotImplementedError): self.framework_utils.field_randomizer(10, default_value=None, enum_list=None) def test_27_field_randomizer_bytes(self): #TODO finish self.framework_utils.field_randomizer(12, default_value=None, enum_list=None) def test_28_field_randomizer_bytes_default(self): #TODO self.framework_utils.field_randomizer(12, default_value=0x23, enum_list=None) def test_29_field_randomizer_uint32(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(13, default_value=None, enum_list=None) if 0 <= result <= 4294967295: is_close = True self.assertTrue(is_close, True) def test_30_field_randomizer_uint32_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False result = self.framework_utils.field_randomizer(13, default_value=429496729, enum_list=None) if 0 <= result <= 4294967295: is_close = True self.assertTrue(is_close, True) def test_31_field_randomizer_enum(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False enum_list = ['foo', 'bar', 'baz'] result = self.framework_utils.field_randomizer(14, default_value=None, enum_list=enum_list) if result.split('+')[0] in enum_list: is_close = True self.assertTrue(is_close, True) def test_32_field_randomizer_enum_default(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) is_close = False enum_list = ['foo', 'bar', 'baz'] result = self.framework_utils.field_randomizer(14, default_value='bar', enum_list=enum_list) if result.split('+')[0] in enum_list: is_close = True self.assertTrue(is_close, True) def test_33_field_randomizer_sfixed32(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) with self.assertRaises(NotImplementedError): self.framework_utils.field_randomizer(15, default_value=None, enum_list=None) def test_34_field_randomizer_sfixed64(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) with self.assertRaises(NotImplementedError): self.framework_utils.field_randomizer(16, default_value=None, enum_list=None) def test_35_field_randomizer_sint32(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) with self.assertRaises(NotImplementedError): self.framework_utils.field_randomizer(17, default_value=None, enum_list=None) def test_36_field_randomizer_sint64(self): # field_randomizer(self, field_type, default_value=None, enum_list=None) with self.assertRaises(NotImplementedError): self.framework_utils.field_randomizer(18, default_value=None, enum_list=None) def test_37_str_to_bool_true(self): result = self.framework_utils.str_to_bool('true') self.assertTrue(result, True) def test_38_str_to_bool_false(self): result = self.framework_utils.str_to_bool('false') self.assertFalse(result, False) def test_39_dir_files_to_archive(self): result = self.framework_utils.dir_files_to_archive() self.assertEqual(len(result), 1) def test_40_md5_checksum_for_proto_files(self): pass
def _parse_proto(self, msg_obj: object, message_name: str) -> OrderedDict: # pylint: disable=R0912,R0915 """ This method is to recursively parse a given protobuf api and produces an OrderedDict representation of it. It's supporters are: - _gather_field function; - _gather_enum function; - _u_dict_list function; :note Its supporter is a list of tuples that maintains an inventory about which level being under process by name and how many elements the actual level has. After an elem has been processed the counter will be decreased, after no more elem left on the actual level, it removes the level from the list. Each level depending on its role will put on to an OrderedDict. Field with label 1,2 will be a nested dict; Field with label 3 will be a list of nested dict; With the update_nested function under the utils class the logic able to append each value to the particular key that has been created before; :param msg_obj: the protobuf api instance :param message_name: the name of the top level message :return: the OrderedDict representation of the protobuf API """ self.context.append((message_name, len(self._gather_field(msg_obj)))) message_dict = OrderedDict() message_dict.update({}) utl = FrameworkUtils() def _recurse( msg_obj: object, message_dict: OrderedDict ) -> OrderedDict: # pylint: disable=R0912,R0915 # traverse the message fields # checking field if not 11, or 14 for ms_na, ms_ob in self._gather_field(msg_obj): if ms_ob.type in [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 17, 18 ]: self.logger.info( f"Checked for non msg non field: {ms_na}, " f"object: {ms_ob}, " f"containtin type {ms_ob.containing_type.name}") if ms_ob.label in [1, 2]: # optional, required field if len(self.context) == 1: message_dict[ms_na] = utl.field_randomizer( ms_ob.type, ms_ob.default_value, None) else: message_dict = utl.update_nested( message_dict, self.context[-1][0], { ms_na: utl.field_randomizer( ms_ob.type, ms_ob.default_value, None) }, self.context) self.context = self._u_dict_list( self.context, self.context[-1][0], self.context[-1][1] - 1) self.logger.debug(f"CONCHECK 1 :{self.context}") elif ms_ob.label == 3: # repeated field message_dict = utl.update_nested( message_dict, self.context[-1][0], { ms_na: [ utl.field_randomizer( ms_ob.type, ms_ob.default_value, None) ] }, self.context) self.context = self._u_dict_list( self.context, self.context[-1][0], self.context[-1][1] - 1) self.logger.debug( f"CONCHECK 1 repeated {ms_ob.label} :{self.context}" ) elif ms_ob.type == 14: if ms_ob.label in [1, 2]: if len(self.context) == 1: message_dict[ms_na] = utl.field_randomizer( ms_ob.type, ms_ob.default_value, None) else: message_dict = utl.update_nested( message_dict, self.context[-1][0], { ms_na: utl.field_randomizer( ms_ob.type, ms_ob.default_value, self._gather_enum(ms_ob)) }, self.context) self.context = self._u_dict_list( self.context, self.context[-1][0], self.context[-1][1] - 1) elif ms_ob.label == 3: if len(self.context) == 1: message_dict[ms_na] = utl.field_randomizer( ms_ob.type, ms_ob.default_value, None) else: message_dict = utl.update_nested( message_dict, self.context[-1][0], { ms_na: [ utl.field_randomizer( ms_ob.type, ms_ob.default_value, self._gather_enum(ms_ob)) ] }, self.context) self.context = self._u_dict_list( self.context, self.context[-1][0], self.context[-1][1] - 1) elif ms_ob.type == 11: self.logger.info( f"Checked for msg field: {ms_na}, object: {ms_ob.message_type.name}" ) if self.context[-1][ 1] != 0: # Not at the end of the current level self.logger.debug(f"CONCHECK 2 :{self.context}") if ms_ob.label in [1, 2]: # 1 optional or 2 required if len(self.context) == 1: message_dict[ms_na] = {} else: message_dict = utl.update_nested( message_dict, self.context[-1][0], {ms_na: {}}, self.context) self.context = self._u_dict_list( self.context, self.context[-1][0], self.context[-1][1] - 1) self.context.append( (ms_na, len(self._gather_field(ms_ob)))) _recurse(ms_ob, message_dict) elif ms_ob.label == 3: # 3 repeated should be list if len(self.context) == 1: message_dict[ms_na] = [ ] # put that list, AND TAKE CARE TO # PLACE THE UPCOMING INCLUDES HERE OBJECTS else: message_dict = utl.update_nested( message_dict, self.context[-1][0], {ms_na: []}, self.context) self.context = self._u_dict_list( self.context, self.context[-1][0], self.context[-1][1] - 1) self.context.append( (ms_na, len(self._gather_field(ms_ob)))) _recurse(ms_ob, message_dict) elif self.context[-1][ 1] == 0: # At the end of the current level if ms_ob.label in [1, 2]: self.logger.debug(f"CONCHECK 3 :{self.context}") self.context = self.context[:-1] self.logger.debug(f"CONCHECK 4 :{self.context}") if len(self.context) == 1: message_dict[ms_na] = {} else: message_dict = utl.update_nested( message_dict, self.context[-1][0], {ms_na: {}}, self.context) self.context.append( (ms_na, len(self._gather_field(ms_ob)))) _recurse(ms_ob, message_dict) elif ms_ob.label == 3: self.context = self.context[:-1] self.logger.debug(f"CONCHECK 5 :{self.context}") if len(self.context) == 1: message_dict[ms_na] = [] else: message_dict = utl.update_nested( message_dict, self.context[-1][0], {ms_na: []}, self.context) self.context.append( (ms_na, len(self._gather_field(ms_ob)))) _recurse(ms_ob, message_dict) self.logger.debug( f"CONTEXTHASNUM 2 :{len(self._gather_field(ms_ob)), self._gather_field(ms_ob)}" ) if self.context[-1][1] == 0: self.context = self.context[:-1] self.logger.debug(f"CONCHECK 6 :{self.context}") return message_dict result = _recurse(msg_obj, message_dict) return result