def test_multi_logs(self): self.assertEqual(parse_logs({ 'firefox': 'INFO:check_stability:Hello World!\nDEBUG:check_stability:Goodbye Cruel World!', 'chrome': 'DEBUG:check_stability:Hello World!\nWARNING:check_stability:Goodbye Cruel World!' }), {'firefox': 'Hello World!', 'chrome': 'Goodbye Cruel World!'}, 'It should include and exclude correctly across multiple logs.')
def webhook_handler(payload, signature): """Respond to Travis webhook.""" travis = Travis() github = GitHub() # The payload comes in the request, but we need to make sure it is # really signed by Travis CI. If not, respond to this request with # an error. verified_payload = travis.get_verified_payload(payload, signature) error = verified_payload.get('error') if error: return error.get('message'), error.get('code') issue_number = int(verified_payload.get('pull_request_number')) logs = travis.get_logs(verified_payload) comments = parse_logs(logs) # Create a separate comment for every job for title, comment in comments.iteritems(): try: github.post_comment(issue_number, comment, title) except requests.RequestException as err: logging.error(err.response.text) return err.response.text, 500 return "OK", 200
def test_multi_logs(self): self.assertEqual( parse_logs([{ 'job_id': 94, 'title': 'firefox', 'data': 'INFO:check_stability:Hello World!\nDEBUG:check_stability:Goodbye Cruel World!' }, { 'job_id': 32, 'title': 'chrome', 'data': 'DEBUG:check_stability:Hello World!\nWARNING:check_stability:Goodbye Cruel World!' }]), [{ 'job_id': 94, 'title': 'firefox', 'text': 'Hello World!' }, { 'job_id': 32, 'title': 'chrome', 'text': 'Goodbye Cruel World!' }], 'It should include and exclude correctly across multiple logs.')
def test_new_line(self): self.assertEqual( parse_logs([{ 'job_id': 83, 'title': 'firefox', 'data': 'INFO:check_stability:Hello World!\nWARNING:check_stability:Goodbye Cruel World!' }, { 'job_id': 88, 'title': 'chrome', 'data': 'INFO:check_stability:Hello World!\nWARNING:check_stability:Goodbye Cruel World!' }]), [{ 'job_id': 83, 'title': 'firefox', 'text': 'Hello World!\nGoodbye Cruel World!' }, { 'job_id': 88, 'title': 'chrome', 'text': 'Hello World!\nGoodbye Cruel World!' }], 'It should include newline characters as appropriate.')
def test_non_check_stability(self): self.assertEqual( parse_logs([{ 'job_id': 8, 'title': 'firefox', 'data': 'Hello World!' }]), [{ 'job_id': 8, 'title': 'firefox', 'text': '' }], 'It should exclude lines that do not include ":check_stability:"')
def test_debug_level(self): self.assertEqual( parse_logs([{ 'job_id': 5, 'title': 'firefox', 'data': 'DEBUG:check_stability:Hello World' }]), [{ 'job_id': 5, 'title': 'firefox', 'text': '' }], 'It should exclude debug log statements that include ":check_stability:".' )
def test_non_debug_level(self): self.assertEqual( parse_logs([{ 'job_id': 3, 'title': 'firefox', 'data': 'ABCDE:check_stability:Hello World' }]), [{ 'job_id': 3, 'title': 'firefox', 'text': 'Hello World' }], 'It should include non-debug log statements that include ":check_stability:".' )
def test_multi_line(self): self.assertEqual( parse_logs([{ 'job_id': 9, 'title': 'firefox', 'data': 'Hello World!\nERROR:check_stability:Goodbye Cruel World!' }]), [{ 'job_id': 9, 'title': 'firefox', 'text': 'Goodbye Cruel World!' }], 'It should include and exclude correctly across multiple lines.')
def run_test_suite(model_dir, configuration, disabled, print_parsed, ignore_test): try: # TODO: assert TF version. Some models may not run on TF1.12 etc model_dir = os.path.abspath(model_dir) test_suites = os.listdir(model_dir) failed_tests = [] passed_tests = [] skipped_tests = [] # download/prepare repo if needed: repo_filename = model_dir + '/repo.txt' repo_based = False # Is this test dir repo based or pb/pbtxt/savedmodel based? if os.path.isfile(repo_filename): repo_based = True repo_info = [ line.strip() for line in open(repo_filename).readlines() if len(line.strip()) > 0 ] repo_name = repo_info[0] repo_version = repo_info[1] if len(repo_info) == 2 else 'master' repo_dl_loc = model_dir + '/downloaded_model' assert not os.path.isdir( repo_dl_loc ), "Did not expect " + repo_dl_loc + " to be present. Maybe a leftover from the last run that was not deleted?" download_repo(repo_dl_loc, repo_name, repo_version) ready_repo(model_dir, repo_dl_loc) # Iterate through each sub-test for flname in test_suites: sub_test_dir = model_dir + '/' + flname # if its directory starting with test, and not containing "disabled" in its name item_is_a_subtest = not os.path.isfile( sub_test_dir) and flname.startswith('test') if item_is_a_subtest: disabled_by_dir_name = 'disabled' in flname disabled_by_cli = flname in disabled if (not disabled_by_dir_name) and (not disabled_by_cli): custom_parser_present = os.path.isfile( sub_test_dir + '/custom_log_parser.py') if repo_based: # TODO: shift the timing inside apply_patch_and_test sub_test_dir = model_dir + '/' + flname tstart = time.time() try: so, se = apply_patch_and_test( sub_test_dir, ('NGRAPH_TF_LOG_PLACEMENT=1', '')[custom_parser_present]) except Exception as e: print(e) failed_tests.append(flname) continue tend = time.time() command_executor.commands += '\n' else: model = [ i for i in os.listdir(sub_test_dir) if '.md' not in i and '.json' not in i ] assert len(model) == 1 model = model[0] split_on_dot = model.split('.') assert len(split_on_dot) <= 2 if len(split_on_dot) == 1: model_format = 'savedmodel' elif split_on_dot[1] in ['pb', 'pbtxt']: model_format = split_on_dot[1] else: assert False, "Unknown input format. Expected savedmodel, pb or pbtxt" # TODO: support checkpoint too later gdef = get_gdef(model_format, sub_test_dir + '/' + model) # TODO: run Level1 tests on gdef. needs another json for that (one which specifies input shapes etc) expected_json_file = sub_test_dir + '/expected.json' expected_json_present = os.path.isfile(expected_json_file) if print_parsed or expected_json_present: # parse logs in this case if custom_parser_present: sys.path.insert(0, os.path.abspath(sub_test_dir)) from custom_log_parser import custom_parse_logs parsed_vals = custom_parse_logs(so) sys.path.pop(0) else: parsed_vals = parse_logs(so) if print_parsed: to_be_printed = { configuration: { 'logparse': parsed_vals, 'time': tend - tstart } } replaced_single_with_double_quotes = json.loads( to_be_printed.__str__().replace("\'", "\"")) print( json.dumps(replaced_single_with_double_quotes, sort_keys=True, indent=4, separators=(',', ': '))) # If expected.json is present, run some extra tests. If not present we deem the test passed if it ran apply_patch_and_test without raising any errors if expected_json_present: try: expected = get_expected_from_json( expected_json_file, configuration, not custom_parser_present) except: assert False, 'Failed to parse ' + expected_json_file assert check_test_types(expected.keys( )), "Got unexpected key in " + expected.keys( ) + ". Should have been " + ','.join(valid_test_types) # We run the test if 'logparse' is present in the expected values to check # for and it is not in the ignore list if ('logparse' in expected) and ('logparse' not in ignore_test): passed, fail_help_string = compare_parsed_values( parsed_vals, expected['logparse']) if not passed: print('Failed in test ' + flname + '. Help message: ' + fail_help_string) failed_tests.append(flname) continue if ('time' in expected) and ('time' not in ignore_test): actual_runtime = tend - tstart # TODO: decide this criteria. time can be pretty variable # TODO: the percentage (0.1) for the time bound might be passed through `expected.json` time_check = (actual_runtime - expected['time'] ) / expected['time'] < 0.1 if not time_check: print("Expected run time for test " + flname + " is " + str(expected['time']) + " but it actually took " + str(actual_runtime)) failed_tests.append(flname) continue passed_tests.append(flname) else: skipped_tests.append(flname) # Make sure the test is exactly one of passed, skipped or failed assert sum([ flname in skipped_tests, flname in passed_tests, flname in failed_tests ]) == 1, str( flname ) + ' does not appear exactly once in passed, skipped or failed test lists' # Clean up if needed cleanup_script = model_dir + '/cleanup.sh' if os.path.isfile(cleanup_script): assert repo_based, 'Did not expect a cleanup script in non-repo based test' command_executor('chmod +x ' + cleanup_script) command_executor(cleanup_script) command_executor.commands += '# Exiting. Done with tests in ' + model_dir.split( '/')[-1] return passed_tests, failed_tests, skipped_tests # TODO: use gdef to run # TODO: add axpy test folders for pb. pbtxt and savedmodel # TODO integrate the if-else paths as much as possible # TODO: check throughput/latency except Exception as e: print(e) return passed_tests, failed_tests, skipped_tests finally: if (os.path.isdir(repo_dl_loc)): command_executor('rm -rf ' + repo_dl_loc)
def test_empty_dict(self): self.assertEqual(parse_logs({}), {}, 'It should return an empty dict if passed an empty dict.')
def test_new_line(self): self.assertEqual(parse_logs({'firefox': 'INFO:check_stability:Hello World!\nWARNING:check_stability:Goodbye Cruel World!', 'chrome': 'INFO:check_stability:Hello World!\nWARNING:check_stability:Goodbye Cruel World!'}), {'firefox': 'Hello World!\nGoodbye Cruel World!', 'chrome': 'Hello World!\nGoodbye Cruel World!'}, 'It should include newline characters as appropriate.')
def test_multi_line(self): self.assertEqual(parse_logs({'firefox': 'Hello World!\nERROR:check_stability:Goodbye Cruel World!'}), {'firefox': 'Goodbye Cruel World!'}, 'It should include and exclude correctly across multiple lines.')
def test_non_check_stability(self): self.assertEqual(parse_logs({'firefox': 'Hello World!'}), {'firefox': ''}, 'It should exclude lines that do not include ":check_stability:"')
def test_debug_level(self): self.assertEqual(parse_logs({'firefox': 'DEBUG:check_stability:Hello World'}), {'firefox': ''}, 'It should exclude debug log statements that include ":check_stability:".')
def test_non_debug_level(self): self.assertEqual(parse_logs({'firefox': 'ABCDE:check_stability:Hello World'}), {'firefox': 'Hello World'}, 'It should include non-debug log statements that include ":check_stability:".')
def test_empty_dict(self): self.assertEqual( parse_logs([]), [], 'It should return an empty dict if passed an empty list.')