def compareOtherOutputs( self, report_name, ref_path, test_path, failures ): ref_md5 = ru.md5_hash_of_file( ref_path ) test_md5 = ru.md5_hash_of_file( test_path ) if ref_md5 == test_md5: # print( "CSV files passed MD5 comparison test." ) return False, "" else: print( self.config_id + " completed but did not match reference! (" + str(self.duration) + ") - " + report_name ) return True, "Failes MD5 check."
def compareOtherOutputs( self, report_name, ref_path, test_path, failures ): ref_md5 = ru.md5_hash_of_file( ref_path ) test_md5 = ru.md5_hash_of_file( test_path ) if ref_md5 == test_md5: # print( "CSV files passed MD5 comparison test." ) return False, "" else: print( self.scenario_path + " completed but did not match reference! (" + str(self.duration) + ") - " + report_name ) return True, "Failes MD5 check."
def compareCsvOutputs(self, ref_path, test_path, failures): # print( "Comparing CSV files: ref = " + ref_path + ", test = " + test_path ) # Do Md5 comp first. ref_md5 = ru.md5_hash_of_file(ref_path) test_md5 = ru.md5_hash_of_file(test_path) if ref_md5 == test_md5: # print( "CSV files passed MD5 comparison test." ) return False, "" fail_validation = False err_msg = "" # print( "CSV files failed MD5 comparison test." ) # First (md5) test failed. Do line length, then line-by-line ref_length = ru.file_len(ref_path) test_length = ru.file_len(test_path) if ref_length != test_length: fail_validation = True err_msg = "Reference output {0} has {1} lines but test output {2} has {3} lines".format( ref_path, ref_length, test_path, test_length) else: with open(ref_path, "r") as ref_file, open(test_path, "r") as test_file: line_num = 0 for ref_line in ref_file: line_num = line_num + 1 test_line = test_file.readline() if ref_line != test_line: ref_line_tokens = ref_line.split(',') test_line_tokens = test_line.split(',') for col_idx in range(len(ref_line_tokens)): if ref_line_tokens[col_idx] != test_line_tokens[ col_idx]: break err_msg = "First mismatch at line {0} of {1} column {2}: reference line...\n{3}vs test line...\n{4}{5} vs {6}".format( line_num, ref_path, col_idx, ref_line, test_line, ref_line_tokens[col_idx], test_line_tokens[col_idx]) fail_validation = True break print(err_msg) failure_txt = err_msg #self.report.addFailingTest( self.scenario_path, failure_txt, test_path, self.scenario_type ) return fail_validation, failure_txt
def hash_for_file(self, filename): if filename not in self.known_files: self.log("Calculating hash for '{0}'.".format(filename)) md5_hash = ru.md5_hash_of_file(filename) self.known_files[filename] = md5_hash else: self.log("Found hash for '{0}' in known files.".format(filename)) return self.known_files[filename]
def hash_for_file(self, filename): if filename not in self.known_files: self.log("Calculating hash for '{0}'.".format(filename)) md5_hash = ru.md5_hash_of_file(filename) self.known_files[filename] = md5_hash else: self.log("Found hash for '{0}' in known files.".format(filename)) return self.known_files[filename]
def compareCsvOutputs( self, ref_path, test_path, failures ): # print( "Comparing CSV files: ref = " + ref_path + ", test = " + test_path ) # Do Md5 comp first. ref_md5 = ru.md5_hash_of_file( ref_path ) test_md5 = ru.md5_hash_of_file( test_path ) if ref_md5 == test_md5: # print( "CSV files passed MD5 comparison test." ) return False, "" fail_validation = False err_msg = "" # print( "CSV files failed MD5 comparison test." ) # First (md5) test failed. Do line length, then line-by-line ref_length = ru.file_len( ref_path ) test_length = ru.file_len( test_path ) if ref_length != test_length: fail_validation = True err_msg = "Reference output {0} has {1} lines but test output {2} has {3} lines".format( ref_path, ref_length, test_path, test_length ) else: ref_file = open( ref_path ) test_file = open( test_path ) line_num = 0 for ref_line in ref_file: line_num = line_num + 1 test_line = test_file.readline() if ref_line != test_line: ref_line_tokens = ref_line.split(',') test_line_tokens = test_line.split(',') for col_idx in range( len( ref_line_tokens) ): if ref_line_tokens[col_idx] != test_line_tokens[col_idx]: break err_msg = "First mismatch at line {0} of {1} column {2}: reference line...\n{3}vs test line...\n{4}{5} vs {6}".format( line_num, ref_path, col_idx, ref_line, test_line, ref_line_tokens[col_idx], test_line_tokens[col_idx] ) fail_validation = True ref_file.close() test_file.close() break print( err_msg ) failure_txt = err_msg #self.report.addFailingTest( self.scenario_path, failure_txt, test_path, self.scenario_type ) return fail_validation, failure_txt
def __init__(self, params): self.params = params self.dtk_hash = ru.md5_hash_of_file(self.params.executable_path) self.sim_dir_sem = threading.Semaphore() self.emodules_map["interventions"] = [] self.emodules_map["disease_plugins"] = [] self.emodules_map["reporter_plugins"] = [] if params.dll_root is not None and params.use_dlls is True: #print( "dll_root (remote) = " + params.dll_root ) self.copyEModulesOver(params) else: print("Not using DLLs")
def __init__(self, params): self.params = params self.dtk_hash = ru.md5_hash_of_file(self.params.executable_path) self.sim_dir_sem = threading.Semaphore() self.emodules_map["interventions"] = [] self.emodules_map["disease_plugins"] = [] self.emodules_map["reporter_plugins"] = [] if params.dll_root is not None and params.use_dlls is True: # print( "dll_root (remote) = " + params.dll_root ) self.copyEModulesOver(params) else: print("Not using DLLs") return
def copyEModulesOver(self, params): print "src_root = " + params.src_root if params.dll_path is not None: emodule_dir = params.dll_path else: if params.scons: emodule_dir = os.path.join(params.src_root, "build") emodule_dir = os.path.join(emodule_dir, "x64") else: emodule_dir = os.path.join(params.src_root, "x64") if params.debug == True: emodule_dir = os.path.join(emodule_dir, "Debug") elif params.quick_start == True: emodule_dir = os.path.join(emodule_dir, "QuickStart") else: emodule_dir = os.path.join(emodule_dir, "Release") print('Assuming emodules (dlls) are in local directory: ' + emodule_dir) if os.path.exists(emodule_dir) == False: print( "Except that directory does not exist! Not copying emodules.") return #print "dll_root = " + params.dll_root dll_dirs = ["disease_plugins", "reporter_plugins", "interventions"] for dll_subdir in dll_dirs: dlls = glob.glob( os.path.join(os.path.join(emodule_dir, dll_subdir), "*.dll")) for dll in dlls: dll_hash = ru.md5_hash_of_file(dll) #print( dll_hash ) # 1) calc md5 of dll # 2) check for existence of rivendell (or whatever) for <root>/emodules/<subdir>/<md5> # 3) if no exist, create and copy # 4) put full path in emodules_json # 5) write out emodules_json when done to target sim dir try: target_dir = os.path.join(params.dll_root, dll_subdir) target_dir = os.path.join(target_dir, dll_hash) if params.sec: print(dll + " will be used without checking 'new-ness'.") elif not (os.path.isdir(target_dir)): print(dll + ": copying to cluster") else: print(dll + ": Already on cluster") if not ( os.path.isdir(target_dir) ) and params.sec == False: # sec = command-line option to skip this os.makedirs(target_dir) shutil.copy( dll, os.path.join(target_dir, os.path.basename(dll))) self.emodules_map[dll_subdir].append( os.path.join(target_dir, os.path.basename(dll))) except IOError: print "Failed to copy dll " + dll + " to " + os.path.join( os.path.join(params.dll_root, dll_dirs[1]), os.path.basename(dll)) ru.final_warnings += "Failed to copy dll " + dll + " to " + os.path.join( os.path.join(params.dll_root, dll_dirs[1]), os.path.basename(dll)) + "\n"
def copyEModulesOver(self, params): print "src_root = " + params.src_root if params.dll_path is not None: emodule_dir = params.dll_path else: if params.scons: emodule_dir = os.path.join(params.src_root, "build") emodule_dir = os.path.join(emodule_dir, "x64") else: emodule_dir = os.path.join(params.src_root, "x64") if params.debug: emodule_dir = os.path.join(emodule_dir, "Debug") elif params.quick_start: emodule_dir = os.path.join(emodule_dir, "QuickStart") else: emodule_dir = os.path.join(emodule_dir, "Release") print('Assuming emodules (dlls) are in local directory: ' + emodule_dir) if not os.path.exists(emodule_dir): print("Except that directory does not exist! Not copying emodules.") return # print "dll_root = " + params.dll_root dll_dirs = ["disease_plugins", "reporter_plugins", "interventions"] for dll_subdir in dll_dirs: dlls = glob.glob(os.path.join(os.path.join(emodule_dir, dll_subdir), "*.dll")) for dll in dlls: dll_hash = ru.md5_hash_of_file(dll) # print( dll_hash ) # 1) calc md5 of dll # 2) check for existence of rivendell (or whatever) for <root>/emodules/<subdir>/<md5> # 3) if no exist, create and copy # 4) put full path in emodules_json # 5) write out emodules_json when done to target sim dir try: target_dir = os.path.join(params.dll_root, dll_subdir) target_dir = os.path.join(target_dir, dll_hash) if params.sec: print(dll + " will be used without checking 'new-ness'.") elif not os.path.isdir(target_dir): print(dll + ": copying to cluster") else: print(dll + ": Already on cluster") if not os.path.isdir(target_dir) and not params.sec: # sec = command-line option to skip this os.makedirs(target_dir) shutil.copy(dll, os.path.join(target_dir, os.path.basename(dll))) self.emodules_map[dll_subdir].append(os.path.join(target_dir, os.path.basename(dll))) except IOError: print "Failed to copy dll " + dll + " to " + os.path.join(os.path.join(params.dll_root, dll_dirs[1]), os.path.basename(dll)) ru.final_warnings += "Failed to copy dll " + dll + " to " + os.path.join(os.path.join(params.dll_root, dll_dirs[1]), os.path.basename(dll)) + "\n" return
def compareJsonOutputs(self, sim_dir, report_name, ref_path, test_path, failures): fail_validation = False failure_txt = "" try: ru.load_json(os.path.join(ru.cache_cwd, ref_path)) except Exception: print("Exception {0} {1} loading json file: {2}.".format( sys.exc_info()[0], sys.exc_info()[1], (os.path.join(ru.cache_cwd, ref_path)))) return ref_json = ru.load_json(os.path.join(sim_dir, ref_path)) if "Channels" not in ref_json.keys(): ref_md5 = ru.md5_hash_of_file(ref_path) test_md5 = ru.md5_hash_of_file(test_path) if ref_md5 == test_md5: return False, "" else: print(self.scenario_path + " completed but did not match reference! (" + str(self.duration) + ") - " + report_name) return True, "Non-Channel JSON failed MD5." else: test_json = ru.load_json(os.path.join(sim_dir, test_path)) if "Channels" not in test_json.keys(): return True, "Reference has Channel data and Test file does not." ref_md5 = self.get_json_data_hash(ref_json["Channels"]) test_md5 = self.get_json_data_hash(test_json["Channels"]) ref_channels = set(ref_json["Channels"]) test_channels = set(test_json["Channels"]) if ref_md5 == test_md5: return False, "" missing_channels = ref_channels - test_channels new_channels = test_channels - ref_channels if len(missing_channels) > 0: fail_validation = True print("ERROR: Missing channels - " + ', '.join(missing_channels)) failure_txt += "Missing channels:\n" + '\n'.join( missing_channels) + "\n" self.report.addFailingTest( self.scenario_path, failure_txt, os.path.join(sim_dir, ("output/" + report_name)), self.scenario_type) if len(new_channels) > 0: print( "WARNING: The test " + report_name + " has " + str(len(new_channels)) + " channels not found in the reference. Please update the reference " + report_name + ".") ru.final_warnings += self.scenario_path + " - New channels not found in reference:\n " + '\n '.join( new_channels ) + "\nPlease update reference from " + os.path.join( sim_dir, os.path.join("output", "InsetChart.json")) + "!\n" self.report.addFailingTest( self.scenario_path, failure_txt, os.path.join(sim_dir, ("output/" + report_name)), self.scenario_type) if "Header" in ref_json.keys() and ref_json["Header"][ "Timesteps"] != test_json["Header"]["Timesteps"]: warning_msg = "WARNING: test " + report_name + " has timesteps " + str( test_json["Header"]["Timesteps"] ) + " DIFFERRING from ref " + report_name + " timesteps " + str( ref_json["Header"]["Timesteps"]) + "!\n" if self.params.hide_graphs: # This is treated as automated running mode (or bamboo nightly build mode) fail_validation = True failure_txt += warning_msg else: # This is treated as manual running mode ru.final_warnings += warning_msg print(warning_msg) if not fail_validation: #print( "Hasn't failed validation on second level review. Time to look channel by channel, timestep by timestep." ) # BinnedReport and its derived classes have "Subchannel_Metadata" in the header if "Header" in ref_json.keys( ) and "Subchannel_Metadata" in ref_json["Header"].keys(): self.compareBinnedReportType(ref_json, test_json, failures) elif "Header" in ref_json.keys( ) and "Report_Type" in ref_json["Header"].keys( ) and ref_json["Header"]["Report_Type"] == "InsetChart": # Assuming a BaseChannelReport self.compareChannelReportType(ref_json, test_json, failures) else: fail_validation = True failures.append( report_name + " - Files are different but cannot do deep dive.") if len(failures) > 0: fail_validation = True failure_txt += "Channel Timestep Reference_Value Test_Value\n" + ''.join( failures) print(self.scenario_path + " completed but did not match reference! (" + str(self.duration) + ") - " + report_name) return fail_validation, failure_txt
def compareJsonOutputs( self, sim_dir, report_name, ref_path, test_path, failures ): fail_validation = False failure_txt = "" try: json.loads( open( os.path.join( ru.cache_cwd, ref_path ) ).read() ) except Exception as ex: print( "Exception {0} loading json file: {1}.".format( ex, ( os.path.join( ru.cache_cwd, ref_path ) ) ) ) return ref_json = json.loads( open( os.path.join( ru.cache_cwd, ref_path ) ).read() ) if "Channels" not in ref_json.keys(): ref_md5 = ru.md5_hash_of_file( ref_path ) test_md5 = ru.md5_hash_of_file( test_path ) if ref_md5 == test_md5: return False, "" else: print( self.scenario_path + " completed but did not match reference! (" + str(self.duration) + ") - " + report_name ) return True, "Non-Channel JSON failed MD5." else: test_json = json.loads( open( os.path.join( sim_dir, test_path ) ).read() ) if "Channels" not in test_json.keys(): return True, "Reference has Channel data and Test file does not." ref_md5 = self.get_json_data_hash( ref_json["Channels"] ) test_md5 = self.get_json_data_hash( test_json["Channels"] ) ref_channels = set(ref_json["Channels"]) test_channels = set(test_json["Channels"]) if ref_md5 == test_md5: return False, "" missing_channels = ref_channels - test_channels new_channels = test_channels - ref_channels if len(missing_channels) > 0: fail_validation = True print("ERROR: Missing channels - " + ', '.join(missing_channels)) failure_txt += "Missing channels:\n" + '\n'.join(missing_channels) + "\n" self.report.addFailingTest( self.scenario_path, failure_txt, os.path.join( sim_dir, ( "output/" + report_name ) ), self.scenario_type ) if len(new_channels) > 0: print("WARNING: The test "+report_name+" has " + str(len(new_channels)) + " channels not found in the reference. Please update the reference "+report_name+".") ru.final_warnings += self.scenario_path + " - New channels not found in reference:\n " + '\n '.join(new_channels) + "\nPlease update reference from " + os.path.join( sim_dir, os.path.join( "output", "InsetChart.json" ) ) + "!\n" self.report.addFailingTest( self.scenario_path, failure_txt, os.path.join( sim_dir, ( "output/" + report_name ) ), self.scenario_type ) if "Header" in ref_json.keys() and ref_json["Header"]["Timesteps"] != test_json["Header"]["Timesteps"]: warning_msg = "WARNING: test "+report_name+" has timesteps " + str(test_json["Header"]["Timesteps"]) + " DIFFERRING from ref "+report_name+" timesteps " + str(ref_json["Header"]["Timesteps"]) + "!\n" if self.params.hide_graphs: # This is treated as automated running mode (or bamboo nightly build mode) fail_validation = True failure_txt += warning_msg else: # This is treated as manual running mode ru.final_warnings += warning_msg print(warning_msg) if not fail_validation: #print( "Hasn't failed validation on second level review. Time to look channel by channel, timestep by timestep." ) # BinnedReport and its derived classes have "Subchannel_Metadata" in the header if "Header" in ref_json.keys() and "Subchannel_Metadata" in ref_json["Header"].keys(): self.compareBinnedReportType( ref_json, test_json, failures ) elif "Header" in ref_json.keys() and "Report_Type" in ref_json["Header"].keys() and ref_json["Header"]["Report_Type"] =="InsetChart": # Assuming a BaseChannelReport self.compareChannelReportType( ref_json, test_json, failures ) else: fail_validation = True failures.append(report_name + " - Files are different but cannot do deep dive.") if len(failures) > 0: fail_validation = True failure_txt += "Channel Timestep Reference_Value Test_Value\n" + ''.join(failures) print( self.scenario_path + " completed but did not match reference! (" + str(self.duration) + ") - " + report_name ) return fail_validation, failure_txt