def evaluate_logwithmodel(logpath): """ Calculate and return evaluation measurements like fitness, precision, simplicity and generalization, given the path of event log. Parameters: logpath (str): Path of event log Returns: fitness (float): Fitness value measured using pm4py precision (float): Precision value measured using pm4py simplicity (float): Simplicity value measured using pm4py generalization (float): Generalization value measured using pm4py """ xes_log = importer.apply(logpath) net, initial_marking, final_marking = inductive_miner.apply(xes_log) fitness = replay_fitness_evaluator.apply( xes_log, net, initial_marking, final_marking, variant=replay_fitness_evaluator.Variants.TOKEN_BASED) prec = precision_evaluator.apply( xes_log, net, initial_marking, final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) simp = simplicity_evaluator.apply(net) gen = generalization_evaluator.apply(xes_log, net, initial_marking, final_marking) return round(fitness['log_fitness'], 3), round(prec, 3), round(simp, 3), round(gen, 3)
def test_tokenreplay(self): log = xes_importer.apply( os.path.join("input_data", "running-example.xes")) from pm4py.algo.discovery.alpha import algorithm as alpha_miner net, im, fm = alpha_miner.apply(log) from pm4py.algo.conformance.tokenreplay import algorithm as token_replay replayed_traces = token_replay.apply( log, net, im, fm, variant=token_replay.Variants.TOKEN_REPLAY) replayed_traces = token_replay.apply( log, net, im, fm, variant=token_replay.Variants.BACKWARDS) from pm4py.evaluation.replay_fitness import evaluator as rp_fitness_evaluator fitness = rp_fitness_evaluator.apply( log, net, im, fm, variant=rp_fitness_evaluator.Variants.TOKEN_BASED) evaluation = rp_fitness_evaluator.evaluate( replayed_traces, variant=rp_fitness_evaluator.Variants.TOKEN_BASED) from pm4py.evaluation.precision import evaluator as precision_evaluator precision = precision_evaluator.apply( log, net, im, fm, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) from pm4py.evaluation.generalization import evaluator as generalization_evaluation generalization = generalization_evaluation.apply( log, net, im, fm, variant=generalization_evaluation.Variants.GENERALIZATION_TOKEN)
def evaluate_precision_tbr(log, petri_net, initial_marking, final_marking): """ Calculates the precision using token-based replay Parameters -------------- log Event log petri_net Petri net object initial_marking Initial marking final_marking Final marking Returns -------------- precision_dictionary Precision dictionary (from TBR) """ from pm4py.evaluation.precision import evaluator as precision_evaluator return precision_evaluator.apply( log, petri_net, initial_marking, final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN)
def check_model_quality(original_log, anonymized_log, result_path): anonymized_model, anonymized_initial_marking, anonymized_final_marking = inductive_miner.apply( anonymized_log) results = dict() fitness = replay_fitness_evaluator.apply( original_log, anonymized_model, anonymized_initial_marking, anonymized_final_marking, variant=replay_fitness_evaluator.Variants.TOKEN_BASED) print("Fitness: " + str(fitness)) results["fitness"] = fitness precision = precision_evaluator.apply( original_log, anonymized_model, anonymized_initial_marking, anonymized_final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) print("Precision: " + str(precision)) results["precision"] = precision gen = generalization_evaluator.apply(original_log, anonymized_model, anonymized_initial_marking, anonymized_final_marking) print("Generalization: " + str(gen)) results["generalization"] = gen simp = simplicity_evaluator.apply(anonymized_model) print("Simplicity: " + str(simp)) results["simplicity"] = simp with open(result_path, 'wb') as file: pickle.dump(results, file)
def model_metrics(model_log_path, metric_log_path, gexp_name): start_time = time() model_log_csv = pd.read_csv(model_log_path, ',') metric_log_csv = pd.read_csv(metric_log_path, ',') parameters = {log_converter.Variants.TO_EVENT_LOG.value.Parameters.CASE_ID_KEY: 'number'} model_log = log_converter.apply(model_log_csv, parameters=parameters, variant=log_converter.Variants.TO_EVENT_LOG) metric_log = log_converter.apply(metric_log_csv, parameters=parameters, variant=log_converter.Variants.TO_EVENT_LOG) parameters = {inductive_miner.Variants.DFG_BASED.value.Parameters.CASE_ID_KEY: 'number', inductive_miner.Variants.DFG_BASED.value.Parameters.ACTIVITY_KEY: 'incident_state', inductive_miner.Variants.DFG_BASED.value.Parameters.TIMESTAMP_KEY: 'sys_updated_at', alignments.Variants.VERSION_STATE_EQUATION_A_STAR.value.Parameters.ACTIVITY_KEY: 'incident_state'} petrinet, initial_marking, final_marking = inductive_miner.apply(model_log, parameters=parameters) gviz = pn_visualizer.apply(petrinet, initial_marking, final_marking) #gviz.render('petrinets\\'+gexp_name+'\\petri_' + model_base + '.png') gviz.render('test_time\\test.png') pn_visualizer.view(gviz) alignments_res = alignments.apply_log(metric_log, petrinet, initial_marking, final_marking, parameters=parameters) fitness = replay_fitness.evaluate(alignments_res, variant=replay_fitness.Variants.ALIGNMENT_BASED, parameters=parameters) precision = calc_precision.apply(metric_log, petrinet, initial_marking, final_marking, parameters=parameters) generaliz = calc_generaliz.apply(metric_log, petrinet, initial_marking, final_marking, parameters=parameters) #generaliz = 0 simplic = calc_simplic.apply(petrinet) f_score = 2 * ((fitness['averageFitness'] * precision) / (fitness['averageFitness'] + precision)) end_time = time() m, s = divmod(end_time - start_time, 60) h, m = divmod(m, 60) print('Fin %02d:%02d:%02d' % (h, m, s)) print(' F:', '%.10f' % fitness['averageFitness'], ' P:', '%.10f' % precision, ' FS:', '%.10f' % f_score, ' G:', '%.10f' % generaliz, ' S:', '%.10f' % simplic, ' T:', '%02d:%02d:%02d' % (h, m, s)) #metrics = pd.Series([model_base, metric_base, '%.10f' % fitness['averageFitness'], # '%.10f' % precision, '%.10f' % f_score, '%.10f' % generaliz, '%.10f' % simplic, # '%02d:%02d:%02d' % (h, m, s)]) return model_base
def evaluate_precision_alignments(log, petri_net, initial_marking, final_marking): """ Calculates the precision using alignments Parameters -------------- log Event log petri_net Petri net object initial_marking Initial marking final_marking Final marking Returns -------------- precision_dictionary Precision dictionary (from alignments) """ from pm4py.evaluation.precision import evaluator as precision_evaluator return precision_evaluator.apply( log, petri_net, initial_marking, final_marking, variant=precision_evaluator.Variants.ALIGN_ETCONFORMANCE)
def evaluate_precision_alignments(log: EventLog, petri_net: PetriNet, initial_marking: Marking, final_marking: Marking) -> float: warnings.warn( 'evaluate_precision_alignments is deprecated, use precision_alignments', DeprecationWarning) """ Calculates the precision using alignments Parameters -------------- log Event log petri_net Petri net object initial_marking Initial marking final_marking Final marking Returns -------------- precision float representing the precision value """ from pm4py.evaluation.precision import evaluator as precision_evaluator return precision_evaluator.apply( log, petri_net, initial_marking, final_marking, variant=precision_evaluator.Variants.ALIGN_ETCONFORMANCE)
def test_alignment(self): log = xes_importer.apply( os.path.join("input_data", "running-example.xes")) from pm4py.algo.discovery.alpha import algorithm as alpha_miner net, im, fm = alpha_miner.apply(log) from pm4py.algo.conformance.alignments import algorithm as alignments aligned_traces = alignments.apply( log, net, im, fm, variant=alignments.Variants.VERSION_STATE_EQUATION_A_STAR) aligned_traces = alignments.apply( log, net, im, fm, variant=alignments.Variants.VERSION_DIJKSTRA_NO_HEURISTICS) from pm4py.evaluation.replay_fitness import evaluator as rp_fitness_evaluator fitness = rp_fitness_evaluator.apply( log, net, im, fm, variant=rp_fitness_evaluator.Variants.ALIGNMENT_BASED) evaluation = rp_fitness_evaluator.evaluate( aligned_traces, variant=rp_fitness_evaluator.Variants.ALIGNMENT_BASED) from pm4py.evaluation.precision import evaluator as precision_evaluator precision = precision_evaluator.apply( log, net, im, fm, variant=rp_fitness_evaluator.Variants.ALIGNMENT_BASED)
def precision_alignments(log: EventLog, petri_net: PetriNet, initial_marking: Marking, final_marking: Marking) -> float: """ Calculates the precision of the model w.r.t. the event log using alignments Parameters -------------- log Event log petri_net Petri net object initial_marking Initial marking final_marking Final marking Returns -------------- precision float representing the precision value """ from pm4py.evaluation.precision import evaluator as precision_evaluator return precision_evaluator.apply( log, petri_net, initial_marking, final_marking, variant=precision_evaluator.Variants.ALIGN_ETCONFORMANCE)
def evaluate_precision_tbr(log: EventLog, petri_net: PetriNet, initial_marking: Marking, final_marking: Marking) -> float: warnings.warn( 'evaluate_precision_tbr is deprecated, use precision_token_based_replay', DeprecationWarning) """ Calculates the precision using token-based replay Parameters -------------- log Event log petri_net Petri net object initial_marking Initial marking final_marking Final marking Returns -------------- precision float representing the precision value """ from pm4py.evaluation.precision import evaluator as precision_evaluator return precision_evaluator.apply( log, petri_net, initial_marking, final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN)
def test_inductiveminer_log(self): log = xes_importer.apply( os.path.join("input_data", "running-example.xes")) net, im, fm = inductive_miner.apply(log) aligned_traces_tr = tr_alg.apply(log, net, im, fm) aligned_traces_alignments = align_alg.apply(log, net, im, fm) evaluation = eval_alg.apply(log, net, im, fm) fitness = rp_fit.apply(log, net, im, fm) precision = precision_evaluator.apply(log, net, im, fm) gen = generalization.apply(log, net, im, fm) sim = simplicity.apply(net)
def test_alphaminer_df(self): log = pd.read_csv(os.path.join("input_data", "running-example.csv")) log = dataframe_utils.convert_timestamp_columns_in_df(log) net, im, fm = alpha_miner.apply(log) aligned_traces_tr = tr_alg.apply(log, net, im, fm) aligned_traces_alignments = align_alg.apply(log, net, im, fm) evaluation = eval_alg.apply(log, net, im, fm) fitness = rp_fit.apply(log, net, im, fm) precision = precision_evaluator.apply(log, net, im, fm) gen = generalization.apply(log, net, im, fm) sim = simplicity.apply(net)
def test_inductiveminer_stream(self): df = pd.read_csv(os.path.join("input_data", "running-example.csv")) df = dataframe_utils.convert_timestamp_columns_in_df(df) stream = log_conversion.apply(df, variant=log_conversion.TO_EVENT_STREAM) net, im, fm = inductive_miner.apply(stream) aligned_traces_tr = tr_alg.apply(stream, net, im, fm) aligned_traces_alignments = align_alg.apply(stream, net, im, fm) evaluation = eval_alg.apply(stream, net, im, fm) fitness = rp_fit.apply(stream, net, im, fm) precision = precision_evaluator.apply(stream, net, im, fm) gen = generalization.apply(stream, net, im, fm) sim = simplicity.apply(net)
def test_etc1(self): # to avoid static method warnings in tests, # that by construction of the unittest package have to be expressed in such way self.dummy_variable = "dummy_value" log = xes_importer.apply( os.path.join(INPUT_DATA_DIR, "running-example.xes")) net, marking, final_marking = inductive_miner.apply(log) precision = etc_alg.apply(log, net, marking, final_marking, variant=etc_alg.ETCONFORMANCE_TOKEN) del precision
def test_evaluation_pm1(self): # to avoid static method warnings in tests, # that by construction of the unittest package have to be expressed in such way self.dummy_variable = "dummy_value" log = xes_importer.apply( os.path.join(INPUT_DATA_DIR, "running-example.xes")) net, marking, final_marking = inductive_miner.apply(log) fitness = fitness_alg.apply(log, net, marking, final_marking) precision = precision_alg.apply(log, net, marking, final_marking) generalization = generalization_alg.apply(log, net, marking, final_marking) simplicity = simplicity_alg.apply(net) del fitness del precision del generalization del simplicity
def evaluation_w_hm(log): # print("evaluation_w_hm() called") net, im, fm = heuristics_miner.apply(log) # print("herusitic miner performed with the given log") # net, im, fm = inductive_miner.apply(log) fitness = replay_fitness_evaluator.apply( log, net, im, fm, variant=replay_fitness_evaluator.Variants.TOKEN_BASED)['log_fitness'] # print("fitness", fitness) prec = precision_evaluator.apply( log, net, im, fm, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) # print("pecision", prec) gen = generalization_evaluator.apply(log, net, im, fm) # print('generality',gen) simp = simplicity_evaluator.apply(net) # print('simplicity',simp) return [len(log), fitness, prec, gen, simp]
def alignment_based_final_cluster_evaluation(clusters): f1_scores = {} weighted_sum_f1 = 0 weighted_sum_fitness = 0 weighted_sum_precision = 0 cluster_lenghts = [] lenghts = 0 for index, cluster in enumerate(clusters): net, im, fm = inductive_miner.apply(cluster) fitness = replay_fitness_evaluator.apply( cluster, net, im, fm, variant=replay_fitness_evaluator.Variants.ALIGNMENT_BASED) precision = precision_evaluator.apply( cluster, net, im, fm, variant=precision_evaluator.Variants.ALIGN_ETCONFORMANCE) f1_score = 2 * (fitness["averageFitness"] * precision) / (fitness["averageFitness"] + precision) f1_scores['cluster ' + str(len(cluster))] = f1_score cluster_lenghts.append(len(cluster)) weighted_sum_f1 += len(cluster) * f1_score weighted_sum_fitness += len(cluster) * fitness["averageFitness"] weighted_sum_precision += len(cluster) * precision lenghts += len(cluster) if len(cluster_lenghts) < 4: cluster_lenghts.append(0) if len(cluster_lenghts) < 5: cluster_lenghts.append(0) weighted_average_fitness = weighted_sum_fitness / lenghts weighted_average_precision = weighted_sum_precision / lenghts weighted_average_f1 = weighted_sum_f1 / lenghts return weighted_average_f1, weighted_average_fitness, weighted_average_precision, f1_scores, cluster_lenghts
def extract_metrics(log, net, im, fm): """ Extracts model quality criteria: fitness, precision, generalization, simplicity Also records time spent in each metric """ start_time = time.time() fitness = replay_fitness_evaluator.apply( log, net, im, fm, variant=replay_fitness_evaluator.Variants.TOKEN_BASED) fitness_time = time.time() - start_time start_time = time.time() precision = precision_evaluator.apply( log, net, im, fm, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) precision_time = time.time() - start_time start_time = time.time() generalization = generalization_evaluator.apply(log, net, im, fm) generalization_time = time.time() - start_time start_time = time.time() simplicity = simplicity_evaluator.apply(net) simplicity_time = time.time() - start_time return [ fitness_time, precision_time, generalization_time, simplicity_time, *fitness.values(), precision, generalization, simplicity, ]
def et_conformance(real_logs, petri_nets): dic = {} for ind, variant in enumerate(variants): net, im, fm = petri_nets[ind] precision = [] for i in range(8): precision.append( precision_evaluator.apply( real_logs[i], net[i], im[i], fm[i], variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN, )) dic[variant] = precision plot( pd.DataFrame(dic, index=list(range(1, 9))), "Precision using ETConformance method", "et_conformance.png", ylabel="Precision", )
def cluster_evaluation(sublog, discovery_technique): if discovery_technique == 'heuristic miner': net, im, fm = heuristics_miner.apply( sublog, parameters={"dependency_thresh": 0.99}) if discovery_technique == 'inductive miner': net, im, fm = inductive_miner.apply(sublog) fitness = replay_fitness_evaluator.apply( sublog, net, im, fm, variant=replay_fitness_evaluator.Variants.TOKEN_BASED) precision = precision_evaluator.apply( sublog, net, im, fm, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) f1_score = 2 * (fitness["log_fitness"] * precision) / (fitness["log_fitness"] + precision) return fitness, precision, f1_score
def calc_and_time_precision(): start_time = time.time() precision = calc_precision.apply(log, *petrinet_res) calc_duration = time.time() - start_time return round(precision, 4), round(calc_duration, 4)
""" fitness_alpha = fitness_evaluator.apply(log, alpha_petri, initial_marking, final_marking) print("fitness_alpha=", fitness_alpha) # Για inductive και heuristic ο υπολογισμός χρειάζεται πάρα πολύ ώρα. #fitness_heuristic = fitness_evaluator.apply(log, heuristic_petri, initial_marking, final_marking) #print("fitness_heuristic=",fitness_heuristic) """ PRECISION """ alpha_prec = precision_evaluator.apply( log_2, alpha_petri, initial_marking, final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) alpha_prec #0.08760792937295869 heuristic_prec = precision_evaluator.apply( log_2, heuristic_petri, initial_marking, final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) heuristic_prec #0.749585222873576 inductive_prec = precision_evaluator.apply( log_2, inductive_petri,
sub = ['3', '4', '15', '65', '92'] for s in sub: # Modello Rete net, initial_marking, final_marking = pnml_importer.apply( '../patterns_file/reti_Fahland/repaired_' + s + '_adjusted.pnml') print("\nValutazione rete sub_" + s + ":") fitness = replay_evaluator.apply( log, net, initial_marking, final_marking, variant=replay_evaluator.Variants.ALIGNMENT_BASED) print("Fitness: ", fitness) precision = precision_evaluator.apply( log, net, initial_marking, final_marking, variant=precision_evaluator.Variants.ALIGN_ETCONFORMANCE) print("Precision: ", precision) generalization = generalization_evaluator.apply(log, net, initial_marking, final_marking) print("Generalization: ", generalization) simplicity = simplicity_evaluator.apply(net) print("Simplicity: ", simplicity)
original_log = xes_importer.apply(original_log_path) model, initial_marking, final_marking = inductive_miner.apply(original_log) fitness = replay_fitness_evaluator.apply( original_log, model, initial_marking, final_marking, variant=replay_fitness_evaluator.Variants.TOKEN_BASED) print(fitness) fitness = fitness["average_trace_fitness"] precision = precision_evaluator.apply( original_log, model, initial_marking, final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) print(str(precision)) fscore = 2 * precision * fitness / (precision + fitness) print("Fscore of: " + str(fscore)) generalization = generalization_evaluator.apply(original_log, model, initial_marking, final_marking) print("Generalization of: " + str(generalization)) log_features, feature_names_log = get_log_representation.get_representation( original_log, str_ev_attr=["concept:name"], str_tr_attr=[], num_ev_attr=[],
def calc_and_time_precision(): start_time = time.time() precision = calc_precision.apply(log, petrinet_res, initial_mark, final_mark) calc_duration = time.time() - start_time return round(precision, 4), round(calc_duration, 4)
from pm4py.algo.discovery.inductive import algorithm as inductive_miner net, im, fm = inductive_miner.apply(log) from pm4py.evaluation.replay_fitness import evaluator as replay_fitness_evaluator fitness = replay_fitness_evaluator.apply( log, net, im, fm, variant=replay_fitness_evaluator.Variants.TOKEN_BASED) #print("fitness is equal to",fitness) sum1 = sum1 + fitness['log_fitness'] from pm4py.evaluation.precision import evaluator as precision_evaluator prec = precision_evaluator.apply( log, net, im, fm, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN) #print("precision is euqal to",prec) sum2 = sum2 + prec array1.append(sum1 / 5) array2.append(sum2 / 5) print(array1) print(array2) print(array1) print(array2)
parameters = { log_converter.Variants.TO_EVENT_LOG.value.Parameters.CASE_ID_KEY: 'number' } log = log_converter.apply(log, parameters=parameters) parameters = { inductive_miner.Variants.DFG_BASED.value.Parameters.CASE_ID_KEY: 'number', inductive_miner.Variants.DFG_BASED.value.Parameters.ACTIVITY_KEY: 'activity', } petrinet_res = inductive_miner.apply(log, parameters=parameters) #fitness = calc_fitness.apply(log, *petrinet_res, parameters=parameters) #print('Conformidade',round(fitness['average_trace_fitness'],4)) precision = calc_precision.apply(log, *petrinet_res, parameters=parameters) print('Precisao', round(precision, 4)) simplic = calc_simplic.apply(petrinet_res[0], parameters=parameters) print('Simplicidade', round(simplic, 4)) generaliz = calc_generaliz.apply(log, *petrinet_res, parameters=parameters) print('Generalização', round(generaliz, 4)) # Precisao 0.1023 # Simplicidade 0.5802 # Generalização 0.575 # ---------------------------------------------------------------------------- # Metrics for kmeans # Generate a csv file with the metrics for all the kmeans results
times_tokenreplay_imdf[logName] = t2 - t1 if ENABLE_ALIGNMENTS: t1 = time.time() fitness_align_imdf[logName] = \ fitness_evaluator.apply(log, inductive_model, inductive_im, inductive_fm, variant=fitness_evaluator.Variants.ALIGNMENT_BASED, parameters=parameters)[ 'percFitTraces'] print(str(time.time()) + " fitness_token_align for " + logName + " succeeded! " + str( fitness_align_imdf[logName])) t2 = time.time() times_alignments_imdf[logName] = t2 - t1 if ENABLE_PRECISION: precision_alpha[logName] = precision_evaluator.apply(log, alpha_model, alpha_initial_marking, alpha_final_marking, variant=precision_evaluator.Variants.ETCONFORMANCE_TOKEN, parameters=parameters) else: precision_alpha[logName] = 0.0 print(str(time.time()) + " precision_alpha for " + logName + " succeeded! " + str(precision_alpha[logName])) generalization_alpha[logName] = generalization_evaluator.apply(log, alpha_model, alpha_initial_marking, alpha_final_marking, parameters=parameters) print(str(time.time()) + " generalization_alpha for " + logName + " succeeded! " + str( generalization_alpha[logName])) simplicity_alpha[logName] = simplicity_evaluator.apply(alpha_model, parameters=parameters) print( str(time.time()) + " simplicity_alpha for " + logName + " succeeded! " + str(simplicity_alpha[logName])) if ENABLE_PRECISION: precision_imdf[logName] = precision_evaluator.apply(log, inductive_model, inductive_im,