def test_filtering_attributes_events(self): # to avoid static method warnings in tests, # that by construction of the unittest package have to be expressed in such way self.dummy_variable = "dummy_value" input_log = os.path.join(INPUT_DATA_DIR, "running-example.xes") log = xes_importer.import_log(input_log) log1 = attributes_filter.apply_events(log, ["reject request"], parameters={"positive": True}) log2 = attributes_filter.apply_events(log, ["reject request"], parameters={"positive": True}) del log1 del log2
def apply(dataframe, filter, parameters=None): """ Apply a filter to the current log (attributes filter) Parameters ------------ log Event log filter Filter to apply parameters Parameters of the algorithm Returns ------------ log Event log """ if parameters is None: parameters = {} parameters[constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = filter[1][0] return attributes_filter.apply_events(dataframe, filter[1][1], parameters=parameters)
def apply(log, list_activities, sample_size, parameters): """ Finds the performance spectrum provided a log and a list of activities Parameters ------------- log Log list_activities List of activities interesting for the performance spectrum (at least two) sample_size Size of the sample parameters Parameters of the algorithm, including the activity key and the timestamp key Returns ------------- points Points of the performance spectrum """ if parameters is None: parameters = {} activity_key = parameters[ constants. PARAMETER_CONSTANT_ACTIVITY_KEY] if constants.PARAMETER_CONSTANT_ACTIVITY_KEY in parameters else xes.DEFAULT_NAME_KEY timestamp_key = parameters[ constants. PARAMETER_CONSTANT_TIMESTAMP_KEY] if constants.PARAMETER_CONSTANT_TIMESTAMP_KEY in parameters else xes.DEFAULT_TIMESTAMP_KEY log = sorting.sort_timestamp_log(log, timestamp_key=timestamp_key) parameters[constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key log = attributes_filter.apply_events(log, list_activities, parameters=parameters) points = [] for trace in log: for i in range(len(trace) - len(list_activities) + 1): acti_comb = [ event[activity_key] for event in trace[i:i + len(list_activities)] ] if acti_comb == list_activities: timest_comb = [ event[timestamp_key].timestamp() for event in trace[i:i + len(list_activities)] ] points.append(timest_comb) points = sorted(points, key=lambda x: x[0]) if len(points) > sample_size: points = points_subset.pick_chosen_points_list(sample_size, points) return points
def filter_events(log, starts, ends, parameters): log_classifier = attributes_filter.apply_events( log, starts + ends, parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: parameters[performance_constants.EVENT_CLASSIFIER], 'positive': True }) return log_classifier
def filter_per_activities_to_keep(chat_id, activities): log = get_current_log(chat_id) tracefilter_log_pos = attributes_filter.apply_events( log, activities, parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name", "positive": True }) xes_exporter.export_log(tracefilter_log_pos, get_log_filename(chat_id, True), parameters={"compress": False}) set_property(chat_id, "current_log", get_log_filename(chat_id, True))
def clean_lifecycle_events(log): '''recives a log and checks if it contains multiple lifecycle events per activity. Returns a log that only contains the starting events. If the log has no lifecycle events it is returned without any changes''' try: if 'lifecycle:transition' in log[0][0].keys(): if len(set([e['lifecycle:transition'] for e in log[0]])) > 1: log = attributes_filter.apply_events( log, ["start"], parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "lifecycle:transition", "positive": True }) except Exception as e: print('An exception occured during cleaning of lifecycle events') print(e) return log
def filter_log_by_relevance(topK, log, relevance_scores): log_new = EventLog() for label in relevance_scores: topK = len(relevance_scores[label]['scores']) if len( relevance_scores[label]['scores']) < topK else topK relevance_scores[label]['scores'] = dict( sorted(relevance_scores[label]['scores'].items(), key=lambda x: x[1], reverse=True)[:topK]) log_dummy = filter_log_by_caseid(log, relevance_scores[label]['traces']) log_dummy = attributes_filter.apply_events( log_dummy, relevance_scores[label]['scores'].keys(), parameters={ attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name", "positive": True }) log_new._list = log_new._list + log_dummy._list return relevance_scores, log_new
- Filtering: - - Used Columns: - """ from eval.evaluation import run_experiment from grm import preprocessing from pm4py.algo.filtering.log.attributes import attributes_filter log_file = "bpi2017.csv" name_of_case_id = "Case ID" name_of_activity = "Activity" name_of_timestamp = "Complete Timestamp" name_of_label = "Accepted" hyper_params = {'num_epochs': 1000} k = 10 log = preprocessing.import_data("../data", log_file, separator=";", quote='', case_id=name_of_case_id, activity=name_of_activity, time_stamp=name_of_timestamp, target=name_of_label) activities = attributes_filter.get_attribute_values(log, "concept:name") # filter out activities representing work items w_activities = [i for i in activities.keys() if i.startswith('W_')] log_filtered = attributes_filter.apply_events(log, w_activities, parameters={ attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name", "positive": True}) run_experiment(log, hyper_params=hyper_params, k=k, ml_flow_run_name_prefix=log_file)
log_file, separator=";", quote='', case_id=name_of_case_id, activity=name_of_activity, time_stamp=name_of_timestamp, target=name_of_label) activities_2017 = attributes_filter.get_attribute_values(log, "concept:name") # filter out activities representing work items w_activities_2017 = [i for i in activities_2017.keys() if i.startswith('W_')] fil_log_17 = attributes_filter.apply_events( log, w_activities_2017, parameters={ attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name", "positive": True }) # instances print("2017 instances: ", len(fil_log_17)) # variants variants = variants_filter.get_variants(fil_log_17) print("2017 variants: ", len(variants)) # instances per variant sum_val = 0 for value in variants.values(): sum_val += len(value)
def apply(df, discovery_algorithm=discover_inductive, parameters=None): if parameters is None: parameters = {} allowed_activities = parameters[ "allowed_activities"] if "allowed_activities" in parameters else None debug = parameters["debug"] if "debug" in parameters else True try: if df.type == "succint": df = succint_mdl_to_exploded_mdl.apply(df) df.type = "exploded" except: pass if len(df) == 0: df = pd.DataFrame({"event_id": [], "event_activity": []}) min_node_freq = parameters[ "min_node_freq"] if "min_node_freq" in parameters else 0 min_edge_freq = parameters[ "min_edge_freq"] if "min_edge_freq" in parameters else 0 df = clean_frequency.apply(df, min_node_freq) df = clean_arc_frequency.apply(df, min_edge_freq) if len(df) == 0: df = pd.DataFrame({"event_id": [], "event_activity": []}) persps = [x for x in df.columns if not x.startswith("event_")] ret = {} ret["nets"] = {} ret["act_count"] = {} ret["replay"] = {} ret["group_size_hist"] = {} ret["act_count_replay"] = {} ret["group_size_hist_replay"] = {} ret["aligned_traces"] = {} ret["place_fitness_per_trace"] = {} ret["aggregated_statistics_frequency"] = {} ret["aggregated_statistics_performance_min"] = {} ret["aggregated_statistics_performance_max"] = {} ret["aggregated_statistics_performance_median"] = {} ret["aggregated_statistics_performance_mean"] = {} diff_log = 0 diff_model = 0 diff_token_replay = 0 diff_performance_annotation = 0 diff_basic_stats = 0 for persp in persps: aa = time.time() if debug: print(persp, "getting log") log = algorithm.apply(df, persp, parameters=parameters) if debug: print(len(log)) if allowed_activities is not None: if persp not in allowed_activities: continue filtered_log = attributes_filter.apply_events( log, allowed_activities[persp]) else: filtered_log = log bb = time.time() diff_log += (bb - aa) # filtered_log = variants_filter.apply_auto_filter(deepcopy(filtered_log), parameters={"decreasingFactor": 0.5}) if debug: print(len(log)) print(persp, "got log") cc = time.time() #net, im, fm = inductive_miner.apply(filtered_log) net, im, fm = discovery_algorithm(filtered_log) """if persp == "items": trans_map = {t.label:t for t in net.transitions} source_place_it = list(trans_map["item out of stock"].in_arcs)[0].source target_place_re = list(trans_map["reorder item"].out_arcs)[0].target skip_trans_1 = PetriNet.Transition(str(uuid.uuid4()), None) net.transitions.add(skip_trans_1) add_arc_from_to(source_place_it, skip_trans_1, net) add_arc_from_to(skip_trans_1, target_place_re, net)""" #net = reduce_petri_net(net) dd = time.time() diff_model += (dd - cc) # net, im, fm = alpha_miner.apply(filtered_log) if debug: print(persp, "got model") xx1 = time.time() activ_count = algorithm.apply(df, persp, variant="activity_occurrence", parameters=parameters) if debug: print(persp, "got activ_count") xx2 = time.time() ee = time.time() variants_idx = variants_module.get_variants_from_log_trace_idx(log) # variants = variants_module.convert_variants_trace_idx_to_trace_obj(log, variants_idx) # parameters_tr = {PARAM_ACTIVITY_KEY: "concept:name", "variants": variants} if debug: print(persp, "got variants") aligned_traces, place_fitness_per_trace, transition_fitness_per_trace, notexisting_activities_in_model = tr_factory.apply( log, net, im, fm, parameters={ "enable_pltr_fitness": True, "disable_variants": True }) if debug: print(persp, "done tbr") element_statistics = performance_map.single_element_statistics( log, net, im, aligned_traces, variants_idx) if debug: print(persp, "done element_statistics") ff = time.time() diff_token_replay += (ff - ee) aggregated_statistics = performance_map.aggregate_statistics( element_statistics) if debug: print(persp, "done aggregated_statistics") element_statistics_performance = performance_map.single_element_statistics( log, net, im, aligned_traces, variants_idx) if debug: print(persp, "done element_statistics_performance") gg = time.time() aggregated_statistics_performance_min = performance_map.aggregate_statistics( element_statistics_performance, measure="performance", aggregation_measure="min") aggregated_statistics_performance_max = performance_map.aggregate_statistics( element_statistics_performance, measure="performance", aggregation_measure="max") aggregated_statistics_performance_median = performance_map.aggregate_statistics( element_statistics_performance, measure="performance", aggregation_measure="median") aggregated_statistics_performance_mean = performance_map.aggregate_statistics( element_statistics_performance, measure="performance", aggregation_measure="mean") hh = time.time() diff_performance_annotation += (hh - ee) if debug: print(persp, "done aggregated_statistics_performance") group_size_hist = algorithm.apply(df, persp, variant="group_size_hist", parameters=parameters) if debug: print(persp, "done group_size_hist") occurrences = {} for trans in transition_fitness_per_trace: occurrences[trans.label] = set() for trace in transition_fitness_per_trace[trans]["fit_traces"]: if not trace in transition_fitness_per_trace[trans][ "underfed_traces"]: case_id = trace.attributes["concept:name"] for event in trace: if event["concept:name"] == trans.label: occurrences[trans.label].add( (case_id, event["event_id"])) # print(transition_fitness_per_trace[trans]) len_different_ids = {} for act in occurrences: len_different_ids[act] = len(set(x[1] for x in occurrences[act])) eid_acti_count = {} for act in occurrences: eid_acti_count[act] = {} for x in occurrences[act]: if not x[0] in eid_acti_count: eid_acti_count[act][x[0]] = 0 eid_acti_count[act][x[0]] = eid_acti_count[act][x[0]] + 1 eid_acti_count[act] = sorted(list(eid_acti_count[act].values())) ii = time.time() diff_basic_stats += (ii - hh) + (xx2 - xx1) ret["nets"][persp] = [net, im, fm] ret["act_count"][persp] = activ_count ret["aligned_traces"][persp] = aligned_traces ret["place_fitness_per_trace"][persp] = place_fitness_per_trace ret["aggregated_statistics_frequency"][persp] = aggregated_statistics ret["aggregated_statistics_performance_min"][ persp] = aggregated_statistics_performance_min ret["aggregated_statistics_performance_max"][ persp] = aggregated_statistics_performance_max ret["aggregated_statistics_performance_median"][ persp] = aggregated_statistics_performance_median ret["aggregated_statistics_performance_mean"][ persp] = aggregated_statistics_performance_mean ret["replay"][persp] = aggregated_statistics ret["group_size_hist"][persp] = group_size_hist ret["act_count_replay"][persp] = len_different_ids ret["group_size_hist_replay"][persp] = eid_acti_count ret["computation_statistics"] = { "diff_log": diff_log, "diff_model": diff_model, "diff_token_replay": diff_token_replay, "diff_performance_annotation": diff_performance_annotation, "diff_basic_stats": diff_basic_stats } return ret
target=name_of_label) # filter out most relevant activity model_path = '../best_models/sp2020/2020-05-05-14-59_best_model.pickle' activities = get_activities(log) grm_model = GRM.GRM(log, activities, restore_file=model_path) filtered_log = EventLog() for trace in log: case_id, pred, rel_scores = grm_model.predict(trace) if len(rel_scores) > 1: most_relevant = max(rel_scores.items(), key=operator.itemgetter(1))[0] log_trace = attributes_filter.apply_events( log, [case_id], parameters={ attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: name_of_case_id, "positive": True }) trace_without_most = attributes_filter.apply_events( log_trace, [most_relevant], parameters={ attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name", "positive": False }) trace_without_most = trace_without_most[0] filtered_log._list.append(trace_without_most)
tracefilter_log_pos = attributes_filter.apply( log, ["Resource10"], parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "org:resource", "positive": True }) tracefilter_log_neg = attributes_filter.apply( log, ["Resource10"], parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "org:resource", "positive": False }) eventsfilter_log = attributes_filter.apply_events( log, ["Resource10"], parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "org:resource", "positive": True }) eventsfilter_log from pm4py.algo.filtering.log.attributes import attributes_filter from pm4py.util import constants filtered_log = attributes_filter.apply_auto_filter( log, parameters={ constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name", "decreasingFactor": 0.6 }) filtered_log #filter on numeric attributes
- Pre-processing - Filtering: - - Used Columns: - """ from eval.evaluation import run_experiment from grm import preprocessing from pm4py.algo.filtering.log.attributes import attributes_filter from pm4py.objects.log.util import sampling logfile = "BPI2020_PermitLog.csv" name_of_case_id = "Case ID" name_of_activity = "Activity" name_of_timestamp = "Complete Timestamp" name_of_label = "(case) Overspent" hyper_params = {'num_epochs': 1000} k = 10 log = preprocessing.import_data("../data", logfile, separator=";", quote='', case_id=name_of_case_id, activity=name_of_activity, time_stamp=name_of_timestamp, target=name_of_label) # log = sampling.sample(log, n=100) # filter only activities representing work items log_filtered = attributes_filter.apply_events(log, ['STAFF MEMBER'], parameters={attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "Resource", "positive": True}) run_experiment(log_filtered, hyper_params=hyper_params, k=k, ml_flow_run_name_prefix=logfile)
if ENABLE_TESTS: # TEST 9: discover performance spectrum from dataframe t0 = time.time() pspectrum.apply(roadtraffic_df, ["Create Fine", "Send Fine"]) t1 = time.time() T9[0] = (t1 - t0) T9[2] = math.ceil(T9[1] / (T9[0] + 0.00000001) * 1000.0) print( "TEST 9 - Discover peformance spectrum from dataframe - %.5f s (test score: %d)" % (T9[0], T9[2])) if ENABLE_TESTS: # TEST 10: filter bpic2017 event log on event attributes t0 = time.time() new_log = attributes_filter.apply_events(bpic2017_log, ["O_Create Offer"]) t1 = time.time() T10[0] = (t1 - t0) T10[2] = math.ceil(T10[1] / (T10[0] + 0.00000001) * 1000.0) print( "TEST 10 - Filter bpic2017 event log on event attributes - %.5f s (test score: %d)" % (T10[0], T10[2])) if DEBUG: F = open("debug.csv", "a") F.write("%.4f;%.4f;%.4f;%.4f;%.4f;%.4f;%.4f;%.4f;%.4f;%.4f\n" % (T1[0], T2[0], T3[0], T4[0], T5[0], T6[0], T7[0], T8[0], T9[0], T10[0])) F.close() scores = [