def get_log_summary(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
events = 0
cases = 0
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
events = events + len(df)
cases = cases + df[CASE_CONCEPT_NAME].nunique()
if count >= no_samples:
break
return {"events": events, "cases": cases}
def get_start_activities(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
overall_sa = Counter()
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
ea = Counter(start_activities_filter.get_start_activities(df, parameters=parameters))
overall_sa = overall_sa + ea
if count >= no_samples:
break
for el in overall_sa:
overall_sa[el] = int(overall_sa[el])
return dict(overall_sa)
def get_events_per_case(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
filters = parameters[FILTERS] if FILTERS in parameters else []
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
length_counter = Counter()
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
case_dict = dict(df.groupby(CASE_CONCEPT_NAME).size())
for case, occ in case_dict.items():
length_counter[int(occ)] += 1
if count >= no_samples:
break
return dict(length_counter)
def load_logs(self):
all_logs = MasterVariableContainer.dbmanager.get_logs_from_db()
for basepath in self.base_folders:
for folder in os.listdir(basepath):
if folder not in self.sublogs_id:
self.sublogs_id[folder] = {}
cpath = os.path.join(basepath, folder)
all_parquets = parquet_importer.get_list_parquet(cpath)
all_parquets_basepath = [
Path(x).name for x in all_parquets
]
for name in all_parquets_basepath:
if name in all_logs:
id = all_logs[name]
else:
id = [
randrange(0, 10),
randrange(0, 10),
randrange(0, 10),
randrange(0, 10),
randrange(0, 10),
randrange(0, 10),
randrange(0, 10)
]
MasterVariableContainer.dbmanager.insert_log_into_db(
name, id)
self.sublogs_id[folder][name] = id
MasterVariableContainer.first_loading_done = True
def get_variants(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
window_size = parameters[PARAMETER_NUM_RET_ITEMS] if PARAMETER_NUM_RET_ITEMS in parameters else DEFAULT_WINDOW_SIZE
start = parameters[PARAMETER_START] if PARAMETER_START in parameters else 0
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY, DEFAULT_TIMESTAMP_KEY],
use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
dictio_variants = {}
events = 0
cases = 0
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters,
force_classifier_insertion=True)
events = events + len(df)
cases = cases + df[CASE_CONCEPT_NAME].nunique()
# dictio = dictio + Counter(dict(df[attribute_key].value_counts()))
stats = case_statistics.get_variant_statistics(df)
d_variants = {x["variant"]: x for x in stats}
for variant in d_variants:
d_variants[variant]["count"] = d_variants[variant]["case:concept:name"]
del d_variants[variant]["case:concept:name"]
if not variant in dictio_variants:
dictio_variants[variant] = d_variants[variant]
else:
dictio_variants[variant]["count"] = dictio_variants[variant]["count"] + d_variants[variant]["count"]
list_variants = sorted(list(dictio_variants.values()), key=lambda x: x["count"], reverse=True)
list_variants = list_variants[start:min(len(list_variants), window_size)]
dictio_variants = {x["variant"]: x for x in list_variants}
if count >= no_samples:
break
list_variants = sorted(list(dictio_variants.values()), key=lambda x: x["count"], reverse=True)
return {"variants": list_variants, "events": events, "cases": cases}
def calculate_performance_dfg(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY, DEFAULT_TIMESTAMP_KEY],
use_transition=use_transition)
if pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY in parameters:
columns.append(parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY])
activity_key, parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = parameters[
pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY], activity_key
else:
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
frequency_dfg = Counter()
performance_dfg = Counter()
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
f_dfg, p_dfg = df_statistics.get_dfg_graph(df, activity_key=activity_key,
sort_timestamp_along_case_id=False, sort_caseid_required=False,
measure="both")
f_dfg = Counter(f_dfg)
for k in p_dfg:
if k not in performance_dfg:
performance_dfg[k] = p_dfg[k]
else:
performance_dfg[k] = (frequency_dfg[k] * performance_dfg[k] + f_dfg[k] * p_dfg[k]) / (
frequency_dfg[k] + f_dfg[k])
frequency_dfg = frequency_dfg + f_dfg
if count >= no_samples:
break
returned_dict = {}
for el in performance_dfg:
returned_dict[el[0] + "@@" + el[1]] = performance_dfg[el]
return returned_dict
def load_log(self, folder_name, log_name):
# print("loading log " + str(log_name)+" into "+str(folder_name))
if not os.path.exists(os.path.join(self.conf, folder_name, log_name)):
for folder in BASE_FOLDER_LIST_OPTIONS:
if folder_name in os.listdir(folder):
list_paths = parquet_importer.get_list_parquet(
os.path.join(folder, folder_name))
list_paths_corr = {}
for x in list_paths:
list_paths_corr[Path(x).name] = x
if log_name in list_paths_corr:
# print("log_name",log_name," in ",os.path.join(folder, folder_name),list_paths_corr[log_name])
shutil.copyfile(
list_paths_corr[log_name],
os.path.join(self.conf, folder_name, log_name))
def get_numeric_attribute_values(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
max_no_of_points_to_sample = parameters[
"max_no_of_points_to_sample"] if "max_no_of_points_to_sample" in parameters else 100000
attribute_key = parameters["attribute_key"]
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [attribute_key], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
overall_list = []
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
df = df.dropna()
if len(df) > max_no_of_points_to_sample:
df = df.sample(n=max_no_of_points_to_sample)
values = list(df[attribute_key])
overall_list = overall_list + values
if count >= no_samples:
break
overall_list = sorted(overall_list)
if len(overall_list) > max_no_of_points_to_sample:
overall_list = points_subset.pick_chosen_points_list(max_no_of_points_to_sample, overall_list)
return overall_list
def get_cases(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
window_size = parameters[PARAMETER_NUM_RET_ITEMS] if PARAMETER_NUM_RET_ITEMS in parameters else DEFAULT_WINDOW_SIZE
start = parameters[PARAMETER_START] if PARAMETER_START in parameters else 0
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY, DEFAULT_TIMESTAMP_KEY],
use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
cases_list = []
events = 0
cases = 0
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters,
force_classifier_insertion=True)
events = events + len(df)
cases = cases + df[CASE_CONCEPT_NAME].nunique()
stats = case_statistics.get_cases_description(df)
c_list = []
for x, y in stats.items():
c_list.append({"caseId": x, "caseDuration": y["caseDuration"], "startTime": y["startTime"],
"endTime": y["endTime"]})
cases_list = sorted(cases_list + c_list, key=lambda x: x["caseDuration"], reverse=True)
cases_list = cases_list[start:min(len(cases_list), window_size)]
if count >= no_samples:
break
return {"cases_list": cases_list, "events": events, "cases": cases}
def get_events(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
case_id = parameters["case_id"]
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
ret = []
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, None, filters, use_transition=use_transition, parameters=parameters)
try:
events = case_statistics.get_events(df, case_id)
if len(events) > 0:
df = parquet_importer.apply(pq)
ret = df[df[CASE_CONCEPT_NAME] == case_id].dropna(how="all", axis=1)
if activity_key != PARAMETER_PM4PYWS_CLASSIFIER:
ret[PARAMETER_PM4PYWS_CLASSIFIER] = ret[activity_key]
ret = ret.to_dict('r')
break
except:
pass
if count >= no_samples:
break
return ret
def get_case_duration(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
max_no_of_points_to_sample = parameters[
"max_no_of_points_to_sample"] if "max_no_of_points_to_sample" in parameters else 100000
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_TIMESTAMP_KEY], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
overall_list = []
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
cases = case_statistics.get_cases_description(df, parameters=parameters)
duration_values = [x["caseDuration"] for x in cases.values()]
overall_list = overall_list + duration_values
if count >= no_samples:
break
overall_list = sorted(overall_list)
if len(overall_list) > max_no_of_points_to_sample:
overall_list = points_subset.pick_chosen_points_list(max_no_of_points_to_sample, overall_list)
return overall_list
def do_caching(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = load_parquet_from_path(pq, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY, DEFAULT_TIMESTAMP_KEY], [],
use_transition=False, force_classifier_insertion=True,
force_timestamp_conversion=True)
PARQUET_CACHE[pq] = df
if count >= no_samples:
break
def get_attribute_names(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
names = set()
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
names = names.union(set(pqq.read_metadata(pq).schema.names))
if count >= no_samples:
break
names = sorted(list(names))
names = [x.replace("AAA", ":") for x in names]
return sorted(list(names))
def get_attribute_values(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
attribute_key = parameters[
pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] if pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY in parameters else DEFAULT_NAME_KEY
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, attribute_key], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
dictio = Counter({})
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
dictio = dictio + Counter(dict(df[attribute_key].value_counts()))
if count >= no_samples:
break
dictio = dict(dictio)
for el in dictio:
dictio[el] = int(dictio[el])
return dictio
def get_list_logs(self):
lp = parquet_importer.get_list_parquet(self.distr_log_path)
return [Path(log).name for log in lp]
def correlation_miner(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
activities = parameters["activities"] if "activities" in parameters else None
complete_timestamp = parameters["complete_timestamp"] if "complete_timestamp" in parameters else DEFAULT_TIMESTAMP_KEY
start_timestamp = parameters["start_timestamp"] if "start_timestamp" in parameters else DEFAULT_TIMESTAMP_KEY
from pm4py.algo.discovery.correlation_mining.variants import classic
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
parameters[classic.Parameters.ACTIVITY_KEY] = activity_key
parameters[classic.Parameters.TIMESTAMP_KEY] = complete_timestamp
parameters[classic.Parameters.START_TIMESTAMP_KEY] = start_timestamp
folder = os.path.join(path, log_name)
columns = get_columns_to_import(filters, [activity_key, complete_timestamp, start_timestamp], use_transition=use_transition)
parquet_list = parquet_importer.get_list_parquet(folder)
PS_matrixes = []
duration_matrixes = []
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
transf_stream, activities_grouped, activities = classic.preprocess_log(df, activities=activities,
parameters=parameters)
PS_matrix, duration_matrix = classic.get_PS_dur_matrix(activities_grouped, activities,
parameters=parameters)
PS_matrixes.append(PS_matrix)
duration_matrixes.append(duration_matrix)
if count >= no_samples:
break
PS_matrix = np.zeros((len(activities), len(activities)))
duration_matrix = np.zeros((len(activities), len(activities)))
z = 0
while z < len(PS_matrixes):
PS_matrix = PS_matrix + PS_matrixes[z]
duration_matrix = np.maximum(duration_matrix, duration_matrixes[z])
z = z + 1
PS_matrix = PS_matrix / float(len(PS_matrixes))
PS_matrix = PS_matrix.tolist()
duration_matrix = duration_matrix.tolist()
return {"PS_matrix": json.dumps(PS_matrix), "duration_matrix": json.dumps(duration_matrix)}
def get_events_per_dotted(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
max_no_events = parameters["max_no_events"] if "max_no_events" in parameters else 10000
attributes = ["@@event_index", parameters["attribute1"], parameters["attribute2"]]
if parameters["attribute3"] is not None:
attributes.append(parameters["attribute3"])
attributes1 = list(set([CASE_CONCEPT_NAME] + [x for x in list(set(attributes)) if not x.startswith("@@")]))
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_TIMESTAMP_KEY] + attributes1,
use_transition=use_transition)
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
df_list = []
no_events = 0
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
df = df[attributes1].dropna()
no_events = no_events + len(df)
df_list.append(df)
if no_events >= max_no_events:
break
if count >= no_samples:
break
df = pd.concat(df_list)
df["@@event_index"] = df.index
df = df.sort_values([DEFAULT_TIMESTAMP_KEY, "@@event_index"])
df = df.reset_index(drop=True)
df["@@case_index"] = df.groupby(CASE_CONCEPT_NAME, sort=False).ngroup()
df = df.sort_values(["@@case_index", DEFAULT_TIMESTAMP_KEY, "@@event_index"])
stream = df.to_dict('r')
stream = sorted(stream, key=lambda x: (x[attributes[2]], x[attributes[1]], x[attributes[0]]))
third_unique_values = []
if len(attributes) > 3:
third_unique_values = sorted(list(set(s[attributes[3]] for s in stream)))
types = {}
if stream:
for attr in attributes:
val = stream[0][attr]
types[attr] = str(type(val))
if type(val) is pd._libs.tslibs.timestamps.Timestamp:
for ev in stream:
ev[attr] = ev[attr].timestamp()
traces = []
if third_unique_values:
for index, v in enumerate(third_unique_values):
traces.append({})
for index2, attr in enumerate(attributes):
if index2 < len(attributes) - 1:
traces[-1][attr] = [s[attr] for s in stream if s[attributes[3]] == v]
else:
third_unique_values.append("UNIQUE")
traces.append({})
for index2, attr in enumerate(attributes):
traces[-1][attr] = [s[attr] for s in stream]
return traces, types, attributes, third_unique_values
def calculate_process_schema_composite_object(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
performance_required = parameters["performance_required"] if "performance_required" in parameters else False
no_samples = parameters[PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
if performance_required:
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY, DEFAULT_TIMESTAMP_KEY],
use_transition=use_transition)
else:
columns = get_columns_to_import(filters, [CASE_CONCEPT_NAME, DEFAULT_NAME_KEY, DEFAULT_TIMESTAMP_KEY], use_transition=use_transition)
if pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY in parameters:
columns.append(parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY])
activity_key, parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = parameters[
pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY], activity_key
else:
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
frequency_dfg = Counter()
performance_dfg = Counter()
overall_ea = Counter()
overall_sa = Counter()
values = Counter({})
events = 0
cases = 0
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq, columns, filters, use_transition=use_transition, parameters=parameters)
if performance_required:
f_dfg, p_dfg = df_statistics.get_dfg_graph(df, activity_key=activity_key,
sort_timestamp_along_case_id=False,
sort_caseid_required=False, measure="both")
else:
f_dfg = df_statistics.get_dfg_graph(df, activity_key=activity_key, sort_timestamp_along_case_id=False,
sort_caseid_required=False)
f_dfg = Counter(f_dfg)
if performance_required:
for k in p_dfg:
if k not in performance_dfg:
performance_dfg[k] = p_dfg[k]
else:
performance_dfg[k] = (frequency_dfg[k] * performance_dfg[k] + f_dfg[k] * p_dfg[k]) / (
frequency_dfg[k] + f_dfg[k])
frequency_dfg = frequency_dfg + f_dfg
ea = Counter(end_activities_filter.get_end_activities(df, parameters=parameters))
overall_ea = overall_ea + ea
sa = Counter(start_activities_filter.get_start_activities(df, parameters=parameters))
overall_sa = overall_sa + sa
values = values + Counter(dict(df[activity_key].value_counts()))
events = events + len(df)
cases = cases + df[CASE_CONCEPT_NAME].nunique()
if count >= no_samples:
break
returned_dict = {}
returned_dict["events"] = events
returned_dict["cases"] = cases
values = dict(values)
for el in values:
values[el] = int(values[el])
returned_dict["activities"] = values
overall_sa = dict(overall_sa)
for el in overall_sa:
overall_sa[el] = int(overall_sa[el])
returned_dict["start_activities"] = overall_sa
overall_ea = dict(overall_ea)
for el in overall_ea:
overall_ea[el] = int(overall_ea[el])
returned_dict["end_activities"] = overall_ea
returned_dict_freq = {}
for el in frequency_dfg:
returned_dict_freq[el[0] + "@@" + el[1]] = int(frequency_dfg[el])
returned_dict["frequency_dfg"] = returned_dict_freq
if performance_required:
returned_dict_perf = {}
for el in performance_dfg:
returned_dict_perf[el[0] + "@@" + el[1]] = float(performance_dfg[el])
returned_dict["performance_dfg"] = returned_dict_perf
return returned_dict