def rebase(
log_obj: Union[EventLog, EventStream, pd.DataFrame],
case_id: str = constants.CASE_CONCEPT_NAME,
activity_key: str = xes_constants.DEFAULT_NAME_KEY,
timestamp_key: str = xes_constants.DEFAULT_TIMESTAMP_KEY,
start_timestamp_key: str = xes_constants.DEFAULT_START_TIMESTAMP_KEY):
"""
Re-base the log object, changing the case ID, activity and timestamp attributes.
Parameters
-----------------
log_obj
Log object
case_id
Case identifier
activity_key
Activity
timestamp_key
Timestamp
start_timestamp_key
Start timestamp
Returns
-----------------
rebased_log_obj
Rebased log object
"""
import pm4py
if isinstance(log_obj, pd.DataFrame):
return format_dataframe(log_obj,
case_id=case_id,
activity_key=activity_key,
timestamp_key=timestamp_key,
start_timestamp_key=start_timestamp_key)
elif isinstance(log_obj, EventLog):
log_obj = pm4py.convert_to_dataframe(log_obj)
log_obj = format_dataframe(log_obj,
case_id=case_id,
activity_key=activity_key,
timestamp_key=timestamp_key,
start_timestamp_key=start_timestamp_key)
from pm4py.objects.conversion.log import converter
return converter.apply(log_obj,
variant=converter.Variants.TO_EVENT_LOG)
elif isinstance(log_obj, EventStream):
log_obj = pm4py.convert_to_dataframe(log_obj)
log_obj = format_dataframe(log_obj,
case_id=case_id,
activity_key=activity_key,
timestamp_key=timestamp_key,
start_timestamp_key=start_timestamp_key)
return pm4py.convert_to_event_stream(log_obj)
def execute_script():
# loads a XES event log
event_log = pm4py.read_xes("../tests/input_data/receipt.xes")
# gets the dataframe out of the event log (through conversion)
dataframe = pm4py.convert_to_dataframe(event_log)
# discovers the log skeleton model
log_skeleton = log_skeleton_discovery.apply(
event_log,
parameters={
log_skeleton_discovery.Variants.CLASSIC.value.Parameters.NOISE_THRESHOLD:
0.03
})
# apply conformance checking
conf_result = log_skeleton_conformance.apply(event_log, log_skeleton)
# gets the diagnostic result out of the dataframe
diagnostics = log_skeleton_conformance.get_diagnostics_dataframe(
event_log, conf_result)
# merges the dataframe containing the events, and the diagnostics dataframe, using the pd.merge method
merged_df = pd.merge(dataframe,
diagnostics,
how="left",
left_on="case:concept:name",
right_on="case_id",
suffixes=('', '_diagn'))
print(merged_df)
def test_csv(self):
df = pd.read_csv("input_data/running-example.csv")
df = pm4py.format_dataframe(df, case_id="case:concept:name", activity_key="concept:name",
timestamp_key="time:timestamp")
log2 = pm4py.convert_to_event_log(df)
stream1 = pm4py.convert_to_event_stream(log2)
df2 = pm4py.convert_to_dataframe(log2)
pm4py.write_xes(log2, "test_output_data/log.xes")
os.remove("test_output_data/log.xes")
def execute_script():
log_path = os.path.join("..", "tests", "input_data",
"interval_event_log.csv")
dataframe = pm4py.read_csv(log_path)
log_path = os.path.join("..", "tests", "input_data", "reviewing.xes")
log = pm4py.read_xes(log_path)
dataframe = pm4py.convert_to_dataframe(log)
parameters = {}
#parameters[constants.PARAMETER_CONSTANT_START_TIMESTAMP_KEY] = "start_timestamp"
parameters[constants.PARAMETER_CONSTANT_TIMESTAMP_KEY] = "time:timestamp"
parameters[constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = "concept:name"
parameters[constants.PARAMETER_CONSTANT_CASEID_KEY] = "case:concept:name"
parameters["strict"] = True
parameters["format"] = "svg"
start_activities = sa_get.get_start_activities(dataframe,
parameters=parameters)
end_activities = ea_get.get_end_activities(dataframe,
parameters=parameters)
att_count = att_get.get_attribute_values(dataframe,
"concept:name",
parameters=parameters)
parameters["start_activities"] = start_activities
parameters["end_activities"] = end_activities
soj_time = soj_time_get.apply(dataframe, parameters=parameters)
print("soj_time")
print(soj_time)
conc_act = conc_act_get.apply(dataframe, parameters=parameters)
print("conc_act")
print(conc_act)
efg = efg_get.apply(dataframe, parameters=parameters)
print("efg")
print(efg)
dfg_freq, dfg_perf = df_statistics.get_dfg_graph(
dataframe, measure="both", start_timestamp_key="start_timestamp")
dfg_gv_freq = dfg_vis_fact.apply(dfg_freq,
activities_count=att_count,
variant=dfg_vis_fact.Variants.FREQUENCY,
soj_time=soj_time,
parameters=parameters)
dfg_vis_fact.view(dfg_gv_freq)
dfg_gv_perf = dfg_vis_fact.apply(dfg_perf,
activities_count=att_count,
variant=dfg_vis_fact.Variants.PERFORMANCE,
soj_time=soj_time,
parameters=parameters)
dfg_vis_fact.view(dfg_gv_perf)
net, im, fm = dfg_conv.apply(dfg_freq)
gviz = pn_vis.apply(net, im, fm, parameters=parameters)
pn_vis.view(gviz)
def execute_script():
ENABLE_VISUALIZATION = True
# reads a XES into an event log
log1 = pm4py.read_xes("../tests/input_data/running-example.xes")
# reads a CSV into a dataframe
df = pd.read_csv("../tests/input_data/running-example.csv")
# formats the dataframe with the mandatory columns for process mining purposes
df = pm4py.format_dataframe(df,
case_id="case:concept:name",
activity_key="concept:name",
timestamp_key="time:timestamp")
# converts the dataframe to an event log
log2 = pm4py.convert_to_event_log(df)
# converts the log read from XES into a stream and dataframe respectively
stream1 = pm4py.convert_to_event_stream(log1)
df2 = pm4py.convert_to_dataframe(log1)
# writes the log1 to a XES file
pm4py.write_xes(log1, "ru1.xes")
dfg, dfg_sa, dfg_ea = pm4py.discover_dfg(log1)
petri_alpha, im_alpha, fm_alpha = pm4py.discover_petri_net_alpha(log1)
petri_inductive, im_inductive, fm_inductive = pm4py.discover_petri_net_inductive(
log1)
petri_heuristics, im_heuristics, fm_heuristics = pm4py.discover_petri_net_heuristics(
log1)
tree_inductive = pm4py.discover_tree_inductive(log1)
heu_net = pm4py.discover_heuristics_net(log1)
pm4py.write_dfg(dfg, dfg_sa, dfg_ea, "ru_dfg.dfg")
pm4py.write_petri_net(petri_alpha, im_alpha, fm_alpha, "ru_alpha.pnml")
pm4py.write_petri_net(petri_inductive, im_inductive, fm_inductive,
"ru_inductive.pnml")
pm4py.write_petri_net(petri_heuristics, im_heuristics, fm_heuristics,
"ru_heuristics.pnml")
pm4py.write_process_tree(tree_inductive, "ru_inductive.ptml")
dfg, dfg_sa, dfg_ea = pm4py.read_dfg("ru_dfg.dfg")
petri_alpha, im_alpha, fm_alpha = pm4py.read_petri_net("ru_alpha.pnml")
petri_inductive, im_inductive, fm_inductive = pm4py.read_petri_net(
"ru_inductive.pnml")
petri_heuristics, im_heuristics, fm_heuristics = pm4py.read_petri_net(
"ru_heuristics.pnml")
tree_inductive = pm4py.read_process_tree("ru_inductive.ptml")
pm4py.save_vis_petri_net(petri_alpha, im_alpha, fm_alpha, "ru_alpha.png")
pm4py.save_vis_petri_net(petri_inductive, im_inductive, fm_inductive,
"ru_inductive.png")
pm4py.save_vis_petri_net(petri_heuristics, im_heuristics, fm_heuristics,
"ru_heuristics.png")
pm4py.save_vis_process_tree(tree_inductive, "ru_inductive_tree.png")
pm4py.save_vis_heuristics_net(heu_net, "ru_heunet.png")
pm4py.save_vis_dfg(dfg, dfg_sa, dfg_ea, "ru_dfg.png")
if ENABLE_VISUALIZATION:
pm4py.view_petri_net(petri_alpha, im_alpha, fm_alpha, format="svg")
pm4py.view_petri_net(petri_inductive,
im_inductive,
fm_inductive,
format="svg")
pm4py.view_petri_net(petri_heuristics,
im_heuristics,
fm_heuristics,
format="svg")
pm4py.view_process_tree(tree_inductive, format="svg")
pm4py.view_heuristics_net(heu_net, format="svg")
pm4py.view_dfg(dfg, dfg_sa, dfg_ea, format="svg")
aligned_traces = pm4py.conformance_alignments(log1, petri_inductive,
im_inductive, fm_inductive)
replayed_traces = pm4py.conformance_tbr(log1, petri_inductive,
im_inductive, fm_inductive)
fitness_tbr = pm4py.evaluate_fitness_tbr(log1, petri_inductive,
im_inductive, fm_inductive)
print("fitness_tbr", fitness_tbr)
fitness_align = pm4py.evaluate_fitness_alignments(log1, petri_inductive,
im_inductive,
fm_inductive)
print("fitness_align", fitness_align)
precision_tbr = pm4py.evaluate_precision_tbr(log1, petri_inductive,
im_inductive, fm_inductive)
print("precision_tbr", precision_tbr)
precision_align = pm4py.evaluate_precision_alignments(
log1, petri_inductive, im_inductive, fm_inductive)
print("precision_align", precision_align)
print("log start activities = ", pm4py.get_start_activities(log2))
print("df start activities = ", pm4py.get_start_activities(df2))
print("log end activities = ", pm4py.get_end_activities(log2))
print("df end activities = ", pm4py.get_end_activities(df2))
print("log attributes = ", pm4py.get_attributes(log2))
print("df attributes = ", pm4py.get_attributes(df2))
print("log org:resource values = ",
pm4py.get_attribute_values(log2, "org:resource"))
print("df org:resource values = ",
pm4py.get_attribute_values(df2, "org:resource"))
print("start_activities len(filt_log) = ",
len(pm4py.filter_start_activities(log2, ["register request"])))
print("start_activities len(filt_df) = ",
len(pm4py.filter_start_activities(df2, ["register request"])))
print("end_activities len(filt_log) = ",
len(pm4py.filter_end_activities(log2, ["pay compensation"])))
print("end_activities len(filt_df) = ",
len(pm4py.filter_end_activities(df2, ["pay compensation"])))
print(
"attributes org:resource len(filt_log) (cases) cases = ",
len(
pm4py.filter_attribute_values(log2,
"org:resource", ["Ellen"],
level="case")))
print(
"attributes org:resource len(filt_log) (cases) events = ",
len(
pm4py.filter_attribute_values(log2,
"org:resource", ["Ellen"],
level="event")))
print(
"attributes org:resource len(filt_df) (events) cases = ",
len(
pm4py.filter_attribute_values(df2,
"org:resource", ["Ellen"],
level="case")))
print(
"attributes org:resource len(filt_df) (events) events = ",
len(
pm4py.filter_attribute_values(df2,
"org:resource", ["Ellen"],
level="event")))
print(
"attributes org:resource len(filt_df) (events) events notpositive = ",
len(
pm4py.filter_attribute_values(df2,
"org:resource", ["Ellen"],
level="event",
retain=False)))
print("variants log = ", pm4py.get_variants(log2))
print("variants df = ", pm4py.get_variants(df2))
print(
"variants filter log = ",
len(
pm4py.filter_variants(log2, [[
"register request", "examine thoroughly", "check ticket",
"decide", "reject request"
]])))
print(
"variants filter df = ",
len(
pm4py.filter_variants(df2, [[
"register request", "examine thoroughly", "check ticket",
"decide", "reject request"
]])))
print("variants filter percentage = ",
len(pm4py.filter_variants_percentage(log2, threshold=0.8)))
print(
"paths filter log len = ",
len(
pm4py.filter_directly_follows_relation(
log2, [("register request", "examine casually")])))
print(
"paths filter dataframe len = ",
len(
pm4py.filter_directly_follows_relation(
df2, [("register request", "examine casually")])))
print(
"timeframe filter log events len = ",
len(
pm4py.filter_time_range(log2,
"2011-01-01 00:00:00",
"2011-02-01 00:00:00",
mode="events")))
print(
"timeframe filter log traces_contained len = ",
len(
pm4py.filter_time_range(log2,
"2011-01-01 00:00:00",
"2011-02-01 00:00:00",
mode="traces_contained")))
print(
"timeframe filter log traces_intersecting len = ",
len(
pm4py.filter_time_range(log2,
"2011-01-01 00:00:00",
"2011-02-01 00:00:00",
mode="traces_intersecting")))
print(
"timeframe filter df events len = ",
len(
pm4py.filter_time_range(df2,
"2011-01-01 00:00:00",
"2011-02-01 00:00:00",
mode="events")))
print(
"timeframe filter df traces_contained len = ",
len(
pm4py.filter_time_range(df2,
"2011-01-01 00:00:00",
"2011-02-01 00:00:00",
mode="traces_contained")))
print(
"timeframe filter df traces_intersecting len = ",
len(
pm4py.filter_time_range(df2,
"2011-01-01 00:00:00",
"2011-02-01 00:00:00",
mode="traces_intersecting")))
# remove the temporary files
os.remove("ru1.xes")
os.remove("ru_dfg.dfg")
os.remove("ru_alpha.pnml")
os.remove("ru_inductive.pnml")
os.remove("ru_heuristics.pnml")
os.remove("ru_inductive.ptml")
os.remove("ru_alpha.png")
os.remove("ru_inductive.png")
os.remove("ru_heuristics.png")
os.remove("ru_inductive_tree.png")
os.remove("ru_heunet.png")
os.remove("ru_dfg.png")
def create_plotting_data(log, file_format, log_information):
"""
Transforms a log, such that it can be easer
used for plotting, removes unnecessary data,
creates df from xes data, and renames columns
input: XES/CSV log ,
file_format str,
log_information django session dict
output: pandas df with pm4py default names for attributes in a two timestamp format
"""
if file_format == "csv":
# Select only the Relevant columns of the Dataframe
if log_information["log_type"] == "noninterval":
# Project the Log onto the Group activities
log = log[[
"case:concept:name",
xes.DEFAULT_TIMESTAMP_KEY,
xes.DEFAULT_TRACEID_KEY,
]]
elif log_information["log_type"] == "lifecycle":
log = log[[
"case:concept:name",
xes.DEFAULT_TIMESTAMP_KEY,
xes.DEFAULT_TRACEID_KEY,
xes.DEFAULT_TRANSITION_KEY,
]]
log = data_transform.transform_lifecycle_csv_to_interval_csv(log)
elif log_information["log_type"] == "timestamp":
log = log[[
"case:concept:name",
xes.DEFAULT_TIMESTAMP_KEY,
xes.DEFAULT_TRACEID_KEY,
xes.DEFAULT_START_TIMESTAMP_KEY,
]]
log = log.rename(
{
log_information["start_timestamp"]:
xes.DEFAULT_START_TIMESTAMP_KEY,
log_information["end_timestamp"]:
xes.DEFAULT_TIMESTAMP_KEY,
},
axis=1,
)
# Simply load the log using XES
elif file_format == "xes":
log = convert_to_dataframe(log)
if log_information["log_type"] == "noninterval":
log[log_information["timestamp"]] = pd.to_datetime(
log[log_information["timestamp"]], utc=True)
log = log[[
"case:concept:name",
xes.DEFAULT_TIMESTAMP_KEY,
xes.DEFAULT_TRACEID_KEY,
]]
# Transform the Timestamp to Datetime, and rename the transition:lifecycle
elif log_information["log_type"] == "lifecycle":
# Convert the Timestamps to Datetime
log[log_information["timestamp"]] = pd.to_datetime(
log[log_information["timestamp"]], utc=True)
# Rename the Columns to the XES defaults
log = log.rename(
{log_information["lifecycle"]: xes.DEFAULT_TRANSITION_KEY},
axis=1)
log = log[[
"case:concept:name",
xes.DEFAULT_TIMESTAMP_KEY,
xes.DEFAULT_TRACEID_KEY,
xes.DEFAULT_TRANSITION_KEY,
]]
log = data_transform.transform_lifecycle_csv_to_interval_csv(log)
elif log_information["log_type"] == "timestamp":
# Convert the Timestamps to Datetime
log[log_information["end_timestamp"]] = pd.to_datetime(
log[log_information["end_timestamp"]], utc=True)
log[log_information["start_timestamp"]] = pd.to_datetime(
log[log_information["start_timestamp"]], utc=True)
log = log[[
"case:concept:name",
xes.DEFAULT_TIMESTAMP_KEY,
xes.DEFAULT_TRACEID_KEY,
xes.DEFAULT_START_TIMESTAMP_KEY,
]]
log = log.rename(
{
log_information["start_timestamp"]:
xes.DEFAULT_START_TIMESTAMP_KEY,
log_information["end_timestamp"]:
xes.DEFAULT_TIMESTAMP_KEY,
},
axis=1,
)
return log
def lstm_algorithm(index, len_of_points, resample_dataset, LSTM_CELLS):
#print('Current path: ',os.getcwd())
if not os.path.exists('results'):
os.makedirs('results')
COLOR_CYCLE = ["#4286f4", "#f44174"]
split_percentage = 0.8
"""
answer = input('Give me the length of window: ')
if answer == 'max':
len_of_points = 0
else:
len_of_points = int(answer)
#len_of_points = 3
"""
fileinfo = {
0: {
'filename': 'base_kasteren.csv',
'separator': ' ',
'columns':
['date', 'time', 'attr1', 'attr2', 'state', 'concept:name']
},
1: {
'filename':
'activity3.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
2: {
'filename': 'activitylog_uci_detailed_labour.xes',
'separator': '',
'columns': []
},
3: {
'filename': 'atmo1.csv',
'separator': ' ',
'columns': ['date', 'time', 'concept:name', 'state', 'activity']
},
4: {
'filename':
'activity1.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
5: {
'filename':
'activity2.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
6: {
'filename': 'espa.xes',
'separator': ';',
'columns': []
},
7: {
'filename':
'activity3.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
8: {
'filename':
'activity4.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
9: {
'filename':
'activity5.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
10: {
'filename':
'activity6.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
11: {
'filename':
'activity7.csv',
'separator':
',',
'columns': [
'id', 'case:concept:name', 'subjectID', 'attr_starttime',
'time:timestamp', 'concept:name', 'label_subactivity'
]
},
12: {
'filename': 'BPI_Challenge_2017.xes',
'separator': '',
'columns': []
},
}
#choose file
filename = fileinfo[index]['filename']
filepath = '../datasets/' + fileinfo[index]['filename']
dataframe = pd.DataFrame()
if not os.path.exists('results/' + filename):
os.makedirs('results/' + filename)
if not os.path.exists('results/' + filename + '/' + str(len_of_points) +
'/'):
os.makedirs('results/' + filename + '/' + str(len_of_points) + '/')
#if it is a csv file
if (filename.find('.csv') != -1):
#load file to dataframe
dataframe = pd.read_csv(filepath,
sep=fileinfo[index]['separator'],
names=fileinfo[index]['columns'],
low_memory=False)
#for Kastern dataset prepare columns
if index in [0, 3, 12]:
dataframe[
'time:timestamp'] = dataframe['date'] + ' ' + dataframe['time']
dataframe['case:concept:name'] = dataframe['date']
#dataframe = dataframe[dataframe['concept:name']!='None']
#print ("file is csv ")
#print(dataframe.head(20))
#drop nan
#convert csv to xes
log = pm4py.convert_to_event_log(dataframe)
else:
#the file is xes
#import log
#xes_importer.iterparse.Parameters.MAX_TRACES = 10
#parameters = {xes_importer.iterparse.Parameters.MAX_TRACES: 50}
#log = xes_importer.apply('datasets/BPI Challenge 2018.xes.gz', parameters=parameters)
log = pm4py.read_xes(filepath)
print(log)
#convert to dataframe
dataframe = pm4py.convert_to_dataframe(log)
print(dataframe)
#print(dataframe['time:timestamp'][0].replace(tzinfo=timezone.utc).astimezone(tz=None))
#dataframe['time:timestamp'] = dataframe['time:timestamp'].dt.tz_convert(None)
if index in [2, 12]:
#process time:timestamp remove zone information
dataframe['time:timestamp'] = dataframe[
'time:timestamp'].dt.tz_convert(None)
#del log
print('Dataframe print\n', dataframe)
#get only start events if lifecycle:transition if column does not exists create it
if 'lifecycle:transition' in dataframe.columns:
dataframe = dataframe[dataframe['lifecycle:transition'] == 'complete']
else:
dataframe['lifecycle:transition'] = 'complete'
#remove Start and End events
dataframe = dataframe[dataframe['concept:name'] != 'Start']
dataframe = dataframe[dataframe['concept:name'] != 'End']
#sort by time
if 'time:timestamp' in dataframe.columns:
dataframe = dataframe.sort_values('time:timestamp')
else:
print('Error: no column time:timestamp in event log')
#print('Sorted dataframe\n',dataframe)
#plot time vs activity
#fig, axes = plt.subplots(1, 1, figsize=(100, 100))
#fig = dataframe.plot(x='time:timestamp', y='concept:name', kind="scatter").get_figure()
#fig.savefig('results/'+filename+'/'+str(len_of_points) +'/conceptname.png', bbox_inches='tight')
#plot time vs trace id
#df.plot(x='col_name_1', y='col_name_2', style='o')
#fig = dataframe.plot(x='time:timestamp', y='case:concept:name', kind="scatter").get_figure()
#fig.savefig('results/'+filename+'/'+str(len_of_points) +'/caseconceptname.png', bbox_inches='tight')
#keep only mandatory columns
dataframe = dataframe[[
'case:concept:name', 'concept:name', 'time:timestamp'
]]
#convert sorted dataframe to log
log = pm4py.convert_to_event_log(dataframe)
#initial_df = dataframe.copy()
#print('Initial dataframe\n',initial_df)
#-----------------------------------------------------------------
############################################################
#-------------- Resample -----------------------------------
###########################################################
if resample_dataset:
#preprocess timestamp to be prepared for resample
#make time:timestamp datetime
dataframe.loc[:, 'time:timestamp'] = pd.to_datetime(
dataframe['time:timestamp'])
#set time:timestamp as index
dataframe = dataframe.set_index(["time:timestamp"])
#remove duplicates
#print('Duplicated\n')
#print(dataframe[dataframe.index.duplicated()])
dataframe = dataframe[~dataframe.index.duplicated(keep='first')]
#------Reasample dataframe every 5min keep last value found if Nan-------------
dataframe = dataframe.resample("5T").fillna("backfill")
print('Resample', dataframe)
#print( dataframe.last())
#save resampled dataframe to csv
dataframe.to_csv('../datasets/resampled_sorted_df.csv')
#dataframe is initial event log sorted by time (start event only)
#convert sorted by time dataframe back to log (xes)
#log = pm4py.convert_to_event_log(dataframe)
#-----------------------------------save to csv-------------------------------------
#uncomment only if you need it
#dataframe.to_csv('datasets/activitylog_uci_detailed_labour.csv')
#print('\nDataframe LOG\n',dataframe)
#--------------- Concat activities of a trace in one row ---------------
#concat events with same case:concept:name (space separated)
print('dataframe\n', dataframe)
df = dataframe.groupby('case:concept:name', sort=False).agg(
{'concept:name': lambda x: ' '.join(x)})
print('df\n', df)
df = df.reset_index()
if len_of_points:
print('--------------------------------------')
df['concept:name'] = df['concept:name'].apply(
lambda x: list(x.split(' ')))
df['concept:name'] = df['concept:name'].apply(
lambda x:
[x[i:i + len_of_points] for i in range(0, len(x), len_of_points)])
df = df.set_index('case:concept:name')['concept:name'].apply(
pd.Series).stack().reset_index(level=0).rename(
columns={0: 'concept:name'})
df['concept:name'] = df['concept:name'].apply(lambda x: ' '.join(x))
print('\ndftest\n', df)
df = df.reset_index() #check here
#del dataframe
#print the activities of the log
activities = pm4py.get_attribute_values(log, 'concept:name')
print('\nActivities:\n', activities)
#split data from event log - 80% for train and 20% for test
#shuffle before split
#df = shuffle(df)
#print('df', df,'\n')
#----------------------- Split Train and Test data ----------------------
#split rows depending on percentage
split_rows = int(df.shape[0] * split_percentage)
print('Split Rows', split_rows, '\n')
#train dataframe
train_df = df[:split_rows]
train_df.to_csv('train.csv')
print('Train Rows', train_df, '\n')
#test dataframe
test_df = df[split_rows:]
test_df.to_csv('test.csv')
#print('Test Rows', test_df,'\n')
# --------------------------------------------------------------
#data = df['concept:name'].copy().to_list()
data = train_df['concept:name'].copy().to_list()
#Just for Eating/Drinking
#data = data.replace('Eating/Drinking','EatDrink')
#print('Data\n',data)
tokenizer = Tokenizer()
#reads the words in data and gives an index for every words based on frequency
tokenizer.fit_on_texts([data])
print('Word index: ')
print(tokenizer.word_index)
#replace every word in the text to correspoding word index - returns list of list with one element so use [0] to get the one and only first list
encoded = tokenizer.texts_to_sequences([data])[0]
#print('encoded: \n')
#print(encoded)
vocab_size = len(tokenizer.word_index) + 1
print('Vocabulary Size: %d' % vocab_size)
#print('list\n',[e for e in encoded])
#print('Min ',min([len(e) for e in encoded]))
# LSTM 3 timesteps - prepare data - encode 2 words -> 1 word
sequences = list()
for i in range(n_input, len(encoded)):
sequence = encoded[i - n_input:i + 1]
sequences.append(sequence)
print('Total Sequences: %d' % len(sequences))
print('Sequences: \n')
print(sequences)
max_length = max([len(seq) for seq in sequences]) #max_length is 3
# Pad sequence to be of the same length
# length of sequence must be 3 (maximum)
# 'pre' or 'post': pad either before or after each sequence
sequences = pad_sequences(sequences, maxlen=max_length, padding='pre')
print('Max Sequence Length: %d' % max_length)
#convert list to array to get X,y train
sequences = array(sequences)
X, y = sequences[:, :-1], sequences[:, -1]
print('X: \n')
print(X)
print('y: \n')
print(y)
#convert y to binary vectors
y = to_categorical(y, num_classes=vocab_size)
print('y: \n')
print(y)
#test data
test_data = test_df['concept:name'].copy().to_list()
test_encoded = tokenizer.texts_to_sequences([test_data])[0]
test_sequences = list()
for i in range(n_input, len(test_encoded)):
test_sequence = test_encoded[i - n_input:i + 1]
test_sequences.append(test_sequence)
max_length = max([len(seq) for seq in test_sequences])
test_sequences = pad_sequences(test_sequences,
maxlen=max_length,
padding='pre')
test_sequences = array(test_sequences)
test_X, test_y = test_sequences[:, :-1], test_sequences[:, -1]
#convert y to binary vectors
test_yl = to_categorical(test_y, num_classes=vocab_size)
model = Sequential()
#the first layer
# - the largest integer (i.e. word index) in the input should be no larger than vocabulary size
# - The Embedding layer is initialized with random weights and will learn an embedding for all of the words in the training dataset.
# - output_dim (50): This is the size of the vector space in which words will be embedded (size of the embedding vectors). It defines the size of the output vectors from this layer for each word. For example, it could be 32 or 100 or even larger. Test different values for your problem.
# - input_length: This is the length of input sequences (here is 2)
# The Embedding layer has weights that are learned. If you save your model to file, this will include weights for the Embedding layer.
# The output of the Embedding layer is a 2D vector with one embedding for each word in the input sequence of words (input document).
# If you wish to connect a Dense layer directly to an Embedding layer, you must first flatten the 2D output matrix to a 1D vector using the Flatten layer.
model.add(
Embedding(vocab_size + 1, LSTM_CELLS, input_length=max_length - 1))
model.add(LSTM(vocab_size))
model.add(Dropout(0.1))
model.add(Dense(vocab_size, activation='softmax'))
opt = Adam(learning_rate=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
history = model.fit(X,
y,
epochs=500,
verbose=0,
batch_size=20,
validation_data=(test_X, test_yl))
print(model.summary())
model.save('lstm_model.h5') # creates a HDF5 file
#del model # deletes the existing model
#predict sequence of n_words activities
def generate_seq(model, tokenizer, max_length, seed_text, n_words):
#get input activity
in_text = seed_text
#print('in_text',in_text,'\n')
#for the number of activities on sequence you want to predict
for _ in range(n_words):
encoded = tokenizer.texts_to_sequences([in_text])[0]
#pad if less than max text length
encoded = pad_sequences([encoded],
maxlen=max_length,
padding='pre')
#print('in text ',in_text)
#predict one activity
#yhat = model.predict_classes(encoded, verbose=0)
yhat = np.argmax(model.predict(encoded), axis=-1)
out_word = ''
for word, index in tokenizer.word_index.items():
#convert predicted activity to word
if index == yhat:
#print('Word',word,'\n')
out_word = word
break
#feed the next input with the sequence of activities
in_text += ' ' + out_word
return in_text
#load trained model
#model = load_model('lstm_model.h5')
# Evaluate network
print('LSTM Network Evaluation:\n')
train_score = model.evaluate(X, y, verbose=0)
print('Train Score\n', train_score)
score = model.evaluate(test_X, test_yl, verbose=0)
print('Test Score\n')
print(score)
print('History\n')
print(history.history.keys())
# plot loss during training
fig = plt.figure()
plt.subplot(211)
plt.title('Loss')
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
fig.savefig('results/' + filename + '/' + str(len_of_points) + '/Loss.png',
bbox_inches='tight')
# plot accuracy during training
fig = plt.figure()
plt.subplot(212)
plt.title('Accuracy')
plt.plot(history.history['accuracy'], label='train')
plt.plot(history.history['val_accuracy'], label='test')
plt.legend()
plt.show()
fig.savefig('results/' + filename + '/' + str(len_of_points) +
'/Accuracy.png',
bbox_inches='tight')
print('LSTM Results: ')
print('\n')
#generated_text = ''
#sequence prediction
for i in tokenizer.word_index:
#print(tokenizer.index_word)
w = generate_seq(model, tokenizer, max_length - 1, i, n_input + 1)
#generated_text = generated_text.join('\n'+w)
print(w)
print('LSTM Results: ')
print('\n')
#for i in tokenizer.word_index:
# print(generate_seq(model, tokenizer, max_length-1, i , 1))
all_data = df['concept:name'].copy().to_list()
all_encoded = tokenizer.texts_to_sequences([all_data])[0]
all_sequences = list()
for i in range(n_input, len(all_encoded)):
all_sequence = all_encoded[i - n_input:i + 1]
all_sequences.append(all_sequence)
max_length = max([len(seq) for seq in all_sequences])
all_sequences = pad_sequences(all_sequences,
maxlen=max_length,
padding='pre')
all_sequences = array(all_sequences)
all_X, all_y = all_sequences[:, :-1], all_sequences[:, -1]
#convert y to binary vectors
all_yl = to_categorical(all_y, num_classes=vocab_size)
#load trained model
#model = load_model('lstm_model.h5')
#print('Tokenizer \n',tokenizer)
print('Tokenizer word index\n', tokenizer.word_index)
np.set_printoptions(suppress=True)
cnt = 0
for i in range(len(all_X)):
#yhat = model.predict_classes(all_X[i].reshape(1,2,1), verbose=0)
yhat = np.argmax(model.predict(all_X[i].reshape(1, n_input, 1)),
axis=-1)
df.loc[i, 'X_input'] = str(all_X[i])
df.loc[i, 'Expected'] = all_y[i]
df.loc[i, 'predicted'] = yhat
#print('Expected:', all_y[i] , 'Predicted', yhat)
prob = model.predict_proba(all_X[i].reshape(1, n_input, 1))[0]
df.loc[i,
'probabilities'] = ' '.join([str(elem) for elem in list(prob)])
if (all_y[i] == yhat):
df.loc[i, 'result'] = 'ok'
cnt += 1
else:
df.loc[i, 'result'] = 'Error'
#print(df['predicted'].replace(tokenizer.word_index))
df.to_csv('results/' + filename + '/' + str(len_of_points) + '/resample_' +
str(resample_dataset) + '_lstm.csv')
print('Total successful: ', cnt, ' out of ', len(all_X), 'Percentage: ',
cnt / len(all_X))
# predict probabilities for test set
yhat_probs = model.predict(test_X, verbose=0)
# predict crisp classes for test set
yhat_classes = model.predict_classes(test_X, verbose=0)
print('yhat_classes\n', yhat_classes)
# reduce to 1d array
#yhat_probs = yhat_probs[:, 0]
#yhat_classes = yhat_classes[:, 0]
# accuracy: (tp + tn) / (p + n)
accuracy = accuracy_score(test_y, yhat_classes)
print('Accuracy: %f' % accuracy)
# precision tp / (tp + fp)
precision = precision_score(test_y, yhat_classes, average='weighted')
print('Precision: %f' % precision)
# recall: tp / (tp + fn)
recall = recall_score(test_y, yhat_classes, average='weighted')
print('Recall: %f' % recall)
# f1: 2 tp / (2 tp + fp + fn)
f1 = f1_score(test_y, yhat_classes, average='weighted')
print('F1 score: %f' % f1)
# kappa
kappa = cohen_kappa_score(test_y, yhat_classes)
print('Cohens kappa: %f' % kappa)
# ROC AUC
#auc = roc_auc_score(test_y, yhat_probs,multi_class='ovr')
#print('ROC AUC: %f' % auc)
# confusion matrix
matrix = confusion_matrix(test_y, yhat_classes)
print(matrix)
fig = plt.figure()
sns.heatmap(matrix, center=True)
plt.show()
fig.savefig('results/' + filename + '/' + str(len_of_points) +
'/ConfusionMatrix.png',
bbox_inches='tight')
#headers
#filename - resample - len of points - train loss + Accuracy - test score
#write results to csv
fd = open("total_results.csv", "a+")
row = filename + '\t' + str(resample_dataset) + '\t' + str(
len_of_points
) + '\t' + str(train_score[0]) + '\t' + str(train_score[1]) + '\t' + str(
score[0]) + '\t' + str(score[1]) + '\t' + str(accuracy) + '\t' + str(
precision) + '\t' + str(recall) + '\t' + str(f1) + '\t' + str(
kappa) + '\t' + '' + '\t' + json.dumps(
tokenizer.word_index) + '\n'
fd.write(row)
fd.close()