def createFile(self, eventLog, fileName="", filePath=""):
destinationPath = self.getDestinationPath()
if filePath != "":
destinationPath = filePath
destinationPath = os.path.join(fileName, filePath)
xes_exporter.export_log(eventLog, destinationPath)
return destinationPath
def process_log(self, new_log_path):
alignments = self.get_alignment()
for trace_no, trace_alignment in enumerate(alignments):
state = True
event_pointer = 0
for trace in trace_alignment["alignment"]:
if (trace[0] == trace[1]):
self.log[trace_no][event_pointer]["concept:state"] = state
self.log[trace_no][event_pointer]["concept:move"] = True
event_pointer += 1
else:
if (trace[0] != None and trace[0] != ">>"):
state = False
self.log[trace_no][event_pointer][
"concept:state"] = state
self.log[trace_no][event_pointer][
"concept:move"] = False
event_pointer += 1
elif (trace[1] != None and trace[1] != ">>"):
state = False
xes_exporter.export_log(self.log, new_log_path)
print("Log exported")
def set_handler_for_process_and_session(self, process, session, handler):
"""
Sets the handler for the current process and session
Parameters
-------------
process
Process
session
Session
handler
Handler
"""
self.load_log_correspondence()
if process in self.logs_correspondence:
if type(handler) is ParquetHandler:
df = handler.dataframe
target_path = os.path.join(
Configuration.temp_logs_path,
str(process) + "_" + str(session) + ".parquet")
parquet_exporter.export_df(df, target_path)
elif type(handler) is XesHandler:
log = handler.log
target_path = os.path.join(
Configuration.temp_logs_path,
str(process) + "_" + str(session) + ".xes")
xes_exporter.export_log(log, target_path)
elif type(handler) is XmlHandler:
pass
def write_sample_logs_to_fs(clus_dict, filepath):
"""
Build separate logs with traces corresponding to each cluster and write them to the filesystem.
Parameters
-----------
clus_dict : dict
Dictionary using the cluster labels as keys and the corresponding list of case ids as values.
filepath
Path to the XES log file
"""
log = xes_importer.import_log(filepath)
for key, value in clus_dict.items():
args = {
'attributes': log.attributes,
'extensions': log.extensions,
'omni_present': log.omni_present,
'classifiers': log.classifiers
}
samplelog = EventLog(**args)
goalpath = filepath[:-4] + "_" + key + ".xes"
for trace in log:
if trace.attributes['concept:name'] in value:
samplelog.append(deepcopy(trace))
xes_exporter.export_log(samplelog, goalpath)
def execute_script():
log_input_directory = "xesinput"
all_logs_names = os.listdir(log_input_directory)
all_logs_names = [log for log in all_logs_names if ".xe" in log]
for logName in all_logs_names:
# logPath = os.path.join("..", "tests", "inputData", logName)
log_path = log_input_directory + "\\" + logName
log = xes_importer.import_log(log_path)
print("\n\n")
print("log loaded")
print("Number of traces - ", len(log))
event_log = log_conv_fact.apply(log,
variant=log_conv_fact.TO_EVENT_STREAM)
print("Number of events - ", len(event_log))
print("Classifiers ", log.classifiers)
exp_log_name = "xescert_exportlogs" + "\\" + "exp_" + logName
print("exporting log", exp_log_name)
xes_exporter.export_log(log, exp_log_name)
print("exported log", exp_log_name)
log, classifier_attr_key = insert_classifier.search_act_class_attr(log)
classifiers = list(log.classifiers.keys())
if classifier_attr_key is None and classifiers:
try:
print(classifiers)
log, classifier_attr_key = insert_classifier.insert_activity_classifier_attribute(
log, classifiers[0])
print(classifier_attr_key)
except:
print("exception in handling classifier")
if classifier_attr_key is None:
classifier_attr_key = "concept:name"
if len(event_log) > 0 and classifier_attr_key in event_log[0]:
parameters = {
constants.PARAMETER_CONSTANT_ACTIVITY_KEY: classifier_attr_key
}
dfg = dfg_factory.apply(log, parameters=parameters)
gviz = dfg_vis_factory.apply(dfg,
log=log,
variant="frequency",
parameters=parameters)
# dfg_vis_factory.view(gviz)
dfg_vis_factory.save(
gviz, "xescert_images\\" + logName.replace("xes", "png"))
print("Reimporting log file just exported - ", exp_log_name)
log = xes_importer.import_log(exp_log_name)
print("log loaded", exp_log_name)
print("Number of traces - ", len(log))
event_log = log_conv_fact.apply(log,
variant=log_conv_fact.TO_EVENT_STREAM)
print("Number of events - ", len(event_log))
print("Classifiers ", log.classifiers)
def test_importExportXESfromGZIP_imp1(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
self.dummy_variable = "dummy_value"
log = xes_importer.import_log(os.path.join(COMPRESSED_INPUT_DATA, "01_running-example.xes.gz"))
xes_exporter.export_log(log, os.path.join(OUTPUT_DATA_DIR, "01-running-example.xes"),
parameters={"compress": True})
os.remove(os.path.join(OUTPUT_DATA_DIR, "01-running-example.xes.gz"))
def export_log(log, filename):
'''
Description: to export xes file using pm4py library function
Used: get the log and call export method of library to
export under provided file name
Input: log file, file name
Output: N/A
'''
xes_exporter.export_log(log, filename)
def prepare_xes_csv(f_train, xes_train, csv_train):
train = readVariantFile(f_train, unique=False)
convertToCsv(train, csv_train)
print("CSV Event Log stored in:", str(csv_train))
train_log = csv_importer.import_event_log(csv_train)
xes_exporter.export_log(train_log, xes_train)
print("XES Event Log stored in:", str(xes_train))
def export_results_to_process_log_xes(self):
"""
Exports the results of the demo to an xes file
with the same name as the previously exported csv file.
We load this csv file and transform it into an xes file.
:return:
"""
# Create filename for current self.min_category_export_score
self.results_xes_path = self.results_log_path_prefix + "_" + \
str(int(round(self.min_category_export_score * 100))) + "_thresh.xes"
# Read previously generated csv file an transform to log
dataframe = csv_import_adapter.import_dataframe_from_path(
self.results_log_csv_path, sep=",")
log = conversion_factory.apply(
dataframe,
parameters={
constants.PARAMETER_CONSTANT_CASEID_KEY: "case:concept:name",
constants.PARAMETER_CONSTANT_ACTIVITY_KEY: "category_name",
constants.PARAMETER_CONSTANT_TIMESTAMP_KEY: "time:timestamp"
})
# Add relevant data for ProM import
log._classifiers = {
'Event Name': ['concept:name'],
'(Event Name AND Lifecycle transition)':
['concept:name', 'lifecycle:transition']
}
log._extensions = {
'Time': {
'prefix': 'time',
'uri': 'http://www.xes-standard.org/time.xesext'
},
'Lifecycle': {
'prefix': 'lifecycle',
'uri': 'http://www.xes-standard.org/lifecycle.xesext'
},
'Concept': {
'prefix': 'concept',
'uri': 'http://www.xes-standard.org/concept.xesext'
}
}
for trace in log._list:
# set trace concept:name to str instead of int, also for ProM import
trace._attributes["concept:name"] = str(
trace._attributes["concept:name"])
# Set org:resource to string as well
for item in trace._list:
item["org:resource"] = str(item["org:resource"])
# Export results to xes
xes_exporter.export_log(log, self.results_xes_path)
logger.info("Exported demo detections to: %s" % self.results_xes_path)
def test_importExportXEStoXES(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
self.dummy_variable = "dummy_value"
log = xes_importer.import_log(os.path.join(INPUT_DATA_DIR, "running-example.xes"))
xes_exporter.export_log(log, os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
log_imported_after_export = xes_importer.import_log(
os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
self.assertEqual(len(log), len(log_imported_after_export))
os.remove(os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
def exportLog(log):
now = datetime.now()
dateTime = now.strftime(" %m-%d-%y %H-%M-%S ")
newName = "anon" + dateTime + settings.EVENT_LOG_NAME[:-3] + "xes"
tmpPath = os.path.join(settings.MEDIA_ROOT, "temp")
newFile = os.path.join(tmpPath, "anonymization", newName)
start_time = time.time()
xes_exporter.export_log(log, newFile)
print("EXPORTING TOOK: --- %s seconds ---" % (time.time() - start_time))
return newName
def create_log(log, name: str, folder='cache/log_cache/'):
print('\tCreating new file (', name, ') in memory')
name = create_unique_name(name)
path = folder + name
if isinstance(log, EventLog):
export_log(log, path)
else:
default_storage.save(path, ContentFile(log.read()))
log = import_log(path)
properties = create_properties(log)
return Log.objects.create(name=name, path=path, properties=properties)
def create_log(log, name: str, folder='cache/log_cache/'):
# just a way to avoid two files with same name shadow each other
name = create_unique_name(name)
path = folder + name
if isinstance(log, EventLog):
export_log(log, path)
else:
default_storage.save(path, ContentFile(log.read()))
log = import_log(path)
properties = create_properties(log)
return Log.objects.create(name=name, path=path, properties=properties)
def filter_per_activities_to_keep(chat_id, activities):
log = get_current_log(chat_id)
tracefilter_log_pos = attributes_filter.apply_events(
log,
activities,
parameters={
constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "concept:name",
"positive": True
})
xes_exporter.export_log(tracefilter_log_pos,
get_log_filename(chat_id, True),
parameters={"compress": False})
set_property(chat_id, "current_log", get_log_filename(chat_id, True))
def createFile(self, eventLog, fileName="", filePath=""):
"""
function that creates an XES file from an event log
:param eventLog: given event log
:param fileName: name of the file that will be created
:param filePath: path where the created file should be stored
:return: path of the stored converted XES file
"""
destinationPath = self.getDestinationPath()
if filePath != "":
destinationPath = filePath
destinationPath = os.path.join(fileName, filePath)
xes_exporter.export_log(eventLog, destinationPath)
return destinationPath
def privatize_df(log, event_int_mapping, epsilon, output):
#get true df frequencies
print("Retrieving Directly Follows Frequencies ", end = '')
df_relations = get_df_frequencies(log, event_int_mapping)
print("Done")
#print(df_relations)
#privatize df frequencies
print("Privatizing Log ", end = '')
df_relations = apply_laplace_noise_df(df_relations, epsilon)
print("Done")
#write to disk
print("Writing privatized Directly Follows Frequencies to disk ", end = '')
#write_to_dfg(df_relations, event_int_mapping, output)
private_log = generate_pm4py_log(df_relations, event_int_mapping)
xes_exporter.export_log(private_log,output)
print("Done")
def test_importExportProblematicLogs(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
self.dummy_variable = "dummy_value"
logs = os.listdir(PROBLEMATIC_XES_DIR)
for log in logs:
log_full_path = os.path.join(PROBLEMATIC_XES_DIR, log)
try:
output_log_path = os.path.join(OUTPUT_DATA_DIR, log)
log = xes_importer.import_log(log_full_path)
xes_exporter.export_log(log, output_log_path)
log_imported_after_export = xes_importer.import_log(output_log_path)
self.assertEqual(len(log), len(log_imported_after_export))
os.remove(output_log_path)
except SyntaxError as e:
logging.info("SyntaxError on log " + str(log) + ": " + str(e))
def create_log(log, name: str, folder='cache/log_cache/', import_in_cache=True):
logger.info('\tCreating new file (' + name + ') in memory')
if import_in_cache:
name = create_unique_name(name)
path = folder + name
if import_in_cache:
if isinstance(log, EventLog):
export_log(log, path)
else:
default_storage.save(path, ContentFile(log.read()))
log = import_log(path)
else: # TODO: this might be risky
if not isinstance(log, EventLog):
log = import_log(path)
properties = create_properties(log)
return Log.objects.create(name=name, path=path, properties=properties)
def test_importExportCSVtoXES(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
self.dummy_variable = "dummy_value"
event_log = csv_importer.import_log(os.path.join(INPUT_DATA_DIR, "running-example.csv"))
event_log.sort()
event_log.sample()
event_log.insert_event_index_as_event_attribute()
trace_log = log_transform.transform_event_log_to_trace_log(event_log)
trace_log.sort()
trace_log.sample()
trace_log.insert_trace_index_as_event_attribute()
xes_exporter.export_log(trace_log, os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
trace_log_imported_after_export = xes_importer.import_log(
os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
self.assertEqual(len(trace_log), len(trace_log_imported_after_export))
os.remove(os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
def export_to_xes(filename):
output = os.path.splitext(filename)[0]
df = csv_import_adapter.import_dataframe_from_path(filename, sep=";")
df = df.rename(
columns={
"Case_ID": "case:concept:name",
"Event": "concept:name",
"Type": "org:type",
"Order": "org:order",
"Is_Chord": "org:is_chord"
})
# create internal XES log from pandas dataframe
log = conversion_factory.apply(df)
# save XES log
xes_exporter.export_log(log, f"{output}.xes")
def test_importExportCSVtoXES(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
self.dummy_variable = "dummy_value"
event_log = csv_importer.import_event_stream(
os.path.join(INPUT_DATA_DIR, "running-example.csv"))
event_log = sorting.sort_timestamp(event_log)
event_log = sampling.sample(event_log)
event_log = index_attribute.insert_event_index_as_event_attribute(
event_log)
log = log_conv_fact.apply(event_log)
log = sorting.sort_timestamp(log)
log = sampling.sample(log)
log = index_attribute.insert_trace_index_as_event_attribute(log)
xes_exporter.export_log(
log, os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
log_imported_after_export = xes_importer.import_log(
os.path.join(OUTPUT_DATA_DIR, "running-example-exported.xes"))
self.assertEqual(len(log), len(log_imported_after_export))
os.remove(os.path.join(OUTPUT_DATA_DIR,
"running-example-exported.xes"))
def test_xes1documentation(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
self.dummy_variable = "dummy_value"
import os
from pm4py.objects.log.importer.xes import factory as xes_importer
log = xes_importer.import_log(os.path.join("input_data","running-example.xes"))
log_lenth = len(log)
first_trace_length = len(log[0])
del log_lenth
del first_trace_length
first_trace_attributes = list(log[0].attributes.keys())
first_event_first_trace_attributes = list(log[0][0].keys())
del first_trace_attributes
del first_event_first_trace_attributes
first_trace_concept_name = log[0].attributes["concept:name"]
first_event_first_trace_concept_name = log[0][0]["concept:name"]
del first_trace_concept_name
del first_event_first_trace_concept_name
from pm4py.objects.log.exporter.xes import factory as xes_exporter
xes_exporter.export_log(log, "exportedLog.xes")
os.remove("exportedLog.xes")
def apply_privacyPreserving(self, technique, resource_aware,
expotPrivacyAwareLog, show_final_result,
hashedActivities, **keyword_param):
utils = Utilities(self.log)
snFull_DF, resourceList, activityList = utils.create_full_matrix_next(
self.log, relation_depth=False, trace_length=False, trace_id=True)
activity_substitutions = utils.make_activitySubstitutions_general(
snFull_DF,
technique,
resource_aware,
NoSubstitutions=keyword_param['NoSubstitutions'],
MinMax=keyword_param['MinMax'],
FixedValue=keyword_param['FixedValue'],
hashedActivities=hashedActivities)
event_attribute2remove = keyword_param['event_attribute2remove'] if 'event_attribute2remove' in keyword_param else []
case_attribute2remove = keyword_param[
'case_attribute2remove'] if 'case_attribute2remove' in keyword_param else []
log_withoutFreq = utils.frequency_elimination(
activity_substitutions,
resource_aware,
True,
True,
event_attribute2remove=event_attribute2remove,
case_attribute2remove=case_attribute2remove)
##adding privacy extension here....
prefix = 'privacy:'
uri = 'paper_version_uri/privacy.xesext'
privacy = privacyExtension(log_withoutFreq, prefix, uri)
eventAttributes = {}
for key in event_attribute2remove:
eventAttributes[key] = 'removed'
if (hashedActivities == True):
eventAttributes['concept:name'] = 'hashed'
if ('concept:name' in eventAttributes):
eventAttributes['concept:name'] += ',perturbed'
traceAttributes = {}
for key in case_attribute2remove:
traceAttributes[key] = 'removed'
privacy.set_anonymizer('substitution', 'event', 'concept:name')
operation_parameters = {}
operation_parameters['techniques'] = technique
operation_parameters['resource_aware'] = resource_aware
layer = privacy.get_last_layer()
privacy.set_optional_anonymizer(
layer, operation_parameters=operation_parameters)
if (hashedActivities):
privacy.set_anonymizer('cryptography', 'event', 'concept:name')
layer = privacy.get_last_layer()
operation_parameters_h = {}
operation_parameters_h['method'] = 'hash'
privacy.set_optional_anonymizer(
layer, operation_parameters=operation_parameters_h)
#End of adding extension
if (expotPrivacyAwareLog):
xes_exporter.export_log(log_withoutFreq,
keyword_param['privacy_aware_log_path'])
if (show_final_result):
read_external_log = False
Event_log_type = 'modified' # original or modified (privacy_aware)
export_intermediate_results = True
threshold = 0.2
encrypted_network = False
self.result_maker(Event_log_type,
read_external_log,
export_intermediate_results,
threshold,
encrypted_network,
log_path=keyword_param['privacy_aware_log_path'],
internal_log=log_withoutFreq)
from pm4py.objects.log.importer.xes import factory as xes_import_factory
from pm4py.objects.log.exporter.xes import factory as xes_exporter
from pm4py.statistics.traces.log import case_statistics
from pm4py.algo.filtering.log.variants import variants_filter
K = [20]
for k in K:
event_log = "Sepsis Cases - Event Log.xes"
log = xes_import_factory.apply(event_log)
var_with_count = case_statistics.get_variant_statistics(log)
variants_count = sorted(var_with_count,
key=lambda x: x['count'],
reverse=True)
to_filter = []
count = 0
for j in range(0, len(variants_count)):
dict = variants_count[j]
if dict["count"] < k:
to_filter.append([dict["variant"]])
else:
count += dict["count"]
for delete in to_filter:
log = variants_filter.apply(log,
delete,
parameters={"positive": False})
xes_exporter.export_log(
log, "baseline" + "_" + str(k) + "-" + "Annonymity" + ".xes")
print("baseline" + "_" + str(k) + "-" + "Annonymity" + ".xes" +
" has been exported!")
times[index_t][activities.index(event["concept:name"])]+=duration.total_seconds()
logfile="30_activities_10k.xes"
#logfile = "test.xes"
#logfile="outliers_30_activities_3k_0.1.xes"
logfile2="datasets/"+logfile
log=xes_import_factory.apply(logfile2)
add_reg_date(log)
activities_stats,log_durations=mean_value_per_Activity(log)
add_randomness_to_executions_times(log,activities_stats,log_durations)
activities_stats,log_durations=mean_value_per_Activity(log)
percentage=0.005
file="datasets/results_"+logfile.split(".")[0]+"_"+str(percentage)+"_description"
with open(file,"w") as f:
for i in range(int(percentage*len(log))):
trace_id=random.randint(0,len(log)-1)
mode=random.random()
outlier=""
if mode<0.45:
act_id=delay_an_activity(log,trace_id,activities_stats)
outlier="delay"
elif mode<0.9:
act_id=end_faster_activity(log,trace_id,activities_stats)
outlier="faster"
else:
act_id=create_measurement_error(log,trace_id,activities_stats)
outlier="measurement"
f.write(",".join([outlier,str(trace_id),str(act_id)])+"\n")
filename="datasets/outliers_"+logfile.split(".")[0]+"_"+str(percentage)
xes_exporter.export_log(log,filename+".xes")
#!/usr/bin/env python3
from time import time
from pm4py.objects.log.importer.xes import factory as xes_import_factory
from pm4py.objects.log.exporter.xes import factory as xes_exporter
in_path = '../static/xes/book/bigger-example.xes'
out_path = '/tmp/out.xes'
print(f'{in_path} --> {out_path}')
t_s = time()
log = xes_import_factory.apply(in_path)
t_m = time()
xes_exporter.export_log(log, out_path)
t_e = time()
print(f'read in {t_m-t_s:.3f}s, wrote in {t_e-t_m:.3f}s')
def export_log_file(log_file, file_path: Path):
xes_file = file_path.with_suffix('.xes')
xes_exporter.export_log(log_file, str(xes_file))
from baseline import Baseline2
from pm4py.objects.log.importer.xes import factory as xes_import_factory
from pm4py.objects.log.exporter.xes import factory as xes_exporter
from pm4py.statistics.traces.log import case_statistics
from pm4py.algo.filtering.log.variants import variants_filter
event_log = "Sepsis Cases - Event Log.xes"
K = [10]
sensitive = ['Age', 'Diagnose']
spectime2 = ["hours", "minutes"]
for k in K:
for t in spectime2:
log = xes_import_factory.apply(event_log)
base2 = Baseline2.Baseline2(log,sensitive,k,t)
log2, d, d_l = base2.suppress_k_annonymity()
var_with_count = case_statistics.get_variant_statistics(log2)
variants_count = sorted(var_with_count, key=lambda x: x['count'], reverse=True)
print(variants_count)
print("deleted elements: " + str(d))
print("deleted traces: " + str(d_l))
xes_exporter.export_log(log2, "xes/baseline2" + "_" + t + str(k) + "-" + "Annonymity" + ".xes")
print("xes/baseline2" + "_" + str(k) + "-" + "Annonymity" + ".xes" + " has been exported!")
privacy = privacyExtension(log, prefix, uri)
privacy.set_anonymizer(operation='suppression', level='event', target='org:resource')
statistics={}
statistics['no_modified_traces'] = 15
statistics['no_modified_events'] = 20
desired_analyses= {}
desired_analyses['1']='process discovery'
desired_analyses['2']='social network discovery'
message = privacy.set_optional_anonymizer(layer = 1, statistics=statistics, desired_analyses=desired_analyses, test='test' )
print(message)
layer = privacy.get_anonymizer(layer=1)
anon = privacy.get_anonymizations()
xes_exporter.export_log(log, 'ext_paper_sample.xes')
# ELA Part----------------------------------------------------------------------------
try:
log_name = log.attributes['concept:name']
except Exception as e:
log_name = "No mame is given for the event log!"
ela = ELA()
ela_desired_analyses = ['analysis 1', 'analysis 2']
data = {'Name': ['Tom', 'nick', 'krish', 'jack'], 'Age': [20, 21, 19, 18]}
df = pd.DataFrame(data)
ela.set_values(origin=log_name, method='method 1', desired_analyses=ela_desired_analyses,data=df.copy())
ela.create_xml('ela_paper_sample.xml')
print(ela.get_values()['data'])
ela = ela.read_xml("ela_paper_sample.xml")
import os
from pm4py.objects.log.importer.xes import factory as xes_importer
from tests.translucent_event_log_new.objects.tel.importer.xes import utils as xes_utils
from tests.translucent_event_log_new.objects.tel.importer.xes import iterparse_tel
from pm4py.objects.log.exporter.xes import factory as xes_exporter
from pm4py.visualization.petrinet import factory as vis_factory
from pm4py.visualization.transition_system import factory as trans_fact
from tests.translucent_event_log.objects.tel import utils
from tests.translucent_event_log_new.algo.discover_petrinet import state_based_region as sb
input_file_path = os.path.join("input_data", "running-example.xes")
log = xes_importer.apply(input_file_path)
tel = iterparse_tel.import_tel(input_file_path)
xes_utils.set_enabled(tel)
for trace in tel:
print(" ")
for event in trace:
print(event['enabled'])
print(event['concept:name'])
output_path = os.path.join("input_data", "running-example_tel.xes")
xes_exporter.export_log(tel, output_path)