def execute_script():
try:
from pm4py.objects.log.importer.parquet import factory as parquet_importer
from pm4py.algo.discovery.dfg.adapters.pandas import df_statistics
log_path = os.path.join("..", "tests", "input_data", log_name)
time1 = time.time()
dataframe = parquet_importer.apply(
log_path, parameters={"columns": allowed_columns})
time2 = time.time()
print(dataframe.columns)
print(
"time interlapsed importing " + log_name + " on columns " +
str(allowed_columns) + ": ", (time2 - time1))
dfg1 = df_statistics.get_dfg_graph(dataframe,
sort_timestamp_along_case_id=False)
time3 = time.time()
print(
"time interlapsed calculating the DFG on columns " +
str(allowed_columns) + " : ", (time3 - time2))
del dataframe
time4 = time.time()
dataframe = parquet_importer.apply(log_path)
print(dataframe.columns)
time5 = time.time()
print("time interlapsed importing " + log_name + " (all columns): ",
(time5 - time4))
dfg2 = df_statistics.get_dfg_graph(dataframe,
sort_timestamp_along_case_id=False)
time6 = time.time()
print("time interlapsed calculating the DFG on all columns : ",
(time6 - time5))
except:
traceback.print_exc()
def load_parquet_from_path(path,
columns,
filters,
use_transition=False,
force_classifier_insertion=False,
force_timestamp_conversion=False,
parameters=None):
if parameters is None:
parameters = {}
if filters is None:
filters = []
if columns is None:
columns = []
df = parquet_importer.apply(path)
else:
df = parquet_importer.apply(path, parameters={"columns": columns})
if DEFAULT_TIMESTAMP_KEY in columns and (filters
or force_timestamp_conversion):
df[DEFAULT_TIMESTAMP_KEY] = pd.to_datetime(df[DEFAULT_TIMESTAMP_KEY],
utc=True)
if use_transition:
df = insert_classifier(df)
elif force_classifier_insertion:
df["@@classifier"] = df[DEFAULT_NAME_KEY]
return df
def test_importExportParquet(self):
# to avoid static method warnings in tests,
# that by construction of the unittest package have to be expressed in such way
from pm4py.objects.log.importer.parquet import factory as parquet_importer
from pm4py.objects.log.exporter.parquet import factory as parquet_exporter
self.dummy_variable = "dummy_value"
df1 = parquet_importer.apply(
os.path.join(INPUT_DATA_DIR, "running-example.parquet"))
parquet_exporter.export_log(
df1, os.path.join(OUTPUT_DATA_DIR, "running-example.parquet"))
df2 = parquet_importer.apply(
os.path.join(OUTPUT_DATA_DIR, "running-example.parquet"))
self.assertEqual(len(df1), len(df2))
os.remove(os.path.join(OUTPUT_DATA_DIR, "running-example.parquet"))
def test_importing_parquet(self):
from pm4py.objects.log.importer.parquet import factory as parquet_importer
df = parquet_importer.apply(os.path.join("input_data",
"receipt.parquet"),
variant="pyarrow")
df = parquet_importer.apply(os.path.join("input_data",
"receipt.parquet"),
variant="fastparquet")
log = parquet_importer.import_log(os.path.join(
"input_data", "running-example.parquet"),
variant="pyarrow")
log = parquet_importer.import_minimal_log(os.path.join(
"input_data", "running-example.parquet"),
variant="pyarrow")
def build_from_path(self, path, parameters=None):
"""
Builds the handler from the specified path to Parquet file
Parameters
-------------
path
Path to the log file
parameters
Parameters of the algorithm
"""
if parameters is None:
parameters = {}
self.dataframe = parquet_importer.apply(path)
# TODO: verify if this is the best way to act
self.dataframe[DEFAULT_TIMESTAMP_KEY] = pd.to_datetime(self.dataframe[DEFAULT_TIMESTAMP_KEY], utc=True)
self.postloading_processing_dataframe()
self.dataframe = self.dataframe.sort_values([DEFAULT_TIMESTAMP_KEY, ws_constants.DEFAULT_EVENT_INDEX_KEY])
if not str(self.dataframe[CASE_CONCEPT_NAME].dtype) == "object":
self.dataframe[CASE_CONCEPT_NAME] = self.dataframe[CASE_CONCEPT_NAME].astype(str)
if not ws_constants.DEFAULT_CASE_INDEX_KEY in self.dataframe:
self.dataframe[ws_constants.DEFAULT_CASE_INDEX_KEY] = self.dataframe.groupby(CASE_CONCEPT_NAME).ngroup()
if not self.is_lazy:
self.sort_dataframe_by_case_id()
self.build_reduced_dataframe()
self.build_variants_df()
self.build_grouped_dataframe()
self.build_reduced_grouped_dataframe()
self.calculate_events_number()
self.calculate_variants_number()
self.calculate_cases_number()
def build_from_path(self, path, parameters=None):
"""
Builds the handler from the specified path to Parquet file
Parameters
-------------
path
Path to the log file
parameters
Parameters of the algorithm
"""
if parameters is None:
parameters = {}
self.dataframe = parquet_importer.apply(path)
# TODO: verify if this is the best way to act
self.dataframe[DEFAULT_TIMESTAMP_KEY] = pd.to_datetime(
self.dataframe[DEFAULT_TIMESTAMP_KEY], utc=True)
self.postloading_processing_dataframe()
self.reduced_dataframe = self.dataframe[[
CASE_CONCEPT_NAME, self.activity_key, DEFAULT_TIMESTAMP_KEY
]]
self.build_variants_df()
self.grouped_dataframe = self.dataframe.groupby(CASE_CONCEPT_NAME)
self.reduced_grouped_dataframe = self.reduced_dataframe.groupby(
CASE_CONCEPT_NAME)
self.calculate_events_number()
self.calculate_variants_number()
self.calculate_cases_number()
def execute_script():
log_path = os.path.join("..", "tests", "input_data", log_name)
time1 = time.time()
dataframe = parquet_importer.apply(log_path, parameters={"columns": allowed_columns})
time2 = time.time()
print(dataframe.columns)
print("time interlapsed importing "+log_name+" on columns "+str(allowed_columns)+": ",(time2-time1))
dfg1 = df_statistics.get_dfg_graph(dataframe, sort_timestamp_along_case_id=False)
time3 = time.time()
print("time interlapsed calculating the DFG on columns "+str(allowed_columns)+" : ",(time3-time2))
del dataframe
time4 = time.time()
dataframe = parquet_importer.apply(log_path)
print(dataframe.columns)
time5 = time.time()
print("time interlapsed importing "+log_name+" (all columns): ",(time5-time4))
dfg2 = df_statistics.get_dfg_graph(dataframe, sort_timestamp_along_case_id=False)
time6 = time.time()
print("time interlapsed calculating the DFG on all columns : ",(time6-time5))
def __init__(self, name, file: str = None, log: pd.DataFrame = None):
if (file is None and log is None) or (file is not None and log is not None):
raise Exception(
"You must either provide a file to load or pass a log object.")
if file:
self.log = parquet_importer.apply(file)
else:
self.log = log
self.name: str = name
self.id: str = str(uuid.uuid4())
def get_events(path, log_name, managed_logs, parameters=None):
if parameters is None:
parameters = {}
no_samples = parameters[
PARAMETER_NO_SAMPLES] if PARAMETER_NO_SAMPLES in parameters else DEFAULT_MAX_NO_SAMPLES
use_transition = parameters[
PARAMETER_USE_TRANSITION] if PARAMETER_USE_TRANSITION in parameters else DEFAULT_USE_TRANSITION
activity_key = DEFAULT_NAME_KEY if not use_transition else PARAMETER_PM4PYWS_CLASSIFIER
filters = parameters[FILTERS] if FILTERS in parameters else []
parameters[pm4py_constants.PARAMETER_CONSTANT_ACTIVITY_KEY] = activity_key
parameters[pm4py_constants.PARAMETER_CONSTANT_ATTRIBUTE_KEY] = activity_key
case_id = parameters["case_id"]
folder = os.path.join(path, log_name)
parquet_list = parquet_importer.get_list_parquet(folder)
ret = []
count = 0
for index, pq in enumerate(parquet_list):
pq_basename = Path(pq).name
if pq_basename in managed_logs:
count = count + 1
df = get_filtered_parquet(pq,
None,
filters,
use_transition=use_transition,
parameters=parameters)
try:
events = case_statistics.get_events(df, case_id)
if len(events) > 0:
df = parquet_importer.apply(pq)
ret = df[df[CASE_CONCEPT_NAME] == case_id].dropna(
how="all", axis=1)
if activity_key != PARAMETER_PM4PYWS_CLASSIFIER:
ret[PARAMETER_PM4PYWS_CLASSIFIER] = ret[activity_key]
ret = ret.to_dict('r')
break
except:
pass
if count >= no_samples:
break
return ret
def import_from_file(file_path, parameters=None):
"""
Apply the deserialization to a file produced by Pyarrow serialization
Parameters
--------------
file_path
File path
parameters
Parameters of the algorithm
Returns
--------------
deser
Deserialized object
"""
if parameters is None:
parameters = {}
from pm4py.objects.log.importer.parquet import factory as parquet_importer
dataframe = parquet_importer.apply(file_path, parameters=parameters)
return dataframe
from analysis.Correlation import Correlation
from analysis.Regression import Regression
from utils.Configuration import Configuration
# from evaluation.Prediction import Prediction
dataset_path = os.path.join('/workspaces/data/BPIC-17',
'BPI_Challenge_2017.parquet')
# dataset_path = os.path.join('/workspaces/data/BPIC-12',
# 'BPI_Challenge_2012.parquet')
# dataset_path = os.path.join('all_wl30min_psInSecMax7200_dt.parquet')
# dataset_path = os.path.join('top20_wl30min_psInSecMax7200_dt_bpi12.parquet')
ALREADY_ANALYSED = False
log = parquet_importer.apply(dataset_path)
# print(log["concept:name"].value_counts())
# print(get_resources(log, as_dict=True))
# exit()
OUTPUT_PATH = "results/"
# Currently we have to use a multiindex due to duplicates in the timestamps (at least pandas says so)
# log.set_index('time:timestamp', inplace=True, append=True, drop=False)
# log.set_index('time:timestamp', inplace=True, verify_integrity=True, append=True, drop=False)
if not ALREADY_ANALYSED:
log.set_index('time:timestamp', inplace=True, drop=False)
log.sort_index(inplace=True)
# Filter for Worklow Events only (Offer and Application do not have a duration)
def transform_csv_dataset_to_parquet_distr_dataset(source_path, target_path, target_num_partitions, activity_key=xes.DEFAULT_NAME_KEY, timestamp_key=xes.DEFAULT_TIMESTAMP_KEY, caseid_key=CASE_CONCEPT_NAME, parameters=None):
"""
Transforms the CSV dataset to a Parquet distributed dataset
Parameters
-------------
source_path
Source path (several CSV dataset)
target_path
Target path (distributed Parquet dataset)
target_num_partitions
Target number of partitions (number of divisions of the output)
activity_key
Column that is the activity
timestamp_key
Column that is the timestamp
caseid_key
Column that is the case ID
parameters
Possible parameters of the algorithm, including:
- sep: the separator
- quotechar: the quotechar
- encoding: the encoding
- timest_columns: the list of column that contain timestamp
- timest_format: the format of ALL the timest_columns
Returns
-------------
void
"""
if parameters is None:
parameters = {}
# create the folder
try:
os.mkdir(target_path)
except:
pass
# create the partitions
dataframe = pd.DataFrame({})
for i in range(target_num_partitions):
tp = os.path.join(target_path, str(i)+".parquet")
parquet_exporter.apply(dataframe, tp)
files = os.listdir(source_path)
for index, file in enumerate(files):
if file.lower().endswith("csv"):
sp = os.path.join(source_path, file)
source_df = csv_importer.import_dataframe_from_path(sp, parameters=parameters)
if activity_key != xes.DEFAULT_NAME_KEY and xes.DEFAULT_NAME_KEY not in source_df.columns:
source_df[xes.DEFAULT_NAME_KEY] = source_df[activity_key]
if timestamp_key != xes.DEFAULT_TIMESTAMP_KEY and xes.DEFAULT_TIMESTAMP_KEY not in source_df.columns:
source_df[xes.DEFAULT_TIMESTAMP_KEY] = source_df[timestamp_key]
if caseid_key != CASE_CONCEPT_NAME and CASE_CONCEPT_NAME not in source_df.columns:
source_df[CASE_CONCEPT_NAME] = source_df[caseid_key]
source_df["@@partition"] = source_df[caseid_key].apply(hash)
source_df["@@partition"] = source_df["@@partition"] % target_num_partitions
for i in range(target_num_partitions):
tp = os.path.join(target_path, str(i)+".parquet")
df2 = source_df[source_df["@@partition"] == i]
del df2["@@partition"]
#df2 = df2.reset_index()
df1 = parquet_importer.apply(tp)
df = pd.concat([df1, df2])
if index == len(files)-1:
df = df.sort_values([caseid_key, timestamp_key])
print("input %d/%d output %d/%d len(df)=" % (index+1,len(files),i+1,target_num_partitions),len(df))
parquet_exporter.apply(df, tp)