def call_with_future(fn, future, args, kwargs):
try:
result = fn(*args, **kwargs)
future.set_result(result)
log('Threading done')
except Exception as exc:
future.set_exception(exc)
def clearFrequency(clear):
log('clearFrequency triggered')
exists, data = existsData()
if exists:
data.set_frequency(None)
data.save()
return []
def setFileChoice(filechoice):
log('setFileChoice triggered')
if isNone(filechoice):
return []
setFile(filechoice)
# clearOptions()
return []
def isHashedAlready(self, cachable):
'''
Function that checks if the function was already calculated and
saved. Returns boolean about the check and the return value of
the function, iff exists, otherwise None.
'''
exists = cachable in self.internalStore
if self.verbose and exists:
log('Reuse result from earlier with')
log('\t{}'.format(cachable))
return exists, self.internalStore[cachable] if exists else None
def insertNewResult(self, cachable, result):
size = sys.getsizeof(self.internalStore)
for key in self.internalStore:
size += sys.getsizeof(self.internalStore[key])
if self.verbose:
log('Current size of internal storage:\t{} Bytes'.format(size))
if size / 1024 / 1024 >= self.storageSize:
if self.verbose:
log('Deleting internalStore due to overrunning storage')
keys = list(self.internalStore.keys())
for key in keys:
del self.internalStore[key]
self.internalStore[cachable] = result
def update_output(list_of_contents, save_button_clicks, list_of_names, list_of_dates):
log('hit')
ctx = dash.callback_context
triggered = []
if not ctx.triggered:
log('Not yet triggered')
return [[], [], []]
else:
triggered = [trig['prop_id'].split('.')[0] for trig in ctx.triggered]
if ids['saveButton'] in triggered:
saveNewFile(*parse_contents(list_of_contents, list_of_names, list_of_dates, True))
if list_of_contents is not None:
children = parse_contents(list_of_contents, list_of_names, list_of_dates)
return children + [{'display': 'block'}]
return [[], [], []]
def saveSettings(_, *args):
log('saveSettings triggered')
exists, data = existsData()
log('Save triggered')
if not exists:
return []
for ind, commands in enumerate([
data.set_column_sort,
data.set_column_id,
data.set_column_outlier,
data.set_relevant_columns,
data.set_has_timestamp_value,
data.set_frequency,
]):
commands(args[ind])
data.save()
return []
def fitSurrogate(self, adjustedToOutlierBlock=False, **kwargs):
# Train a random Forrest Classifier on Features
if 'seed' in kwargs:
np.random.seed(**specificKwargs(kwargs, {'seed': 1}))
for _, bchar in self.outlierBlocks:
if bchar[1] not in self.surrogates:
self.surrogates[
bchar[1]] = sklearn.ensemble.RandomForestClassifier(
**specificKwargs(kwargs, {'n_estimators': 500}))
self.surrogates[bchar[1]].fit(
self.getDataframe(
bchar[1],
adjustedToOutlierBlock=adjustedToOutlierBlock),
self.npmap({
True: 'outlier',
False: 'inlier'
})(self.getOutlierAdjusted(
adjustedToOutlierBlock=adjustedToOutlierBlock)))
log(f'Surrogates for len {bchar[1]} trained')
def wrapper_do_function(*args, **kwargs):
try:
parent = args[0]
except:
raise
assert isinstance(parent, Cachable), 'Only Cachable datatypes are possible'
kwargsToCheck = [l for l in list(func.__code__.co_varnames) if l != 'self']
interestingKwargs = {}
if type(kwargsToCheck) != type(None):
interestingKwargs = {kwarg: kwargs[kwarg] if kwarg in kwargs else None for kwarg in kwargsToCheck}
else:
interestingKwargs = kwargs
parDict = inspectDict(parent)
localAttribute = 'attr-loc-'
if payAttentionTo != None:
for attr in castlist(payAttentionTo):
interestingKwargs['{}{}'.format(localAttribute, attr)] = '{}'.format(parDict[attr] if attr in parDict else 'None')
ignored = list(ignore) if type(ignore) != type(None) else []
globalAttribute = 'attr-glo-'
if parent.alwaysCheck != None:
for attr in [att for att in castlist(parent.alwaysCheck) if att not in ignored]:
if attr not in interestingKwargs and '{}{}'.format(localAttribute, attr) not in interestingKwargs:
interestingKwargs['{}{}'.format(globalAttribute,attr)] = '{}'.format(parDict[attr] if attr in parDict else 'None')
cachable = '{}__{}'.format(func.__name__, '__'.join(['{}_{}'.format(k, interestingKwargs[k]) for k in interestingKwargs]))
if len(args) > 1:
cachable += '__args__{}'.format('_'.join(str(arg) for arg in args[1:]))
already, ret = parent.isHashedAlready(cachable)
if already:
return ret
result = func(*args, **kwargs)
parent.insertNewResult(cachable, result)
if hasattr(parent, 'save') and callable(parent.save):
if parent.verbose:
log(f'Saving file after {cachable}')
parent.save()
return result
def explainAll(self, index):
ob = self.outlierBlocks
index = index % len(ob)
index = ob[index]
index = index[0][0] + (index[1][1] // 2)
exp = []
exp.append(self.explainLimeInstance(index))
log(exp[0])
exp.append(self.explainShapInstance(index))
log(exp[1])
exp.append(self.explainContrastiveFoilInstance(index))
log(exp[2])
return exp
def readUcrAndRealworld(path=ucrAndRealworldData):
files = [os.path.abspath(os.path.join(path, fil)) for fil in os.listdir(path) if '.csv' in fil]
for fil in files:
log('Reading', fil)
df = pd.read_csv(fil)
if np.any([file2.endswith(re.sub('.csv', '', os.path.basename(fil))) for file2 in os.listdir(dir_datafiles)]):
continue
data = Data(df, column_sort='tsid',
filename = re.sub('.csv', '', os.path.basename(fil)),
originalfilename = os.path.basename(fil))
log('Set columns for', fil)
data.set_column_sort('time')
data.set_has_timestamp_value(False)
data.set_column_outlier([col for col in df.columns.tolist() if col in ['lof', 'isof', 'ee']])
data.set_relevant_columns([col for col in df.columns.tolist() if col not in ['tsid', 'time', 'lof', 'isof', 'ee']])
data.save()
log('Precalculate', fil)
data.precalculate()
data.precalculatePlots()
data.save()
log('Done')
def uid():
if 'uid' not in session:
log(session)
session['uid'] = uuid.uuid4().__str__()
def precalculate(self):
self.initOutlierExplanations()
for oek in self.outlierExplanations:
oe = self.outlierExplanations[oek]
log('Make featureframes for', oek)
oe.makeFeatureFrames()
def calc_feature_frame(self, l):
log('Feature frame calculation length', l)
if l not in self.featureFrames:
ret = self.extract_features(windowsize=l, roll = False, removeNa = True)
# TODO: Investigate again, why you need the decrementation of -1
self.featureFrames[l] = ret
def deleteFile(_):
log('deleteFile triggered')
deleteCurrentFile()
return [[], None]
def wrapper(*args, **kwargs):
future = Future()
log('Lets thread this')
Thread(target=call_with_future,
args=(fn, future, args, kwargs)).start()
return future
def readArtificialFiles(path=artificialData):
files = [os.path.abspath(os.path.join(path, fil)) for fil in os.listdir(path) if '.csv' in fil]
for fil in files:
log('Reading', fil)
df = pd.read_csv(fil)
data = Data(df, column_sort='idx',
filename = re.sub('.csv', '', os.path.basename(fil)) + '_signalChange',
originalfilename = re.sub('.csv', '_signalChange.csv', os.path.basename(fil)))
log('Set columns for', fil)
data.set_column_sort('time')
data.set_has_timestamp_value(True)
if 'yclean-signalChange' in df.columns.tolist():
data.set_relevant_columns(['yclean-signalChange'])
data.set_column_outlier(['outlierclean'])
# yclean-noise': y, 'yclean-signalChange' : y2, 'outlierclean': out,
# 'ynoisy-noise': yNoisy, 'ynoisy-signalChange' : y2Noisy, 'outliernoisy': outNois
else:
data.set_relevant_columns(['ynoisy-signalChange'])
data.set_column_outlier(['outliernoisy'])
data.save()
log('Precalculate1', fil)
data.precalculate()
data.precalculatePlots()
data.save()
log('Done1')
if np.any([file2.endswith(re.sub('.csv', '', os.path.basename(fil))) for file2 in os.listdir(dir_datafiles)]):
continue
data = Data(df, column_sort='idx',
filename = re.sub('.csv', '', os.path.basename(fil)),
originalfilename = os.path.basename(fil))
log('Set columns for', fil)
data.set_column_sort('time')
data.set_has_timestamp_value(True)
if 'yclean-noise' in df.columns.tolist():
data.set_relevant_columns(['yclean-noise'])
data.set_column_outlier(['outlierclean'])
# yclean-noise': y, 'yclean-signalChange' : y2, 'outlierclean': out,
# 'ynoisy-noise': yNoisy, 'ynoisy-signalChange' : y2Noisy, 'outliernoisy': outNois
else:
data.set_relevant_columns(['ynoisy-noise'])
data.set_column_outlier(['outliernoisy'])
data.save()
log('Precalculate2', fil)
data.precalculate()
data.precalculatePlots()
data.save()
log('Done2')
return data