def __init__(self):
self.Helpers = Helpers()
self.Logging = Logging()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"]+"Client/")
def __init__(self):
self.Helpers = Helpers()
self.Logging = Logging()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"NLU/")
self.ChatLogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"Chat/")
self.Logging.logMessage(self.LogFile, "NLU", "INFO",
"NLU Classifier LogFile Set")
self.startMQTT()
def __init__(self, user):
self.Helpers = Helpers()
self.Logging = Logging()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"Client/")
self.apiUrl = self._confs["AI"]["FQDN"] + "/communicate/infer/" + user
self.headers = {"content-type": 'application/json'}
self.Logging.logMessage(self.LogFile, "CLIENT", "INFO",
"GeniSys AI Client Ready")
def __init__(self):
self.Helpers = Helpers()
self.Logging = Logging()
self.JumpWayREST = JumpWayREST()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"]+"Client/")
self.Logging.logMessage(
self.LogFile,
"CLIENT",
"INFO",
"GeniSys AI JumpWay REST Client Ready")
def __init__(self, jumpWay):
self.Helpers = Helpers()
self.Logging = Logging()
self.jumpwayCl = jumpWay
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"Train/")
self.Logging.logMessage(self.LogFile, "LogFile", "INFO",
"NLU Trainer LogFile Set")
self.Model = Model()
self.Data = Data(self.Logging, self.LogFile)
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.setupData()
self.setupEntities()
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.JumpWay = JumpWay()
self.MySql = MySql()
self.MySql.setMysqlCursorRows()
self.Logging = Logging()
self.LogFile = self.Logging.setLogFile(self._confs["aiCore"]["Logs"] +
"Client/")
def _scale(self, train_df):
''' centers the data around 0 and scales it
:param: train_df: Dataframe that contains the training data
:return: dataframe with additional column scaled_FEATURE_X containing scaled features
:return: trained_scalers: dictionary - Per feature stores scaler object that is needed in the testing
phase to perform identical scaling, with key as column name
'''
Logging().log("Scaling Features...")
trained_scalers = {}
for col in train_df.columns:
# 1. consider only relevant columns
if not col.startswith("FEATURE"):
continue
# 2. standard scaler
scaler = preprocessing.StandardScaler()
scaler = scaler.fit(train_df[col])
train_df['scaled_' + col] = scaler.transform(train_df[col])
trained_scalers[col] = copy.deepcopy(scaler)
return train_df, trained_scalers
def _outlier_removal(self, train_df, remove_empty, nr_iterations,
split_windows, std_threshold):
''' outliers are removed from the training dataframe per feature by windowing and removing
all values per window that are further away than std_threshold times the standard
deviation
:param: train_df: Dataframe that contains the training data
:param: remove_empty: Boolean - if true empty features are removed
:param: nr_iterations: Number of iterations that are repeated to remove outliers per window
:param: split_windows: Data is split into split_windows equal length window that are between minimal risk and 1
:param: std_threshold: data that is further away than std_threshold * std of the feature is removed
:return: output_dfs: list of dataframes with each having a column scaled_FEATURE_X that is outlierfree now and a column risk which is the risk
for that feature at its row
'''
if not self._remove_outliers:
print("Outlier removal disabled!")
# 1. Initialize
output_dfs = []
iteration = range(nr_iterations)
first = True
# Per feature and window
for col in train_df.columns:
# 2. only scaled features are considered
if not col.startswith("scaled_FEATURE"): continue
#Logging().log("CURRENT -> "+ col)
result_df = train_df.sort_values("RISK")
# 3. iterate multiple times over window
# on each iteration remove outliers
for i in iteration:
sub_dfs = []
indices = []
rs = 0
# 4. iterate over windows
for r in np.linspace(result_df["RISK"].min(), 1, 10):
sub_df = result_df[(rs <= result_df["RISK"])
& (r > result_df["RISK"])]
if self._remove_outliers:
sub_df = sub_df[(
(sub_df[col] - sub_df[col].mean()) /
sub_df[col].std()).abs() < std_threshold]
sub_dfs.append(sub_df)
rs = r
result_df = pd.concat(sub_dfs)
# 5. Merge result to common dataframe
output_dfs.append(result_df[["RISK", col]])
# 6. Remove empty
if (remove_empty and len(result_df[col].unique()) < 2):
continue
# 7. Plot results
if self._visualize_outlier:
Logging().log("Pre - Standard Deviation vorher: " +
str(train_df[col].std()))
Visual().plot_scatter(train_df["RISK"],
train_df[col]) #, "RISK", "feature")
Logging().log("Post - Standard Deviation nachher: " +
str(result_df[col].std()))
Visual().plot_scatter(result_df["RISK"], result_df[col])
return output_dfs
def _build_heat_map(self, output_dfs):
''' using convolution for each point of a 2d array risk vs. feature value per feature
a heat map is generated
:param output_dfs: list of dataframes with each having a column scaled_FEATURE_X (that is outlierfree and scaled now) and a column
risk which is the risk for that feature at its row
:return a dictionary is returned that contains the feature name as key and its 2d heatmap as output
'''
dimensions = {}
for feature_df in output_dfs: # each output_df has one risk and value
Logging().log("Processing Feature: " + feature_df.columns[1])
# Testmode
if self._test_mode and (feature_df.columns[1]
== "scaled_FEATURE_5"):
print("Testing thus, break now!")
break
try:
values = np.empty(len(feature_df))
values.fill(1)
# Assign X Y Z
X = feature_df.RISK.as_matrix()
Y = feature_df[feature_df.columns[1]].as_matrix()
Z = values
# create x-y points to be used in heatmap of identical size
risk_min = min([
rm for rm in [df.RISK.min() for df in output_dfs]
if not math.isnan(rm)
])
risk_max = 1
feature_min = min([
rm
for rm in [df[df.columns[1]].min() for df in output_dfs]
if not math.isnan(rm)
])
feature_max = max([
rm
for rm in [df[df.columns[1]].max() for df in output_dfs]
if not math.isnan(rm)
])
xi = np.linspace(risk_min, risk_max, self._grid_area)
yi = np.linspace(feature_min, feature_max, self._grid_area)
# Z is a matrix of x-y values interpolated (!)
zi = griddata((X, Y),
Z, (xi[None, :], yi[:, None]),
method=self._interpol_method)
zmin = 0
zmax = 1
zi[(zi < zmin) | (zi > zmax)] = None
# Convolve each point with a gaussian kernel giving the heat value at point xi,yi being Z
# Advantage: kee horizontal and vertical influence
grid_cur = np.nan_to_num(zi)
# Smooth with a Gaussian kernel
kernel = Gaussian2DKernel(stddev=self._std_gaus,
x_size=self._kernel_size,
y_size=self._kernel_size)
grad = scipy_convolve(grid_cur,
kernel,
mode='same',
method='direct')
# Store the model in memory
dimensions[feature_df.columns[1]] = [
copy.deepcopy(np.absolute(grad)),
copy.deepcopy(xi),
copy.deepcopy(yi)
]
#print("GRAD")
#print(str(grad))
if self._visualize_heatmap:
fig, (ax_orig, ax_mag) = plt.subplots(1, 2)
ax_orig.imshow(grid_cur[::-1, ::-1], cmap='RdYlGn')
ax_orig.set_title('Original')
ax_mag.imshow(
np.absolute(grad)[::-1, ::-1], cmap='RdYlGn'
) # https://matplotlib.org/examples/color/colormaps_reference.html
ax_mag.set_title('Heat')
fig.show()
plt.show()
except:
Logging().log("No chance")
#traceback.print_exc()
dimensions[feature_df.columns[1]] = None
return dimensions, xi
def _build_one_heat_map(self, feature_df, risk_min, feature_min,
feature_max):
Logging().log("Processing Feature: " + feature_df.columns[1])
try:
values = np.empty(len(feature_df))
values.fill(1)
# Assign X Y Z
X = feature_df.RISK.as_matrix()
Y = feature_df[feature_df.columns[1]].as_matrix()
Z = values
# create x-y points to be used in heatmap of identical size
#risk_min = min([rm for rm in [df.RISK.min() for df in output_dfs] if not math.isnan(rm)])
risk_max = 1
xi = np.linspace(risk_min, risk_max, self._grid_area)
yi = np.linspace(feature_min, feature_max, self._grid_area)
# Z is a matrix of x-y values interpolated (!)
zi = griddata((X, Y),
Z, (xi[None, :], yi[:, None]),
method=self._interpol_method)
zmin = 0
zmax = 1
zi[(zi < zmin) | (zi > zmax)] = None
# Convolve each point with a gaussian kernel giving the heat value at point xi,yi being Z
# Advantage: kee horizontal and vertical influence
grid_cur = np.nan_to_num(zi)
# Smooth with a Gaussian kernel
kernel = Gaussian2DKernel(stddev=self._std_gaus,
x_size=self._kernel_size,
y_size=self._kernel_size)
grad = scipy_convolve(grid_cur,
kernel,
mode='same',
method='direct')
# Store the model in memory
feature_name = feature_df.columns[1]
result = [
feature_name,
[
copy.deepcopy(np.absolute(grad)),
copy.deepcopy(xi),
copy.deepcopy(yi)
], grid_cur
]
except:
#traceback.print_exc()
feature_name = feature_df.columns[1]
Logging().log(
str(feature_df.columns[1]) +
": No heat map created due to error")
result = [feature_name, None, None]
return result