def get_is_inside_boundary_function():
is_inside_boundary = pml.DefineFunction(
name=_IS_INSIDE_BOUNDARY,
optype=OPTYPE.CATEGORICAL.value,
dataType=DATATYPE.BOOLEAN.value,
ParameterField=[
pml.ParameterField(name=_TAG,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value),
pml.ParameterField(name=_TAG_UPPER_BOUNDARY,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value),
pml.ParameterField(name=_TAG_LOWER_BOUNDARY,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value)
],
Apply=pml.Apply(
function=FUNCTION.AND.value,
Apply_member=[
pml.Apply(function=FUNCTION.GREATER_THAN.value,
FieldRef=[
pml.FieldRef(field=_TAG),
pml.FieldRef(field=_TAG_LOWER_BOUNDARY)
]),
pml.Apply(function=FUNCTION.LESS_OR_EQUAL.value,
FieldRef=[
pml.FieldRef(field=_TAG),
pml.FieldRef(field=_TAG_UPPER_BOUNDARY)
])
]))
return is_inside_boundary
def get_local_transformation(index):
derived_fields = []
derived_field_names = []
hull = self._hulls[index]
if self._use_lag:
for i in range(1, self._length_of_fingerprint):
name = hull["name"] + _UNDERSCORE + str(i - 1)
derived_fields.append(
pml.DerivedField(
name=name,
optype=OPTYPE.CONTINUOUS,
dataType=DATATYPE.DOUBLE,
Lag=pml.Lag(field=hull["name"], n=self._length_of_fingerprint - i)
)
)
last_derived_name = hull["name"] + _UNDERSCORE + str(self._length_of_fingerprint - 1)
derived_fields.append(
pml.DerivedField(
name=last_derived_name,
optype=OPTYPE.CONTINUOUS,
dataType=DATATYPE.DOUBLE,
FieldRef=pml.FieldRef(field=hull["name"])
)
)
for idx, val in enumerate(hull["values"]):
name = "distance_tag_" + str(idx)
derived_field_names.append(name)
derived_fields.append(
pml.DerivedField(
name=name,
optype=OPTYPE.CONTINUOUS,
dataType=DATATYPE.DOUBLE,
Apply=pml.Apply(
function=_CALCULATE_DISTANCE,
FieldRef=[pml.FieldRef(field=hull["name"] + _UNDERSCORE + str(idx))],
Constant=[
pml.Constant(valueOf_=val["maxValue"]),
pml.Constant(valueOf_=val["minValue"])
]
)
)
)
derived_fields.append(
pml.DerivedField(
name=_SUM_OF_DISTANCE,
optype=OPTYPE.CONTINUOUS,
dataType=DATATYPE.DOUBLE,
Apply=pml.Apply(
function=FUNCTION.SUM,
FieldRef=[
pml.FieldRef(
field=field
)
for field in derived_field_names
]
)
)
)
return pml.LocalTransformations(DerivedField=derived_fields)
def __init__(self, keras_model, dataSet, script_args):
ny.LocalTransformations.__init__(self)
if script_args:
ret_type = script_args['return_type'].lower()
def_name = script_args['def_name'] if script_args[
'content'].__class__.__name__ == 'str' else script_args[
'content'].__name__
ny.LocalTransformations.add_DerivedField(
self,
ny.DerivedField(name="base64String",
optype="categorical"
if ret_type == "string" else "continuous",
dataType=ret_type,
Apply=ny.Apply(
function='customFunc',
FieldRef=[ny.FieldRef(field=dataSet)])))
else:
ny.LocalTransformations.add_DerivedField(
self,
ny.DerivedField(name="base64String",
optype="categorical",
dataType="string",
Apply=ny.Apply(
function="CNN:getBase64String",
FieldRef=[ny.FieldRef(field="image")])))
def pca(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's PCA
Parameters
----------
trfm :
Contains the Sklearn's PCA preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to PCA preprocessing.
"""
pca.counter += 1
pp_dict = dict()
derived_flds = list()
derived_colnames = list()
val = trfm.mean_
zero = 0.0
for preprocess_idx in range(trfm.n_components_):
add = list()
for pca_idx in range(trfm.n_features_):
apply_inner = pml.Apply(
function=FUNCTION.SUBSTRACTTION.value,
Constant=[
pml.Constant(dataType=DATATYPE.DOUBLE.value,
valueOf_="{:.16f}".format(val[pca_idx]))
],
FieldRef=[pml.FieldRef(field=col_names[pca_idx])])
apply_outer = pml.Apply(
function=FUNCTION.MULTIPLICATION.value,
Apply_member=[apply_inner],
Constant=[
pml.Constant(
dataType=DATATYPE.DOUBLE.value,
valueOf_=zero
if trfm.components_[preprocess_idx][pca_idx] == 0.0
else "{:.16f}".format(
trfm.components_[preprocess_idx][pca_idx]))
])
add.append(apply_outer)
app0 = pml.Apply(function=FUNCTION.SUM.value, Apply_member=add)
derived_flds.append(
pml.DerivedField(Apply=app0,
dataType=DATATYPE.DOUBLE.value,
optype=OPTYPE.CONTINUOUS.value,
name="PCA" + str(pca.counter) + "-" +
str(preprocess_idx)))
name = derived_flds[preprocess_idx].get_name()
derived_colnames.append(name)
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def tfidf_vectorizer(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's TfIdfVectorizer
Parameters
----------
trfm :
Contains the Sklearn's TfIdfVectorizer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to TfIdfVectorizer preprocessing.
"""
pp_dict = dict()
features = [
str(feat.encode("utf8"))[2:-1] for feat in trfm.get_feature_names()
]
idfs = trfm.idf_
extra_features = list(trfm.vocabulary_.keys())
derived_flds = list()
derived_colnames = get_derived_colnames('tfidf@[' + col_names[0] + ']',
features)
if trfm.lowercase:
derived_flds.append(
pml.DerivedField(name='lowercase(' + col_names[0] + ')',
optype=OPTYPE.CATEGORICAL.value,
dataType=DATATYPE.STRING.value,
Apply=pml.Apply(
function=FUNCTION.LOWERCASE.value,
FieldRef=[pml.FieldRef(field=col_names[0])])))
for feat_idx, idf in zip(range(len(features)), idfs):
derived_flds.append(
pml.DerivedField(
name=derived_colnames[feat_idx],
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value,
Apply=pml.Apply(
function=FUNCTION.MULTIPLICATION.value,
TextIndex=[
pml.TextIndex(
textField='lowercase(' + col_names[0] + ')',
wordSeparatorCharacterRE='\\s+',
tokenize='true',
Constant=pml.Constant(valueOf_=features[feat_idx]),
Extension=[
pml.Extension(value=extra_features[feat_idx])
])
],
Constant=[pml.Constant(valueOf_="{:.16f}".format(idf))])))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_name'] = col_names[0]
pp_dict['pp_feat_class_lbl'] = list()
return pp_dict
def pca(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's PCA preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to PCA preprocessing.
"""
pca.counter += 1
pp_dict = dict()
derived_flds = list()
derived_colnames = list()
val = trfm.mean_
zero = 0.0
for preprocess_idx in range(trfm.n_components_):
add = list()
for pca_idx in range(trfm.n_features_):
apply_inner = pml.Apply(
function='-',
Constant=[
pml.Constant(dataType="double",
valueOf_="{:.16f}".format(val[pca_idx]))
],
FieldRef=[pml.FieldRef(field=col_names[pca_idx])])
apply_outer = pml.Apply(
function="*",
Apply_member=[apply_inner],
Constant=[
pml.Constant(
dataType="double",
valueOf_=zero
if trfm.components_[preprocess_idx][pca_idx] == 0.0
else "{:.16f}".format(
trfm.components_[preprocess_idx][pca_idx]))
])
add.append(apply_outer)
app0 = pml.Apply(function="sum", Apply_member=add)
derived_flds.append(
pml.DerivedField(Apply=app0,
dataType="double",
optype="continuous",
name="PCA" + str(pca.counter) + "-" +
str(preprocess_idx)))
name = derived_flds[preprocess_idx].get_name()
derived_colnames.append(name)
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def lag(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Nyoka's Lag instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Lag preprocessing.
"""
derived_flds = list()
pp_dict = dict()
derived_colnames = get_derived_colnames(trfm.aggregation, col_names)
if trfm.aggregation == 'stddev':
for idx, name in enumerate(col_names):
applyies = list()
for i in range(trfm.value):
lags = list()
lags.append(pml.Lag(field=name, n=i + 1))
lags.append(pml.Lag(field=name, n=trfm.value, aggregate="avg"))
sub_func = pml.Apply(function="-", Lag=lags)
pow_func = pml.Apply(
function="pow",
Apply_member=[sub_func],
Constant=[pml.Constant(dataType="integer", valueOf_=2)])
applyies.append(pow_func)
add_func = pml.Apply(function="+", Apply_member=applyies)
div_func = pml.Apply(function="/",
Apply_member=[add_func],
Constant=[
pml.Constant(dataType="double",
valueOf_=float(trfm.value))
])
sqrt_func = pml.Apply(function="sqrt", Apply_member=[div_func])
derived_fld = pml.DerivedField(name=derived_colnames[idx],
Apply=sqrt_func,
optype="continuous",
dataType="double")
derived_flds.append(derived_fld)
else:
for idx, name in enumerate(col_names):
lag = pml.Lag(field=name, n=trfm.value, aggregate=trfm.aggregation)
derived_fld = pml.DerivedField(name=derived_colnames[idx],
Lag=lag,
optype="continuous",
dataType="double")
derived_flds.append(derived_fld)
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def rbst_scaler(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's RobustScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to RobustScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames('robustScaler', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_inner = list()
apply_inner.append(
pml.Apply(
function='-',
Constant=[
pml.Constant(
dataType="double", # <---------------------
valueOf_="{:.16f}".format(
trfm.center_[col_name_idx]))
],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])],
Extension=[
pml.Extension(name='scaling',
anytypeobjs_=['RobustScaler'])
]))
apply_outer = pml.Apply(
Apply_member=apply_inner,
function='/',
Constant=[
pml.Constant(
dataType="double", # <----------------------------
valueOf_="{:.16f}".format(trfm.scale_[col_name_idx]))
])
derived_flds.append(
pml.DerivedField(Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def std_scaler(trfm, col_names, **kwargs):
"""
Parameters
----------
trfm :
Contains the Sklearn's Standard Scaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Standard Scaler preprocessing.
"""
derived_flds = list()
pp_dict = dict()
if is_present("labelBinarizer", col_names):
derived_flds_hidden = kwargs['derived_fld']
if derived_flds_hidden:
derived_flds.extend(derived_flds_hidden)
derived_colnames = get_derived_colnames('standardScaler', col_names)
for col_name_idx in range(len(col_names)):
apply_inner = list()
apply_inner.append(
pml.Apply(
function='-',
Constant=[
pml.Constant(
dataType="double", # <---------------------
valueOf_="{:.16f}".format(trfm.mean_[col_name_idx]))
],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]))
apply_outer = pml.Apply(
Apply_member=apply_inner,
function='/',
Constant=[
pml.Constant(
dataType="double", # <----------------------------
valueOf_="{:.16f}".format(trfm.scale_[col_name_idx]))
])
derived_flds.append(
pml.DerivedField(Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def tfidf_vectorizer(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's TfIdfVectorizer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to TfIdfVectorizer preprocessing.
"""
pp_dict = dict()
features = [
str(feat.encode("utf8"))[2:-1] for feat in trfm.get_feature_names()
]
idfs = trfm.idf_
derived_flds = list()
derived_colnames = get_derived_colnames('tfidf@[' + col_names[0] + ']',
features)
if trfm.lowercase:
derived_flds.append(
pml.DerivedField(name='lowercase(' + col_names[0] + ')',
optype='categorical',
dataType='string',
Apply=pml.Apply(
function='lowercase',
FieldRef=[pml.FieldRef(field=col_names[0])])))
for feat_idx, idf in zip(range(len(features)), idfs):
derived_flds.append(pml.DerivedField(
name = derived_colnames[feat_idx],
optype='continuous',
dataType='double',
Apply=pml.Apply(function='*',
TextIndex=[pml.TextIndex(textField='lowercase(' + col_names[0] + ')' if trfm.lowercase \
else col_names[0],
wordSeparatorCharacterRE=trfm.token_pattern,
tokenize='true',
Constant=pml.Constant(valueOf_=features[feat_idx]),
)],
Constant=[pml.Constant(valueOf_="{:.16f}".format(idf))])
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_name'] = col_names[0]
pp_dict['pp_feat_class_lbl'] = list()
return pp_dict
def polynomial_features(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's PolynomialFeatures
Parameters
----------
trfm :
Contains the Sklearn's PolynomialFeatures preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to PolynomialFeatures preprocessing.
"""
polynomial_features.poly_ctr += 1
pp_dict = dict()
derived_flds = []
derived_colnames = []
for polyfeat_idx in range(trfm.powers_.shape[0]):
apply_inner_container = []
for col_name_idx in range(len(col_names)):
val = int(trfm.powers_[polyfeat_idx][col_name_idx])
apply_inner = pml.Apply(
function='pow',
Constant=[pml.Constant(
dataType="integer",
valueOf_=val
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])])
apply_inner_container.append(apply_inner)
apply_outer = pml.Apply(function="product",
Apply_member=apply_inner_container
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
dataType="double",
optype="continuous",
name="poly" + str(polynomial_features.poly_ctr) + '-' + "x" + str(polyfeat_idx)
))
name = derived_flds[polyfeat_idx].get_name()
derived_colnames.append(name)
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def rbst_scaler(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's RobustScaler
Parameters
----------
trfm :
Contains the Sklearn's RobustScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to RobustScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames('robustScaler', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_inner = list()
apply_inner.append(
pml.Apply(
function=FUNCTION.SUBSTRACTTION.value,
Constant=[
pml.Constant(dataType=DATATYPE.DOUBLE.value,
valueOf_="{:.16f}".format(
trfm.center_[col_name_idx]))
],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]))
apply_outer = pml.Apply(Apply_member=apply_inner,
function=FUNCTION.DIVISION.value,
Constant=[
pml.Constant(
dataType=DATATYPE.DOUBLE.value,
valueOf_="{:.16f}".format(
trfm.scale_[col_name_idx]))
])
derived_flds.append(
pml.DerivedField(Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def min_max_scaler(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's MinMaxScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to MinMaxScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
# col_names = list(filter(lambda x: x not in exception_cols, col_names))
derived_colnames = get_derived_colnames("minMaxScaler", col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_inner = list()
apply_inner.append(
pml.Apply(
function='*',
Constant=[
pml.Constant(dataType="double",
valueOf_="{:.16f}".format(
trfm.scale_[col_name_idx]))
],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]))
apply_outer = pml.Apply(Apply_member=apply_inner,
function='+',
Constant=[
pml.Constant(
dataType="double",
valueOf_="{:.16f}".format(
trfm.min_[col_name_idx]))
])
derived_flds.append(
pml.DerivedField(Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def __init__(self, dataSet, script_args):
if script_args['content'].__class__.__name__ == 'str':
content = script_args['content']
def_name = script_args['def_name']
else:
import inspect
content = inspect.getsource(script_args['content'])
def_name = script_args['content'].__name__
encode = True
if "encode" in script_args:
encode = script_args['encode']
if encode:
content = base64.b64encode(content.encode()).decode()
return_type = script_args['return_type'].lower()
extension = [
ny.Extension(extender='ADAPA',
name=def_name,
value=return_type,
anytypeobjs_=[content])
]
def_func = ny.DefineFunction(
name='customFunc',
optype='categorical' if return_type == 'string' else 'continous',
dataType=return_type,
ParameterField=[
ny.ParameterField(name=dataSet, dataType='binary')
],
Apply=ny.Apply(function='python',
Extension=extension,
FieldRef=[ny.FieldRef(field=dataSet)]),
)
ny.TransformationDictionary.__init__(self)
ny.TransformationDictionary.add_DefineFunction(self, def_func)
def count_vectorizer(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's CountVectorizer preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to CountVectorizer preprocessing.
"""
pp_dict = dict()
features = [
str(feat.encode("utf8"))[2:-1] for feat in trfm.get_feature_names()
]
# extra_features = [str(feat.encode("utf8"))[2:-1] for feat in list(trfm.vocabulary_.keys())]
# features = trfm.get_feature_names()
# extra_features = list(trfm.vocabulary_.keys())
derived_flds = list()
derived_colnames = get_derived_colnames('count_vec@[' + col_names[0] + ']',
features)
# derived_colnames = list()
if trfm.lowercase:
derived_flds.append(
pml.DerivedField(name='lowercase(' + col_names[0] + ')',
optype='categorical',
dataType='string',
Apply=pml.Apply(
function='lowercase',
FieldRef=[pml.FieldRef(field=col_names[0])])))
for imp_features, index in zip(features, range(len(features))):
# no_punct_word = remove_punctuation(imp_features)
# if len(no_punct_word) == 0:
# df_name = 'count_vec@[' + col_names[0] + ']('+ imp_features+')'
# derived_colnames.append(df_name)
df_name = derived_colnames[index]
derived_flds.append(pml.DerivedField(name=df_name,
optype='continuous',
dataType='double',
TextIndex=pml.TextIndex(textField='lowercase(' + col_names[0] + ')' if trfm.lowercase \
else col_names[0],
# wordSeparatorCharacterRE='\s+',
wordSeparatorCharacterRE=trfm.token_pattern,
tokenize='true',
Constant=pml.Constant(dataType="string",
valueOf_=imp_features),
# Extension=[pml.Extension(
# anytypeobjs_=[extra_features[index]])]
)))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_name'] = col_names[0]
pp_dict['pp_feat_class_lbl'] = list()
return pp_dict
def count_vectorizer(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's CountVectorizer
Parameters
----------
trfm :
Contains the Sklearn's CountVectorizer preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to CountVectorizer preprocessing.
"""
pp_dict = dict()
features = [
str(feat.encode("utf8"))[2:-1] for feat in trfm.get_feature_names()
]
extra_features = list(trfm.vocabulary_.keys())
derived_flds = list()
derived_colnames = get_derived_colnames('count_vec@[' + col_names[0] + ']',
features)
if trfm.lowercase:
derived_flds.append(
pml.DerivedField(name='lowercase(' + col_names[0] + ')',
optype=OPTYPE.CATEGORICAL,
dataType=DATATYPE.STRING,
Apply=pml.Apply(
function=FUNCTION.LOWERCASE,
FieldRef=[pml.FieldRef(field=col_names[0])])))
for imp_features, index in zip(features, range(len(features))):
df_name = derived_colnames[index]
derived_flds.append(pml.DerivedField(name=df_name,
optype=OPTYPE.CONTINUOUS,
dataType=DATATYPE.DOUBLE,
TextIndex=pml.TextIndex(textField='lowercase(' + col_names[0] + ')' if trfm.lowercase \
else col_names[0],
wordSeparatorCharacterRE='\\s+',
tokenize='true',
Constant=pml.Constant(dataType=DATATYPE.STRING,
valueOf_=imp_features),
Extension=[pml.Extension(value=extra_features[index])]
)))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_name'] = col_names[0]
pp_dict['pp_feat_class_lbl'] = list()
return pp_dict
def custom_function(trfm, col_names):
derived_flds = list()
pp_dict = dict()
input_cols = trfm.input_cols
output_cols = trfm.output_cols
modified_cols = trfm.modified_cols
derived_colnames = [
'customFunction(' + col + ')' if col in modified_cols else col
for col in output_cols
]
ext = pml.Extension(extender="ADAPA",
name=trfm.func.__name__,
value="python",
anytypeobjs_=[trfm.source_code])
apply = pml.Apply(function="custom", Extension=[ext])
def_func = pml.DefineFunction(name=trfm.func.__name__,
optype="continous",
dataType="Double-Array")
for col in trfm.input_cols:
def_func.add_ParameterField(pml.ParameterField(name=col))
def_func.Apply = apply
fld_refs = list()
for fld in input_cols:
fld_refs.append(pml.FieldRef(field=fld))
for col in modified_cols:
apply = pml.Apply(function=trfm.func.__name__ + ":" +
str(output_cols.index(col)),
FieldRef=fld_refs)
derived_flds.append(pml.DerivedField(name='customFunction('+col+')', optype="continous",\
dataType="double", Apply=apply))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['def_func'] = def_func
return pp_dict
def get_normalization_function():
if len(self._hulls) == 1:
max_distance = pml.Constant(valueOf_=self._max_distances[0])
else:
max_distance = pml.Constant(
valueOf_=self._length_of_fingerprint * len(self._hulls))
max_distance.original_tagname_ = "Constant"
constant_100 = pml.Constant(valueOf_=100)
constant_100.original_tagname_ = "Constant"
# constant_max_distance = pml.Constant(valueOf_=self._max_distances[0])
# constant_max_distance.original_tagname_ = "Constant"
substraction_function = pml.Apply(
function=FUNCTION.MULTIPLICATION.value,
Apply_member=[
pml.Apply(function=FUNCTION.DIVISION.value,
Apply_member=[
pml.Apply(
function=FUNCTION.SUBSTRACTTION.value,
FieldRef=[
max_distance,
pml.FieldRef(field="totalDistance")
])
],
Constant=[max_distance]), constant_100
])
substraction_function.original_tagname_ = "Apply"
equal_function = pml.Apply(
function=FUNCTION.IF.value,
Apply_member=[
pml.Apply(function=FUNCTION.EQUAL.value,
FieldRef=[pml.FieldRef(field="totalDistance")],
Constant=[pml.Constant(valueOf_=0)])
],
Constant=[
pml.Constant(valueOf_=100),
substraction_function,
])
equal_function.original_tagname_ = "Apply"
return pml.Apply(
function=FUNCTION.IF.value,
Apply_member=[
pml.Apply(function=FUNCTION.GREATER_OR_EQUAL.value,
FieldRef=[pml.FieldRef(field="totalDistance")],
Constant=[max_distance])
],
Constant=[pml.Constant(valueOf_=0), equal_function])
def get_calculate_distance_function():
value_for_true = pml.Constant(valueOf_=0)
value_for_true.original_tagname_ = _CONSTANT
calculate_distance = pml.DefineFunction(
name=_CALCULATE_DISTANCE,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value,
ParameterField=[
pml.ParameterField(name=_TAG,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value),
pml.ParameterField(name=_TAG_UPPER_BOUNDARY,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value),
pml.ParameterField(name=_TAG_LOWER_BOUNDARY,
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value)
],
Apply=pml.Apply(
function=FUNCTION.IF.value,
Apply_member=[
pml.Apply(function=_IS_INSIDE_BOUNDARY,
FieldRef=[
pml.FieldRef(field=_TAG),
pml.FieldRef(field=_TAG_UPPER_BOUNDARY),
pml.FieldRef(field=_TAG_LOWER_BOUNDARY)
]), value_for_true,
pml.Apply(
function=FUNCTION.IF.value,
Apply_member=[
pml.Apply(
function=FUNCTION.LESS_OR_EQUAL.value,
FieldRef=[
pml.FieldRef(field=_TAG),
pml.FieldRef(field=_TAG_LOWER_BOUNDARY)
]),
pml.Apply(
function=FUNCTION.SUBSTRACTTION.value,
FieldRef=[
pml.FieldRef(
field=_TAG_LOWER_BOUNDARY),
pml.FieldRef(field=_TAG)
]),
pml.Apply(
function=FUNCTION.SUBSTRACTTION.value,
FieldRef=[
pml.FieldRef(field=_TAG),
pml.FieldRef(field=_TAG_UPPER_BOUNDARY)
])
])
]))
return calculate_distance
def __init__(self, keras_model, dataSet):
arch_name = 'mobilenet'
if 'vgg' in keras_model.name:
arch_name = 'vgg'
elif 'xception' in keras_model.name:
arch_name = 'xception'
elif 'inception' in keras_model.name:
arch_name = 'inception'
elif 'resnet' in keras_model.name:
arch_name = 'resnet'
ny.LocalTransformations.__init__(self)
ny.LocalTransformations.add_DerivedField(self, ny.DerivedField(
name="base64String", optype="categorical", dataType="string",
trainingBackend="tensorflowChannelLast", architectureName=arch_name,
Apply=ny.Apply(function="CNN:getBase64String",
FieldRef=[ny.FieldRef(field="image")])))
def max_abs_scaler(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's MaxAbsScaler
Parameters
----------
trfm :
Contains the Sklearn's MaxabsScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to MaxabsScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames('maxAbsScaler', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_outer = pml.Apply(
function='/',
Constant=[pml.Constant(
dataType="double",
valueOf_="{:.16f}".format(trfm.max_abs_[col_name_idx])
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def binarizer(trfm, col_names):
"""
Generates pre-processing elements for Scikit-Learn's Binarizer
Parameters
----------
trfm :
Contains the Sklearn's Binarizer preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Binarizer preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames("binarizer", col_names)
for col_name_idx in range(len(col_names)):
apply_outer = pml.Apply(
function='threshold',
Constant=[pml.Constant(
dataType="double",
valueOf_=trfm.threshold
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])])
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def imputer(trfm, col_names, **kwargs):
"""
Generates pre-processing elements for Scikit-Learn's Imputer
Parameters
----------
trfm :
Contains the Sklearn's Imputer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Imputer preprocessing.
"""
original_col_names = imputer.col_names
derived_colnames = col_names
pp_dict = dict()
derived_flds = list()
model = kwargs['model']
mining_strategy = trfm.strategy
if "mean" in mining_strategy:
mining_strategy = MISSING_VALUE_TREATMENT_METHOD.AS_MEAN.value
elif "median" in mining_strategy:
mining_strategy = MISSING_VALUE_TREATMENT_METHOD.AS_MEDIAN.value
elif "most_frequent" in mining_strategy:
mining_strategy = MISSING_VALUE_TREATMENT_METHOD.AS_MODE.value
mining_replacement_val = trfm.statistics_
if not any_in(original_col_names, col_names):
derived_colnames = get_derived_colnames('imputer', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
const_list = list()
apply_inner = list()
apply_inner.append(
pml.Apply(function=FUNCTION.IS_MISSING.value,
FieldRef=[
pml.FieldRef(field=col_names[col_name_idx])
]))
const_obj = pml.Constant(
dataType=DATATYPE.DOUBLE.value,
valueOf_=mining_replacement_val[col_name_idx]),
fieldref_obj = pml.FieldRef(field=col_names[col_name_idx])
fieldref_obj.original_tagname_ = "FieldRef"
const_list.append(const_obj[0])
const_list.append(fieldref_obj)
apply_outer = pml.Apply(Apply_member=apply_inner,
function=FUNCTION.IF.value,
Constant=const_list)
derived_flds.append(
pml.DerivedField(Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype=OPTYPE.CONTINUOUS.value,
dataType=DATATYPE.DOUBLE.value))
else:
pp_dict['mining_strategy'] = mining_strategy
pp_dict['mining_replacement_val'] = mining_replacement_val
pp_dict['mining_attributes'] = col_names
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def imputer(trfm, col_names, **kwargs):
"""
Parameters
----------
trfm :
Contains the Sklearn's Imputer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Imputer preprocessing.
"""
original_col_names = imputer.col_names
derived_colnames = col_names
pp_dict = dict()
derived_flds = list()
model = kwargs['model']
mining_strategy = trfm.strategy
if "mean" in mining_strategy:
mining_strategy = "asMean"
elif "median" in mining_strategy:
mining_strategy = "asMedian"
elif "most_frequent" in mining_strategy:
mining_strategy = "asMode"
mining_replacement_val = trfm.statistics_
if not any_in(original_col_names, col_names):
derived_colnames = get_derived_colnames('imputer', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
const_list = list()
apply_inner = list()
apply_inner.append(
pml.Apply(function='isMissing',
FieldRef=[
pml.FieldRef(field=col_names[col_name_idx])
]))
const_obj = pml.Constant(
dataType="double", # <---------------------
valueOf_=mining_replacement_val[col_name_idx]),
fieldref_obj = pml.FieldRef(field=col_names[col_name_idx])
fieldref_obj.original_tagname_ = "FieldRef"
const_list.append(const_obj[0])
const_list.append(fieldref_obj)
apply_outer = pml.Apply(Apply_member=apply_inner,
function='if',
Constant=const_list)
derived_flds.append(
pml.DerivedField(Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"))
else:
pp_dict['mining_strategy'] = mining_strategy
pp_dict['mining_replacement_val'] = mining_replacement_val
pp_dict['mining_attributes'] = col_names
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict