def lbl_binarizer(trfm, col_names, **kwargs):
"""
Parameters
----------
trfm :
Contains the Sklearn's Label Binarizer preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Label Binarizer preprocessing.
"""
derived_flds = list()
derived_colnames = list()
pp_dict = dict()
categoric_lbls = trfm.classes_.tolist()
model_exception_list = ["LinearRegression", "LogisticRegression", "SVR", "SVC"]
model = kwargs['model']
for col_name_idx in range(len(col_names)):
if len(categoric_lbls) == 2:
derived_colnames = get_derived_colnames("labelBinarizer(" + str(col_names[col_name_idx]),
[categoric_lbls[-1]], ")")
norm_descr = pml.NormDiscrete(field=str(col_names[-1]), value=str(categoric_lbls[-1]))
derived_flds.append(pml.DerivedField(NormDiscrete=norm_descr,
name=derived_colnames[-1],
optype="categorical",
dataType="double"))
else:
derived_colnames = get_derived_colnames("labelBinarizer(" + str(col_names[col_name_idx]),
categoric_lbls, ")")
for attribute_name in col_names:
for class_name, class_idx in zip(categoric_lbls, range(len(categoric_lbls))):
norm_descr = pml.NormDiscrete(field=str(attribute_name), value=str(class_name))
derived_flds.append(
pml.DerivedField(NormDiscrete=norm_descr,
name=derived_colnames[class_idx],
optype="categorical",
dataType="double"))
if any_in([model.__class__.__name__], model_exception_list):
pp_dict['hidden_lb_der_flds'] = derived_flds
exception_cols.extend(derived_colnames)
derived_flds = list()
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_class_lbl'] = categoric_lbls
pp_dict['pp_feat_name'] = col_names[0]
return pp_dict
def nyObjOfModel(self, pmmlObj, singMod):
import nyokaBase.PMML43Ext as ny
if singMod['pmmlModelObject'].__dict__[
'original_tagname_'] == 'MiningModel':
nyokaObj = ny.PMML(MiningBuildTask=pmmlObj.MiningBuildTask,
DataDictionary=pmmlObj.DataDictionary,
MiningModel=[singMod['pmmlModelObject']])
elif singMod['pmmlModelObject'].__dict__[
'original_tagname_'] == 'DeepNetwork':
nyokaObj = ny.PMML(DataDictionary=pmmlObj.DataDictionary,
DeepNetwork=[singMod['pmmlModelObject']])
else:
nyokaObj = None
return nyokaObj
def polynomial_features(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's PolynomialFeatures preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to PolynomialFeatures preprocessing.
"""
polynomial_features.poly_ctr += 1
pp_dict = dict()
derived_flds = []
derived_colnames = []
for polyfeat_idx in range(trfm.powers_.shape[0]):
apply_inner_container = []
for col_name_idx in range(len(col_names)):
val = int(trfm.powers_[polyfeat_idx][col_name_idx])
apply_inner = pml.Apply(
function='pow',
Constant=[pml.Constant(
dataType="integer",
valueOf_=val
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])])
apply_inner_container.append(apply_inner)
apply_outer = pml.Apply(function="product",
Apply_member=apply_inner_container
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
dataType="double",
optype="continuous",
name="poly" + str(polynomial_features.poly_ctr) + '-' + "x" + str(polyfeat_idx)
))
name = derived_flds[polyfeat_idx].get_name()
derived_colnames.append(name)
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def __init__(self, pmml):
self.nyoka_pmml = ny.parse(pmml,True)
self.image_input = None
self.layer_input = None
self.model = None
self.layers_outputs = {}
self.model = self._build_model()
def get_ensemble_models(model, derived_col_names, col_names, target_name, mining_imp_val,categoric_values):
"""
It returns the Mining Model element of the model
Parameters
----------
model :
Contains Xgboost model object.
derived_col_names : List
Contains column names after preprocessing.
col_names : List
Contains list of feature/column names.
target_name : String
Name of the Target column.
mining_imp_val : tuple
Contains the mining_attributes,mining_strategy, mining_impute_value.
categoric_values : tuple
Contains Categorical attribute names and its values
Returns
-------
mining_models :
Returns the MiningModel of the respective Xgboost model
"""
model_kwargs = sklToPmml.get_model_kwargs(model, col_names, target_name, mining_imp_val)
if 'XGBRegressor' in str(model.__class__):
model_kwargs['Targets'] = sklToPmml.get_targets(model, target_name)
mining_models = list()
mining_models.append(pml.MiningModel(
modelName="XGBoostModel",
Segmentation=get_outer_segmentation(model, derived_col_names, col_names, target_name, mining_imp_val,categoric_values),
**model_kwargs
))
return mining_models
def get_outer_segmentation(model, derived_col_names, col_names, target_name, mining_imp_val,categoric_values):
"""
It returns the Segmentation element of the model.
Parameters
----------
model :
Contains Xgboost model object.
derived_col_names : List
Contains column names after preprocessing.
col_names : List
Contains list of feature/column names.
target_name : String
Name of the Target column.
mining_imp_val : tuple
Contains the mining_attributes,mining_strategy, mining_impute_value
categoric_values : tuple
Contains Categorical attribute names and its values
Returns
-------
segmentation :
Get the outer most Segmentation of an xgboost model
"""
if 'XGBRegressor' in str(model.__class__):
segmentation=get_segments(model, derived_col_names, col_names, target_name, mining_imp_val,categoric_values)
else:
segmentation = pml.Segmentation(
multipleModelMethod=get_multiple_model_method(model),
Segment=get_segments(model, derived_col_names, col_names, target_name, mining_imp_val,categoric_values)
)
return segmentation
def selectArchitecture(self, checkTemplateID):
if checkTemplateID == 'mobilenetArch':
pmmlObj = ny.parse(open(settingFilePath + 'MobilenetArch.pmml',
'r'),
silence=True)
templateArch = self.pmmlToJson(settingFilePath +
'MobilenetArch.pmml')
elif checkTemplateID == 'vgg16Arch':
pmmlObj = ny.parse(open(settingFilePath + 'vGG16Arch.pmml', 'r'),
silence=True)
templateArch = self.pmmlToJson(settingFilePath + 'vGG16Arch.pmml')
elif checkTemplateID == 'vgg19Arch':
pmmlObj = ny.parse(open(settingFilePath + 'vGG19Arch.pmml', 'r'),
silence=True)
templateArch = self.pmmlToJson(settingFilePath + 'vGG19Arch.pmml')
return templateArch, pmmlObj
def get_segments_for_xgbr(model, derived_col_names, feature_names, target_name, mining_imp_val,categorical_values):
"""
It returns all the Segments element of the model
Parameters
----------
model :
Contains Xgboost model object.
derived_col_names : List
Contains column names after preprocessing.
feature_names : List
Contains list of feature/column names.
target_name : List
Name of the Target column.
mining_imp_val : tuple
Contains the mining_attributes,mining_strategy, mining_impute_value
categoric_values : tuple
Contains Categorical attribute names and its values
Returns
-------
segment :
Get the Segmentation element which contains inner segments.
"""
segments = list()
get_nodes_in_json_format = []
for i in range(model.n_estimators):
get_nodes_in_json_format.append(json.loads(model._Booster.get_dump(dump_format='json')[i]))
segmentation = pml.Segmentation(multipleModelMethod="sum",
Segment=generate_Segments_Equal_To_Estimators(get_nodes_in_json_format, derived_col_names,
feature_names))
return segmentation
def writePmml(pmmlObj, filepath, lockForPMML):
try:
lockForPMML.acquire()
pmmlObj = removeExtraNewLinesFromWeights(pmmlObj)
scrptVal2 = []
scrptVal = pmmlObj.script
if len(scrptVal) > 0:
for num, sc in enumerate(scrptVal):
urlOfScript = sc.class_
useFor = sc.for_
code = None
scripCode = sc.get_valueOf_()
code = scripCode.lstrip('\n')
lines = []
code = scripCode.lstrip('\n')
leading_spaces = len(code) - len(code.lstrip(' '))
for line in code.split('\n'):
lines.append(line[leading_spaces:])
code = '\n'.join(lines)
scriptCode = code.replace('<', '<')
# print (scriptCode)
scrp = pml.script(content=scriptCode,
class_=urlOfScript,
for_=useFor)
scrptVal2.append(scrp)
pmmlObj.script = scrptVal2
# print ('Code Step 10.1')
pmmlObj.export(open(filepath, 'w'), 0)
# print('>>>>>>>>>>>, PMML written')
except Exception as e:
print('>>>>>>>>>>>> ', str(e))
finally:
lockForPMML.release()
def writePMML(self, model, predictedClass, fileName, dataSet):
try:
from nyokaBase.keras.keras_model_to_pmml import KerasToPmml
pmmlToBack = KerasToPmml(model,
model_name="TrainedModel",
description="Keras Models in PMML",
dataSet=dataSet,
predictedClasses=predictedClass)
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Error while converting Keras to PMML >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return -1
scriptCode = self.pmmlObj['script']
if scriptCode == []:
scriptCode = None
else:
for sc in scriptCode:
sc.__dict__['valueOf_'] = sc.get_valueOf_().replace(
'<', '<')
pmmlObjNew = pmmlToBack.__dict__
dDict = pmmlObjNew['DataDictionary']
netw = pmmlObjNew['DeepNetwork']
netw = self.updateSectionInfo(netw)
extensionInfoForData = [
ny.Extension(value=self.hdExtDet, anytypeobjs_=[''])
]
hd = ny.Header(copyright="Copyright (c) 2018 Software AG",
Extension=extensionInfoForData,
description="Neural Network Model",
Timestamp=ny.Timestamp(datetime.now()))
with open(fileName, 'w') as filetosave:
jj = ny.PMML(version="4.3Ext",
DeepNetwork=netw,
DataDictionary=dDict,
Header=hd,
script=scriptCode)
jj.export(filetosave, 0)
def mining_Field_For_First_Segment(feature_names):
"""
It returns the Mining Schema of the First Segment.
Parameters
----------
feature_names: List
Contains list of feature/column names.
Returns:
-------
mining_schema_for_1st_segment:
Returns the MiningSchema for the main segment.
"""
mining_fields_1st_segment = []
for name in feature_names:
mining_fields_1st_segment.append(pml.MiningField(name=name))
mining_schema_for_1st_segment = pml.MiningSchema(MiningField=mining_fields_1st_segment)
return mining_schema_for_1st_segment
def lbl_encoder(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's LabelEncoder preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to LabelEncoder preprocessing.
"""
pp_dict = dict()
derived_flds = list()
field_column_pair = list()
rows = []
categoric_lbls = trfm.classes_.tolist()
categoric_lbls_num = trfm.transform(trfm.classes_.tolist()).tolist()
derived_colnames = get_derived_colnames('labelEncoder', col_names)
for row_idx in range(len(categoric_lbls_num)):
row_main = pml.row()
row_main.elementobjs_ = ['input', 'output']
row_main.input = categoric_lbls[row_idx]
row_main.output = str(categoric_lbls_num[row_idx])
rows.append(row_main)
field_column_pair.append(pml.FieldColumnPair(field=str(col_names[0]), column="input"))
inline_table = pml.InlineTable(row=rows)
map_values = pml.MapValues(outputColumn="output", FieldColumnPair=field_column_pair, InlineTable=inline_table)
derived_flds.append(
pml.DerivedField(MapValues=map_values, name=derived_colnames[0], optype="continuous", dataType="double"))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_class_lbl'] = categoric_lbls
pp_dict['pp_feat_name'] = col_names[0]
return pp_dict
def post(self,requests):
filePath = requests.POST.get('filePath')
from nyokaBase import PMML43Ext as ny
pmmlObj = ny.parse(open(filePath,'r'),silence=True)
nn = NeuralNetworkModelTrainer()
nn.pmmlfileObj = pmmlObj
returnVal = nn.generateAndCompileModel('mean_squared_error','adam',0.1,['accuracy','f1'],compileTestOnly=True)
if returnVal.__class__.__name__ == 'dict':
return JsonResponse(returnVal)
else:
return JsonResponse({'status':'Model Compiled Successfully'},status=200)
def xgboost_to_pmml(pipeline, col_names, target_name, pmml_f_name='from_xgboost.pmml'):
"""
Exports xgboost pipeline object into pmml
Parameters
----------
pipeline :
Contains an instance of Pipeline with preprocessing and final estimator
col_names : List
Contains list of feature/column names.
target_name : String
Name of the target column.
pmml_f_name : String
Name of the pmml file. (Default='from_xgboost.pmml')
Returns
-------
Returns a pmml file
"""
try:
model = pipeline.steps[-1][1]
except:
raise TypeError("Exporter expects pipeleine_instance and not an estimator_instance")
else:
if isinstance(col_names, np.ndarray):
col_names = col_names.tolist()
ppln_sans_predictor = pipeline.steps[:-1]
trfm_dict_kwargs = dict()
derived_col_names = col_names
categoric_values = tuple()
mining_imp_val = tuple()
if ppln_sans_predictor:
pml_pp = pp.get_preprocess_val(ppln_sans_predictor, col_names, model)
trfm_dict_kwargs['TransformationDictionary'] = pml_pp['trfm_dict']
derived_col_names = pml_pp['derived_col_names']
col_names = pml_pp['preprocessed_col_names']
categoric_values = pml_pp['categorical_feat_values']
mining_imp_val = pml_pp['mining_imp_values']
PMML_kwargs = get_PMML_kwargs(model,
derived_col_names,
col_names,
target_name,
mining_imp_val,
categoric_values)
pmml = pml.PMML(
version=sklToPmml.get_version(),
Header=sklToPmml.get_header(),
MiningBuildTask=sklToPmml.get_mining_buildtask(pipeline),
DataDictionary=sklToPmml.get_data_dictionary(model, col_names, target_name, categoric_values),
**trfm_dict_kwargs,
**PMML_kwargs
)
pmml.export(outfile=open(pmml_f_name, "w"), level=0)
def tfidf_vectorizer(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's TfIdfVectorizer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to TfIdfVectorizer preprocessing.
"""
pp_dict = dict()
features = trfm.get_feature_names()
idfs = trfm.idf_
extra_features = list(trfm.vocabulary_.keys())
derived_flds = list()
derived_colnames = get_derived_colnames('tfidf@[' + col_names[0] + ']', features)
derived_flds.append(
pml.DerivedField(name='lowercase(' + col_names[0] + ')',
optype='categorical', dataType='string',
Apply=pml.Apply(function='lowercase',
FieldRef=[pml.FieldRef(field=col_names[0])])))
for feat_idx, idf in zip(range(len(features)), idfs):
derived_flds.append(pml.DerivedField(
name=derived_colnames[feat_idx],
optype='continuous',
dataType='double',
Apply=pml.Apply(function='*',
TextIndex=[pml.TextIndex(textField='lowercase(' + col_names[0] + ')',
wordSeparatorCharacterRE='\s+',
tokenize='true',
Constant=pml.Constant(valueOf_=features[feat_idx]),
Extension=[pml.Extension(anytypeobjs_=[extra_features[feat_idx]])])],
Constant=[pml.Constant(valueOf_=idf)])
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_name'] = col_names[0]
pp_dict['pp_feat_class_lbl'] = list()
return pp_dict
def max_abs_scaler(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's MaxabsScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to MaxabsScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames('maxAbsScaler', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_outer = pml.Apply(
function='/',
Constant=[pml.Constant(
dataType="double", # <---------------------
valueOf_=unround_scalers(trfm.max_abs_[col_name_idx])
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def add_segmentation(model,segments_equal_to_estimators,mining_schema_for_1st_segment,out,id):
"""
It returns the First Segments for a binary classifier and returns number of Segments equls to number of values
target class for multiclass classifier
Parameters
----------
model:
Contains Xgboost model object.
segments_equal_to_estimators: List
Contains List Segements equals to the number of the estimators of the model.
mining_schema_for_1st_segment:
Contains Mining Schema for the First Segment
out:
Contains the Output element
id: Integer
Index of the Segements
Returns:
-------
segments_equal_to_estimators:
Returns list of segments equal to number of estimator of the model
"""
segmentation = pml.Segmentation(multipleModelMethod="sum", Segment=segments_equal_to_estimators)
mining_model = pml.MiningModel(functionName='regression', modelName="MiningModel", MiningSchema=mining_schema_for_1st_segment,
Output=out, Segmentation=segmentation)
if model.n_classes_==2:
First_segment = pml.Segment(True_=pml.True_(), id=id, MiningModel=mining_model)
return First_segment
else:
segments_equal_to_class = pml.Segment(True_=pml.True_(), id=id + 1, MiningModel=mining_model)
return segments_equal_to_class
def binarizer(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's Binarizer preprocessing instance.
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Binarizer preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames("binarizer", col_names)
for col_name_idx in range(len(col_names)):
apply_outer = pml.Apply(
function='threshold',
Constant=[pml.Constant(
dataType="double",
valueOf_=trfm.threshold
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])])
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def dump_config(self):
config_vars = [var for var in self.config.__dir__() if var.isupper()]
config_dict = {}
for var in config_vars:
val = getattr(self.config, var)
if val.__class__.__name__ == "ndarray":
val = val.tolist()
config_dict[var] = val
info_dict = {}
info_dict["config"] = config_dict
info_dict["mode"] = self.mode
exten_obj = pml.Extension(anytypeobjs_=[str(info_dict)], name="config")
self.pmml_obj.DeepNetwork[0].Extension = [exten_obj]
def test_8_compileModel(self):
logging.info("Test Case : Compile a model.(2)")
filePath = 'testUseCase/supportdata/from_sklearn.pmml'
from nyokaBase import PMML43Ext as ny
pmmlObj = ny.parse(open(filePath, 'r'), silence=True)
from trainModel.mergeTrainingNN import NeuralNetworkModelTrainer
nn = NeuralNetworkModelTrainer()
nn.pmmlfileObj = pmmlObj
returnVal = nn.generateAndCompileModel('mean_squared_error',
'adam',
0.1, ['accuracy', 'f1'],
compileTestOnly=True)
self.assertEqual('status' in returnVal, True)
self.assertEqual('errorMessage' in returnVal, True)
self.assertEqual('errorTraceback' in returnVal, True)
self.assertEqual(returnVal['status'], 'Model Compilation Failed')
def test_7_compileModel(self):
logging.info("Test Case : Compile a model.(1)")
filePath = 'testUseCase/supportdata/irisNN.pmml'
from nyokaBase import PMML43Ext as ny
pmmlObj = ny.parse(open(filePath, 'r'), silence=True)
from trainModel.mergeTrainingNN import NeuralNetworkModelTrainer
nn = NeuralNetworkModelTrainer()
nn.pmmlfileObj = pmmlObj
returnVal = nn.generateAndCompileModel('mean_squared_error',
'adam',
0.1, ['accuracy', 'f1'],
compileTestOnly=True)
self.assertEqual('nyoka_pmml' in returnVal.__dict__, True)
self.assertEqual('model' in returnVal.__dict__, True)
self.assertEqual(returnVal.nyoka_pmml.__class__.__name__, 'PMML')
self.assertEqual(returnVal.__class__.__name__, 'GenerateKerasModel')
def writePmml(pmmlObj, filepath, lockForPMML):
_deepNetworkObj = pmmlObj.DeepNetwork[0]
_deepNetworkObj.modelName = 'model1'
_deepNetworkObj.taskType = "trainAndscore"
pmmlObj.DeepNetwork[0] = _deepNetworkObj
# print ('came to write')
try:
lockForPMML.acquire()
pmmlObj = removeExtraNewLinesFromWeights(pmmlObj)
scrptVal2 = []
scrptVal = pmmlObj.script
if len(scrptVal) > 0:
for num, sc in enumerate(scrptVal):
scriptPurpose = sc.scriptPurpose
modelVal = sc.for_
classVal = sc.class_
filePathUrl = sc.filePath
scriptOutput = sc.scriptOutput
code = None
scripCode = sc.get_valueOf_()
code = scripCode.lstrip('\n')
lines = []
code = scripCode.lstrip('\n')
leading_spaces = len(code) - len(code.lstrip(' '))
for line in code.split('\n'):
lines.append(line[leading_spaces:])
code = '\n'.join(lines)
scriptCode = code.replace('<', '<')
# scrp=pml.script(content=scriptCode,for_=modelVal,class_=taskTypeVal,scriptPurpose=scriptPurpose,scriptOutput=scriptOutput,filePath=filePathUrl)
scrp = pml.script(content=scriptCode,
for_=modelVal,
class_=classVal,
scriptPurpose=scriptPurpose,
scriptOutput=scriptOutput,
filePath=filePathUrl)
scrptVal2.append(scrp)
pmmlObj.script = scrptVal2
# print ('Code Step 10.1')
pmmlObj.export(open(filepath, 'w'), 0)
# print('>>>>>>>>>>>, PMML written')
except Exception as e:
print('>>>>>>>>>>>> ', str(e))
finally:
lockForPMML.release()
def rbst_scaler(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's RobustScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to RobustScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames('robustScaler', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_inner = list()
apply_inner.append(pml.Apply(
function='-',
Constant=[pml.Constant(
dataType="double", # <---------------------
valueOf_=unround_scalers(trfm.center_[col_name_idx])
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])],
Extension=[pml.Extension(name='scaling', anytypeobjs_=['RobustScaler'])]
))
apply_outer = pml.Apply(
Apply_member=apply_inner,
function='/',
Constant=[pml.Constant(
dataType="double", # <----------------------------
valueOf_=unround_scalers(trfm.scale_[col_name_idx])
)]
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def std_scaler(trfm, col_names, **kwargs):
"""
Parameters
----------
trfm :
Contains the Sklearn's Standard Scaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Standard Scaler preprocessing.
"""
derived_flds = list()
pp_dict = dict()
if is_present("labelBinarizer",col_names):
derived_flds_hidden = kwargs['derived_fld']
if derived_flds_hidden:
derived_flds.extend(derived_flds_hidden)
derived_colnames = get_derived_colnames('standardScaler', col_names)
for col_name_idx in range(len(col_names)):
apply_inner = list()
apply_inner.append(pml.Apply(
function='-',
Constant=[pml.Constant(
dataType="double", # <---------------------
valueOf_=unround_scalers(trfm.mean_[col_name_idx])
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]
))
apply_outer = pml.Apply(
Apply_member=apply_inner,
function='/',
Constant=[pml.Constant(
dataType="double", # <----------------------------
valueOf_=unround_scalers(trfm.scale_[col_name_idx])
)]
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def pca(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's PCA preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to PCA preprocessing.
"""
pca.counter += 1
pp_dict = dict()
derived_flds = list()
derived_colnames = list()
val = trfm.mean_
zero = 0.0
for preprocess_idx in range(trfm.n_components):
add = list()
for pca_idx in range(trfm.n_features_):
apply_inner = pml.Apply(function='-',
Constant=[pml.Constant(dataType="double",
valueOf_=val[pca_idx])],
FieldRef=[pml.FieldRef(field=col_names[pca_idx])])
apply_outer = pml.Apply(function="*",
Apply_member=[apply_inner],
Constant=[pml.Constant(dataType="double",
valueOf_=zero if trfm.components_[preprocess_idx][
pca_idx] == 0.0 else
trfm.components_[preprocess_idx][pca_idx])])
add.append(apply_outer)
app0 = pml.Apply(function="sum", Apply_member=add)
derived_flds.append(pml.DerivedField(Apply=app0,
dataType="double",
optype="continuous",
name="PCA" + str(pca.counter) + "-" + str(preprocess_idx)))
name = derived_flds[preprocess_idx].get_name()
derived_colnames.append(name)
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def getDetailsOfPMML(filepath):
# print ('Enter this world')
pmmlObj = pml.parse(filepath, silence=True)
tempObj = pmmlObj.__dict__
listOfObjectstogetData = []
for j in tempObj.keys():
if (tempObj[j] is None):
pass
elif (isinstance(tempObj[j], typing.List)):
if (len(tempObj[j]) == 0):
pass
else:
listOfObjectstogetData.append(j)
else:
listOfObjectstogetData.append(j)
allInfo = {}
for towork in listOfObjectstogetData:
if towork == 'version':
allInfo['Version'] = tempObj['version']
elif towork == 'Header':
allInfo.update(nyokaUtilities.getHeaderInfo(tempObj))
elif towork == 'DataDictionary':
allInfo.update(nyokaUtilities.getDataFields(tempObj))
elif towork == 'NearestNeighborModel':
allInfo.update(
nyokaUtilities.getInfoNearestNeighborModel(tempObj))
elif towork == 'DeepNetwork':
allInfo.update(nyokaUtilities.getInfoOfDeepNetwork(tempObj))
elif towork == 'MiningModel':
allInfo.update(nyokaUtilities.getInfoMiningModel(tempObj))
elif towork == 'SupportVectorMachineModel':
allInfo.update(
nyokaUtilities.getInfoSupportVectorMachineModel(tempObj))
elif towork == 'TreeModel':
allInfo.update(nyokaUtilities.getInfoTreeModel(tempObj))
elif towork == 'RegressionModel':
allInfo.update(nyokaUtilities.getInfoLinearModel(tempObj))
elif towork == 'NaiveBayesModel':
allInfo.update(
nyokaUtilities.getInfoOfNaiveBayesModel(tempObj))
allInfo = nyokaUtilities.changeStructure(allInfo)
# print('response sent')
return JsonResponse(allInfo)
def generate_Segments_Equal_To_Estimators(val, derived_col_names, col_names):
"""
It returns number of Segments equal to the estimator of the model.
Parameters
----------
val: List
Contains a list of well structured node for binary classification/inner segments for multi-class classification
derived_col_names: List
Contains column names after preprocessing.
col_names: List
Contains list of feature/column names.
Returns:
-------
segments_equal_to_estimators:
Returns list of segments equal to number of estimator of the model
"""
segments_equal_to_estimators = []
for i in range(len(val)):
main_node = pml.Node(True_=pml.True_())
m_flds = []
mining_field_for_innner_segments = col_names
create_node(val[i], main_node, derived_col_names)
for name in mining_field_for_innner_segments:
m_flds.append(pml.MiningField(name=name))
segments_equal_to_estimators.append((pml.Segment(id=i + 1, True_=pml.True_(),
TreeModel=pml.TreeModel(functionName="regression",
modelName="DecisionTreeModel",
missingValueStrategy="none",
noTrueChildStrategy="returnLastPrediction",
splitCharacteristic="multiSplit",
Node=main_node,
MiningSchema=pml.MiningSchema(
MiningField=m_flds)))))
return segments_equal_to_estimators
def get_preprocess_val(ppln_sans_predictor, initial_colnames, model):
"""
Parameters
----------
model :
Contains an instance of Sklearn model
ppln_sans_predictor :
Contains an instance of Sklearn Pipeline
initial_colnames : list
Contains list of feature/column names.
Returns
-------
pml_pp: dictionary
Returns a dictionary that contains data related to pre-processing
"""
pml_pp = dict()
pml_derived_flds = list()
initial_colnames = [col_name for col_name in initial_colnames]
updated_colnames = initial_colnames.copy()
dtd_feat_names = list()
classes = list()
class_attribute = list()
mining_strategy = list()
mining_replacement_val = list()
mining_attributes = list()
derived_flds_hidden = list()
pml_trfm_dict = list()
polynomial_features.poly_ctr = 0
pca.counter = 0
imputer.col_names = initial_colnames
for ppln_step in ppln_sans_predictor:
ppln_step_inst = ppln_step[1]
if "DataFrameMapper" == get_class_name(ppln_step_inst):
dfm_steps = ppln_step_inst.features
dfm_col_names = list()
for dfm_step in dfm_steps:
dfm_step_col_names = dfm_step[0]
dfm_step_trfms = dfm_step[1]
if not hasattr(dfm_step_col_names, "__len__") or isinstance(dfm_step_col_names, str):
dfm_step_col_names = [dfm_step_col_names]
if not hasattr(dfm_step_trfms, "__len__") or isinstance(dfm_step_trfms, str):
dfm_step_trfms = [dfm_step_trfms]
for name in dfm_step_col_names:
if name not in dtd_feat_names:
dtd_feat_names.append(name)
for trfm in dfm_step_trfms:
pp_dict = get_pml_derived_flds(trfm, dfm_step_col_names, derived_fld=derived_flds_hidden,
model=model)
derived_flds = pp_dict['der_fld']
derived_names = pp_dict['der_col_names']
if 'pp_feat_class_lbl' in pp_dict.keys():
classes.append(pp_dict['pp_feat_class_lbl'])
class_attribute.append(pp_dict['pp_feat_name'])
if 'pp_feat_class_ohe' in pp_dict.keys():
classes.append(pp_dict['pp_feat_class_ohe'])
class_attribute.append(pp_dict['pp_feat_name'])
if 'mining_strategy' in pp_dict.keys():
mining_attributes.append(pp_dict['der_col_names'])
mining_strategy.append(pp_dict['mining_strategy'])
mining_replacement_val.append(pp_dict['mining_replacement_val'])
if 'hidden_lb_der_flds' in pp_dict.keys():
derived_flds_hidden.extend(pp_dict['hidden_lb_der_flds'])
if 'hidden_ohe_der_flds' in pp_dict.keys():
derived_flds_hidden.extend(pp_dict['hidden_ohe_der_flds'])
pml_derived_flds.extend(derived_flds)
dfm_step_col_names = derived_names
dfm_col_names.extend(derived_names)
updated_colnames = dfm_col_names
else:
if not dtd_feat_names:
dtd_feat_names = initial_colnames
updated_colnames = initial_colnames
if not hasattr(ppln_step_inst, "__len__") or isinstance(ppln_step_inst, str):
ppln_step_inst = [ppln_step_inst]
for trfm in ppln_step_inst:
pp_dict = get_pml_derived_flds(trfm, updated_colnames, derived_fld=derived_flds_hidden, model=model)
derived_flds = pp_dict['der_fld']
derived_names = pp_dict['der_col_names']
if 'pp_feat_class_lbl' in pp_dict.keys():
classes.append(pp_dict['pp_feat_class_lbl'])
class_attribute.append(pp_dict['pp_feat_name'])
if 'pp_feat_class_ohe' in pp_dict.keys():
classes.append(pp_dict['pp_feat_class_ohe'])
class_attribute.append(pp_dict['pp_feat_name'])
if 'mining_strategy' in pp_dict.keys():
mining_attributes.append(pp_dict['der_col_names'])
mining_strategy.append(pp_dict['mining_strategy'])
mining_replacement_val.append(pp_dict['mining_replacement_val'])
pml_derived_flds.extend(derived_flds)
updated_colnames = derived_names
if pml_derived_flds:
pml_trfm_dict = [pml.TransformationDictionary(DerivedField=pml_derived_flds)]
pml_pp['trfm_dict'] = pml_trfm_dict
pml_pp['derived_col_names'] = updated_colnames
pml_pp['preprocessed_col_names'] = dtd_feat_names
pml_pp['categorical_feat_values'] = classes, class_attribute
pml_pp['mining_imp_values'] = mining_attributes, mining_strategy, mining_replacement_val
return pml_pp
def get_segments_for_xgbc(model, derived_col_names, feature_names, target_name, mining_imp_val,categoric_values):
"""
It returns all the segments of the Xgboost classifier.
Parameters
----------
model :
Contains Xgboost model object.
derived_col_names : List
Contains column names after preprocessing.
feature_names: List
Contains list of feature/column names.
target_name : String
Name of the Target column.
mining_imp_val : tuple
Contains the mining_attributes,mining_strategy, mining_impute_value
categoric_values : tuple
Contains Categorical attribute names and its values
Returns
-------
regrs_models :
Returns all the segments of the xgboost model.
"""
segments = list()
if model.n_classes_ == 2:
get_nodes_in_json_format=[]
for i in range(model.n_estimators):
get_nodes_in_json_format.append(json.loads(model._Booster.get_dump(dump_format='json')[i]))
mining_schema_for_1st_segment = mining_Field_For_First_Segment(feature_names)
outputField = list()
outputField.append(pml.OutputField(name="xgbValue", optype="continuous", dataType="float",
feature="predictedValue", isFinalResult="true"))
out = pml.Output(OutputField=outputField)
oField=list()
oField.append('xgbValue')
segments_equal_to_estimators = generate_Segments_Equal_To_Estimators(get_nodes_in_json_format, derived_col_names,
feature_names)
First_segment = add_segmentation(model,segments_equal_to_estimators, mining_schema_for_1st_segment, out, 1)
reg_model=sklToPmml.get_regrs_models(model, oField, oField, target_name,mining_imp_val,categoric_values)[0]
reg_model.normalizationMethod='logit'
last_segment = pml.Segment(True_=pml.True_(), id=2,
RegressionModel=reg_model)
segments.append(First_segment)
segments.append(last_segment)
else:
get_nodes_in_json_format = []
for i in range(model.n_estimators * model.n_classes_):
get_nodes_in_json_format.append(json.loads(model._Booster.get_dump(dump_format='json')[i]))
oField = list()
for index in range(0, model.n_classes_):
inner_segment = []
for in_seg in range(index, len(get_nodes_in_json_format), model.n_classes_):
inner_segment.append(get_nodes_in_json_format[in_seg])
mining_schema_for_1st_segment = mining_Field_For_First_Segment(feature_names)
outputField = list()
outputField.append(pml.OutputField(name='xgbValue(' + str(index) + ')', optype="continuous",
feature="predictedValue", dataType="float", isFinalResult="true"))
out = pml.Output(OutputField=outputField)
oField.append('xgbValue(' + str(index) + ')')
segments_equal_to_estimators = generate_Segments_Equal_To_Estimators(inner_segment, derived_col_names,
feature_names)
segments_equal_to_class = add_segmentation(model,segments_equal_to_estimators,
mining_schema_for_1st_segment, out, index)
segments.append(segments_equal_to_class)
last_segment = pml.Segment(True_=pml.True_(), id=model.n_classes_ + 1,
RegressionModel=sklToPmml.get_regrs_models(model,oField,oField,target_name,
mining_imp_val,categoric_values)[0])
segments.append(last_segment)
return segments
def imputer(trfm, col_names, **kwargs):
"""
Parameters
----------
trfm :
Contains the Sklearn's Imputer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to Imputer preprocessing.
"""
original_col_names = imputer.col_names
derived_colnames = col_names
pp_dict = dict()
derived_flds = list()
model = kwargs['model']
mining_strategy = trfm.strategy
if "mean" in mining_strategy:
mining_strategy = "asMean"
elif "median" in mining_strategy:
mining_strategy = "asMedian"
elif "most_frequent" in mining_strategy:
mining_strategy = "asMode"
mining_replacement_val = trfm.statistics_
if not any_in(original_col_names, col_names):
derived_colnames = get_derived_colnames('imputer', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
const_list = list()
apply_inner = list()
apply_inner.append(pml.Apply(function='isMissing', FieldRef=[pml.FieldRef(field=col_names[col_name_idx])]))
const_obj = pml.Constant(
dataType="double", # <---------------------
valueOf_=mining_replacement_val[col_name_idx]
),
fieldref_obj = pml.FieldRef(field=col_names[col_name_idx])
fieldref_obj.original_tagname_ = "FieldRef"
const_list.append(const_obj[0])
const_list.append(fieldref_obj)
apply_outer = pml.Apply(
Apply_member=apply_inner,
function='if',
Constant=const_list
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
else:
pp_dict['mining_strategy'] = mining_strategy
pp_dict['mining_replacement_val'] = mining_replacement_val
pp_dict['mining_attributes'] = col_names
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict