def dump_config(self):
config_vars = [var for var in self.config.__dir__() if var.isupper()]
config_dict = {}
for var in config_vars:
val = getattr(self.config, var)
if val.__class__.__name__ == "ndarray":
val = val.tolist()
config_dict[var] = val
info_dict = {}
info_dict["config"] = config_dict
info_dict["mode"] = self.mode
exten_obj = pml.Extension(anytypeobjs_=[str(info_dict)], name="config")
self.pmml_obj.DeepNetwork[0].Extension = [exten_obj]
def rbst_scaler(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's RobustScaler preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to RobustScaler preprocessing.
"""
pp_dict = dict()
derived_flds = list()
derived_colnames = get_derived_colnames('robustScaler', col_names)
for col_name_idx in range(len(col_names)):
if (col_names[col_name_idx] not in exception_cols):
apply_inner = list()
apply_inner.append(pml.Apply(
function='-',
Constant=[pml.Constant(
dataType="double", # <---------------------
valueOf_=unround_scalers(trfm.center_[col_name_idx])
)],
FieldRef=[pml.FieldRef(field=col_names[col_name_idx])],
Extension=[pml.Extension(name='scaling', anytypeobjs_=['RobustScaler'])]
))
apply_outer = pml.Apply(
Apply_member=apply_inner,
function='/',
Constant=[pml.Constant(
dataType="double", # <----------------------------
valueOf_=unround_scalers(trfm.scale_[col_name_idx])
)]
)
derived_flds.append(pml.DerivedField(
Apply=apply_outer,
name=derived_colnames[col_name_idx],
optype="continuous",
dataType="double"
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
return pp_dict
def writePMML(self, model, predictedClass, fileName, dataSet):
try:
from nyokaBase.keras.keras_model_to_pmml import KerasToPmml
pmmlToBack = KerasToPmml(model,
model_name="TrainedModel",
description="Keras Models in PMML",
dataSet=dataSet,
predictedClasses=predictedClass)
except Exception as e:
data_details = self.upDateStatus()
data_details['status'] = 'Training Failed'
data_details[
'errorMessage'] = 'Error while converting Keras to PMML >> ' + str(
e)
data_details['errorTraceback'] = traceback.format_exc()
with open(self.statusFile, 'w') as filetosave:
json.dump(data_details, filetosave)
# sys.exit()
return -1
scriptCode = self.pmmlObj['script']
if scriptCode == []:
scriptCode = None
else:
for sc in scriptCode:
sc.__dict__['valueOf_'] = sc.get_valueOf_().replace(
'<', '<')
pmmlObjNew = pmmlToBack.__dict__
dDict = pmmlObjNew['DataDictionary']
netw = pmmlObjNew['DeepNetwork']
netw = self.updateSectionInfo(netw)
extensionInfoForData = [
ny.Extension(value=self.hdExtDet, anytypeobjs_=[''])
]
hd = ny.Header(copyright="Copyright (c) 2018 Software AG",
Extension=extensionInfoForData,
description="Neural Network Model",
Timestamp=ny.Timestamp(datetime.now()))
with open(fileName, 'w') as filetosave:
jj = ny.PMML(version="4.3Ext",
DeepNetwork=netw,
DataDictionary=dDict,
Header=hd,
script=scriptCode)
jj.export(filetosave, 0)
def tfidf_vectorizer(trfm, col_names):
"""
Parameters
----------
trfm :
Contains the Sklearn's TfIdfVectorizer preprocessing instance
col_names : list
Contains list of feature/column names.
The column names may represent the names of preprocessed attributes.
Returns
-------
pp_dict : dictionary
Returns a dictionary that contains attributes related to TfIdfVectorizer preprocessing.
"""
pp_dict = dict()
features = trfm.get_feature_names()
idfs = trfm.idf_
extra_features = list(trfm.vocabulary_.keys())
derived_flds = list()
derived_colnames = get_derived_colnames('tfidf@[' + col_names[0] + ']', features)
derived_flds.append(
pml.DerivedField(name='lowercase(' + col_names[0] + ')',
optype='categorical', dataType='string',
Apply=pml.Apply(function='lowercase',
FieldRef=[pml.FieldRef(field=col_names[0])])))
for feat_idx, idf in zip(range(len(features)), idfs):
derived_flds.append(pml.DerivedField(
name=derived_colnames[feat_idx],
optype='continuous',
dataType='double',
Apply=pml.Apply(function='*',
TextIndex=[pml.TextIndex(textField='lowercase(' + col_names[0] + ')',
wordSeparatorCharacterRE='\s+',
tokenize='true',
Constant=pml.Constant(valueOf_=features[feat_idx]),
Extension=[pml.Extension(anytypeobjs_=[extra_features[feat_idx]])])],
Constant=[pml.Constant(valueOf_=idf)])
))
pp_dict['der_fld'] = derived_flds
pp_dict['der_col_names'] = derived_colnames
pp_dict['pp_feat_name'] = col_names[0]
pp_dict['pp_feat_class_lbl'] = list()
return pp_dict
def addLayerToPMML(_positionOfLayer, toUpdateLayer, processedOutput,
pmmlObject):
def checkIfFlushRequired(_oldPosition, _newPositon, _pmmlLayer,
_networkLayers):
if _oldPosition == _newPositon:
if (_pmmlLayer.LayerParameters.inputDimension
== _networkLayers[_oldPosition].LayerParameters.
inputDimension) & (
_pmmlLayer.LayerParameters.outputDimension
== _networkLayers[_oldPosition].
LayerParameters.outputDimension):
return False
else:
return True
else:
return True
if processedOutput['itemType'] == 'LAYER':
_inputForPMML = nyokaPMMLUtilities.convertToStandardJson(
processedOutput)
# print ('',_inputForPMML)
_pmmlOfLayer = nyokaPMMLUtilities.addLayer(_inputForPMML)
# print('>>>>>>>',_pmmlOfLayer.__dict__)
_deepNetworkObj = pmmlObject.DeepNetwork[0]
# print ('step 1')
_NetworkLayersObject = _deepNetworkObj.NetworkLayer
# print ('step 2')
_idsOfNetworklayer = [i.layerId for i in _NetworkLayersObject]
flushMemory = False
if toUpdateLayer == True:
_oldPositionOFLayer = _idsOfNetworklayer.index(
processedOutput['layerId'])
# print('>>>> layer id is ',processedOutput['layerId'])
# print('>>>> old position to delete',_oldPositionOFLayer)
# print('>>>> new position ',_positionOfLayer)
# print ('_idsOfNetworklayer',_idsOfNetworklayer)
flushMemory = checkIfFlushRequired(_oldPositionOFLayer,
_positionOfLayer,
_pmmlOfLayer,
_NetworkLayersObject)
# print('flushmemory is ',flushMemory)
if flushMemory == False:
_NetworkLayersObject[
_oldPositionOFLayer].Extension = _pmmlOfLayer.Extension
_pmmlOfLayer = _NetworkLayersObject[
_oldPositionOFLayer]
del _NetworkLayersObject[_oldPositionOFLayer]
_NetworkLayersObject.insert(_positionOfLayer, _pmmlOfLayer)
# print ('step 3')
if flushMemory == True:
_NetworkLayersObject = resetNetworkLayer(
_NetworkLayersObject, _positionOfLayer)
# print ('step 6')
_NetworkLayersObject = reorderIdsOfPmml(_NetworkLayersObject)
_deepNetworkObj.NetworkLayer = _NetworkLayersObject
_deepNetworkObj.numberOfLayers = len(_NetworkLayersObject)
pmmlObject.DeepNetwork[0] = _deepNetworkObj
elif processedOutput['itemType'] == 'DATA':
# print ("DATA layer came")
extensionInfoForData = [
pml.Extension(value=[], anytypeobjs_=[''])
]
dataVal = {}
try:
dataVal['dataUrl'] = processedOutput['filePath']
extensionInfoForData = [
pml.Extension(value=dataVal, anytypeobjs_=[''])
]
pmmlObject.Header.Extension = extensionInfoForData
# print ('Data Step 3')
# pmmlObject.export(sys.stdout,0)
except:
pass
elif processedOutput['itemType'] == 'CODE':
# print ("CODE layer came")
try:
scrptVal = pmmlObject.script
urlOfScript = processedOutput['url']
scriptFile = open(processedOutput['filePath'], 'r')
scriptCode = scriptFile.read()
scriptCode = scriptCode.replace('<', '<')
# print (scriptCode)
useFor = processedOutput['useFor']
scrp = pml.script(content=scriptCode,
class_=urlOfScript,
for_=useFor)
# scrp.export(sys.stdout,0)
scrptVal.append(scrp)
pmmlObject.script = scrptVal
# print ('Code Step 10')
# pmmlObject.export(sys.stdout,0)
except:
pass
return (pmmlObject)
def getPmml(self, architecture):
fName = 'classification'
lenOfArch = len(architecture)
mName = 'Keras Model'
netWorkInfo = []
scriptVal = []
extensionInfoForData = [pml.Extension(value=[], anytypeobjs_=[''])]
dataVal = {}
for counta, j in enumerate(architecture):
if counta == 0:
someConectionId = 'na'
else:
someConectionId = tempConId
if j['itemType'] in ['CODE']:
# print ('##################',j)
scriptFile = open(j['filePath'], 'r')
scriptCode = scriptFile.read()
scriptCode = scriptCode.replace('<', '<')
scriptInfo = {}
scrptVal = []
# dataVal['scriptUrl']=j['url']
useFor = j['useFor']
extensionInfoForScript = [
pml.Extension(value=scrptVal, anytypeobjs_=[''])
]
scrp = pml.script(content=scriptCode,
Extension=extensionInfoForScript,
for_=useFor)
scriptVal.append(scrp)
tempConId = None
elif j['itemType'] in ['DATA']:
try:
dataVal['dataUrl'] = j['filePath']
extensionInfoForData = [
pml.Extension(value=dataVal, anytypeobjs_=[''])
]
except:
pass
tempConId = None
elif j['itemType'] == 'FOLDING':
# print (j)
for k in j['children']:
tempdata7 = self.convertToStandardJson(k)
tempdata7['connectionLayerId'] = someConectionId
tempConId = tempdata7['layerId']
pp = addLayer(tempdata7)
netWorkInfo.append(pp)
someConectionId = tempConId
else:
# print ('Start of tamasha$$$$$$$$$$',j)
tempdata7 = self.convertToStandardJson(j)
tempdata7['connectionLayerId'] = someConectionId
tempConId = tempdata7['layerId']
# print ('pakda', tempdata7)
pp = self.addLayer(tempdata7)
netWorkInfo.append(pp)
kk = pml.DeepNetwork(modelName=mName,
functionName=fName,
NetworkLayer=netWorkInfo,
numberOfLayers=lenOfArch)
tt = pml.Timestamp(datetime.now())
hd = pml.Header(copyright="Copyright (c) 2018 Software AG",
Extension=extensionInfoForData,
description="Neural Network Model",
Timestamp=pml.Timestamp(datetime.now()))
jj = pml.PMML(version="4.3Ext",
script=scriptVal,
Header=hd,
DeepNetwork=[kk])
return jj
def addLayer(self, tempData):
# print ('addLayerPMML 00000111>>>> ',tempData)
tempLayerBiasobject = pml.LayerBias(content=[])
tempBia = tempLayerBiasobject.__dict__
tempBia['weightsFlattenAxis'] = "0"
tempLayerWeightobject = pml.LayerWeights(content=[])
tempWei = tempLayerWeightobject.__dict__
tempWei['weightsFlattenAxis'] = "0"
if tempData['layerType'] in ['Input']:
tempData['properties']['outputDimension'] = self.outputForInput(
tempData)
if tempData['layerType'] in ['Dropout']:
if tempData['properties']['dropoutRate'] == '':
tempData['properties']['dropoutRate'] = '0'
tempData['properties']['outputDimension'] = self.outputForInput(
tempData)
elif tempData['layerType'] in ['BatchNormalization']:
tempData['properties']['outputDimension'] = self.outputForInput(
tempData)
tempWei['weightsShape'] = self.weightBatchNormalization(tempData)
elif tempData['layerType'] in ['Activation']:
tempData['properties']['outputDimension'] = tempData['properties'][
'inputDimension']
elif tempData['layerType'] in ['Conv2D', 'DepthwiseConv2D']:
tempData['properties']['outputDimension'] = self.outputForFlatten(
tempData)
tempWei['weightsShape'] = self.weightConvo(tempData)
elif tempData['layerType'] in ['Flatten', 'GlobalAveragePooling2D']:
tempData['properties']['outputDimension'] = self.outputForFlatten(
tempData)
elif tempData['layerType'] in ['MaxPooling2D', 'AveragePooling2D']:
tempData['properties']['outputDimension'] = self.outputForFlatten(
tempData)
elif tempData['layerType'] in ['MaxPooling1D', 'AveragePooling1D']:
tempData['properties']['outputDimension'] = self.outputForFlatten(
tempData)
tempData['properties']['poolSize'] = str(
tempData['properties']['poolSize'])
tempData['properties']['stride'] = str(
tempData['properties']['stride'])
elif tempData['layerType'] in ['Reshape']:
tempData['properties']['outputDimension'] = self.outputForFlatten(
tempData)
tempData['properties']['reshapeTarget'] = str(
tuple(tempData['properties']['reshapeTarget']))
elif tempData['layerType'] in ['ZeroPadding2D', 'ZeroPadding1D']:
tempData['properties']['outputDimension'] = self.outputForFlatten(
tempData)
if len(tempData['properties']['paddingDims']) == 4:
top_pad, bottom_pad, left_pad, right_pad = tempData[
'properties']['paddingDims']
tempData['properties']['paddingDims'] = ((top_pad, bottom_pad),
(left_pad, right_pad))
elif len(tempData['properties']['paddingDims']) == 1:
pad = tempData['properties']['paddingDims'][0]
tempData['properties']['paddingDims'] = (pad, pad)
tempData['properties']['paddingDims'] = str(
tuple(tempData['properties']['paddingDims']))
elif tempData['layerType'] in ['Dense']:
if tempData['properties']['activationFunction'] == "relu":
tempData['properties']['activationFunction'] = "rectifier"
elif tempData['properties']['activationFunction'] == "tanh":
tempData['properties']['activationFunction'] = "tanch"
tempData['properties']['outputDimension'] = self.outputForDense(
tempData)
tempWei['weightsShape'] = self.weightBiasDense(tempData)[0]
tempBia['weightsShape'] = self.weightBiasDense(tempData)[1]
# print ('$$$$$$$$$$$$$$$tempData',tempData)
tempLayerobject = pml.LayerParameters()
tempvals = tempLayerobject.__dict__
for j in list(tempvals.keys()):
try:
tempvals[j] = tempData['properties'][j]
except:
pass
if tempData['sectionId'] == None:
if tempData['layerType'] == 'Input':
input_filed_name = "base64String"
kk = pml.NetworkLayer(
inputFieldName=input_filed_name,
layerType=tempData['layerType'],
layerId=tempData['layerId'],
connectionLayerId=tempData['connectionLayerId'],
LayerParameters=tempLayerobject)
elif tempData['layerType'] == 'Dense':
kk=pml.NetworkLayer(layerType=tempData['layerType'],layerId=tempData['layerId'],\
connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject,LayerWeights=tempLayerWeightobject,LayerBias=tempLayerBiasobject)
elif tempData['layerType'] in [
'Conv2D', 'DepthwiseConv2D', 'BatchNormalization'
]:
kk=pml.NetworkLayer(layerType=tempData['layerType'],layerId=tempData['layerId'],\
connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject,LayerWeights=tempLayerWeightobject)
else:
kk=pml.NetworkLayer(layerType=tempData['layerType'],layerId=tempData['layerId'],\
connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject)
else:
extensionInfo = [
pml.Extension(value={'sectionId': tempData['sectionId']},
anytypeobjs_=[''])
]
if tempData['layerType'] == 'Input':
input_filed_name = "base64String"
kk=pml.NetworkLayer(inputFieldName=input_filed_name,Extension=[extensionInfo], layerType=tempData['layerType'],\
layerId=tempData['layerId'],connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject)
elif tempData['layerType'] == 'Dense':
kk=pml.NetworkLayer(layerType=tempData['layerType'],layerId=tempData['layerId'],Extension=extensionInfo,\
connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject,LayerWeights=tempLayerWeightobject,LayerBias=tempLayerBiasobject)
elif tempData['layerType'] in [
'Conv2D', 'DepthwiseConv2D', 'BatchNormalization'
]:
kk=pml.NetworkLayer(layerType=tempData['layerType'],layerId=tempData['layerId'],Extension=extensionInfo,\
connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject,LayerWeights=tempLayerWeightobject)
else:
kk=pml.NetworkLayer(layerType=tempData['layerType'],layerId=tempData['layerId'],Extension=extensionInfo,\
connectionLayerId=tempData['connectionLayerId'],\
LayerParameters=tempLayerobject)
return kk