def validation(self, preprocessor_type):
# IF PREPROCESSOR_TYPE IS NONE:
if preprocessor_type is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', str)
raise MissingMandatoryFieldException('Given:', None, 'Required:', str)
# IF PREPROCESSOR_TYPE IS NOT STRING:
if not isinstance(preprocessor_type, str):
self.logger.error(InvalidInfoException.__name__,
'Given:', type(preprocessor_type), 'Required:', str)
raise InvalidInfoException('Given:', type(preprocessor_type), 'Required:', str)
# IF PREPROCESSOR_TYPE NOT A CHILD OF ABSTRACT_DIALOGUE_PREPROCESSOR:
if preprocessor_type not in [y.__name__ for y in AbstractDialoguePreProcessor().__class__.__subclasses__()]:
self.logger.error(InvalidInfoException.__name__,
'Given:', preprocessor_type,
'Required Value from :', [y.__name__ for y in
AbstractDialoguePreProcessor().__class__.__subclasses__()])
raise InvalidInfoException('Given:', preprocessor_type,
'Required Value from :', [y.__name__ for y in
AbstractDialoguePreProcessor().__class__.__subclasses__()])
# ALL CASES POSITIVE
return True
def validation(self, args):
if args is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', type(None))
raise MissingMandatoryFieldException('Given:', type(None))
if not isinstance(args, dict):
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(args), 'Required:', type(dict))
raise InvalidInfoException('Given:', type(args), 'Required:',
type(dict))
if StandardFlowDialogueFeatureEngineerHandlerImpl.__name__ not in args:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
args.items(), 'Required:',
StandardFlowDialogueFeatureEngineerHandlerImpl.__name__)
raise MissingMandatoryFieldException(
'Given:', args.items(), 'Required:',
StandardFlowDialogueFeatureEngineerHandlerImpl.__name__)
if args[StandardFlowDialogueFeatureEngineerHandlerImpl.
__name__] is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', type(None), 'Required:', type(list))
raise MissingMandatoryFieldException('Given:', type(None),
'Required:', type(list))
if not isinstance(
args[StandardFlowDialogueFeatureEngineerHandlerImpl.__name__],
list):
self.logger.error(
InvalidInfoException.__name__, 'Given:',
type(args[
StandardFlowDialogueFeatureEngineerHandlerImpl.__name__]),
'Required:', type(list))
raise InvalidInfoException(
'Given:',
type(args[
StandardFlowDialogueFeatureEngineerHandlerImpl.__name__]),
'Required:', type(list))
for config in args[
StandardFlowDialogueFeatureEngineerHandlerImpl.__name__]:
if not isinstance(config, AbstractConfig):
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(config), 'Required:',
type(AbstractConfig))
raise InvalidInfoException('Given:', type(config), 'Required:',
type(AbstractConfig))
if PandasDAOImpl.__name__ not in args:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', args.items(), 'Required:',
PandasDAOImpl.__name__)
raise MissingMandatoryFieldException('Given:', args.items(),
'Required:',
PandasDAOImpl.__name__)
if args[PandasDAOImpl.__name__] is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', type(None), 'Required:',
type(pd.DataFrame))
raise MissingMandatoryFieldException('Given:',
type(None), 'Required:',
type(pd.DataFrame))
return True
def __argument_type_validation(self, test_data, model_params):
if not isinstance(test_data, DataFrame):
raise InvalidInfoException(
"argument 'test_data' is not of type Pandas Dataframe")
if not isinstance(model_params, dict):
raise InvalidInfoException(
"argument 'model_params' is not of type dictionary")
def __argument_type_validation(self, data, model_params,
model_hyper_params,
model_cross_validator_params):
if not isinstance(data, DataFrame):
self.logger.error(
"InvalidInfoException : argument 'data' is not of type 'Pandas DataFrame'"
)
raise InvalidInfoException(
"argument 'data' is not of type 'Pandas DataFrame'")
if not isinstance(model_params, dict):
self.logger.error(
"InvalidInfoException : argument 'model_params' is not of type dictionary"
)
raise InvalidInfoException(
"argument 'model_params' is not of type dictionary")
if not isinstance(model_hyper_params, dict):
self.logger.error(
"InvalidInfoException : argument 'model_hyper_params' is not of type dictionary"
)
raise InvalidInfoException(
"argument 'model_hyper_params' is not of type dictionary")
if not isinstance(model_cross_validator_params, dict):
self.logger.error(
"InvalidInfoException : argument 'model_cross_validator_params' is not of type dictionary"
)
raise InvalidInfoException(
"argument 'model_cross_validator_params' is not of type dictionary"
)
def preprocess_validation(self, args):
# TRY THIS:
try:
# IF INITIAL VALIDATION SUCCESSFUL:
if super().preprocess_validation(args):
# FOR ALL REQ_INPUT:
if self.config_pattern.properties.req_input is not None:
for arr in self.config_pattern.properties.req_input:
for elem in arr:
# IF ROW-WISE ELEMENT IN ARGS IS NOT OF REQUIRED DATA TYPE:
if args[elem].dtype != str:
# ERROR:
self.logger.error(
InvalidInfoException.__name__, 'Given:',
args[elem].dtype, 'Required:', str)
raise InvalidInfoException(
'Given:', args[elem].dtype, 'Required:',
str)
# FOR ALL REQ_DATA:
if self.config_pattern.properties.req_data is not None:
for arr in self.config_pattern.properties.req_data:
for elem in arr:
# IF ROW-WISE ELEMENT IN ARGS IS NOT OF REQUIRED DATA TYPE:
if args[elem].dtype != str:
# ERROR:
self.logger.error(
InvalidInfoException.__name__, 'Given:',
args[elem].dtype, 'Required:', str)
raise InvalidInfoException(
'Given:', args[elem].dtype, 'Required:',
str)
# ALL CASES POSITIVE
return True
# CATCH ERRORS:
except (MissingMandatoryFieldException, InvalidInfoException) as exp:
raise CommonBaseException(exp)
def engineer_feature_validation(self, args):
if args is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', type(None))
raise MissingMandatoryFieldException('Given:', type(None))
if not isinstance(args, dict):
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(args), 'Required:', type(dict))
raise InvalidInfoException('Given:', type(args), 'Required:',
type(dict))
if self.config_pattern.properties.req_input is not None:
for arr in self.config_pattern.properties.req_input:
for elem in arr:
if elem not in args:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
args.items(), 'Required:', elem)
raise MissingMandatoryFieldException(
'Given:', args.items(), 'Required:', elem)
if args[elem] is None:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
type(None), 'Required:', type(pd.Series))
raise MissingMandatoryFieldException(
'Given:', type(None), 'Required:', type(pd.Series))
if self.config_pattern.properties.req_data is not None:
for arr in self.config_pattern.properties.req_data:
for elem in arr:
if elem not in args:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
args.items(), 'Required:', elem)
raise MissingMandatoryFieldException(
'Given:', args.items(), 'Required:', elem)
if args[elem] is None:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
type(None), 'Required:', type(pd.Series))
raise MissingMandatoryFieldException(
'Given:', type(None), 'Required:', type(pd.Series))
if self.config_pattern.properties.req_args is not None:
for arr in self.config_pattern.properties.req_args:
for elem in arr:
if elem not in args:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
args.items(), 'Required:', elem)
raise MissingMandatoryFieldException(
'Given:', args.items(), 'Required:', elem)
if args[elem] is None:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
type(None), 'Required:', type(AbstractUtils))
raise MissingMandatoryFieldException(
'Given:', type(None), 'Required:',
type(AbstractUtils))
return True
def validate_dict_for_empty_values(cls, _dict):
for key, value in _dict.items():
if value is None:
cls.logger.error(
'InvalidInfoException : Null or empty value received against dictionary key: '
+ str(key))
raise InvalidInfoException(
'Null or empty value received against dictionary key: ' +
str(key))
def __argument_type_validation(self, evaluation_metric_list,
predicted_target, actual_target):
if not isinstance(evaluation_metric_list, list):
self.logger.error(
"InvalidInfoException : argument 'evaluation_metric_list' is not of type 'list'"
)
raise InvalidInfoException(
"argument 'evaluation_metric_list' is not of type 'list'")
if not isinstance(predicted_target, np.ndarray):
self.logger.error(
"InvalidInfoException : argument 'predicted_target' is not of type 'np.ndarray'"
)
raise InvalidInfoException(
"argument 'predicted_target' is not of type 'np.ndarray'")
if not isinstance(actual_target, np.ndarray):
self.logger.error(
"InvalidInfoException : argument 'actual_target' is not of type 'np.ndarray'"
)
raise InvalidInfoException(
"argument 'actual_target' is not of type 'np.ndarray'")
def validate_data_features_against_model(cls, data, features_list):
model_features_not_in_data = CommonUtilities.get_list_difference(
features_list, data.columns)
if len(model_features_not_in_data) > 0:
cls.logger.error("InvalidInfoException : Features: " +
str(model_features_not_in_data) +
" not found in received data " +
"but required by the model")
raise InvalidInfoException("Features: " +
str(model_features_not_in_data) +
" not found in received data " +
"but required by the model")
return
def validate_model_params(cls, model_params):
if CommonConstants.MODEL_NAME_TAG not in model_params or not model_params.get(
CommonConstants.MODEL_NAME_TAG):
cls.logger.error("MissingMandatoryFieldException : " +
str(CommonConstants.MODEL_NAME_TAG) +
" key does not exist in " +
"'model_params' or has no value assigned ")
raise MissingMandatoryFieldException(
str(CommonConstants.MODEL_NAME_TAG) +
" key does not exist in " +
"'model_params' or has no value assigned ")
elif not isinstance(model_params[CommonConstants.MODEL_NAME_TAG], str):
cls.logger.error("InvalidInfoException : " +
str(CommonConstants.MODEL_NAME_TAG) +
' value is not of type string')
raise InvalidInfoException(
str(CommonConstants.MODEL_NAME_TAG) +
' value is not of type string')
if CommonConstants.FEATURE_LIST_TAG not in model_params or not model_params[
CommonConstants.FEATURE_LIST_TAG]:
cls.logger.error("MissingMandatoryFieldException : " +
str(CommonConstants.FEATURE_LIST_TAG) +
" key does not exist in " +
"'model_params' or has no value assigned ")
raise MissingMandatoryFieldException(
str(CommonConstants.FEATURE_LIST_TAG) +
" key does not exist in " +
"'model_params' or has no value assigned ")
elif not isinstance(model_params[CommonConstants.FEATURE_LIST_TAG],
list):
cls.logger.error("InvalidInfoException : " +
str(CommonConstants.FEATURE_LIST_TAG) +
' value is not of type list')
raise InvalidInfoException(
str(CommonConstants.FEATURE_LIST_TAG) +
' value is not of type list')
return
def load_and_validate_new_features_list(
self, feature_engineering_model_specific_dict):
self.logger.info(
"In Class: CommonUtilities, Function: load_and_validate_new_features_list"
)
exception_str = " does not exist in service param dictionary"
self.logger.info("Checking if mandatory key, " +
CommonConstants.NEW_FEATURE_LIST_TAG +
" exists in param dictionary against flow: " +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
if CommonConstants.NEW_FEATURE_LIST_TAG not in feature_engineering_model_specific_dict:
self.logger.error('MissingMandatoryFieldException : Key Error: ' +
CommonConstants.NEW_FEATURE_LIST_TAG +
exception_str)
raise MissingMandatoryFieldException(
'Key Error: ' + CommonConstants.NEW_FEATURE_LIST_TAG +
exception_str)
features_list = feature_engineering_model_specific_dict[
CommonConstants.NEW_FEATURE_LIST_TAG]
self.logger.info("Checking if a value exists against mandatory key, " +
CommonConstants.NEW_FEATURE_LIST_TAG +
" in param dictionary against flow: " +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
if features_list is None:
self.logger.error(
'MissingMandatoryFieldException : Value Error: No value exists for tag: '
+ CommonConstants.NEW_FEATURE_LIST_TAG + ' for flow: ' +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
raise MissingMandatoryFieldException(
'Value Error: No value exists for tag: ' +
CommonConstants.NEW_FEATURE_LIST_TAG + ' for flow: ' +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
self.logger.info("Checking data type of value against key, " +
CommonConstants.NEW_FEATURE_LIST_TAG +
" in param dictionary against flow: " +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
if not isinstance(features_list, dict):
self.logger.error(
'InvalidInfoException : Invalid value for tag: ' +
CommonConstants.NEW_FEATURE_LIST_TAG + ' for flow: ' +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
raise InvalidInfoException(
'Invalid value for tag: ' +
CommonConstants.NEW_FEATURE_LIST_TAG + ' for flow: ' +
CommonConstants.FEATURE_ENGINEERING_FLOW_STANDARD)
return features_list.keys()
def validate_pipeline_model(cls, model_name, pipeline_model):
"""
author: [email protected]
Validates that input pipeline model is not null and belongs to the correct datatype
:param pipeline_model_name: Name of given pipeline model
:param pipeline_model: Pipeline
:return: --
"""
if pipeline_model is None:
cls.logger.error("MissingMandatoryFieldException" + model_name +
" is Missing")
raise MissingMandatoryFieldException(model_name +
" argument is None")
if not isinstance(pipeline_model, PipelineModel) and not isinstance(
pipeline_model, Pipeline):
cls.logger.error(
'InvalidInfoException : "Invalid type for argument ' +
model_name)
raise InvalidInfoException("Invalid type for argument " +
model_name)
def validation(self, args):
# IF ARGS IS NONE:
if args is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', dict)
raise MissingMandatoryFieldException('Given:', None, 'Required:',
dict)
# IF ARGS IS NOT A DICT:
if not isinstance(args, dict):
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(args), 'Required:', dict)
raise InvalidInfoException('Given:', type(args), 'Required:', dict)
# IF ABSTRACT_DIALOGUE_PREPROCESSOR IS NOT IN ARGS:
if AbstractDialoguePreProcessor.__name__ not in args:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', args.keys(), 'Required:',
AbstractDialoguePreProcessor.__name__)
raise MissingMandatoryFieldException(
'Given:', args.keys(), 'Required:',
AbstractDialoguePreProcessor.__name__)
# IF ABSTRACT_DIALOGUE_PREPROCESSOR IN ARGS IS NONE:
if args[AbstractDialoguePreProcessor.__name__] is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', list)
raise MissingMandatoryFieldException('Given:', None, 'Required:',
list)
# ABSTRACT_DIALOGUE_PREPROCESSOR IN ARGS IS NOT OF REQUIRED DATA TYPE:
if not isinstance(args[AbstractDialoguePreProcessor.__name__], list):
self.logger.error(
InvalidInfoException.__name__, 'Given:',
type(args[AbstractDialoguePreProcessor.__name__]), 'Required:',
list)
raise InvalidInfoException(
'Given:', type(args[AbstractDialoguePreProcessor.__name__]),
'Required:', list)
# FOR CONFIG IN ABSTRACT_DIALOGUE_PREPROCESSOR IN ARGS:
for config in args[AbstractDialoguePreProcessor.__name__]:
# IF CONFIG IS NONE:
if config is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', AbstractConfig)
raise MissingMandatoryFieldException('Given:', None,
'Required:',
AbstractConfig)
# IF CONFIG IN ARGS IS NOT OF REQUIRED DATA TYPE:
if not isinstance(config, AbstractConfig):
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(config), 'Required:', AbstractConfig)
raise InvalidInfoException('Given:', type(config), 'Required:',
AbstractConfig)
# IF PANDAS_DAO_IMPL NOT IN ARGS:
if PandasDAOImpl.__name__ not in args:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', args.keys(), 'Required:',
PandasDAOImpl.__name__)
raise MissingMandatoryFieldException('Given:', args.keys(),
'Required:',
PandasDAOImpl.__name__)
# IF PANDAS_DAO_IMPL IN ARGS IS NONE:
if args[PandasDAOImpl.__name__] is None:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', pd.DataFrame)
raise MissingMandatoryFieldException('Given:', None, 'Required:',
pd.DataFrame)
# IF PANDAS_DAO_IMPL IN ARGS IS NOT OF REQUIRED DATA TYPE:
if not isinstance(args[PandasDAOImpl.__name__], pd.DataFrame):
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(args[PandasDAOImpl.__name__]), 'Required:',
pd.DataFrame)
raise InvalidInfoException('Given:',
type(args[PandasDAOImpl.__name__]),
'Required:', pd.DataFrame)
# ALL CASES POSITIVE
return True
def preprocess_validation(self, args):
# IF ARGS IS NONE:
if args is None:
# ERROR:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', dict)
raise MissingMandatoryFieldException('Given:', None, 'Required:',
dict)
# IF ARGS IS NOT A DICT:
if not isinstance(args, dict):
# ERROR:
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(args), 'Required:', dict)
raise InvalidInfoException('Given:', type(args), 'Required:', dict)
# FOR ALL REQ_INPUT:
if self.config_pattern.properties.req_input is not None:
for arr in self.config_pattern.properties.req_input:
for elem in arr:
# IF ELEMENT IS MISSING FROM ARGS:
if elem not in args:
# ERROR:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
args.keys(), 'Required:', elem)
raise MissingMandatoryFieldException(
'Given:', args.keys(), 'Required:', elem)
# IF ELEMENT IN ARGS IS NONE:
if args[elem] is None:
# ERROR:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
None, 'Required:', pd.Series)
raise MissingMandatoryFieldException(
'Given:', None, 'Required:', pd.Series)
# IF ELEMENT IN ARGS IS NOT OF REQUIRED DATA TYPE:
if not isinstance(args[elem], pd.Series):
# ERROR:
self.logger.error(InvalidInfoException.__name__,
'Given:', type(args[elem]),
'Required:', pd.Series)
raise InvalidInfoException('Given:', type(args[elem]),
'Required:', pd.Series)
# FOR ALL REQ_DATA:
if self.config_pattern.properties.req_data is not None:
for arr in self.config_pattern.properties.req_data:
for elem in arr:
# IF ELEMENT IS MISSING FROM ARGS:
if elem not in args:
# ERROR:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
args.keys(), 'Required:', elem)
raise MissingMandatoryFieldException(
'Given:', args.keys(), 'Required:', elem)
# IF ELEMENT IN ARGS IS NONE:
if args[elem] is None:
# ERROR:
self.logger.error(
MissingMandatoryFieldException.__name__, 'Given:',
None, 'Required:', pd.Series)
raise MissingMandatoryFieldException(
'Given:', None, 'Required:', pd.Series)
# IF ELEMENT IN ARGS IS NOT OF REQUIRED DATA TYPE:
if not isinstance(args[elem], pd.Series):
# ERROR:
self.logger.error(InvalidInfoException.__name__,
'Given:', type(args[elem]),
'Required:', pd.Series)
raise InvalidInfoException('Given:', type(args[elem]),
'Required:', pd.Series)
# FOR ALL REQ_ARGS:
if self.config_pattern.properties.req_args is not None:
elem = self.config_pattern.properties.req_args
# IF ELEMENT IS NOT IN ARGS:
if elem not in args:
# ERROR:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', args.keys(), 'Required:', elem)
raise MissingMandatoryFieldException('Given:', args.keys(),
'Required:', elem)
# IF ELEMENT IN ARGS IS NONE:
if args[elem] is None:
# ERROR:
self.logger.error(MissingMandatoryFieldException.__name__,
'Given:', None, 'Required:', AbstractUtils)
raise MissingMandatoryFieldException('Given:', None,
'Required:',
AbstractUtils)
# IF ELEMENT IN ARGS IS NOT OF REQUIRED DATA TYPE
if not isinstance(args[elem], type(UtilsFactory.get_utils(elem))):
# ERROR:
self.logger.error(InvalidInfoException.__name__, 'Given:',
type(args[elem]), 'Required:',
type(UtilsFactory.get_utils(elem)))
raise InvalidInfoException('Given:', type(args[elem]),
'Required:',
type(UtilsFactory.get_utils(elem)))
# ALL CASES POSITIVE
return True