def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.core.groupby.generic', 'agg') if not hasattr(self, '_mlinspect_dag_node'): raise NotImplementedError("TODO: Support agg if groupby happened in external code") input_dag_node = get_dag_node_for_id(self._mlinspect_dag_node) # pylint: disable=no-member operator_context = OperatorContext(OperatorType.GROUP_BY_AGG, function_info) input_infos = PandasBackend.before_call(operator_context, []) result = original(self, *args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result) if len(args) > 0: description = "Groupby '{}', Aggregate: '{}'".format(result.index.name, args) else: description = "Groupby '{}', Aggregate: '{}'".format(result.index.name, kwargs) columns = [result.index.name] + list(result.columns) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(description, columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_dag_node], backend_result) new_return_value = backend_result.annotated_dfobject.result_data return new_return_value
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('statsmodel.api', 'add_constant') input_info = get_input_info(args[0], caller_filename, lineno, function_info, optional_code_reference, optional_source_code) operator_context = OperatorContext(OperatorType.PROJECTION_MODIFY, function_info) input_infos = SklearnBackend.before_call( operator_context, [input_info.annotated_dfobject]) result = original(input_infos[0].result_data, *args[1:], **kwargs) backend_result = SklearnBackend.after_call(operator_context, input_infos, result) new_return_value = backend_result.annotated_dfobject.result_data dag_node = DagNode( op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails("Adds const column", ["array"]), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_info.dag_node], backend_result) return new_return_value
def patched_fit_transform(self, *args, **kwargs): """ Patch for ('example_pipelines.healthcare.healthcare_utils.MyW2VTransformer', 'fit_transform') """ # pylint: disable=no-method-argument self.mlinspect_fit_transform_active = True # pylint: disable=attribute-defined-outside-init original = gorilla.get_original_attribute( healthcare_utils.MyW2VTransformer, 'fit_transform') function_info = FunctionInfo( 'example_pipelines.healthcare.healthcare_utils', 'MyW2VTransformer') input_info = get_input_info(args[0], self.mlinspect_caller_filename, self.mlinspect_lineno, function_info, self.mlinspect_optional_code_reference, self.mlinspect_optional_source_code) operator_context = OperatorContext(OperatorType.TRANSFORMER, function_info) input_infos = SklearnBackend.before_call( operator_context, [input_info.annotated_dfobject]) result = original(self, input_infos[0].result_data, *args[1:], **kwargs) backend_result = SklearnBackend.after_call(operator_context, input_infos, result) new_return_value = backend_result.annotated_dfobject.result_data assert isinstance(new_return_value, MlinspectNdarray) dag_node = DagNode( singleton.get_next_op_id(), BasicCodeLocation(self.mlinspect_caller_filename, self.mlinspect_lineno), operator_context, DagNodeDetails("Word2Vec: fit_transform", ['array']), get_optional_code_info_or_none( self.mlinspect_optional_code_reference, self.mlinspect_optional_source_code)) add_dag_node(dag_node, [input_info.dag_node], backend_result) self.mlinspect_fit_transform_active = False # pylint: disable=attribute-defined-outside-init return new_return_value
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.core.frame', 'merge') input_info_a = get_input_info(self, caller_filename, lineno, function_info, optional_code_reference, optional_source_code) input_info_b = get_input_info(args[0], caller_filename, lineno, function_info, optional_code_reference, optional_source_code) operator_context = OperatorContext(OperatorType.JOIN, function_info) input_infos = PandasBackend.before_call(operator_context, [input_info_a.annotated_dfobject, input_info_b.annotated_dfobject]) # No input_infos copy needed because it's only a selection and the rows not being removed don't change result = original(input_infos[0].result_data, input_infos[1].result_data, *args[1:], **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result) result = backend_result.annotated_dfobject.result_data description = "on '{}'".format(kwargs['on']) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(description, list(result.columns)), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_info_a.dag_node, input_info_b.dag_node], backend_result) return result
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.core.frame', 'replace') input_info = get_input_info(self, caller_filename, lineno, function_info, optional_code_reference, optional_source_code) operator_context = OperatorContext(OperatorType.PROJECTION_MODIFY, function_info) input_infos = PandasBackend.before_call(operator_context, [input_info.annotated_dfobject]) # No input_infos copy needed because it's only a selection and the rows not being removed don't change result = original(input_infos[0].result_data, *args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result) result = backend_result.annotated_dfobject.result_data if isinstance(args[0], dict): raise NotImplementedError("TODO: Add support for replace with dicts") description = "Replace '{}' with '{}'".format(args[0], args[1]) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(description, list(result.columns)), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_info.dag_node], backend_result) return result
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ # pylint: disable=too-many-locals function_info = FunctionInfo('pandas.core.frame', '__setitem__') operator_context = OperatorContext(OperatorType.PROJECTION_MODIFY, function_info) input_info = get_input_info(self, caller_filename, lineno, function_info, optional_code_reference, optional_source_code) if isinstance(args[0], str): input_infos = PandasBackend.before_call(operator_context, [input_info.annotated_dfobject]) input_infos = copy.deepcopy(input_infos) result = original(self, *args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, self) columns = list(self.columns) # pylint: disable=no-member description = "modifies {}".format([args[0]]) else: raise NotImplementedError("TODO: Handling __setitem__ for key type {}".format(type(args[0]))) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(description, columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_info.dag_node], backend_result) assert hasattr(self, "_mlinspect_annotation") return result
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.core.frame', '__getitem__') input_info = get_input_info(self, caller_filename, lineno, function_info, optional_code_reference, optional_source_code) if isinstance(args[0], str): # Projection to Series columns = [args[0]] operator_context = OperatorContext(OperatorType.PROJECTION, function_info) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails("to {}".format(columns), columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) elif isinstance(args[0], list) and isinstance(args[0][0], str): # Projection to DF columns = args[0] operator_context = OperatorContext(OperatorType.PROJECTION, function_info) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails("to {}".format(columns), columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) elif isinstance(args[0], pandas.Series): # Selection operator_context = OperatorContext(OperatorType.SELECTION, function_info) columns = list(self.columns) # pylint: disable=no-member if optional_source_code: description = "Select by Series: {}".format(optional_source_code) else: description = "Select by Series" dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(description, columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) else: raise NotImplementedError() input_infos = PandasBackend.before_call(operator_context, [input_info.annotated_dfobject]) result = original(input_infos[0].result_data, *args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result) result = backend_result.annotated_dfobject.result_data add_dag_node(dag_node, [input_info.dag_node], backend_result) return result
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.core.frame', 'DataFrame') operator_context = OperatorContext(OperatorType.DATA_SOURCE, function_info) input_infos = PandasBackend.before_call(operator_context, []) original(self, *args, **kwargs) result = self backend_result = PandasBackend.after_call(operator_context, input_infos, result) columns = list(self.columns) # pylint: disable=no-member dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(None, columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [], backend_result)
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.io.parsers', 'read_csv') operator_context = OperatorContext(OperatorType.DATA_SOURCE, function_info) input_infos = PandasBackend.before_call(operator_context, []) result = original(*args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result) description = "{}".format(args[0].split(os.path.sep)[-1]) dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(description, list(result.columns)), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [], backend_result) return result
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('numpy.random', 'random') operator_context = OperatorContext(OperatorType.DATA_SOURCE, function_info) input_infos = SklearnBackend.before_call(operator_context, []) result = original(*args, **kwargs) backend_result = SklearnBackend.after_call(operator_context, input_infos, result) dag_node = DagNode( op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails("random", ['array']), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [], backend_result) new_return_value = backend_result.annotated_dfobject.result_data return new_return_value
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('statsmodels.datasets', 'get_rdataset') operator_context = OperatorContext(OperatorType.DATA_SOURCE, function_info) input_infos = PandasBackend.before_call(operator_context, []) result = original(*args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result.data) result.data = backend_result.annotated_dfobject.result_data dag_node = DagNode( op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails(result.title, list(result.data.columns)), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [], backend_result) return result
def patched__getitem__(self, *args, **kwargs): """ Patch for ('pandas.core.series', 'Series') """ original = gorilla.get_original_attribute( pandas.core.indexing._LocIndexer, '__getitem__') # pylint: disable=protected-access if call_info_singleton.column_transformer_active: op_id = singleton.get_next_op_id() caller_filename = call_info_singleton.transformer_filename lineno = call_info_singleton.transformer_lineno function_info = call_info_singleton.transformer_function_info optional_code_reference = call_info_singleton.transformer_optional_code_reference optional_source_code = call_info_singleton.transformer_optional_source_code if isinstance(args[0], tuple) and not args[0][0].start and not args[0][0].stop \ and isinstance(args[0][1], list) and isinstance(args[0][1][0], str): # Projection to one or multiple columns, return value is df columns = args[0][1] else: raise NotImplementedError() operator_context = OperatorContext(OperatorType.PROJECTION, function_info) input_info = get_input_info(self.obj, caller_filename, # pylint: disable=no-member lineno, function_info, optional_code_reference, optional_source_code) input_infos = PandasBackend.before_call(operator_context, [input_info.annotated_dfobject]) result = original(self, *args, **kwargs) backend_result = PandasBackend.after_call(operator_context, input_infos, result) result = backend_result.annotated_dfobject.result_data dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails("to {}".format(columns), columns), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_info.dag_node], backend_result) else: result = original(self, *args, **kwargs) return result
def patched_fit(self, *args, **kwargs): """ Patch for ('statsmodel.api.OLS', 'fit') """ # pylint: disable=no-method-argument, too-many-locals original = gorilla.get_original_attribute(api.OLS, 'fit') function_info = FunctionInfo('statsmodel.api.OLS', 'fit') # Train data # pylint: disable=no-member data_backend_result, train_data_node, train_data_result = add_train_data_node( self, self.data.exog, function_info) self.data.exog = train_data_result # pylint: disable=no-member label_backend_result, train_labels_node, train_labels_result = add_train_label_node( self, self.data.endog, function_info) self.data.endog = train_labels_result # Estimator operator_context = OperatorContext(OperatorType.ESTIMATOR, function_info) input_dfs = [ data_backend_result.annotated_dfobject, label_backend_result.annotated_dfobject ] input_infos = SklearnBackend.before_call(operator_context, input_dfs) result = original(self, *args, **kwargs) estimator_backend_result = SklearnBackend.after_call( operator_context, input_infos, None) dag_node = DagNode( singleton.get_next_op_id(), BasicCodeLocation(self.mlinspect_caller_filename, self.mlinspect_lineno), operator_context, DagNodeDetails("Decision Tree", []), get_optional_code_info_or_none( self.mlinspect_optional_code_reference, self.mlinspect_optional_source_code)) add_dag_node(dag_node, [train_data_node, train_labels_node], estimator_backend_result) return result
def execute_inspections(op_id, caller_filename, lineno, optional_code_reference, optional_source_code): """ Execute inspections, add DAG node """ function_info = FunctionInfo('pandas.core.frame', 'dropna') input_info = get_input_info(self, caller_filename, lineno, function_info, optional_code_reference, optional_source_code) operator_context = OperatorContext(OperatorType.SELECTION, function_info) input_infos = PandasBackend.before_call(operator_context, [input_info.annotated_dfobject]) # No input_infos copy needed because it's only a selection and the rows not being removed don't change result = original(input_infos[0].result_data, *args[1:], **kwargs) if result is None: raise NotImplementedError("TODO: Support inplace dropna") backend_result = PandasBackend.after_call(operator_context, input_infos, result) result = backend_result.annotated_dfobject.result_data dag_node = DagNode(op_id, BasicCodeLocation(caller_filename, lineno), operator_context, DagNodeDetails("dropna", list(result.columns)), get_optional_code_info_or_none(optional_code_reference, optional_source_code)) add_dag_node(dag_node, [input_info.dag_node], backend_result) return result