def commit(self): matching_output = self.data non_matching_output = None if self.data: domain = self.data.domain conditions = [] for attr_name, oper, values in self.conditions: attr_index = domain.index(attr_name) attr = domain[attr_index] if isinstance(attr, ContinuousVariable): if any(not v for v in values): continue filter = data_filter.FilterContinuous( attr_index, oper, *[float(v) for v in values]) elif isinstance(attr, StringVariable): if any(v for v in values): continue filter = data_filter.FilterString( attr_index, oper, *[str(v) for v in values]) else: if oper == 2: f_values = None else: if not values or not values[0]: continue values = [attr.values[i - 1] for i in values] if oper == 0: f_values = {values[0]} else: f_values = set(attr.values) f_values.remove(values[0]) filter = data_filter.FilterDiscrete(attr_index, f_values) conditions.append(filter) if conditions: filters = data_filter.Values(conditions) matching_output = filters(self.data) filters.negate = True non_matching_output = filters(self.data) # if hasattr(self.data, "name"): # matching_output.name = self.data.name # non_matching_output.name = self.data.name # # if self.purge_attributes or self.purge_classes: # remover = orange.RemoveUnusedValues(removeOneValued=True) # # newDomain = remover(matching_output, 0, True, self.purge_classes) # if newDomain != matching_output.domain: # matching_output = orange.ExampleTable(newDomain, matching_output) # # newDomain = remover(non_matching_output, 0, True, self.purge_classes) # if newDomain != non_matching_output.domain: # nonmatchingOutput = orange.ExampleTable(newDomain, non_matching_output) self.send("Matching Data", matching_output) self.send("Unmatched Data", non_matching_output) self.update_info(matching_output, self.data_out_rows)
def test_continuous_value_filter_isdefined(self): filtered_data = filter.Values(conditions=[ filter.FilterContinuous(1, filter.FilterContinuous.IsDefined) ])(self.table) correct_data = [row for row in self.data if row[1] is not None] self.assertEqual(len(filtered_data), len(correct_data)) self.assertSequenceEqual(filtered_data, correct_data)
def test_continuous_value_filter_not_equal(self): filtered_data = filter.Values(conditions=[ filter.FilterContinuous(0, filter.FilterContinuous.NotEqual, 1) ])(self.table) correct_data = [row for row in self.data if row[0] != 1] self.assertEqual(len(filtered_data), len(correct_data)) self.assertSequenceEqual(filtered_data, correct_data)
def output_data(self): matching_output = self.data non_matching_output = None if self.data: domain = self.data.domain filters = data_filter.Values() for attr_name, oper, values in self.conditions: attr_index = domain.index(attr_name) attr = domain[attr_index] if isinstance(attr, ContinuousVariable): if any(not v for v in values): continue filter = data_filter.FilterContinuous( attr_index, oper, *[float(v) for v in values]) elif isinstance(attr, StringVariable): if any(v for v in values): continue filter = data_filter.FilterString( attr_index, oper, *[str(v) for v in values]) else: if oper in [2, 3]: raise NotImplementedError( "subset filters for discrete attributes are not " "implemented yet") elif oper == 4: f_values = None else: if not values or not values[0]: continue if oper == 0: f_values = {values[0] - 1} else: f_values = set(range(len(attr.values))) f_values.remove(values[0] - 1) filter = data_filter.FilterDiscrete(attr_index, f_values) filters.conditions.append(filter) matching_output = filters(self.data) filters.negate = True non_matching_output = filters(self.data) if hasattr(self.data, "name"): matching_output.name = self.data.name non_matching_output.name = self.data.name """ if self.purge_attributes or self.purge_classes: remover = orange.RemoveUnusedValues(removeOneValued=True) newDomain = remover(matching_output, 0, True, self.purge_classes) if newDomain != matching_output.domain: matching_output = orange.ExampleTable(newDomain, matching_output) newDomain = remover(non_matching_output, 0, True, self.purge_classes) if newDomain != non_matching_output.domain: nonmatchingOutput = orange.ExampleTable(newDomain, non_matching_output) """ self.send("Matching Data", matching_output) self.send("Unmatched Data", non_matching_output)
def test_continuous_value_filter_outside(self): filtered_data = filter.Values(conditions=[ filter.FilterContinuous(0, filter.FilterContinuous.Outside, 2, 3) ])(self.table) correct_data = [row for row in self.data if row[0] is not None and not 2 <= row[0] <= 3] self.assertEqual(len(filtered_data), len(correct_data)) self.assertSequenceEqual(filtered_data, correct_data)
def test_continuous_value_filter_between(self): filtered_data = filter.Values(conditions=[ filter.FilterContinuous(0, filter.FilterContinuous.Between, 1, 2) ])(self.table) correct_data = [row for row in self.data if row[0] is not None and 1 <= row[0] <= 2] self.assertEqual(len(filtered_data), len(correct_data)) self.assertSequenceEqual(filtered_data, correct_data)
def test_continuous_value_filter_greater(self): filtered_data = filter.Values(conditions=[ filter.FilterContinuous(0, filter.FilterContinuous.Greater, 1) ])(self.table) correct_data = [row for row in self.data if row[0] is not None and row[0] > 1] self.assertEqual(len(filtered_data), len(correct_data)) self.assertSequenceEqual(filtered_data, correct_data)
def commit(self): matching_output = self.data non_matching_output = None annotated_output = None self.Error.clear() if self.data: domain = self.data.domain conditions = [] for attr_name, oper_idx, values in self.conditions: attr_index = domain.index(attr_name) attr = domain[attr_index] operators = self.Operators[type(attr)] opertype, _ = operators[oper_idx] if attr.is_continuous: try: floats = self._values_to_floats(attr, values) except ValueError as e: self.Error.parsing_error(e.args[0]) return if floats is None: continue filter = data_filter.FilterContinuous( attr_index, opertype, *floats) elif attr.is_string: filter = data_filter.FilterString( attr_index, opertype, *[str(v) for v in values]) else: if opertype == FilterDiscreteType.IsDefined: f_values = None else: if not values or not values[0]: continue values = [attr.values[i - 1] for i in values] if opertype == FilterDiscreteType.Equal: f_values = {values[0]} elif opertype == FilterDiscreteType.NotEqual: f_values = set(attr.values) f_values.remove(values[0]) elif opertype == FilterDiscreteType.In: f_values = set(values) else: raise ValueError("invalid operand") filter = data_filter.FilterDiscrete(attr_index, f_values) conditions.append(filter) if conditions: self.filters = data_filter.Values(conditions) matching_output = self.filters(self.data) self.filters.negate = True non_matching_output = self.filters(self.data) row_sel = np.in1d(self.data.ids, matching_output.ids) annotated_output = create_annotated_table(self.data, row_sel) # if hasattr(self.data, "name"): # matching_output.name = self.data.name # non_matching_output.name = self.data.name purge_attrs = self.purge_attributes purge_classes = self.purge_classes if (purge_attrs or purge_classes) and \ not isinstance(self.data, SqlTable): attr_flags = sum([ Remove.RemoveConstant * purge_attrs, Remove.RemoveUnusedValues * purge_attrs ]) class_flags = sum([ Remove.RemoveConstant * purge_classes, Remove.RemoveUnusedValues * purge_classes ]) # same settings used for attributes and meta features remover = Remove(attr_flags, class_flags, attr_flags) matching_output = remover(matching_output) non_matching_output = remover(non_matching_output) annotated_output = remover(annotated_output) if matching_output is not None and not len(matching_output): matching_output = None if non_matching_output is not None and not len(non_matching_output): non_matching_output = None if annotated_output is not None and not len(annotated_output): annotated_output = None self.Outputs.matching_data.send(matching_output) self.Outputs.unmatched_data.send(non_matching_output) self.Outputs.annotated_data.send(annotated_output) self.match_desc = report.describe_data_brief(matching_output) self.nonmatch_desc = report.describe_data_brief(non_matching_output) self.update_info(matching_output, self.data_out_rows, "Out: ")
def commit(self): matching_output = self.data non_matching_output = None self.error() if self.data: domain = self.data.domain conditions = [] for attr_name, oper_idx, values in self.conditions: attr_index = domain.index(attr_name) attr = domain[attr_index] operators = self.Operators[type(attr)] opertype, _ = operators[oper_idx] if attr.is_continuous: if any(not v for v in values): continue # Parse datetime strings into floats if isinstance(attr, TimeVariable): try: values = [attr.parse(v) for v in values] except ValueError as e: self.error(e.args[0]) return filter = data_filter.FilterContinuous( attr_index, opertype, *[float(v) for v in values]) elif attr.is_string: filter = data_filter.FilterString( attr_index, opertype, *[str(v) for v in values]) else: if opertype == FilterDiscreteType.IsDefined: f_values = None else: if not values or not values[0]: continue values = [attr.values[i - 1] for i in values] if opertype == FilterDiscreteType.Equal: f_values = {values[0]} elif opertype == FilterDiscreteType.NotEqual: f_values = set(attr.values) f_values.remove(values[0]) elif opertype == FilterDiscreteType.In: f_values = set(values) else: raise ValueError("invalid operand") filter = data_filter.FilterDiscrete(attr_index, f_values) conditions.append(filter) if conditions: self.filters = data_filter.Values(conditions) matching_output = self.filters(self.data) self.filters.negate = True non_matching_output = self.filters(self.data) # if hasattr(self.data, "name"): # matching_output.name = self.data.name # non_matching_output.name = self.data.name purge_attrs = self.purge_attributes purge_classes = self.purge_classes if (purge_attrs or purge_classes) and \ not isinstance(self.data, SqlTable): attr_flags = sum([ Remove.RemoveConstant * purge_attrs, Remove.RemoveUnusedValues * purge_attrs ]) class_flags = sum([ Remove.RemoveConstant * purge_classes, Remove.RemoveUnusedValues * purge_classes ]) # same settings used for attributes and meta features remover = Remove(attr_flags, class_flags, attr_flags) matching_output = remover(matching_output) non_matching_output = remover(non_matching_output) self.send("Matching Data", matching_output) self.send("Unmatched Data", non_matching_output) self.match_desc = report.describe_data_brief(matching_output) self.nonmatch_desc = report.describe_data_brief(non_matching_output) self.update_info(matching_output, self.data_out_rows, "Out: ")
def commit(self): matching_output = self.data non_matching_output = None if self.data: domain = self.data.domain conditions = [] for attr_name, oper, values in self.conditions: attr_index = domain.index(attr_name) attr = domain[attr_index] if attr.is_continuous: if any(not v for v in values): continue filter = data_filter.FilterContinuous( attr_index, oper, *[float(v) for v in values]) elif attr.is_string: filter = data_filter.FilterString( attr_index, oper, *[str(v) for v in values]) else: if oper == 3: f_values = None else: if not values or not values[0]: continue values = [attr.values[i - 1] for i in values] if oper == 0: f_values = {values[0]} elif oper == 1: f_values = set(attr.values) f_values.remove(values[0]) elif oper == 2: f_values = set(values) else: raise ValueError("invalid operand") filter = data_filter.FilterDiscrete(attr_index, f_values) conditions.append(filter) if conditions: filters = data_filter.Values(conditions) matching_output = filters(self.data) filters.negate = True non_matching_output = filters(self.data) # if hasattr(self.data, "name"): # matching_output.name = self.data.name # non_matching_output.name = self.data.name purge_attrs = self.purge_attributes purge_classes = self.purge_classes if (purge_attrs or purge_classes) and \ not isinstance(self.data, SqlTable): attr_flags = sum([ Remove.RemoveConstant * purge_attrs, Remove.RemoveUnusedValues * purge_attrs ]) class_flags = sum([ Remove.RemoveConstant * purge_classes, Remove.RemoveUnusedValues * purge_classes ]) # same settings used for attributes and meta features remover = Remove(attr_flags, class_flags, attr_flags) matching_output = remover(matching_output) non_matching_output = remover(non_matching_output) self.send("Matching Data", matching_output) self.send("Unmatched Data", non_matching_output) self.match_desc = report.describe_data_brief(matching_output) self.nonmatch_desc = report.describe_data_brief(non_matching_output) self.update_info(matching_output, self.data_out_rows)