def commit(self):
matching_output = self.data
non_matching_output = None
if self.data:
domain = self.data.domain
conditions = []
for attr_name, oper, values in self.conditions:
attr_index = domain.index(attr_name)
attr = domain[attr_index]
if isinstance(attr, ContinuousVariable):
if any(not v for v in values):
continue
filter = data_filter.FilterContinuous(
attr_index, oper, *[float(v) for v in values])
elif isinstance(attr, StringVariable):
if any(v for v in values):
continue
filter = data_filter.FilterString(
attr_index, oper, *[str(v) for v in values])
else:
if oper == 2:
f_values = None
else:
if not values or not values[0]:
continue
values = [attr.values[i - 1] for i in values]
if oper == 0:
f_values = {values[0]}
else:
f_values = set(attr.values)
f_values.remove(values[0])
filter = data_filter.FilterDiscrete(attr_index, f_values)
conditions.append(filter)
if conditions:
filters = data_filter.Values(conditions)
matching_output = filters(self.data)
filters.negate = True
non_matching_output = filters(self.data)
# if hasattr(self.data, "name"):
# matching_output.name = self.data.name
# non_matching_output.name = self.data.name
#
# if self.purge_attributes or self.purge_classes:
# remover = orange.RemoveUnusedValues(removeOneValued=True)
#
# newDomain = remover(matching_output, 0, True, self.purge_classes)
# if newDomain != matching_output.domain:
# matching_output = orange.ExampleTable(newDomain, matching_output)
#
# newDomain = remover(non_matching_output, 0, True, self.purge_classes)
# if newDomain != non_matching_output.domain:
# nonmatchingOutput = orange.ExampleTable(newDomain, non_matching_output)
self.send("Matching Data", matching_output)
self.send("Unmatched Data", non_matching_output)
self.update_info(matching_output, self.data_out_rows)
def test_continuous_value_filter_isdefined(self):
filtered_data = filter.Values(conditions=[
filter.FilterContinuous(1, filter.FilterContinuous.IsDefined)
])(self.table)
correct_data = [row for row in self.data if row[1] is not None]
self.assertEqual(len(filtered_data), len(correct_data))
self.assertSequenceEqual(filtered_data, correct_data)
def test_continuous_value_filter_not_equal(self):
filtered_data = filter.Values(conditions=[
filter.FilterContinuous(0, filter.FilterContinuous.NotEqual, 1)
])(self.table)
correct_data = [row for row in self.data if row[0] != 1]
self.assertEqual(len(filtered_data), len(correct_data))
self.assertSequenceEqual(filtered_data, correct_data)
def output_data(self):
matching_output = self.data
non_matching_output = None
if self.data:
domain = self.data.domain
filters = data_filter.Values()
for attr_name, oper, values in self.conditions:
attr_index = domain.index(attr_name)
attr = domain[attr_index]
if isinstance(attr, ContinuousVariable):
if any(not v for v in values):
continue
filter = data_filter.FilterContinuous(
attr_index, oper, *[float(v) for v in values])
elif isinstance(attr, StringVariable):
if any(v for v in values):
continue
filter = data_filter.FilterString(
attr_index, oper, *[str(v) for v in values])
else:
if oper in [2, 3]:
raise NotImplementedError(
"subset filters for discrete attributes are not "
"implemented yet")
elif oper == 4:
f_values = None
else:
if not values or not values[0]:
continue
if oper == 0:
f_values = {values[0] - 1}
else:
f_values = set(range(len(attr.values)))
f_values.remove(values[0] - 1)
filter = data_filter.FilterDiscrete(attr_index, f_values)
filters.conditions.append(filter)
matching_output = filters(self.data)
filters.negate = True
non_matching_output = filters(self.data)
if hasattr(self.data, "name"):
matching_output.name = self.data.name
non_matching_output.name = self.data.name
"""
if self.purge_attributes or self.purge_classes:
remover = orange.RemoveUnusedValues(removeOneValued=True)
newDomain = remover(matching_output, 0, True, self.purge_classes)
if newDomain != matching_output.domain:
matching_output = orange.ExampleTable(newDomain, matching_output)
newDomain = remover(non_matching_output, 0, True, self.purge_classes)
if newDomain != non_matching_output.domain:
nonmatchingOutput = orange.ExampleTable(newDomain, non_matching_output)
"""
self.send("Matching Data", matching_output)
self.send("Unmatched Data", non_matching_output)
def test_continuous_value_filter_outside(self):
filtered_data = filter.Values(conditions=[
filter.FilterContinuous(0, filter.FilterContinuous.Outside, 2, 3)
])(self.table)
correct_data = [row for row in self.data
if row[0] is not None and not 2 <= row[0] <= 3]
self.assertEqual(len(filtered_data), len(correct_data))
self.assertSequenceEqual(filtered_data, correct_data)
def test_continuous_value_filter_between(self):
filtered_data = filter.Values(conditions=[
filter.FilterContinuous(0, filter.FilterContinuous.Between, 1, 2)
])(self.table)
correct_data = [row for row in self.data
if row[0] is not None and 1 <= row[0] <= 2]
self.assertEqual(len(filtered_data), len(correct_data))
self.assertSequenceEqual(filtered_data, correct_data)
def test_continuous_value_filter_greater(self):
filtered_data = filter.Values(conditions=[
filter.FilterContinuous(0, filter.FilterContinuous.Greater, 1)
])(self.table)
correct_data = [row for row in self.data
if row[0] is not None and row[0] > 1]
self.assertEqual(len(filtered_data), len(correct_data))
self.assertSequenceEqual(filtered_data, correct_data)
def commit(self):
matching_output = self.data
non_matching_output = None
annotated_output = None
self.Error.clear()
if self.data:
domain = self.data.domain
conditions = []
for attr_name, oper_idx, values in self.conditions:
attr_index = domain.index(attr_name)
attr = domain[attr_index]
operators = self.Operators[type(attr)]
opertype, _ = operators[oper_idx]
if attr.is_continuous:
try:
floats = self._values_to_floats(attr, values)
except ValueError as e:
self.Error.parsing_error(e.args[0])
return
if floats is None:
continue
filter = data_filter.FilterContinuous(
attr_index, opertype, *floats)
elif attr.is_string:
filter = data_filter.FilterString(
attr_index, opertype, *[str(v) for v in values])
else:
if opertype == FilterDiscreteType.IsDefined:
f_values = None
else:
if not values or not values[0]:
continue
values = [attr.values[i - 1] for i in values]
if opertype == FilterDiscreteType.Equal:
f_values = {values[0]}
elif opertype == FilterDiscreteType.NotEqual:
f_values = set(attr.values)
f_values.remove(values[0])
elif opertype == FilterDiscreteType.In:
f_values = set(values)
else:
raise ValueError("invalid operand")
filter = data_filter.FilterDiscrete(attr_index, f_values)
conditions.append(filter)
if conditions:
self.filters = data_filter.Values(conditions)
matching_output = self.filters(self.data)
self.filters.negate = True
non_matching_output = self.filters(self.data)
row_sel = np.in1d(self.data.ids, matching_output.ids)
annotated_output = create_annotated_table(self.data, row_sel)
# if hasattr(self.data, "name"):
# matching_output.name = self.data.name
# non_matching_output.name = self.data.name
purge_attrs = self.purge_attributes
purge_classes = self.purge_classes
if (purge_attrs or purge_classes) and \
not isinstance(self.data, SqlTable):
attr_flags = sum([
Remove.RemoveConstant * purge_attrs,
Remove.RemoveUnusedValues * purge_attrs
])
class_flags = sum([
Remove.RemoveConstant * purge_classes,
Remove.RemoveUnusedValues * purge_classes
])
# same settings used for attributes and meta features
remover = Remove(attr_flags, class_flags, attr_flags)
matching_output = remover(matching_output)
non_matching_output = remover(non_matching_output)
annotated_output = remover(annotated_output)
if matching_output is not None and not len(matching_output):
matching_output = None
if non_matching_output is not None and not len(non_matching_output):
non_matching_output = None
if annotated_output is not None and not len(annotated_output):
annotated_output = None
self.Outputs.matching_data.send(matching_output)
self.Outputs.unmatched_data.send(non_matching_output)
self.Outputs.annotated_data.send(annotated_output)
self.match_desc = report.describe_data_brief(matching_output)
self.nonmatch_desc = report.describe_data_brief(non_matching_output)
self.update_info(matching_output, self.data_out_rows, "Out: ")
def commit(self):
matching_output = self.data
non_matching_output = None
self.error()
if self.data:
domain = self.data.domain
conditions = []
for attr_name, oper_idx, values in self.conditions:
attr_index = domain.index(attr_name)
attr = domain[attr_index]
operators = self.Operators[type(attr)]
opertype, _ = operators[oper_idx]
if attr.is_continuous:
if any(not v for v in values):
continue
# Parse datetime strings into floats
if isinstance(attr, TimeVariable):
try:
values = [attr.parse(v) for v in values]
except ValueError as e:
self.error(e.args[0])
return
filter = data_filter.FilterContinuous(
attr_index, opertype, *[float(v) for v in values])
elif attr.is_string:
filter = data_filter.FilterString(
attr_index, opertype, *[str(v) for v in values])
else:
if opertype == FilterDiscreteType.IsDefined:
f_values = None
else:
if not values or not values[0]:
continue
values = [attr.values[i - 1] for i in values]
if opertype == FilterDiscreteType.Equal:
f_values = {values[0]}
elif opertype == FilterDiscreteType.NotEqual:
f_values = set(attr.values)
f_values.remove(values[0])
elif opertype == FilterDiscreteType.In:
f_values = set(values)
else:
raise ValueError("invalid operand")
filter = data_filter.FilterDiscrete(attr_index, f_values)
conditions.append(filter)
if conditions:
self.filters = data_filter.Values(conditions)
matching_output = self.filters(self.data)
self.filters.negate = True
non_matching_output = self.filters(self.data)
# if hasattr(self.data, "name"):
# matching_output.name = self.data.name
# non_matching_output.name = self.data.name
purge_attrs = self.purge_attributes
purge_classes = self.purge_classes
if (purge_attrs or purge_classes) and \
not isinstance(self.data, SqlTable):
attr_flags = sum([
Remove.RemoveConstant * purge_attrs,
Remove.RemoveUnusedValues * purge_attrs
])
class_flags = sum([
Remove.RemoveConstant * purge_classes,
Remove.RemoveUnusedValues * purge_classes
])
# same settings used for attributes and meta features
remover = Remove(attr_flags, class_flags, attr_flags)
matching_output = remover(matching_output)
non_matching_output = remover(non_matching_output)
self.send("Matching Data", matching_output)
self.send("Unmatched Data", non_matching_output)
self.match_desc = report.describe_data_brief(matching_output)
self.nonmatch_desc = report.describe_data_brief(non_matching_output)
self.update_info(matching_output, self.data_out_rows, "Out: ")
def commit(self):
matching_output = self.data
non_matching_output = None
if self.data:
domain = self.data.domain
conditions = []
for attr_name, oper, values in self.conditions:
attr_index = domain.index(attr_name)
attr = domain[attr_index]
if attr.is_continuous:
if any(not v for v in values):
continue
filter = data_filter.FilterContinuous(
attr_index, oper, *[float(v) for v in values])
elif attr.is_string:
filter = data_filter.FilterString(
attr_index, oper, *[str(v) for v in values])
else:
if oper == 3:
f_values = None
else:
if not values or not values[0]:
continue
values = [attr.values[i - 1] for i in values]
if oper == 0:
f_values = {values[0]}
elif oper == 1:
f_values = set(attr.values)
f_values.remove(values[0])
elif oper == 2:
f_values = set(values)
else:
raise ValueError("invalid operand")
filter = data_filter.FilterDiscrete(attr_index, f_values)
conditions.append(filter)
if conditions:
filters = data_filter.Values(conditions)
matching_output = filters(self.data)
filters.negate = True
non_matching_output = filters(self.data)
# if hasattr(self.data, "name"):
# matching_output.name = self.data.name
# non_matching_output.name = self.data.name
purge_attrs = self.purge_attributes
purge_classes = self.purge_classes
if (purge_attrs or purge_classes) and \
not isinstance(self.data, SqlTable):
attr_flags = sum([
Remove.RemoveConstant * purge_attrs,
Remove.RemoveUnusedValues * purge_attrs
])
class_flags = sum([
Remove.RemoveConstant * purge_classes,
Remove.RemoveUnusedValues * purge_classes
])
# same settings used for attributes and meta features
remover = Remove(attr_flags, class_flags, attr_flags)
matching_output = remover(matching_output)
non_matching_output = remover(non_matching_output)
self.send("Matching Data", matching_output)
self.send("Unmatched Data", non_matching_output)
self.match_desc = report.describe_data_brief(matching_output)
self.nonmatch_desc = report.describe_data_brief(non_matching_output)
self.update_info(matching_output, self.data_out_rows)
