def run_implementation(self):
# Se obtienen los resultados del data fusion
dal = DALMongo(self.project_id)
records = dal.get_fused_records()
if not self.only_matches:
records += dal.get_non_matches()
schema = dal.get_global_schema()
return self._load_module(records=records, schema=schema).run()
def config_json(project_id):
dal = DALMongo(project_id)
cols = [{
"label": c['custom_name'],
"value": c['name'],
"id": c['name'],
"config": {
"key": {
'type': 'hidden',
'value': c['name'],
}
}
} for c in dal.get_global_schema() if c['name'].startswith('__new__')]
# Above checking of __new__ prefix is unncessary I think...
encoding_configs = dynamic_loading.list_modules('encoding')
rowmodel = {
'type': 'row',
'cols': {
'1_key': {
'type': 'dropdown',
'label': 'Select a column',
'selectedoption': {},
'options': cols
},
'encoding': {
"type": "dropdown",
'label': 'Select encoding',
'selectedoption': {},
'options': encoding_configs
}
}
}
return {
'keys': {
'type': 'rows',
'rows': [],
'label': 'Keys',
"rowmodel": rowmodel
}
}
def run_implementation(self):
# Se obtienen los grupos de registros
dal = DALMongo(self.project_id)
groups = dal.get_indexing_groups()
segmented_schema = dal.get_global_schema()
output_fields_schema = {}
matched_cols = []
for column in segmented_schema:
if column['name'].startswith("__new__"):
output_fields_schema[column['name']] = column['fields']
matched_cols.append(column['name'])
simils = []
max_weight = max(
[float(module['weight']) for idx, module in self.config.items()])
for group in groups:
for r1 in group.records1:
for r2 in group.records2:
# Initialize similarity vector
sv = SimilarityVector(r1._id, r2._id, group=group.key)
for col in matched_cols: # could be r2.matched_cols() as well (they return the same)
if not self.segmentation_skipped:
for out_field, comparison_module in self.config.items(
):
# Check that the output field exists in the column, otherwise it wont create an entrance
# in the similarity vector
if out_field in [
f['output_field']
for f in output_fields_schema[col]
]:
# Se obienen los valores a comparar y se comparan
out_field_value1 = r1.get_output_field_col(
out_field, col)
out_field_value2 = r2.get_output_field_col(
out_field, col)
module = self._load_module(
comparison_module)
weight = float(comparison_module['weight'])
# Actualiza el valor de la comparacion en el vector
sim_value = module.run(
out_field_value1, out_field_value2)
sim_value_weighted = sim_value * weight / max_weight
sv.vector.append(sim_value_weighted)
# sv.comparisons.append([out_field_value1, out_field_value2])
sv.comparisons.append({
'values':
[out_field_value1, out_field_value2],
'output_field':
out_field
})
else:
comparison_module = self.config[col]
# Se obienen los valores completos de la columna
column_value_s1 = r1.get_field_col(col)
column_value_s2 = r2.get_field_col(col)
module = self._load_module(comparison_module)
weight = float(comparison_module['weight'])
# Actualiza el valor de la comparacion en el vector
sim_value = module.run(column_value_s1,
column_value_s2)
sim_value_weighted = sim_value * weight / max_weight
sv.vector.append(sim_value_weighted)
sv.comparisons.append(
{'values': [column_value_s1, column_value_s2]})
simils.append(sv)
self._append_result_collection(simils)