def schema(request, project_id):
dal = dal_mongo.DALMongo(project_id)
# project = Project.objects.get(id=project_id)
schema1 = [c.name for c in dal.get_schema(1)]
schema2 = [c.name for c in dal.get_schema(2)]
return JsonResponse({'source1': schema1, 'source2': schema2}, safe=False)
def indexingdata(request, project_id):
project = Project.objects.get(id=project_id)
dal = dal_mongo.DALMongo(project_id)
return JsonResponse(
{
'results':
dal.get_limited_indexing_keys(25),
'cant_idx_groups':
dal.get_count_indexing_groups(),
'cant_comparisons':
dal.get_number_of_comparisons_to_do(),
'cant_comparisons_full_index':
dal.get_extracted_data_count(1) * dal.get_extracted_data_count(2)
},
safe=False)
def output_fields(request, project_id):
"""
Method to return the output fields
:param request:
:param project_id:
:return:
"""
project = Project.objects.get(id=project_id)
dal = dal_mongo.DALMongo(project_id)
ret = {}
if project.segmentation_skipped:
ret['col_or_outputfield'] = "column"
ret['values'] = dal.get_matched_cols()
else:
ret['col_or_outputfield'] = "output field"
ret['values'] = dal.get_output_fields_matched_cols()
return JsonResponse(ret, safe=False)
def previewdata(request, project_id, step):
dal = dal_mongo.DALMongo(project_id)
previewdata1 = dal.get_aggregated_records(step,
1,
pipeline=[{
'$limit': 15
}],
json_format=True)
previewdata2 = dal.get_aggregated_records(step,
2,
pipeline=[{
'$limit': 15
}],
json_format=True)
new_previewdata1 = []
new_previewdata2 = []
for r in previewdata1:
new_row = {}
for col in r['columns']:
new_row[col['name']] = col['fields']
new_previewdata1.append(new_row)
for r in previewdata2:
new_row = {}
for col in r['columns']:
new_row[col['name']] = col['fields']
new_previewdata2.append(new_row)
return JsonResponse(
{
'source1': {
'results': new_previewdata1,
'total_count': dal.get_extracted_data_count(1)
},
'source2': {
'results': new_previewdata2,
'total_count': dal.get_extracted_data_count(2)
}
},
safe=False)
def segmentedschema(request, project_id):
dal = dal_mongo.DALMongo(project_id)
# project = Project.objects.get(id=project_id)
schema1 = []
schema2 = []
for c in dal.get_segmented_schema(1):
new_col = {'colname': c.name, 'segments': []}
for segment in c.fields:
new_col['segments'].append(segment.output_field)
schema1.append(new_col)
for c in dal.get_segmented_schema(2):
new_col = {'colname': c.name, 'segments': []}
for segment in c.fields:
new_col['segments'].append(segment.output_field)
schema2.append(new_col)
return JsonResponse({'source1': schema1, 'source2': schema2}, safe=False)
def comparisondata(request, project_id):
dal = dal_mongo.DALMongo(project_id)
data = dal.get_comparison_info()
ret_data = []
for d in data:
new_d = {}
for i in range(len(d['comparisons'])):
d['comparisons'][i]['vector_value'] = d['vector'][i]
new_d['comparisons'] = {'record1': [], 'record2': []}
if 'output_field' in d['comparisons'][0]:
d['comparisons'].sort(key=lambda c: c['output_field'])
for c in d['comparisons']:
new_d['comparisons']['record1'].append(c['values'][0])
new_d['comparisons']['record2'].append(c['values'][1])
new_d['vector'] = [c['vector_value'] for c in d['comparisons']]
ret_data.append(new_d)
response = {
'results': ret_data,
'total_comparisons_made': dal.get_total_comparisons_made()
}
return JsonResponse(response, safe=False)
def matchesresult(request, project_id):
dal = dal_mongo.DALMongo(project_id)
data = dal.get_matches_info()
matches = []
non_matches = []
potential_matches = []
for d in data:
if 'output_field' in d['record1'][0]:
r1 = [
o['value']
for o in sorted(d['record1'], key=lambda o: o['output_field'])
]
r2 = [
o['value']
for o in sorted(d['record2'], key=lambda o: o['output_field'])
]
else:
r1 = [o['value'] for o in d['record1']]
r2 = [o['value'] for o in d['record2']]
new_d = {'match_type': d['match_type'], 'record1': r1, 'record2': r2}
if d['match_type'] == 0:
non_matches.append(new_d)
elif d['match_type'] == 1:
matches.append(new_d)
elif d['match_type'] == 2:
potential_matches.append(new_d)
response = {
'results': matches + potential_matches + non_matches,
'total_data': dal.get_total_comparisons_made(),
'total_matches': dal.get_matches_count(),
'total_potential_matches': dal.get_potential_matches_count(),
'total_non_matches': dal.get_non_matches_count()
}
return JsonResponse(response, safe=False)
def finalschema(request, project_id):
dal = dal_mongo.DALMongo(project_id)
# project = Project.objects.get(id=project_id)
schema = _transform_final_schema(dal.get_global_schema())
return JsonResponse(schema, safe=False)
def destroy(self, request, *args, **kwargs):
dal = dal_mongo.DALMongo(kwargs['pk'])
dal.drop_database()
return super(ProjectViewSet, self).destroy(request, *args, **kwargs)
def fuseddata(request, project_id):
dal = dal_mongo.DALMongo(project_id)
data = dal.get_fused_preview()
return JsonResponse(data, safe=False)