def detail_scoring(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
matches = db.indivscores(fid)
return render(req, 'doit/pop_scores.html', {
'matches': matches, 'attr_name': attr_name, 'fid': fid,
'db': dbname, })
def source_data(req, dbname, sid):
db = DoitDB(dbname)
data = {
'fields': db.source_fields(sid),
'entities': db.source_entities(sid, 10)
}
return HttpResponse(simplejson.dumps(data), mimetype='application/json')
def mapper_results(req, dbname):
db = DoitDB(dbname)
mappings = simplejson.loads(req.POST['mappings'])
rejects = simplejson.loads(req.POST['rejects'])
s = db.create_mappings(mappings)
t = db.create_mappings(rejects, anti=True)
return HttpResponse(s)
def detail_summary(req, sid, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
meta = db.field_meta(fid)
vals = db.fieldexamples(fid, 1000, distinct=False)
histo = bucketize(vals)
return render_to_response('doit/pop_summary.html', {'histo': histo, 'attr_name': attr_name, 'source': sid, 'fid': fid, 'metadata': meta,})
def detail_scoring(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
matches = db.indivscores(fid)
return render_to_response('doit/pop_scores.html', {
'matches': matches, 'attr_name': attr_name, 'fid': fid,
'db': dbname,})
def detail_shared(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
shared = []
matches = db.field_candidates(fid)
shared_value = False
for match in matches[:4]:
shared.append({
'name': match['name'],
'values': db.sharedvalues(fid, match['id'])
})
table = [[]]
for match in shared:
table[0].append(match['name'])
for i in range(0, 10):
table.append([])
for match in shared:
try:
table[i + 1].append(match['values'][i])
if match['values'][i] is not None:
shared_value = True
except IndexError:
table[i + 1].append(' ')
return render(
req, 'doit/pop_shared.html', {
'shared': table,
'attr_name': attr_name,
'fid': fid,
'db': dbname,
'at_least_one': shared_value,
})
def lowscoremapper(req, dbname):
db = DoitDB(dbname)
matchscores = db.lowscorers(25)
attr_list = []
for name in matchscores:
cand = sorted(matchscores[name], key=itemgetter(2), reverse=True)
attr_list.append({'name': name, 'candidates': cand})
return render(req, 'doit/mapper.html', {'attr_list': attr_list})
def source_processor(req, dbname, sid, method_index):
db = DoitDB(dbname)
method_name = db.process_source(sid, method_index)
r = {
'method': method_name,
'source': sid,
'redirect': '/doit/' + dbname + '/' + sid + '/'
}
return HttpResponse(simplejson.dumps(r), mimetype='application/json')
def suggest_new_attribute(req, dbname):
db = DoitDB(dbname)
field_id = req.POST['fid']
suggestion = req.POST['suggestion']
username = req.POST['user']
comment = req.POST['comment']
success = db.new_attribute(field_id, suggestion, username, comment)
return HttpResponse(simplejson.dumps({'success': success}),
mimetype='application/json')
def suggest_new_attribute(req, dbname):
db = DoitDB(dbname)
field_id = req.POST['fid']
suggestion = req.POST['suggestion']
username = req.POST['user']
comment = req.POST['comment']
success = db.new_attribute(field_id, suggestion, username, comment)
return HttpResponse(simplejson.dumps({'success': success}),
mimetype='application/json')
def lowscoremapper(req, dbname):
db = DoitDB(dbname)
matchscores = db.lowscorers(25)
attr_list = []
for name in matchscores:
cand = sorted(matchscores[name],
key=itemgetter(2), reverse=True)
attr_list.append({'name': name, 'candidates': cand})
return render(req, 'doit/mapper.html', {'attr_list': attr_list})
def detail_summary(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
meta = db.field_meta(fid)
vals = db.fieldexamples(fid, 1000, distinct=False)
histo = bucketize(vals)
return render_to_response('doit/pop_summary.html', {
'histo': histo, 'attr_name': attr_name, 'source': '', 'fid': fid,
'metadata': meta, 'db': dbname,})
def mapper_by_field_name(req, dbname, field_name, comp_op):
db = DoitDB(dbname)
meta = {'category': 'Fields named "%s"' % field_name}
field_mappings = db.field_mappings_by_name(field_name,
exact_match=(comp_op != 'like'))
# egs = db.examplevalues(sid)
# for fid in field_mappings:
# egs.setdefault(int(fid), None)
# field_mappings[fid]['example'] = egs[int(fid)]
return render(req, 'doit/mapper.html', {
'attr_list': field_mappings.values(), 'field_name': field_name,
'meta': meta, })
def mapper_by_field_name(req, dbname, field_name, comp_op):
db = DoitDB(dbname)
meta = {'category': 'Fields named "%s"' % field_name}
field_mappings = db.field_mappings_by_name(field_name,
exact_match=(comp_op != 'like'))
# egs = db.examplevalues(sid)
# for fid in field_mappings:
# egs.setdefault(int(fid), None)
# field_mappings[fid]['example'] = egs[int(fid)]
return render_to_response('doit/mapper.html', {
'attr_list': field_mappings.values(), 'field_name': field_name,
'meta': meta,})
def compare_entities(req, dbname):
db = DoitDB(dbname)
save_entity_comparison_feedback(req, db)
target_similarity = req.GET['sim'] if 'sim' in req.GET \
else '0.' + str(random.randint(0, 9))
eid1, eid2, sim = db.get_entities_to_compare(target_similarity)
e1 = {'id': eid1, 'data': db.entity_data(eid1)}
e2 = {'id': eid2, 'data': db.entity_data(eid2)}
guess = 'Yes' if sim > 0.6 else 'No'
attr = pretty_order_entity_attributes(e1, e2)
return render_to_response('doit/compare-entities.html', {
'attributes': attr, 'similarity': sim, 'guess': guess,
'e1id': eid1, 'e2id': eid2,})
def mapper(req, sid, dbname):
db = DoitDB(dbname)
meta = dict()
meta['data'] = db.source_meta(sid)
meta['category'] = 'Source %s' % sid
field_mappings = db.field_mappings_by_source2(sid)
egs = db.examplevalues(sid)
for fid in field_mappings:
egs.setdefault(int(fid), None)
field_mappings[fid]['example'] = egs[int(fid)]
attr_list = sorted(field_mappings.values(), key=lambda f: f['match']['score'])
return render(req, 'doit/mapper.html', {
'attr_list': attr_list, 'source_id': sid, 'meta': meta, })
def compare_entities(req, dbname):
db = DoitDB(dbname)
save_entity_comparison_feedback(req, db)
target_similarity = req.GET['sim'] if 'sim' in req.GET \
else '0.' + str(random.randint(0, 9))
eid1, eid2, sim = db.get_entities_to_compare(target_similarity)
e1 = {'id': eid1, 'data': db.entity_data(eid1)}
e2 = {'id': eid2, 'data': db.entity_data(eid2)}
guess = 'Yes' if sim > 0.6 else 'No'
attr = pretty_order_entity_attributes(e1, e2)
return render(req, 'doit/compare-entities.html', {
'attributes': attr, 'similarity': sim, 'guess': guess,
'e1id': eid1, 'e2id': eid2, })
def mapper(req, sid, dbname):
db = DoitDB(dbname)
meta = dict()
meta['data'] = db.source_meta(sid)
meta['category'] = 'Source %s' % sid
field_mappings = db.field_mappings_by_source2(sid)
egs = db.examplevalues(sid)
for fid in field_mappings:
egs.setdefault(int(fid), None)
field_mappings[fid]['example'] = egs[int(fid)]
attr_list = sorted(field_mappings.values(), key=lambda f: f['match']['score'])
return render_to_response('doit/mapper.html', {
'attr_list': attr_list, 'source_id': sid,
'meta': meta,})
def detail_distro(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
vals = db.fieldexamples(fid, 1000, distinct=False)
histos = [bucketize(vals)]
histos[0]['name'] = attr_name
matches = db.field_candidates(fid)[:4]
for match in matches:
histo = bucketize(db.globalfieldexamples(int(match['id']), n=1000, distinct=False))
histo['name'] = match['name']
histos.append(histo)
return render(req, 'doit/pop_distro.html', {
'histos': histos, 'attr_name': attr_name, 'fid': fid,
'db': dbname, })
def source_table(req, dbname, sid):
db = DoitDB(dbname)
fields = db.source_fields(sid)
entities = db.source_entities(sid, 10)
for entity in entities:
vals = []
for field in fields:
try:
vals.append(entity['fields'][field['id']])
except KeyError:
vals.append('')
if vals[-1] is None: vals[-1] = ''
entity['fields'] = vals
return render_to_response('doit/viewTable_template.html', {
'fields': fields, 'entities': entities})
def detail_distro(req, dbname, sid, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
vals = db.fieldexamples(fid, 1000, distinct=False)
histos = [bucketize(vals)]
histos[0]['name'] = attr_name
matches = sorted(db.field_mappings(sid)[int(fid)]['matches'], key=itemgetter('score'), reverse=True)[:4]
for match in matches:
histo = bucketize(db.globalfieldexamples(int(match['id']), n=1000, distinct=False))
histo['name'] = match['name']
histos.append(histo)
return render_to_response('doit/pop_distro.html', {'histos': histos, 'attr_name': attr_name, 'fid': fid,})
def mapper(req, sid, dbname):
db = DoitDB(dbname)
meta = db.source_meta(sid)
field_mappings = db.field_mappings(sid)
egs = db.examplevalues(sid)
attr_list = []
for fid in field_mappings:
egs.setdefault(int(fid), None)
cand = sorted(field_mappings[fid]['matches'],
key=itemgetter('score'), reverse=True)
field_info = {'name': field_mappings[fid]['name'], 'fid': fid,
'candidates': cand, 'example': egs[int(fid)]}
if field_mappings[fid]['mapping'] is not None:
field_info['mapping'] = field_mappings[fid]['mapping']
attr_list.append(field_info)
return render_to_response('doit/mapper.html', {
'attr_list': attr_list, 'source_id': sid, 'metadata': meta,})
def detail_shared(req, sid, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
shared = []
matches = db.field_mappings(sid)[int(fid)]['matches']
for match in matches:
shared.append({'name': match['name'],
'values': db.sharedvalues(fid, match['id'])})
table = [[]]
for match in shared:
table[0].append(match['name'])
for i in range(0, 10):
table.append([])
for match in shared:
try:
table[i+1].append(match['values'][i])
except IndexError:
table[i+1].append(' ')
return render_to_response('doit/pop_shared.html', {'shared': table, 'attr_name': attr_name, 'fid': fid,})
def detail_examples(req, sid, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
egs = [{'name': attr_name, 'values': db.fieldexamples(fid, 10)}]
matches = db.field_mappings(sid)[int(fid)]['matches'][:5]
for match in matches:
egs.append({'name': match['name'], 'values': db.globalfieldexamples(match['id'], 10)})
transpose = [[]]
for eg in egs:
transpose[0].append(eg['name'])
for i in range(0, 10):
transpose.append([])
for eg in egs:
try:
transpose[i+1].append(eg['values'][i])
except IndexError:
transpose[i+1].append(' ')
return render_to_response('doit/pop_egs.html', {'examples': transpose, 'attr_name': attr_name, 'fid': fid,})
def detail_examples(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
egs = [{'name': attr_name, 'values': db.fieldexamples(fid, 10)}]
matches = db.field_candidates(fid)[:5]
for match in matches:
egs.append({
'name': match['name'],
'values': db.globalfieldexamples(match['id'], 10)
})
transpose = [[]]
for eg in egs:
transpose[0].append(eg['name'])
for i in range(0, 10):
transpose.append([])
for eg in egs:
try:
transpose[i + 1].append(eg['values'][i])
except IndexError:
transpose[i + 1].append(' ')
return render(req, 'doit/pop_egs.html', {
'examples': transpose,
'attr_name': attr_name,
'fid': fid,
'db': dbname
})
def mapper_by_field_set(req, dbname):
db = DoitDB(dbname)
answerer_id = req.GET.get('answerer_id', False)
fields = req.GET.get('fields', False)
domain_id = req.GET.get('domain_id', False)
assert all((fields, answerer_id, domain_id, auth_user(answerer_id, fields),))
# hack to make sure that ids are really ints
# this should raise an integer parse error if the ids are
# tampered with.
field_ids = map(int, fields.split(','))
field_mappings = db.field_mappings_by_id_list(field_ids=field_ids, answerer_id=answerer_id)
attr_list = sorted(field_mappings.values(), key=lambda f: f['match']['score'], reverse=True)
source_name = ''
if len(attr_list) > 0:
# we assume that all mappings are from same source
source_name = attr_list[0]['source_name']
c = {'source_name': source_name,
'attr_list': attr_list,
'expertsrc': True,
'expertsrc_url': settings.EXPERTSRC_URL,
'answerer_id': answerer_id,
'domain_id': domain_id}
return render(req, 'doit/expertsrc-mapper.html', c, context_instance=RequestContext(req))
def mapper_by_field_set(req, dbname):
db = DoitDB(dbname)
answerer_id = req.GET.get('answerer_id', False)
fields = req.GET.get('fields', False)
domain_id = req.GET.get('domain_id', False)
assert all((
fields,
answerer_id,
domain_id,
auth_user(answerer_id, fields),
))
# hack to make sure that ids are really ints
# this should raise an integer parse error if the ids are
# tampered with.
field_ids = map(int, fields.split(','))
field_mappings = db.field_mappings_by_id_list(field_ids=field_ids,
answerer_id=answerer_id)
attr_list = sorted(field_mappings.values(),
key=lambda f: f['match']['score'],
reverse=True)
source_name = ''
if len(attr_list) > 0:
# we assume that all mappings are from same source
source_name = attr_list[0]['source_name']
c = {
'source_name': source_name,
'attr_list': attr_list,
'expertsrc': True,
'expertsrc_url': settings.EXPERTSRC_URL,
'answerer_id': answerer_id,
'domain_id': domain_id
}
return render(req,
'doit/expertsrc-mapper.html',
c,
context_instance=RequestContext(req))
def detail_shared(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
shared = []
matches = db.field_candidates(fid)
shared_value = False
for match in matches[:4]:
shared.append({'name': match['name'],
'values': db.sharedvalues(fid, match['id'])})
table = [[]]
for match in shared:
table[0].append(match['name'])
for i in range(0, 10):
table.append([])
for match in shared:
try:
table[i + 1].append(match['values'][i])
if match['values'][i] is not None:
shared_value = True
except IndexError:
table[i + 1].append(' ')
return render(req, 'doit/pop_shared.html', {
'shared': table, 'attr_name': attr_name, 'fid': fid,
'db': dbname, 'at_least_one': shared_value, })
def detail_distro(req, dbname, fid):
db = DoitDB(dbname)
attr_name = db.fieldname(fid)
vals = db.fieldexamples(fid, 1000, distinct=False)
histos = [bucketize(vals)]
histos[0]['name'] = attr_name
matches = db.field_candidates(fid)[:4]
for match in matches:
histo = bucketize(db.globalfieldexamples(int(match['id']), n=1000, distinct=False))
histo['name'] = match['name']
histos.append(histo)
return render_to_response('doit/pop_distro.html', {
'histos': histos, 'attr_name': attr_name, 'fid': fid,
'db': dbname,})
def field_candidates(req, fid, dbname):
db = DoitDB(dbname)
return render(req, 'doit/candidate_list.html', {
'fid': fid, 'candidates': db.field_candidates(fid)})
def source_index(req, dbname):
db = DoitDB(dbname)
return render(req, 'doit/source_index.html', {
'source_list': db.sources(),
'dbname': dbname,
})
def mapper_results(req, sid, dbname): db = DoitDB(dbname) s = db.create_mappings(req.POST.items()) return HttpResponse(s)
def source_meta(req, dbname, sid):
db = DoitDB(dbname)
meta = dict()
meta['data'] = db.source_meta(sid)
meta['category'] = 'Source %s' % sid
return render(req, 'doit/source_meta.html', {'meta': meta})
def source_processor(req, dbname, sid, method_index):
db = DoitDB(dbname)
method_name = db.process_source(sid, method_index)
r = {'method': method_name, 'source': sid,
'redirect': '/doit/' + dbname + '/' + sid + '/'}
return HttpResponse(simplejson.dumps(r), mimetype='application/json')
def source_index(req, dbname):
db = DoitDB(dbname)
return render_to_response('doit/source_index.html', {
'source_list': db.sources(), 'dbname': dbname,})
def field_candidates(req, fid, dbname):
db = DoitDB(dbname)
return render(req, 'doit/candidate_list.html', {
'fid': fid,
'candidates': db.field_candidates(fid)
})
def source_meta(req, dbname, sid):
db = DoitDB(dbname)
meta = dict()
meta['data'] = db.source_meta(sid)
meta['category'] = 'Source %s' % sid
return render(req, 'doit/source_meta.html', {'meta': meta})
def source_entities(req, dbname, sid):
db = DoitDB(dbname)
data = {'entities': db.source_entities(sid, 10)}
return HttpResponse(simplejson.dumps(data),
mimetype='application/json')