def test_add_bite(self):
table_name = "volleyball_single"
table_col = 0
data = {'table': table_name, 'subject_col_id': table_col, 'slice': 0, 'total': 1, 'technique': 'found-majority'}
Bite.delete().execute() # delete all Bites
Apple.delete().execute() # delete all Apples
result = self.app.post('/add', data=data, content_type='multipart/form-data')
self.assertIn('apple', result.json)
apple_id = result.json['apple']
self.assertEqual(result.status_code, 200, msg=result.data)
database.connect(reuse_if_open=True)
self.assertEqual(len(Bite.select()), 1)
sleep(sleep_time)
result = self.app.get('/status')
self.assertEqual(result.status_code, 200, msg=result.data)
self.assertTrue(result.is_json)
j = {
"apples": [
{
"apple": table_name,
"status": STATUS_COMPLETE,
"complete": True,
"agreement": 1.0,
"elected": 0,
}
]
}
self.assertEqual(elect(apple_id, 'majority'), None) # just for the coverage
self.assertEqual(elect(apple_id, 'found-majority'), None) # just for the coverage
def add_bite():
table_name = request.values.get('table').strip()
column = int(request.values.get('subject_col_id'))
# if 'subject_col_id' in request.values:
# column = int(request.values.get('subject_col_id'))
# else:
# column = int(request.values.get('col_id'))
slice = int(request.values.get('slice'))
tot = int(request.values.get('total')) # total number of slices
agg_technique = request.values.get('technique') # aggregation technique
if agg_technique not in AGGREGATION_TECHNIQUES:
return jsonify(error="Invalid aggregation technique"), 400
apples = Apple.select().where(Apple.table==table_name)
if len(apples) == 0:
logger.log(LOG_LVL, "\nNew apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
try:
apple = Apple(table=table_name, total=tot)
apple.save()
except DatabaseError:
logger.log(LOG_LVL, "\nCatch existing: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
apple = apples[0]
else:
apple = apples[0]
logger.log(LOG_LVL, "\nExisting apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
if apple.complete:
return jsonify(error="The apple is already complete, your request will not be processed"), 400
b = Bite(apple=apple, slice=slice, col_id=column)
b.save()
slices = []
for bite in apple.bites:
slices.append(bite.slice)
if sorted(slices) == range(apple.total):
apple.complete = True
apple.save()
if apple.complete:
# if app.testing:
# combine_graphs(apple.id)
# else:
# g.db.close()
# p = Process(target=combine_graphs, args=(apple.id,))
# p.start()
g.db.close()
p = Process(target=elect, args=(apple.id, agg_technique))
p.start()
return jsonify({"apple": apple.id, "bite": b.id})
def score(slice_id, endpoint, onlydomain):
"""
# annotate each cell
# build graph
# compute scores
# send scored graph to combine
:param slice_id:
:param endpoint:
:param onlydomain:
:return:
"""
bites = Bite.select().where(Bite.id == slice_id)
if len(bites) == 0:
logger.warning("No bite with id: %s" % slice_id)
return False
else:
bite = bites[0]
logger.debug("The bite is found")
bite.status = STATUS_INPROGRESS
bite.save()
annotate_column(bite, endpoint, onlydomain)
build_graph(bite=bite, endpoint=endpoint)
graph_dir, m = compute_scores(bite=bite, endpoint=endpoint)
logger.info("m = %d" % m)
send_scored_graph(bite=bite, graph_file_dir=graph_dir, m=m)
bite.status = STATUS_COMPLETE
bite.save()
return True
def test_add_multiple_bite(self):
table_name = "volleyball_double"
table_col = 0
data = {'table': table_name, 'subject_col_id': table_col, 'slice': 0, 'total': 2, 'technique': 'majority'}
Bite.delete().execute() # delete all Bites
Apple.delete().execute() # delete all Apples
result = self.app.post('/add', data=data, content_type='multipart/form-data')
self.assertEqual(result.status_code, 200, msg=result.data)
database.connect(reuse_if_open=True)
self.assertEqual(len(Bite.select()), 1)
sleep(sleep_time)
result = self.app.get('/status')
self.assertEqual(result.status_code, 200, msg=result.data)
self.assertTrue(result.is_json)
j = {
"apples": [
{
"apple": table_name,
"status": STATUS_NEW,
"complete": False,
"elected": None,
"agreement": None,
}
]
}
self.assertDictEqual(result.get_json(), j)
data = {'table': table_name, 'subject_col_id': table_col, 'slice': 1, 'total': 2, 'technique': 'majority'}
result = self.app.post('/add', data=data, content_type='multipart/form-data')
self.assertEqual(result.status_code, 200, msg=result.data)
database.connect(reuse_if_open=True)
self.assertEqual(len(Bite.select()), 2)
sleep(sleep_time)
result = self.app.get('/status')
self.assertEqual(result.status_code, 200, msg=result.data)
self.assertTrue(result.is_json)
j = {
"apples": [
{
"apple": table_name,
"status": STATUS_COMPLETE,
"complete": True,
"agreement": 1.0,
"elected": table_col,
}
]
}
def score():
logger.debug("\nin score")
if request.method == 'GET':
return render_template('score.html')
uploaded_file = request.files['file_slice']
table_name = request.form['table']
column = request.form['column']
fname = table_name + "__" + get_random() + "__" + str(column) + ".tsv"
fname = secure_filename(fname)
total = int(request.form['total'])
b = Bite(table=table_name,
slice=request.form['slice'],
column=column,
addr=request.form['addr'],
fname=fname,
total=total)
b.save()
uploaded_file.save(os.path.join('local_uploads', fname))
if app.testing or 'test' in request.form:
from score import parse_args
logger.debug("will wait for the scoring to be done")
logger.debug("id: %d" % b.id)
b.status = STATUS_INPROGRESS
b.save()
parse_args(args=["--id", "%d" % b.id, "--testing"])
return jsonify({'msg': 'scored'})
else:
logger.debug(
"will return and the scoring will run in a different thread")
comm = "python score.py --id %s" % str(b.id)
subprocess.Popen(comm, shell=True)
return jsonify({'msg': 'scoring in progress'})
def get_graph():
bite_id = request.values.get('id')
bites = Bite.select().where(Bite.id == bite_id)
if len(bites) == 1:
bite = bites[0]
graph_file_name = graph_fname_from_bite(bite)
return send_from_directory(UPLOAD_DIR,
graph_file_name,
as_attachment=True)
abort(404)
def test_score_with_extra_space(self):
table_name = 'players'
players = ["Anouer Taouerghi\n"]
content = "\t".join(players)
data = {'table': table_name, 'column': 0, 'slice': 0, 'total': 1, 'addr': ''}
data['file_slice'] = (StringIO(content), "players.csv")
Bite.delete().execute() # delete all instances
print("after deletiong: "+str(len(Bite.select())))
result = self.app.post('/score', data=data, content_type='multipart/form-data')
database.connect(reuse_if_open=True)
self.assertEqual(result.status_code, 200)
bites = Bite.select().where(Bite.table==table_name, Bite.column==0)
self.assertEqual(len(bites), 1)
data_dir = os.path.join(DATA_DIR, bites[0].fname)
print("data_dir: "+data_dir)
data_file_exists = os.path.isfile(data_dir)
self.assertTrue(data_file_exists, msg="The data file is not found")
annotated_cells = {"data":
{
'Anouer Taouerghi': {u'http://dbpedia.org/resource/Anouer_Taouerghi': [
u'http://dbpedia.org/ontology/Person',
u'http://dbpedia.org/ontology/Agent',
u'http://dbpedia.org/ontology/Athlete',
u'http://dbpedia.org/ontology/VolleyballPlayer']},
}
}
f = open(data_dir)
computed_data = json.load(f)
self.assertDictEqual(annotated_cells, computed_data)
bite = Bite.select()[0]
graph_file_name = graph_fname_from_bite(bite)
graph_file_dir = os.path.join(UPLOAD_DIR, graph_file_name)
tgraph = type_graph.TypeGraph()
tgraph.load_from_file(graph_file_dir)
tgraph.set_score_for_graph(coverage_weight=0.1, m=1, fsid=3)
results = [n for n in tgraph.get_scores()]
results.sort(key=lambda node: node.path_specificity)
self.assertEqual(results[0].title, "http://dbpedia.org/ontology/VolleyballPlayer")
self.assertEqual(bite.status, "complete")
def add_bite():
if request.method == 'GET':
return render_template('combine.html')
table_name = request.values.get('table').strip()
column = int(request.values.get('column'))
slice = int(request.values.get('slice'))
m = int(request.values.get('m'))
tot = int(request.values.get('total')) # total number of slices
uploaded_file = request.files['graph']
apples = Apple.select().where(Apple.table==table_name, Apple.column==column)
if len(apples) == 0:
logger.log(LOG_LVL, "\nNew apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
apple = Apple(table=table_name, column=column, total=tot)
apple.save()
else:
apple = apples[0]
logger.log(LOG_LVL, "\nExisting apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
if apple.complete:
return jsonify(error="The apple is already complete, your request will not be processed"), 400
b = Bite(apple=apple, slice=slice, m=m)
b.save()
fname = "%d-%s-%d-%d.json" % (b.id, b.apple.table.replace(" ", "_"), b.apple.column, b.slice)
uploaded_file.save(os.path.join(UPLOAD_DIR, fname))
b.fname = fname
b.save()
slices = []
for bite in apple.bites:
slices.append(bite.slice)
if sorted(slices) == range(apple.total):
apple.complete = True
apple.save()
if apple.complete:
if app.testing:
combine_graphs(apple.id)
else:
g.db.close()
p = Process(target=combine_graphs, args=(apple.id,))
p.start()
return jsonify({"apple": apple.id, "bite": b.id})
def fetch():
bites = """
Bites
<table>
<tr>
<td>Table</td>
<td>Column</td>
<td>Slice</td>
<td>Fname</td>
<td>Address</td>
</tr>
"""
for bite in Bite.select():
bites += "<tr>"
bites += "<td>%s</td>\n" % bite.table
bites += "<td>%d</td>\n" % bite.column
bites += "<td>%d</td>\n" % bite.slice
bites += "<td>%s</td>\n" % bite.fname
bites += "<td>%s</td>\n" % bite.addr
bites += "</tr>"
bites += "</table>"
return bites
def list_bites():
return jsonify(bites=[b.json() for b in Bite.select()])
def register():
table_name = request.args.get('table')
b = Bite(table=table_name, slice=0, column=0, addr="default")
b.save()
return 'Table: %s is added' % table_name