def add_bite():
table_name = request.values.get('table').strip()
column = int(request.values.get('subject_col_id'))
# if 'subject_col_id' in request.values:
# column = int(request.values.get('subject_col_id'))
# else:
# column = int(request.values.get('col_id'))
slice = int(request.values.get('slice'))
tot = int(request.values.get('total')) # total number of slices
agg_technique = request.values.get('technique') # aggregation technique
if agg_technique not in AGGREGATION_TECHNIQUES:
return jsonify(error="Invalid aggregation technique"), 400
apples = Apple.select().where(Apple.table==table_name)
if len(apples) == 0:
logger.log(LOG_LVL, "\nNew apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
try:
apple = Apple(table=table_name, total=tot)
apple.save()
except DatabaseError:
logger.log(LOG_LVL, "\nCatch existing: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
apple = apples[0]
else:
apple = apples[0]
logger.log(LOG_LVL, "\nExisting apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
if apple.complete:
return jsonify(error="The apple is already complete, your request will not be processed"), 400
b = Bite(apple=apple, slice=slice, col_id=column)
b.save()
slices = []
for bite in apple.bites:
slices.append(bite.slice)
if sorted(slices) == range(apple.total):
apple.complete = True
apple.save()
if apple.complete:
# if app.testing:
# combine_graphs(apple.id)
# else:
# g.db.close()
# p = Process(target=combine_graphs, args=(apple.id,))
# p.start()
g.db.close()
p = Process(target=elect, args=(apple.id, agg_technique))
p.start()
return jsonify({"apple": apple.id, "bite": b.id})
def get_graph():
apple_id = request.values.get('id')
#apple_id = int(apple_id)
apples = Apple.select().where(Apple.id==apple_id)
print("apples: ")
print(apples)
print("len: ")
print(len(apples))
if len(apples) == 1:
apple = apples[0]
fname = apple.fname
if fname != "":
return send_from_directory(UPLOAD_DIR, fname, as_attachment=True)
abort(404)
def add_bite():
if request.method == 'GET':
return render_template('combine.html')
table_name = request.values.get('table').strip()
column = int(request.values.get('column'))
slice = int(request.values.get('slice'))
m = int(request.values.get('m'))
tot = int(request.values.get('total')) # total number of slices
uploaded_file = request.files['graph']
apples = Apple.select().where(Apple.table==table_name, Apple.column==column)
if len(apples) == 0:
logger.log(LOG_LVL, "\nNew apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
apple = Apple(table=table_name, column=column, total=tot)
apple.save()
else:
apple = apples[0]
logger.log(LOG_LVL, "\nExisting apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot))
if apple.complete:
return jsonify(error="The apple is already complete, your request will not be processed"), 400
b = Bite(apple=apple, slice=slice, m=m)
b.save()
fname = "%d-%s-%d-%d.json" % (b.id, b.apple.table.replace(" ", "_"), b.apple.column, b.slice)
uploaded_file.save(os.path.join(UPLOAD_DIR, fname))
b.fname = fname
b.save()
slices = []
for bite in apple.bites:
slices.append(bite.slice)
if sorted(slices) == range(apple.total):
apple.complete = True
apple.save()
if apple.complete:
if app.testing:
combine_graphs(apple.id)
else:
g.db.close()
p = Process(target=combine_graphs, args=(apple.id,))
p.start()
return jsonify({"apple": apple.id, "bite": b.id})
def elect(apple_id, technique):
"""
:param apple_id:
:return: most_frequent, agreement (how many agreed on this).
"""
database = get_database()
database.connect(reuse_if_open=True)
apples = Apple.select().where(Apple.id==apple_id)
if len(apples) == 0:
logger.error("Apple with id: %s does not exists" % str(apple_id))
database.close()
return
# return None, None
apple = apples[0]
apple.status = STATUS_PROCESSING
apple.save()
try:
col_ids = [b.col_id for b in apple.bites]
# c = Counter(col_ids)
# sorted_cols_counts = sorted(c.items(), key=lambda item: item[1])
# most_frequent_col_id = sorted_cols_counts[-1][0]
# agreement = sorted_cols_counts[-1][1]*1.0/len(col_ids)
if technique =="majority":
elected_col_id, agreement = majority_aggregation(col_ids)
elif technique == "found-majority":
elected_col_id, agreement = found_majority_aggregation(col_ids)
else:
logger.error("unvalid technique: <%s>" % technique)
apple.status = STATUS_STOPPED
apple.save()
return
apple.elected = elected_col_id
apple.agreement = agreement
apple.status = STATUS_COMPLETE
apple.save()
except Exception as e:
logger.error(str(e))
apple.status = STATUS_STOPPED
apple.save()
database.close()
def combine_graphs(apple_id):
database = get_database()
database.connect(reuse_if_open=True)
apple = Apple.select().where(Apple.id==apple_id)[0]
apple.status = STATUS_PROCESSING
apple.save()
graphs = []
for bite in apple.bites:
print("bite: %d" % bite.id)
f = open(os.path.join(UPLOAD_DIR, bite.fname))
j = json.load(f)
graphs.append(j)
f.close()
graph = merge_graphs(graphs=graphs)
fname = "%d-%s-%d-merged.json" % (apple.id, apple.table.replace(" ", "_"), apple.column)
f = open(os.path.join(UPLOAD_DIR, fname), "w")
f.write(json.dumps(graph))
f.close()
apple.fname = fname
apple.status = STATUS_COMPLETE
apple.save()
database.close()
def all_apples():
return jsonify(apples=[apple.json() for apple in Apple.select()])
def status():
apples = [{"apple": apple.table, "status": apple.status, "complete": apple.complete} for apple in Apple.select()]
return jsonify(apples=apples)
def status():
# apples = [{"apple": apple.table, "status": apple.status, "complete": apple.complete} for apple in Apple.select()]
apples = [{"apple": apple.table, "status": apple.status, "complete": apple.complete, "elected": apple.elected, "agreement": apple.agreement} for apple in Apple.select()]
return jsonify(apples=apples)
