def add_bite(): table_name = request.values.get('table').strip() column = int(request.values.get('subject_col_id')) # if 'subject_col_id' in request.values: # column = int(request.values.get('subject_col_id')) # else: # column = int(request.values.get('col_id')) slice = int(request.values.get('slice')) tot = int(request.values.get('total')) # total number of slices agg_technique = request.values.get('technique') # aggregation technique if agg_technique not in AGGREGATION_TECHNIQUES: return jsonify(error="Invalid aggregation technique"), 400 apples = Apple.select().where(Apple.table==table_name) if len(apples) == 0: logger.log(LOG_LVL, "\nNew apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot)) try: apple = Apple(table=table_name, total=tot) apple.save() except DatabaseError: logger.log(LOG_LVL, "\nCatch existing: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot)) apple = apples[0] else: apple = apples[0] logger.log(LOG_LVL, "\nExisting apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot)) if apple.complete: return jsonify(error="The apple is already complete, your request will not be processed"), 400 b = Bite(apple=apple, slice=slice, col_id=column) b.save() slices = [] for bite in apple.bites: slices.append(bite.slice) if sorted(slices) == range(apple.total): apple.complete = True apple.save() if apple.complete: # if app.testing: # combine_graphs(apple.id) # else: # g.db.close() # p = Process(target=combine_graphs, args=(apple.id,)) # p.start() g.db.close() p = Process(target=elect, args=(apple.id, agg_technique)) p.start() return jsonify({"apple": apple.id, "bite": b.id})
def get_graph(): apple_id = request.values.get('id') #apple_id = int(apple_id) apples = Apple.select().where(Apple.id==apple_id) print("apples: ") print(apples) print("len: ") print(len(apples)) if len(apples) == 1: apple = apples[0] fname = apple.fname if fname != "": return send_from_directory(UPLOAD_DIR, fname, as_attachment=True) abort(404)
def add_bite(): if request.method == 'GET': return render_template('combine.html') table_name = request.values.get('table').strip() column = int(request.values.get('column')) slice = int(request.values.get('slice')) m = int(request.values.get('m')) tot = int(request.values.get('total')) # total number of slices uploaded_file = request.files['graph'] apples = Apple.select().where(Apple.table==table_name, Apple.column==column) if len(apples) == 0: logger.log(LOG_LVL, "\nNew apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot)) apple = Apple(table=table_name, column=column, total=tot) apple.save() else: apple = apples[0] logger.log(LOG_LVL, "\nExisting apple: table=%s, columns=%d, slice=%d ,total=%d" % (table_name, column, slice, tot)) if apple.complete: return jsonify(error="The apple is already complete, your request will not be processed"), 400 b = Bite(apple=apple, slice=slice, m=m) b.save() fname = "%d-%s-%d-%d.json" % (b.id, b.apple.table.replace(" ", "_"), b.apple.column, b.slice) uploaded_file.save(os.path.join(UPLOAD_DIR, fname)) b.fname = fname b.save() slices = [] for bite in apple.bites: slices.append(bite.slice) if sorted(slices) == range(apple.total): apple.complete = True apple.save() if apple.complete: if app.testing: combine_graphs(apple.id) else: g.db.close() p = Process(target=combine_graphs, args=(apple.id,)) p.start() return jsonify({"apple": apple.id, "bite": b.id})
def elect(apple_id, technique): """ :param apple_id: :return: most_frequent, agreement (how many agreed on this). """ database = get_database() database.connect(reuse_if_open=True) apples = Apple.select().where(Apple.id==apple_id) if len(apples) == 0: logger.error("Apple with id: %s does not exists" % str(apple_id)) database.close() return # return None, None apple = apples[0] apple.status = STATUS_PROCESSING apple.save() try: col_ids = [b.col_id for b in apple.bites] # c = Counter(col_ids) # sorted_cols_counts = sorted(c.items(), key=lambda item: item[1]) # most_frequent_col_id = sorted_cols_counts[-1][0] # agreement = sorted_cols_counts[-1][1]*1.0/len(col_ids) if technique =="majority": elected_col_id, agreement = majority_aggregation(col_ids) elif technique == "found-majority": elected_col_id, agreement = found_majority_aggregation(col_ids) else: logger.error("unvalid technique: <%s>" % technique) apple.status = STATUS_STOPPED apple.save() return apple.elected = elected_col_id apple.agreement = agreement apple.status = STATUS_COMPLETE apple.save() except Exception as e: logger.error(str(e)) apple.status = STATUS_STOPPED apple.save() database.close()
def combine_graphs(apple_id): database = get_database() database.connect(reuse_if_open=True) apple = Apple.select().where(Apple.id==apple_id)[0] apple.status = STATUS_PROCESSING apple.save() graphs = [] for bite in apple.bites: print("bite: %d" % bite.id) f = open(os.path.join(UPLOAD_DIR, bite.fname)) j = json.load(f) graphs.append(j) f.close() graph = merge_graphs(graphs=graphs) fname = "%d-%s-%d-merged.json" % (apple.id, apple.table.replace(" ", "_"), apple.column) f = open(os.path.join(UPLOAD_DIR, fname), "w") f.write(json.dumps(graph)) f.close() apple.fname = fname apple.status = STATUS_COMPLETE apple.save() database.close()
def all_apples(): return jsonify(apples=[apple.json() for apple in Apple.select()])
def status(): apples = [{"apple": apple.table, "status": apple.status, "complete": apple.complete} for apple in Apple.select()] return jsonify(apples=apples)
def status(): # apples = [{"apple": apple.table, "status": apple.status, "complete": apple.complete} for apple in Apple.select()] apples = [{"apple": apple.table, "status": apple.status, "complete": apple.complete, "elected": apple.elected, "agreement": apple.agreement} for apple in Apple.select()] return jsonify(apples=apples)