def save_exit_resolve_conflicts(): """ during record linkage, save and exit the current page 1. save answered responses to redis 2. save kapr to mongodb """ user = current_user pid = r.get(user.username + '_working_pid') assignment_id = pid + '-' + user.username user_data_raw = request.form['user_data'] data_list = user_data_raw.split(';') user_data = '' for line in data_list: if line: user_data += ('uid:' + user.username + ',' + line + ';') formatted_data = ud.parse_user_data(user_data) storage_model.save_working_answers(assignment_id, formatted_data, r) # update kapr to db KAPR_key = assignment_id + '_KAPR' kapr = r.get(KAPR_key) storage_model.update_kapr_conflicts(mongo=mongo, username=user.username, pid=pid, kapr=kapr) return "data saved."
def save_data(): user = current_user pid = r.get(user.username + '_working_pid') assignment_id = pid + '-' + user.username user_data_raw = request.form['user_data'] #print(user_data_raw) data_list = user_data_raw.split(';') user_data = '' for line in data_list: if line: user_data += ('uid:' + user.username + ',' + line + ';') formatted_data = ud.parse_user_data(user_data) storage_model.save_answers(mongo, pid, user.username, formatted_data) #user_data_key = assignment_id + '_user_data' #r.append(user_data_key, formatted_data) log_data = { 'username': user.username, 'timestamp': time.time(), 'url': '/save_data', 'pid': str(pid), 'assignment_id': str(assignment_id), 'log': user_data_raw, } storage_model.mlog(mongo=mongo, data=log_data) return 'data_saved.'
def open_cell(): user = current_user pid = r.get(user.username + '_working_pid') assignment_id = pid + '-' + user.username is_rc = r.get(user.username + '_working_pid_rc') pair_datafile = storage_model.get_project_pair_datafile(mongo=mongo, user=user, pid=pid) working_data = dm.DataPairList( data_pairs=dl.load_data_from_csv(pair_datafile)) project_pairfile = storage_model.get_project_pair_datafile( mongo=mongo, user=user.username, pid=pid) full_project_pairs = storage_model.get_total_pairs_from_pairfile( project_pairfile) working_data.set_kapr_size(full_project_pairs) full_data = dl.load_data_from_csv(project_pairfile) id1 = request.args.get('id1') id2 = request.args.get('id2') mode = request.args.get('mode') pair_num = str(id1.split('-')[0]) attr_num = str(id1.split('-')[2]) if is_rc != '1': assignment_status = storage_model.get_assignment_status( mongo=mongo, username=user.username, pid=pid) kapr_limit = float(assignment_status['kapr_limit']) # is resolve conflict if is_rc == '1': kapr_limit = 100 ret = dm.open_cell(assignment_id, full_data, working_data, pair_num, attr_num, mode, r, kapr_limit) log_data = { 'username': user.username, 'timestamp': time.time(), 'url': '/get_cell', 'pid': str(pid), 'assignment_id': str(assignment_id), 'log': json.dumps(ret) } storage_model.mlog(mongo=mongo, data=log_data) return jsonify(ret)
def open_big_cell(): user = current_user pid = r.get(user.username + '_working_pid') assignment_id = pid + '-' + user.username pair_datafile = storage_model.get_pair_datafile(mongo=mongo, user=user, pid=pid) full_data = dl.load_data_from_csv(pair_datafile) working_data = dm.DataPairList( data_pairs=dl.load_data_from_csv(pair_datafile)) assignment_status = storage_model.get_assignment_status( mongo=mongo, username=user.username, pid=pid) kapr_limit = float(assignment_status['kapr_limit']) id1 = request.args.get('id1') id2 = request.args.get('id2') id3 = request.args.get('id3') id4 = request.args.get('id4') mode = request.args.get('mode') pair_num1 = str(id1.split('-')[0]) attr_num1 = str(id1.split('-')[2]) ret1 = dm.open_cell(assignment_id, full_data, working_data, pair_num1, attr_num1, mode, r, kapr_limit) pair_num2 = str(id3.split('-')[0]) attr_num2 = str(id3.split('-')[2]) ret2 = dm.open_cell(assignment_id, full_data, working_data, pair_num2, attr_num2, mode, r, kapr_limit) if ret2['result'] == 'fail': return jsonify(ret2) ret = { 'value1': ret1['value1'], 'value2': ret1['value2'], 'value3': ret2['value1'], 'value4': ret2['value2'], 'id': ret1['id'], 'mode': ret2['mode'], 'KAPR': ret2['KAPR'], 'result': ret2['result'], 'new_delta': ret2['new_delta'] } log_data = { 'username': user.username, 'timestamp': time.time(), 'url': '/get_big_cell', 'pid': str(pid), 'assignment_id': str(assignment_id), 'log': json.dumps(ret) } storage_model.mlog(mongo=mongo, data=log_data) return jsonify(ret)
def record_linkage_next(pid): """ update page number to db update kapr to db flush related cache in redis """ user = current_user assignment_id = pid + '-' + user.username # find if this project exist project = storage_model.get_assignment(mongo=mongo, username=user.username, pid=pid) if not project: return page_not_found('page_not_found') # increase page number to db storage_model.increase_assignment_page(mongo=mongo, username=user.username, pid=pid) # increase pair index to db storage_model.increase_pair_idx(mongo=mongo, pid=pid, username=user.username) # update kapr to db KAPR_key = assignment_id + '_KAPR' kapr = r.get(KAPR_key) storage_model.update_kapr(mongo=mongo, username=user.username, pid=pid, kapr=kapr) # flush related cache in redis # dont flush yet, because resolve conflicts need these data # TODO: flush these data when resolve conflict finished # storage_model.clear_working_page_cache(assignment_id, r) # check if the project is completed completed = storage_model.is_project_completed(mongo=mongo, pid=pid) if completed: storage_model.combine_result(mongo, pid) # don't update the result yet, because we use ajax to write result, the result might not been updated # if there are conflicts, the result is updated after the resolve_conflict indices = storage_model.detect_result_conflicts(mongo, pid) if len(indices) == 0: storage_model.update_result(mongo=mongo, pid=pid) else: create_resolve_conflict_project(pid) flask.flash('You have completed the project.', 'alert-success') return redirect(url_for('project')) return redirect(url_for('record_linkage', pid=pid))
def resolve_conflicts2_next(pid): """ update page number to db update kapr to db flush related cache in redis """ user = current_user assignment_id = pid + '-' + user.username # find if this project exist assignment = storage_model.get_conflict_project(mongo=mongo, username=user.username, pid=pid) if not assignment: return page_not_found('page_not_found') # increase page number to db storage_model.increase_conflict_page_pairidx(mongo=mongo, username=user.username, pid=pid) # update kapr to db KAPR_key = assignment_id + '_KAPR' kapr = r.get(KAPR_key) storage_model.update_kapr_conflicts(mongo=mongo, username=user.username, pid=pid, kapr=kapr) # flush related cache in redis # dont flush yet, because resolve conflicts need these data # TODO: flush these data when resolve conflict finished # storage_model.clear_working_page_cache(assignment_id, r) # check if the project is completed completed = storage_model.is_conflict_project_completed(mongo=mongo, pid=pid) if completed: # combine resolve_conflict_result with result storage_model.update_resolve_conflicts(mongo, pid) # update result file to int_file storage_model.update_result(mongo=mongo, pid=pid) storage_model.delete_resolve_conflict(mongo, pid) # flush redis data storage_model.clear_working_page_cache(assignment_id, r) flask.flash('You have completed resolve conflicts of this project.', 'alert-success') return redirect(url_for('project', pid=pid)) return redirect(url_for('resolve_conflicts2', pid=pid))
def save_data_resolve_conflicts(): user = current_user pid = r.get(user.username + '_working_pid') user_data_raw = request.form['user_data'] data_list = user_data_raw.split(';') user_data = '' for line in data_list: if line: user_data += ('uid:' + user.username + ',' + line + ';') formatted_data = ud.parse_user_data(user_data) storage_model.save_resolve_conflicts(mongo, pid, user.username, formatted_data) #storage_model.update_result(mongo=mongo, pid=pid) return 'data_saved.'
def save_data(): user = current_user pid = r.get(user.username + '_working_pid') assignment_id = pid + '-' + user.username user_data_raw = request.form['user_data'] data_list = user_data_raw.split(';') user_data = '' for line in data_list: if line: user_data += ('uid:' + user.username + ',' + line + ';') formatted_data = ud.parse_user_data(user_data) storage_model.save_answers(mongo, pid, user.username, formatted_data) user_data_key = assignment_id + '_user_data' r.append(user_data_key, formatted_data) return 'data_saved.'
def create_resolve_conflict_project(pid): project = storage_model.get_project_by_pid(mongo, pid) owner = project['owner'] assignment_id = pid + '-' + owner # get pair_num of conflicts conflict_indices = storage_model.detect_result_conflicts(mongo, pid) # get block information of the project project = storage_model.get_project_by_pid(mongo=mongo, pid=pid) # arrange conflict pairs by block block_id = project['block_id'] # arrange pairs by block id conflicts = list() for block in block_id: cur_block = list() for idx in conflict_indices: if idx in block: cur_block.append(idx) if cur_block: conflicts.append(cur_block) # simulate open cells for those opened by assignees pair_datafile = storage_model.get_pair_datafile_by_owner(mongo=mongo, owner=owner, pid=pid) working_data = dm.DataPairList( data_pairs=dl.load_data_from_csv(pair_datafile), indices=conflict_indices) project_pairfile = storage_model.get_project_pair_datafile(mongo=mongo, user=owner, pid=pid) full_project_pairs = storage_model.get_total_pairs_from_pairfile( project_pairfile) working_data.set_kapr_size(full_project_pairs) full_data = dl.load_data_from_csv(project_pairfile) KAPR_key = assignment_id + '_KAPR' r.set(KAPR_key, 0.0) ids_list = working_data.get_ids() ids = list(zip(ids_list[0::2], ids_list[1::2])) data_mode_list = storage_model.get_conflict_data_mode( pid, ids, mongo, r, assignment_id) dm.batched_open_cell(assignment_id, full_data, working_data, ids, data_mode_list, r, kapr_limit=100) KAPR_key = assignment_id + '_KAPR' current_kapr = r.get(KAPR_key) result_path = os.path.join( config.DATA_DIR, 'internal', project['owner'] + '_' + project['project_name'] + '_conflict_result.csv') # create result file f = open(result_path, 'w+') f.close() isfull = storage_model.has_full_assignee(mongo, pid) conflict_project = { 'pid': pid, 'project_name': project['project_name'], 'pair_num': conflicts, 'current_page': 0, 'page_size': len(conflicts), 'kapr_limit': 100, 'current_kapr': current_kapr, 'pair_idx': 0, 'total_pairs': len(conflict_indices), 'result_path': result_path, 'isfull': isfull, } storage_model.save_conflict_project(mongo, conflict_project) return 'block_id'