def get_previous_worker_code(job_id, worker_id, treatment): """ Generate a code for the user in case he already took the main task """ resp_table = get_table(resp_BASE, job_id=job_id, schema="result", treatment=treatment) prop_table = get_table(prop_BASE, job_id=job_id, schema="result", treatment=treatment) worker_code = None if is_worker_available(worker_id, resp_table): worker_code = get_db().execute( f"SELECT completion_code from {resp_table} where worker_id=?", [worker_id]).fetchone()[0] if not worker_code: worker_code = generate_completion_code(resp_BASE, job_id) if is_worker_available(worker_id, prop_table): worker_code = get_db().execute( f"SELECT completion_code from {prop_table} where worker_id=?", [worker_id]).fetchone()[0] if not worker_code: worker_code = generate_completion_code(prop_BASE, job_id) return worker_code
def handle_webhook(treatment): """ request.args: - job_id: job's id - worker_id: worker's id - synchron: Directly process data without puting in a queue for another thread """ app.logger.debug("handle_webhook") sync_process = False auto_finalize = False sync_process = request.args.get("synchron", False) form = request.form.to_dict() if "signal" in form: signal = form['signal'] if signal in {'unit_complete', 'new_judgments'}: payload_raw = form['payload'] signature = form['signature'] payload = json.loads(payload_raw) job_id = payload['job_id'] job_config = get_job_config(get_db("DB"), job_id) payload_ext = payload_raw + job_config["api_key"] verif_signature = hashlib.sha1(payload_ext.encode()).hexdigest() if signature == verif_signature: args = (signal, payload, job_id, job_config, treatment, auto_finalize) if sync_process: _process_judgments(*args) else: app.config["THREADS_POOL"].starmap_async( _process_judgments, [args]) else: job_id = request.args.get("job_id") worker_id = request.args.get("worker_id") job_config = get_job_config(get_db("DB"), job_id) auto_finalize = True payload = { "judgments_count": 1, "job_id": job_id, "results": { "judgments": [{ "job_id": job_id, "worker_id": worker_id }] } } args = ("new_judgments", payload, job_id, job_config, treatment, auto_finalize) if sync_process: _process_judgments(*args) else: app.config["THREADS_POOL"].starmap_async(_process_judgments, [args]) # flash("You may close this tab now and continue with the survey.") # return render_template("info.html", job_id=job_id, webhook=True) return Response(status=200)
def approve_and_reject_assignments(job_id, treatment): base = "survey" table_assignment = get_table("txx", "DEPRECATED_JOB_ID_PARAMETER", schema=None) table_survey = get_table(base, "DEPRECATED_JOB_ID_PARAMETER", schema="result", treatment=treatment) # sql = f""" # select a.job_id as job_id, a.assignment_id as assignment_id, a.worker_id as worker_id, s.worker_code as worker_code # from {table_assignment} as a left join {table_survey} as s on a.worker_id=s.worker_id # where {table_assignment}.job_id like ?""" sql = f"select * from {table_assignment} where {table_assignment}.job_id like ?" with get_db() as con: df = pd.read_sql(sql, con=get_db(), params=(job_id, )) api = MTurk(job_id, sandbox=app.config["MTURK_SANDBOX"]) payment_count = 0 validation_count = 0 rejection_count = 0 for idx in range(df.shape[0]): row = df.iloc[idx].to_dict() print("ROW: ", row) worker_id = row["worker_id"] assignment_id = row["assignment_id"] print("worker_id", worker_id, type(worker_id), "assignment_id", assignment_id, type(assignment_id)) success = True with get_db("DB") as con: if row["worker_code"] != WORKER_CODE_DROPPED and has_worker_submitted( con, job_id, worker_id, treatment): success &= api.approve_assignment(row["assignment_id"], "Thank you for your work.") validation_count += success if success: success &= pay_worker_bonus(job_id, worker_id, api=api, con=con, assignment_id=assignment_id) payment_count += success else: #if row["worker_code"] == WORKER_CODE_DROPPED: success &= api.reject_assignment( row["assignment_id"], f"You exceeded the number of {MAXIMUM_CONTROL_MISTAKES} mistakes on the control questions." ) rejection_count += success app.logger.info( f"validations: {validation_count}, rejections: {rejection_count}, payments: {payment_count}, rows: {df.shape[0]}" )
def handle_index_feedback(treatment, base_treatment): job_id = request.args.get("job_id", "na") worker_id = request.args.get("worker_id", "na") max_judgments = None try: max_judgments = int(request.args.get("max_judgments", "0")) except ValueError: pass previous_worker_code = get_previous_worker_code(job_id, worker_id, base_treatment) app.logger.debug(f"handle_index: job_id: {job_id}, worker_id: {worker_id}") is_proposer = check_is_proposer_next(job_id, worker_id, treatment, max_judgments=max_judgments) table_all = get_table(BASE, "all", schema=None) con = get_db() if table_exists(con, table_all): with con: res = con.execute(f"SELECT * from {table_all} WHERE worker_id=?", (worker_id, )).fetchone() # if res: # flash(f"You already took part on this survey. Thank you for your participation") # return render_template("error.html") if previous_worker_code is None: if is_proposer: return redirect(url_for(f"{treatment}.prop.index", **request.args)) else: return redirect(url_for(f"{treatment}.resp.index", **request.args)) else: if prop_BASE in previous_worker_code: return redirect(url_for(f"{treatment}.prop.index", **request.args)) else: return redirect(url_for(f"{treatment}.resp.index", **request.args))
def process_insert_row(job_id, resp_row, treatment, overwrite=False): """ Insert a new row for the proposer assuming no model is available """ app.logger.debug("process_insert_row") resp_row = dict(resp_row) _, features_dict = get_features(job_id, resp_worker_id=resp_row[WORKER_KEY], treatment=treatment) resp_row = dict(resp_row) resp_row.update(features_dict) df = pd.DataFrame(data=[resp_row]) df[STATUS_KEY] = RowState.JUDGEABLE df[LAST_MODIFIED_KEY] = time.time() df[WORKER_KEY] = None df["resp_worker_id"] = resp_row[WORKER_KEY] table = get_table(BASE, job_id=job_id, schema="data", treatment=treatment) con = get_db("DATA") insert(df, table, con=con, overwrite=overwrite, unique_fields=["resp_worker_id"]) app.logger.debug("process_insert_row - done")
def has_worker_and_features(base, task_module, job_id, worker_id): con = get_db() table = get_table(base, job_id, "result") features = [feature for feature in task_module.FEATURES] sql = f"SELECT {','.join(features)} FROM {table} WHERE job_id=? AND worker_id=?" res = con.execute(sql, [job_id, worker_id]) return res is not None
def test_index(client): path = f"/admin" job_id = "test_index_job" api_key = "api_key" secret = app.config["ADMIN_SECRET"] base_code = "base_code" expected_judgments = 16 with app.test_request_context(path): # if clear_session: # with client: # with client.session_transaction() as sess: # sess.clear() client.get(path, follow_redirects=True) return client.post(path, data={ "job_id": job_id, api_key: "api_key", "secret": secret, "base_code": base_code, "expected_judgments": expected_judgments }, follow_redirects=True) job_config = get_job_config(get_db(), job_id) assert job_config["job_id"] == job_id assert job_config["base_code"] == base_code assert job_config["api_key"] == api_key assert job_config["expected_judgments"] == expected_judgments expected_judgments = 32 base_code = "super_base_code" job_config["expected_judgments"] = expected_judgments job_config["base_code"] = base_code with app.test_request_context(path): client.get(path, follow_redirects=True) return client.post(path, data=job_config, follow_redirects=True)
def finalize_resp(job_id, worker_id, treatment): app.logger.debug("finalize_resp") table = get_table(base="resp", job_id=job_id, schema="result", treatment=treatment) with get_db("RESULT") as con: res = con.execute( f"SELECT * from {table} where job_id=? and worker_id=?", (job_id, worker_id)).fetchone() if res: resp_result = dict(res) insert_row(job_id, resp_result, treatment) else: app.logger.warning( f"finalize_resp: worker_id {worker_id} not found - job_id: {job_id}" ) app.logger.debug( f"Treatment: {treatment}, auto_dss: {TREATMENTS_AUTO_DSS}") if treatment.upper() in TREATMENTS_AUTO_DSS: time.sleep(WEBHOOK_DELAY) worker_id = f"auto_prop_{uuid.uuid4()}" args = [ None, treatment, job_id, worker_id, "auto", True, True, True, None ] app.logger.debug(f"auto-proposal! {args}") # client = app.test_client() app.config["THREADS_POOL"].starmap_async(_process_prop_round, [args])
def handle_done(treatment, template=None, response_to_result_func=None): app.logger.debug("handle_done") if template is None: template = f"txx/{BASE}.done.html" if response_to_result_func is None: response_to_result_func = prop_to_prop_result worker_code_key = f"{BASE}_worker_code" worker_bonus_key = f"{BASE}_worker_bonus" cookie_obj = get_cookie_obj(BASE) ai_cookie_obj = get_cookie_obj(AI_COOKIE_KEY) if not cookie_obj.get(BASE, None): flash("Sorry, you are not allowed to use this service. ^_^") return render_template("error.html") if not (cookie_obj.get("worker_code")): job_id = cookie_obj["job_id"] worker_code = generate_completion_code(base=BASE, job_id=job_id) proposal = cookie_obj["proposal"] proposal["completion_code"] = worker_code proposal.update(ai_cookie_obj.get("ai_proposal", {})) row_info = cookie_obj["row_info"] worker_id = cookie_obj["worker_id"] row_info = get_row(get_db(), job_id, worker_id, TREATMENT) offer = proposal["offer_dss"] min_offer = row_info["min_offer"] worker_bonus = 0 if offer >= min_offer: worker_bonus = 5 prop_result = response_to_result_func(proposal, job_id=job_id, worker_id=worker_id, row_data=row_info) prop_result["worker_bonus"] = worker_bonus try: save_result2file( get_output_filename(base=BASE, job_id=job_id, treatment=treatment), prop_result) except Exception as err: app.log_exception(err) try: save_result2db(table=get_table(base=BASE, job_id=job_id, schema="result", treatment=treatment), response_result=prop_result, unique_fields=["worker_id"]) #increase_worker_bonus(job_id=job_id, worker_id=worker_id, bonus_cents=worker_bonus, con=get_db("DB")) except Exception as err: app.log_exception(err) cookie_obj.clear() cookie_obj[BASE] = True cookie_obj["worker_id"] = worker_id cookie_obj[worker_bonus_key] = cents_repr(worker_bonus) cookie_obj[worker_code_key] = worker_code req_response = make_response( render_template(template, worker_code=cookie_obj[worker_code_key])) set_cookie_obj(req_response, BASE, cookie_obj) return req_response
def create_resp_data_auto_prop_table(treatment, ref, use_ai_offer=None): app.logger.debug(f"create_resp_data_auto_prop_table - {ref}, {treatment}") con = get_db() table = get_table(BASE, None, "data", treatment=treatment) assert len(ref) == 4, "expected references of the form <txyz>" required_columns = """ai_calls_acceptance_probability,ai_calls_best_offer_probability,ai_calls_count_repeated,ai_calls_offers,ai_calls_pauses,ai_nb_calls,ai_offer,feedback_accuracy,feedback_explanation,feedback_understanding,job_id,model_type,offer,offer_dss,offer_final,prop_time_spent,prop_worker_id,timestamp,worker_id""".split( ",") columns_to_clear = """ai_calls_acceptance_probability,ai_calls_best_offer_probability,ai_calls_count_repeated,ai_calls_offers,ai_calls_pauses,ai_nb_calls,feedback_accuracy,feedback_explanation,feedback_understanding,job_id,prop_time_spent,timestamp,worker_id""".split( ",") if not table_exists(con, table): df = pd.read_csv( os.path.join(CODE_DIR, 'data', ref, 'export', f'result__{ref}_prop.csv')) for col in required_columns: if col not in df.columns: df[col] = None columns = [col for col in required_columns if col in df.columns] df = df[columns] if use_ai_offer: df["offer_final"] = df["ai_offer"] df["offer"] = df["ai_offer"] df["offer_dss"] = df["ai_offer"] df[[col for col in columns_to_clear]] = None df["job_id"] = f"REFAUTO{ref.upper()}" df[STATUS_KEY] = RowState.JUDGEABLE df[WORKER_KEY] = None df["updated"] = 0 with con: df.to_sql(table, con, index=False) app.logger.debug("create_table_data: table created")
def index(): app.logger.debug(f"admin.index") if request.method == "POST": job_id = request.form.get('job_id') api_key = request.form.get('api_key') secret = request.form.get('secret') base_code = request.form.get('base_code') expected_judgments = 0 try: expected_judgments = int(request.form.get("expected_judgments")) except: pass if secret != app.config['ADMIN_SECRET']: flash('Incorrect secret, please try again') return redirect(request.url) job_config = JobConfig(job_id=job_id, api_key=api_key, base_code=base_code, expected_judgments=expected_judgments) app.logger.debug( f"Job_config: {job_config}, request_form: {request.form.to_dict()}" ) update_job(get_db('DB'), job_id, job_config) flash("Job successfully configured") return render_template("admin.html")
def process_insert_row_dss(job_id, resp_row, treatment, overwrite=False): """ Insert a new row for the proposer with the ai predicted min_offer """ app.logger.debug("process_insert_row_dss") ENABLED_FAKE_MODEL_KEY = f"{treatment.upper()}_FAKE_MODEL" MODEL_KEY = f"{TREATMENTS_MODEL_REFS[treatment.upper()]}_MODEL" FAKE_MODEL_TYPES = [WEAK_FAKE_MODEL, STRONG_FAKE_MODEL] model_type = None ai_offer = 0 features, features_dict = get_features(job_id, resp_worker_id=resp_row[WORKER_KEY], treatment=treatment) resp_row = dict(resp_row) resp_row.update(features_dict) df = pd.DataFrame(data=[resp_row]) df[STATUS_KEY] = RowState.JUDGEABLE df[LAST_MODIFIED_KEY] = time.time() df[WORKER_KEY] = None df["resp_worker_id"] = resp_row[WORKER_KEY] df["ai_offer"] = ai_offer df["model_type"] = model_type table = get_table(BASE, job_id=job_id, schema="data", treatment=treatment) con = get_db("DATA") insert(df, table, con=con, overwrite=overwrite, unique_fields=["resp_worker_id"]) with con: rowid = con.execute( f"SELECT {PK_KEY} FROM {table} where resp_worker_id=?", (resp_row[WORKER_KEY], )).fetchone()[PK_KEY] # if true, fake models should be used for this treatment if app.config[ENABLED_FAKE_MODEL_KEY]: if rowid is not None: model_type = FAKE_MODEL_TYPES[rowid % len(FAKE_MODEL_TYPES)] else: model_type = REAL_MODEL else: model_type = REAL_MODEL if model_type == WEAK_FAKE_MODEL: ai_offer = predict_weak(resp_row["min_offer"]) elif model_type == STRONG_FAKE_MODEL: ai_offer = predict_strong(resp_row["min_offer"]) else: # Models predict a vector ai_offer = app.config[MODEL_KEY].predict(features)[0] ai_offer = int(ai_offer) with con: update( sql=f"UPDATE {table} SET ai_offer=?, model_type=? where rowid=?", args=(ai_offer, model_type, rowid), con=con) app.logger.debug("process_insert_row_dss - done MODEL_TYPE: " + str(rowid % len(FAKE_MODEL_TYPES)) + " " + str(rowid))
def get_features(job_id, resp_worker_id, treatment, tasks=None, tasks_features=None): """ :returns: (numpy.array) features :returns: (dict) features_rows untransformed """ app.logger.debug("get_features") MODEL_INFOS_KEY = f"{TREATMENTS_MODEL_REFS[treatment.upper()]}_MODEL_INFOS" if tasks is None: tasks = app.config["TASKS"] con = get_db("RESULT") if tasks_features is None: tasks_features = app.config["TASKS_FEATURES"] row_features = dict() for task_name, features in tasks_features.items(): if task_name in tasks: task_table = get_table(task_name, job_id=job_id, schema="result", is_task=True) if table_exists(con, task_table): with con: sql = f"SELECT {','.join(features)} FROM {task_table} WHERE worker_id=?" res = con.execute(sql, (resp_worker_id, )).fetchone() # task tables are shared. when using a REF, there the table may exists but without valid rows if res is not None: row_features.update(dict(res)) resp_features = {"resp": ["resp_time_spent"]} for name, features in resp_features.items(): table = get_table(name, job_id=job_id, treatment=treatment, schema="result", is_task=False) if table_exists(con, table): with con: sql = f"SELECT {','.join(features)} FROM {table} WHERE worker_id=?" res = con.execute(sql, (resp_worker_id, )).fetchone() # task tables are shared. when using a REF, there the table may exists but without valid rows if res is not None: row_features.update(dict(res)) tmp_df = pd.DataFrame(data=[row_features]) REF_MODEL_KEY = f"{TREATMENTS_MODEL_REFS[treatment.upper()]}_MODEL" dss_available = bool(app.config.get(REF_MODEL_KEY)) if dss_available: x, _ = df_to_xy( tmp_df, select_columns=app.config[MODEL_INFOS_KEY]["top_columns"]) else: x = None app.logger.debug("get_features - done") return x, row_features
def index(): app.logger.debug(f"dashboard.index") treatments = [ treatment.lower() for treatment in reversed(app.config["TREATMENTS"]) ] infos = get_treaments_infos(get_db('DB'), treatments) return render_template("dashboard.html", treatments=treatments, infos=infos)
def handle_index(treatment, template=None, messages=None, has_dss_component=False): app.logger.debug("handle_index") if messages is None: messages = [] if template is None: template = f"txx/resp.html" cookie_obj = get_cookie_obj(BASE) worker_code_key = f"{BASE}_worker_code" worker_id = request.args.get("worker_id", "na") job_id = request.args.get("job_id", "na") close_row(get_db(), job_id, 2, treatment) # The task was already completed, so we skip to the completion code display if cookie_obj.get(BASE) and cookie_obj.get( worker_code_key) and cookie_obj.get("worker_id") == worker_id: req_response = redirect(url_for(f"{treatment}.resp.done")) return req_response if request.method == "GET": app.logger.debug( f"handle_index: job_id:{job_id}, worker_id:{worker_id} ") cookie_obj['response'] = HHI_Resp_ADM() cookie_obj["worker_id"] = worker_id cookie_obj["job_id"] = job_id cookie_obj["auto_finalize"] = request.args.get("auto_finalize") for message in messages: flash(message) if request.method == "POST": response = cookie_obj["response"] response["time_stop"] = time.time() response["min_offer"] = int(request.form["min_offer"]) cookie_obj['response'] = response if has_dss_component: req_response = make_response( redirect(url_for(f"{treatment}.resp.index_dss", **request.args))) else: req_response = make_response( redirect(url_for(f"{treatment}.resp.done", **request.args))) set_cookie_obj(req_response, BASE, cookie_obj) return req_response cookie_obj[BASE] = True req_response = make_response( render_template(template, offer_values=OFFER_VALUES, form=ResponderForm())) set_cookie_obj(req_response, BASE, cookie_obj) return req_response
def test_get_job_config(): job_id = "--------" with app.app_context(): job_config = get_job_config(get_db(), job_id=job_id) assert job_config["job_id"] == job_id assert "api_key" in job_config assert "nb_rows" in job_config assert "unique_worker" in job_config assert "base_code" in job_config assert "expected_judgments" in job_config assert "payment_max_cents" in job_config assert len(job_config) == 7
def generate_completion_code(base, job_id): """ :param base: :param job_id: """ app.logger.debug("generate_completion_code") job_config = get_job_config(get_db("DB"), job_id) base_completion_code = job_config["base_code"] part1 = f"{base}:" part2 = base_completion_code part3 = "".join(random.choices(string.ascii_letters + string.digits, k=5)) return "".join([part1, part2, part3])
def pay_bonus_assignments(job_id): table = get_table("txx", "all", schema=None) df = pd.read_sql(f"select * from {table}", get_db("result")) df = df[df["job_id"] == job_id] api = MTurk(job_id, sandbox=app.config["MTURK_SANDBOX"]) con = get_db("DB") payment_count = 0 for idx in range(df.shape[0]): row = df.iloc[idx] worker_id = row["worker_id"] assignment_id = row["assignment_id"] success = True if row["worker_code"] != WORKER_CODE_DROPPED: success &= pay_worker_bonus(job_id, worker_id, api=api, con=con, assignment_id=assignment_id) payment_count += success app.logger.info(f"payments: {payment_count}, rows: {df.shape[0]}")
def finalize_round(job_id, prop_worker_id, treatment): app.logger.debug("finalize_round") # TODO: At this point, all features have been gathered resp_worker_id = "na" con = get_db("RESULT") resp_bonus = 0 prop_bonus = 0 offer, min_offer = 0, 0 with con: table = get_table(BASE, job_id, schema="result", treatment=treatment) try: # min_offer_final was introduce late, so it may be missing on some tables... # in case of absence, a fallback to min_offer is perfectly fine. res = con.execute( f"SELECT offer_final as offer, min_offer_final as min_offer, resp_worker_id from {table} WHERE prop_worker_id=?", (prop_worker_id, )).fetchone() except Exception as err: app.logger.warn(f"{err}") res = con.execute( f"SELECT offer_final as offer, min_offer, resp_worker_id from {table} WHERE prop_worker_id=?", (prop_worker_id, )).fetchone() offer, min_offer = res["offer"], res["min_offer"] resp_worker_id = res["resp_worker_id"] offer = max(min(offer, MAX_GAIN), 0) min_offer = max(min(min_offer, MAX_GAIN), 0) if offer >= min_offer: resp_bonus += offer prop_bonus += (MAX_GAIN - offer) assert (resp_bonus + prop_bonus) == 0 or (resp_bonus + prop_bonus) == MAX_GAIN con2 = get_db("DB") with con2: if not is_fake_worker(resp_worker_id): increase_worker_bonus(job_id, resp_worker_id, resp_bonus, con2) if not is_fake_worker(prop_worker_id): increase_worker_bonus(job_id, prop_worker_id, prop_bonus, con2)
def is_resp_in_prop_result(resp_worker_id, job_id, treatment): con = get_db() table = get_table("prop", job_id, "result", treatment=treatment) sql = f"SELECT * FROM {table} WHERE job_id=? and resp_worker_id=?" print("SQL: ", sql) try: with con: res = con.execute(sql, [job_id, resp_worker_id]).fetchone() if res is None: return False else: return True except Exception: pass return False
def is_worker_available(worker_id, table): """ Returns True if <table> exist and contains a column "worker_id" which has the value <worker_id> """ con = get_db() sql = f"SELECT * FROM {table} WHERE {WORKER_KEY}=?" try: res = con.execute(sql, [worker_id]).fetchone() if res is None: return False else: return True except Exception as err: pass return False
def save_result2db(table, response_result, overwrite=False, unique_fields=None): """ :param table: (str) :param response_result: (dict) :param overwrite: (bool) :param unique_fields: (str|list) """ df = pd.DataFrame(data=[response_result]) insert(df, table=table, con=get_db("RESULT"), overwrite=overwrite, unique_fields=unique_fields)
def get_resp_worker_id(base, job_id, prop_worker_id, treatment=None): """ :param base: :param job_id: :param prop_worker_id: :param treament: """ con = get_db("RESULT") with con: table = get_table(base, job_id=job_id, schema="result", treatment=treatment) res = con.execute( f"SELECT resp_worker_id from {table} WHERE prop_worker_id=?", (prop_worker_id, )).fetchone() resp_worker_id = res["resp_worker_id"] return resp_worker_id
def create_prop_data_table(treatment, ref): app.logger.debug(f"create_table_data - {ref}, {treatment}") con = get_db() table_data = get_table(BASE, None, "data", treatment=treatment) table_resp = get_table("resp", None, "result", treatment=treatment) assert len(ref) == 4, "expected references of the form <txyz>" job_id = f"REF{ref.upper()}" if not table_exists(con, table_data): df_resp = None try: # normaly, we expect an exported data-table to be available with all required features df = pd.read_csv( os.path.join(CODE_DIR, 'data', ref, 'export', f'data__{ref}_prop.csv')) df[STATUS_KEY] = RowState.JUDGEABLE df[WORKER_KEY] = None df["updated"] = 0 df["job_id"] = f"REF{ref.upper()}" with con: df.to_sql(table_data, con, index=False) app.logger.debug("create_table_data: table created") df_resp = pd.read_csv( os.path.join(CODE_DIR, 'data', ref, 'export', f'result__{ref}_resp.csv')) df["job_id"] = f"REF{ref.upper()}" except Exception as err: app.logger.warn(f"silenced-error: {err}") # otherwise, we work with the resp. table, which means the model does/should not expect any features. df = pd.read_csv( os.path.join(CODE_DIR, 'data', ref, 'export', f'result__{ref}_resp.csv')) df["job_id"] = f"REF{ref.upper()}" for idx in range(len(df)): resp_row = df.iloc[idx] resp_row = dict(**resp_row) insert_row(job_id, resp_row, treatment) app.logger.debug("create_table_data: table created") df_resp = df df_resp.to_sql(table_resp, con, index=False, if_exists='replace') app.logger.debug("resp-table-cloned: table created")
def increase_worker_bonus(job_id, worker_id, bonus_cents, con=None): """ :param job_id: :param worker_id: :param bonus_cents: (int) :param con: """ app.logger.debug( f"increase_worker_bonus - job_id: {job_id}, worker_id: {worker_id}, bonus_cents: {bonus_cents}" ) if con is None: con = get_db("DB") row_data = { 'job_id': job_id, 'worker_id': worker_id, 'timestamp': str(datetime.datetime.now()), 'bonus_cents': bonus_cents, 'paid_bonus_cents': 0 } table = _get_payment_table(job_id) worker_row_exists = False if table_exists(con, table): with con: row = con.execute( f'select *, rowid from {table} WHERE job_id==? and worker_id==?', (job_id, worker_id)).fetchone() if row: worker_row_exists = True row_data["bonus_cents"] += row["bonus_cents"] row_data["paid_bonus_cents"] += row["paid_bonus_cents"] update( f"UPDATE {table} SET bonus_cents=?, paid_bonus_cents=? WHERE rowid=?", (row_data["bonus_cents"], row_data["paid_bonus_cents"], row["rowid"]), con=con) if not worker_row_exists: app.logger.debug( f"increase_worker_bonus: {job_id}, {worker_id}, {bonus_cents}") df = pd.DataFrame(data=[row_data]) insert(df, table, con, unique_fields=["worker_id"]) con.commit()
def handle_index(treatment, resp_only=None, prop_only=None): job_id = request.args.get("job_id", "na") worker_id = request.args.get("worker_id", "na") max_judgments = None try: max_judgments = int(request.args.get("max_judgments", "0")) except ValueError: pass previous_worker_code = get_previous_worker_code(job_id, worker_id, treatment) app.logger.debug(f"handle_index: job_id: {job_id}, worker_id: {worker_id}") is_proposer = check_is_proposer_next(job_id, worker_id, treatment, max_judgments=max_judgments, resp_only=resp_only, prop_only=prop_only) table_all = get_table(BASE, "all", schema=None) con = get_db() if previous_worker_code is None: if resp_only: return redirect(url_for(f"{treatment}.resp.index", **request.args)) elif prop_only: return redirect(url_for(f"{treatment}.prop.index", **request.args)) else: if is_proposer: return redirect( url_for(f"{treatment}.prop.index", **request.args)) else: return redirect( url_for(f"{treatment}.resp.index", **request.args)) else: flash("You already completed the main task!") if resp_BASE in previous_worker_code: return render_template("txx/resp.done.html", worker_code=previous_worker_code) else: return render_template("txx/prop.done.html", worker_code=previous_worker_code)
def _get_worker_bonus_row(job_id, worker_id, con=None): """ :param job_id: :param worker_id: :param con: """ app.logger.debug( f"_get_worker_bonus_row: job_id: {job_id}, worker_id: {worker_id}") if con is None: con = get_db("DB") table = _get_payment_table(job_id) if table_exists(con, table): with con: row = con.execute( f'select *, rowid from {table} WHERE job_id==? and worker_id==?', (job_id, worker_id)).fetchone() if row: row_dict = dict(row) return row_dict else: app.logger.error( f"_get_worker_bonus_row: worker not found! job_id: {job_id}, worker_id: {worker_id}" ) return None
def handle_survey_feedback(treatment=None, template=None, code_prefixes=None, form_class=None, overview_url=None): app.logger.info("handle_survey_feedback") if code_prefixes is None: code_prefixes = { "code_cpc": "cpc:", "code_exp": "exp:", "code_risk": "risk:", "code_cc": "cc:", "code_ras": "ras:" } if form_class is None: form_class = MainFormFeeback if template is None: template = "txx/survey_feedback.html" if overview_url is None: if treatment and treatment.upper() in TREATMENTS_AUTO_DSS: overview_url = url_for("overview_auto_prop_feedback") else: overview_url = url_for("overview_feedback") cookie_obj = get_cookie_obj(BASE) adapter_cookie = get_adapter_from_dict(cookie_obj.get("adapter", {})) adapter_args = get_adapter_from_dict(request.args) adapter_referrer = get_adapter() if adapter_referrer.job_id not in {"", "na", None}: adapter = adapter_referrer elif adapter_cookie.job_id not in {"", "na", None}: adapter = adapter_cookie else: adapter = adapter_args app.logger.debug(f"adapter: {adapter.to_dict()}") worker_code_key = f"{BASE}_worker_code" arg_job_id = adapter.get_job_id() arg_worker_id = adapter.get_worker_id() job_id = arg_job_id or f"na" worker_id = arg_worker_id if worker_id in {"", "na", None}: worker_id = str(uuid.uuid4()) adapter.worker_id = worker_id max_judgments = 0 #not set if adapter.has_api(): api = adapter.get_api() max_judgments = api.get_max_judgments() else: try: max_judgments = int( request.args.get("max_judgments", max_judgments)) except: pass job_config = get_job_config(get_db(), job_id) max_judgments = max(max_judgments, job_config.expected_judgments) next_player = check_is_proposer_next(job_id, worker_id, treatment, max_judgments=max_judgments) table_all = get_table("txx", "all", schema=None) # The task was already completed, so we skip to the completion code display if cookie_obj.get(BASE) and cookie_obj.get(worker_code_key): req_response = make_response( redirect(url_for(f"{treatment}.survey.survey_done", **request.args))) set_cookie_obj(req_response, BASE, cookie_obj) return req_response if treatment is None: treatment = get_latest_treatment() cookie_obj["job_id"] = job_id cookie_obj["worker_id"] = worker_id cookie_obj["assignment_id"] = adapter.get_assignment_id() cookie_obj["treatment"] = treatment cookie_obj["adapter"] = adapter.to_dict() cookie_obj[BASE] = True form = form_class(request.form) drop = request.form.get("drop") con = get_db() # if table_exists(con, table_all): # with con: # res = con.execute(f"SELECT * from {table_all} WHERE worker_id=?", (worker_id,)).fetchone() # if res: # flash(f"You already took part on this survey. You can just ignore this HIT for the assignment to be RETURNED later to another worker or you can submit right now for a REJECTION using the survey code provided.") # req_response = make_response(render_template("error.html", worker_code=WORKER_CODE_DROPPED)) # return req_response #The next player should be a proposer but some responders may still be processing data ## Should not matter in feedback surveys if next_player == NEXT_IS_WAITING: resp_table = get_table(base="resp", job_id=job_id, schema="result", treatment=treatment) prop_table = get_table(base="prop", job_id=job_id, schema="result", treatment=treatment) # We make sure the user didn't accidentaly refreshed the page after processing the main task if (not is_worker_available(worker_id, resp_table) and not is_worker_available(worker_id, prop_table)): flash( "Unfortunately there is no task available right now. Please check again in 15 minutes. Otherwise you can just ignore this HIT for the assignment to be RETURNED later to another worker or you can submit right now for a REJECTION using the survey code provided." ) return render_template("error.html", worker_code=WORKER_CODE_DROPPED) if adapter.is_preview() or job_id == "na": flash( "Please do note that you are currently in the preview mode of the survey. You SHOULD NOT FILL NEITHER SUBMIT the survey in this mode. Please go back to Mturk and read the instructions about how to correctly start this survey." ) if request.method == "POST" and (drop == "1" or form.validate_on_submit()): form = form_class(request.form) response = request.form.to_dict() response["timestamp"] = str(datetime.datetime.now()) cookie_obj["response"] = response is_codes_valid = True # Responders have to fill and submit tasks if is_codes_valid and job_id != "na" and worker_id != "na": cookie_obj["response"] = response #NOTE: the url should be pointing to handle_survey_feedback_done req_response = make_response( redirect( url_for(f"{treatment}.survey.survey_done", **request.args))) set_cookie_obj(req_response, BASE, cookie_obj) return req_response elif is_codes_valid: flash( "Your data was submitted and validated but not save as you are currently in the preview mode of the survey. Please go back to Mturk and read the instructions about how to correctly start this survey." ) req_response = make_response( render_template(template, job_id=job_id, worker_id=worker_id, treatment=treatment, form=form, max_judgments=max_judgments, max_gain=MAX_GAIN, maximum_control_mistakes=MAXIMUM_CONTROL_MISTAKES, overview_url=overview_url, tasks=app.config["TASKS"])) set_cookie_obj(req_response, BASE, cookie_obj) return req_response
def check_is_proposer_next(job_id, worker_id, treatment, max_judgments=None, resp_only=None, prop_only=None): app.logger.debug("check_is_proposer_next") resp_table = get_table(resp_BASE, job_id=job_id, schema="result", treatment=treatment) prop_table = get_table(prop_BASE, job_id=job_id, schema="result", treatment=treatment) prop_table_data = get_table(prop_BASE, job_id=job_id, schema="data", treatment=treatment) job_config = get_job_config(get_db("DB"), job_id) con = get_db("DATA") nb_resp = 0 nb_prop = 0 nb_prop_open = 0 if table_exists(con, resp_table): with con: tmp = con.execute( f"SELECT COUNT(*) as count from {resp_table} where job_id=?", (job_id, )).fetchone() if tmp: nb_resp = tmp["count"] if table_exists(con, prop_table_data): with con: judging_timeout = time.time() - JUDGING_TIMEOUT_SEC tmp = con.execute( f"SELECT COUNT(*) as count from {prop_table_data} where (job_id=? OR job_id like 'REF%') and ({STATUS_KEY}=? OR ({STATUS_KEY}=? and {LAST_MODIFIED_KEY}<?) OR ({WORKER_KEY}=?))", (job_id, RowState.JUDGEABLE, RowState.JUDGING, judging_timeout, worker_id)).fetchone() if tmp: nb_prop_open = tmp["count"] if table_exists(con, prop_table): with con: tmp = con.execute( f"SELECT COUNT(*) as count from {prop_table} where (job_id=? OR job_id like 'REF%')", (job_id, )).fetchone() if tmp: nb_prop = tmp["count"] #TODO: if nb_resp >= expected row/2, should only take props if max_judgments is None or max_judgments == 0: max_judgments = job_config["expected_judgments"] max_resp = (max_judgments // 2) max_prop = (max_judgments // 2) if resp_only: max_resp = max_judgments elif prop_only: max_prop = max_judgments if max_judgments > 0: #if (max_judgments // 2) <= nb_resp and (max_judgments // 2) > nb_prop: if max_resp <= nb_resp and max_prop > nb_prop: if nb_prop_open > 0: is_proposer = NEXT_IS_PROPOSER else: is_proposer = NEXT_IS_WAITING elif nb_prop_open > 0: is_proposer = NEXT_IS_PROPOSER else: if resp_only or prop_only: is_proposer = NEXT_IS_WAITING else: is_proposer = NEXT_IS_RESPONDER if resp_only: if max_judgments > nb_resp: is_proposer = NEXT_IS_RESPONDER else: is_proposer = NEXT_IS_WAITING elif prop_only: if max_judgments > nb_prop: is_proposer = NEXT_IS_PROPOSER else: is_proposer = NEXT_IS_WAITING elif nb_prop_open > 0: is_proposer = NEXT_IS_PROPOSER else: is_proposer = NEXT_IS_RESPONDER app.logger.debug( f"max_judgments: {max_judgments}, nb_prop: {nb_prop}, nb_resp: {nb_resp}, nb_prop_open: {nb_prop_open}, is_proposer: {is_proposer}" ) return is_proposer
def _process_judgments(signal, payload, job_id, job_config, treatment, auto_finalize=False): """ :param signal: (str) :param payload: (dict) :param job_id: (int|str) :param job_config: (JobConfig) :param auto_finalize (bool) """ error_happened = False app.logger.debug( f"_process_judgments: {signal}, job_id: {job_id}, auto_finalize: {auto_finalize}" ) with app.app_context(): try: if signal == "new_judgments": judgments_count = payload['judgments_count'] fig8 = FigureEight(job_id, job_config["api_key"]) for idx in range(judgments_count): if auto_finalize == True: try: con = get_db("RESULT") worker_judgment = payload['results']['judgments'][ idx] worker_id = worker_judgment["worker_id"] app.logger.debug( f"_process_judgments: {signal}, job_id: {job_id}, worker_id: {worker_id}" ) is_responder = False is_proposer = False table_resp = get_table(resp_BASE, job_id=job_id, schema="result", treatment=treatment) table_prop = get_table(prop_BASE, job_id=job_id, schema="result", treatment=treatment) with con: if table_exists(con, table_resp): res = con.execute( f"SELECT * from {table_resp} WHERE job_id=? and worker_id=?", (job_id, worker_id)).fetchone() if res: is_responder = True if not is_responder and table_exists( con, table_prop): res = con.execute( f"SELECT * from {table_prop} WHERE job_id=? and worker_id=?", (job_id, worker_id)).fetchone() if res: is_proposer = True if is_responder: finalize_resp(job_id=job_id, worker_id=worker_id, treatment=treatment) elif is_proposer: finalize_round(job_id=job_id, prop_worker_id=worker_id, treatment=treatment) else: app.logger.error( f"Error: unknown worker_id: {worker_id} for job_id: {job_id}" ) except Exception as err: if not error_happened: app.log_exception(err) error_happened = True else: worker_judgment = payload['results']['judgments'][idx] worker_id = worker_judgment["worker_id"] pay_worker_bonus(job_id, worker_id, fig8) elif signal == "unit_complete": judgments_count = payload['judgments_count'] fig8 = FigureEight(job_id, job_config["api_key"]) for idx in range(judgments_count): if auto_finalize == False: worker_judgment = payload['results']['judgments'][idx] worker_id = worker_judgment["worker_id"] # PAY_WORKER won't pay someone twice. pay_worker_bonus(job_id, worker_id, fig8) #TODO: may process the whole unit here pass except Exception as err: app.log_exception(err) app.logger.debug(f"_process_judgments: {signal}, job_id: {job_id} - done")