def get_previous_worker_code(job_id, worker_id, treatment):
"""
Generate a code for the user in case he already took the main task
"""
resp_table = get_table(resp_BASE,
job_id=job_id,
schema="result",
treatment=treatment)
prop_table = get_table(prop_BASE,
job_id=job_id,
schema="result",
treatment=treatment)
worker_code = None
if is_worker_available(worker_id, resp_table):
worker_code = get_db().execute(
f"SELECT completion_code from {resp_table} where worker_id=?",
[worker_id]).fetchone()[0]
if not worker_code:
worker_code = generate_completion_code(resp_BASE, job_id)
if is_worker_available(worker_id, prop_table):
worker_code = get_db().execute(
f"SELECT completion_code from {prop_table} where worker_id=?",
[worker_id]).fetchone()[0]
if not worker_code:
worker_code = generate_completion_code(prop_BASE, job_id)
return worker_code
def test_index_auto(client, treatment, prefix=""):
#takes the role assigned by the system and solves the corresponding tasks
client = None
job_id = "test_auto"
for _ in range(TASK_REPETITION):
auto_worker_id = generate_worker_id(f"{prefix}index_auto")
path = f"/{treatment}/?worker_id={auto_worker_id}&job_id={job_id}"
with app.test_request_context(path):
client = get_client()
res = client.get(path, follow_redirects=True)
time.sleep(0.001)
# Play as responder
# print(res.data)
if b"role of a RESPONDER" in res.data:
# res = _process_resp(client, treatment, job_id=job_id, worker_id=auto_worker_id, min_offer=MIN_OFFER)
# process_tasks(client, job_id=job_id, worker_id=auto_worker_id, bonus_mode="full", url_kwargs={"auto_finalize": 1, "treatment": treatment})
res = _process_resp_tasks(client, treatment, job_id=job_id, worker_id=auto_worker_id, min_offer=get_min_offer())
assert b"resp:" in res.data
assert is_worker_available(auto_worker_id, get_table("resp", job_id=job_id, schema="result"))
# Play a proposer
elif b"role of a PROPOSER" in res.data:
app.logger.warning("PROPOSER")
res = client.get(path, follow_redirects=True)
res = _process_prop_round(client, treatment, job_id=job_id, worker_id=auto_worker_id, offer=get_offer(), response_available=True)
assert b"prop:" in res.data
assert is_worker_available(auto_worker_id, get_table("resp", job_id=job_id, schema="result"))
else:
assert False, "Nooooooooooooooo"
time.sleep(WEBHOOK_DELAY)
def get_features(job_id,
resp_worker_id,
treatment,
tasks=None,
tasks_features=None):
"""
:returns: (numpy.array) features
:returns: (dict) features_rows untransformed
"""
app.logger.debug("get_features")
MODEL_INFOS_KEY = f"{TREATMENTS_MODEL_REFS[treatment.upper()]}_MODEL_INFOS"
if tasks is None:
tasks = app.config["TASKS"]
con = get_db("RESULT")
if tasks_features is None:
tasks_features = app.config["TASKS_FEATURES"]
row_features = dict()
for task_name, features in tasks_features.items():
if task_name in tasks:
task_table = get_table(task_name,
job_id=job_id,
schema="result",
is_task=True)
if table_exists(con, task_table):
with con:
sql = f"SELECT {','.join(features)} FROM {task_table} WHERE worker_id=?"
res = con.execute(sql, (resp_worker_id, )).fetchone()
# task tables are shared. when using a REF, there the table may exists but without valid rows
if res is not None:
row_features.update(dict(res))
resp_features = {"resp": ["resp_time_spent"]}
for name, features in resp_features.items():
table = get_table(name,
job_id=job_id,
treatment=treatment,
schema="result",
is_task=False)
if table_exists(con, table):
with con:
sql = f"SELECT {','.join(features)} FROM {table} WHERE worker_id=?"
res = con.execute(sql, (resp_worker_id, )).fetchone()
# task tables are shared. when using a REF, there the table may exists but without valid rows
if res is not None:
row_features.update(dict(res))
tmp_df = pd.DataFrame(data=[row_features])
REF_MODEL_KEY = f"{TREATMENTS_MODEL_REFS[treatment.upper()]}_MODEL"
dss_available = bool(app.config.get(REF_MODEL_KEY))
if dss_available:
x, _ = df_to_xy(
tmp_df, select_columns=app.config[MODEL_INFOS_KEY]["top_columns"])
else:
x = None
app.logger.debug("get_features - done")
return x, row_features
def finalize_resp(job_id, worker_id, treatment):
app.logger.debug("finalize_resp")
table = get_table(base="resp",
job_id=job_id,
schema="result",
treatment=treatment)
with get_db("RESULT") as con:
res = con.execute(
f"SELECT * from {table} where job_id=? and worker_id=?",
(job_id, worker_id)).fetchone()
if res:
resp_result = dict(res)
insert_row(job_id, resp_result, treatment)
else:
app.logger.warning(
f"finalize_resp: worker_id {worker_id} not found - job_id: {job_id}"
)
app.logger.debug(
f"Treatment: {treatment}, auto_dss: {TREATMENTS_AUTO_DSS}")
if treatment.upper() in TREATMENTS_AUTO_DSS:
time.sleep(WEBHOOK_DELAY)
worker_id = f"auto_prop_{uuid.uuid4()}"
args = [
None, treatment, job_id, worker_id, "auto", True, True, True, None
]
app.logger.debug(f"auto-proposal! {args}")
# client = app.test_client()
app.config["THREADS_POOL"].starmap_async(_process_prop_round, [args])
def handle_done(treatment, template=None, response_to_result_func=None):
app.logger.debug("handle_done")
if template is None:
template = f"txx/{BASE}.done.html"
if response_to_result_func is None:
response_to_result_func = prop_to_prop_result
worker_code_key = f"{BASE}_worker_code"
worker_bonus_key = f"{BASE}_worker_bonus"
cookie_obj = get_cookie_obj(BASE)
ai_cookie_obj = get_cookie_obj(AI_COOKIE_KEY)
if not cookie_obj.get(BASE, None):
flash("Sorry, you are not allowed to use this service. ^_^")
return render_template("error.html")
if not (cookie_obj.get("worker_code")):
job_id = cookie_obj["job_id"]
worker_code = generate_completion_code(base=BASE, job_id=job_id)
proposal = cookie_obj["proposal"]
proposal["completion_code"] = worker_code
proposal.update(ai_cookie_obj.get("ai_proposal", {}))
row_info = cookie_obj["row_info"]
worker_id = cookie_obj["worker_id"]
row_info = get_row(get_db(), job_id, worker_id, TREATMENT)
offer = proposal["offer_dss"]
min_offer = row_info["min_offer"]
worker_bonus = 0
if offer >= min_offer:
worker_bonus = 5
prop_result = response_to_result_func(proposal,
job_id=job_id,
worker_id=worker_id,
row_data=row_info)
prop_result["worker_bonus"] = worker_bonus
try:
save_result2file(
get_output_filename(base=BASE,
job_id=job_id,
treatment=treatment), prop_result)
except Exception as err:
app.log_exception(err)
try:
save_result2db(table=get_table(base=BASE,
job_id=job_id,
schema="result",
treatment=treatment),
response_result=prop_result,
unique_fields=["worker_id"])
#increase_worker_bonus(job_id=job_id, worker_id=worker_id, bonus_cents=worker_bonus, con=get_db("DB"))
except Exception as err:
app.log_exception(err)
cookie_obj.clear()
cookie_obj[BASE] = True
cookie_obj["worker_id"] = worker_id
cookie_obj[worker_bonus_key] = cents_repr(worker_bonus)
cookie_obj[worker_code_key] = worker_code
req_response = make_response(
render_template(template, worker_code=cookie_obj[worker_code_key]))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
def handle_survey_cpc_done():
if not session.get("txx"):
flash("Sorry, you are not allowed to use this service. ^_^")
return render_template("error.html")
job_id = session["job_id"]
worker_id = session["worker_id"]
treatment = session["treatment"]
response = session["response"]
response["worker_id"] = worker_id
result = {k: v for k, v in response.items() if k not in {'csrf_token'}}
try:
save_result2file(
get_output_filename(base="survey_cpc",
job_id=job_id,
treatment=treatment), result)
except Exception as err:
app.log_exception(err)
try:
save_result2db(table=get_table(base="survey_cpc",
job_id=job_id,
schema="result",
treatment=treatment),
response_result=result,
unique_fields=["worker_id"])
except Exception as err:
app.log_exception(err)
return render_template("info.html", job_id=job_id)
def has_worker_and_features(base, task_module, job_id, worker_id):
con = get_db()
table = get_table(base, job_id, "result")
features = [feature for feature in task_module.FEATURES]
sql = f"SELECT {','.join(features)} FROM {table} WHERE job_id=? AND worker_id=?"
res = con.execute(sql, [job_id, worker_id])
return res is not None
def handle_done(treatment, template=None, no_features=None):
app.logger.debug("handle_done")
if template is None:
template = f"txx/resp.done.html"
if no_features is None:
completion_code_base = BASE
else:
# survey should not require tasks
completion_code_base = BASE + "NF"
cookie_obj = get_cookie_obj(BASE)
worker_code_key = f"{BASE}_worker_code"
if not cookie_obj.get(BASE, None):
flash("Sorry, you are not allowed to use this service. ^_^")
return render_template("error.html")
if not cookie_obj.get(worker_code_key):
job_id = cookie_obj["job_id"]
worker_code = generate_completion_code(completion_code_base, job_id)
response = cookie_obj["response"]
response["completion_code"] = worker_code
worker_id = cookie_obj["worker_id"]
resp_result = resp_to_resp_result(response,
job_id=job_id,
worker_id=worker_id)
try:
#save_resp_result(TUBE_RES_FILENAME, resp_result)
save_result2file(
get_output_filename(BASE, job_id, treatment=treatment),
resp_result)
except Exception as err:
app.log_exception(err)
try:
#save_resp_result2db(get_db("RESULT"), resp_result, job_id)
save_result2db(table=get_table(base=BASE,
job_id=job_id,
schema="result",
treatment=treatment),
response_result=resp_result,
unique_fields=["worker_id"])
increase_worker_bonus(job_id=job_id,
worker_id=worker_id,
bonus_cents=0)
except Exception as err:
app.log_exception(err)
cookie_obj.clear()
cookie_obj[BASE] = True
cookie_obj["worker_id"] = worker_id
cookie_obj[worker_code_key] = worker_code
auto_finalize = cookie_obj.get("auto_finalize",
request.args.get("auto_finalize"))
if auto_finalize and no_features: # and base=="hexaco":
#NOTE: there is an import here ^_^
finalize_resp(job_id, worker_id, treatment)
req_response = make_response(
render_template(template, worker_code=cookie_obj[worker_code_key]))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
def get_row_ignore_job(con, job_id, worker_id, treatment):
"""
Get a row and change it's state to Judging with a last modified time
:param con: sqlite3|sqlalchemy connection
:param job_id: job id
:param worker_id: worker's id
:param treatment:
"""
app.logger.debug("get_row")
table = get_table(base=BASE,
job_id=job_id,
schema="data",
treatment=treatment)
if not table_exists(con, table):
app.logger.warning(f"table: {table} does not exist")
return None
res = None
dep_change_time = time.time() - JUDGING_TIMEOUT_SEC
# Let's check if the same worker is looking for a row (then give him the one he is working on back)
free_rowid = con.execute(
f'select {PK_KEY} from {table} where {STATUS_KEY}==? and {WORKER_KEY}==? and {LAST_MODIFIED_KEY} > ?',
(RowState.JUDGING, worker_id, dep_change_time)).fetchone()
if not free_rowid:
# let's search for a row that remained too long in the 'judging' state
free_rowid = con.execute(
f'select {PK_KEY} from {table} where {STATUS_KEY}==? and {LAST_MODIFIED_KEY} < ?',
(RowState.JUDGING, dep_change_time)).fetchone()
if not free_rowid:
# Let's check for a judgeable untouched row
free_rowid = con.execute(
f'select {PK_KEY} from {table} where {STATUS_KEY}==?',
(RowState.JUDGEABLE, )).fetchone()
if free_rowid:
free_rowid = free_rowid[PK_KEY]
## let's search for a rowid that hasn't been processed yetf
res = dict(
con.execute(
f'select {PK_KEY}, * from {table} where {PK_KEY}=?',
(free_rowid, )).fetchone())
with con:
update(
f'update {table} set {LAST_MODIFIED_KEY}=?, {STATUS_KEY}=?, {WORKER_KEY}=? where {PK_KEY}=?',
(time.time(), RowState.JUDGING, worker_id, free_rowid),
con=con)
return res
if free_rowid:
free_rowid = free_rowid[PK_KEY]
res = dict(
con.execute(f'select {PK_KEY}, * from {table} where {PK_KEY}=?',
(free_rowid, )).fetchone())
with con:
update(
f'UPDATE {table} set {LAST_MODIFIED_KEY}=?, {STATUS_KEY}=?, {WORKER_KEY}=? where {PK_KEY}=?',
(time.time(), RowState.JUDGING, worker_id, free_rowid),
con=con)
else:
app.logger.warning(
f"no row available! job_id: {job_id}, worker_id: {worker_id}")
return res
def handle_index_feedback(treatment, base_treatment):
job_id = request.args.get("job_id", "na")
worker_id = request.args.get("worker_id", "na")
max_judgments = None
try:
max_judgments = int(request.args.get("max_judgments", "0"))
except ValueError:
pass
previous_worker_code = get_previous_worker_code(job_id, worker_id,
base_treatment)
app.logger.debug(f"handle_index: job_id: {job_id}, worker_id: {worker_id}")
is_proposer = check_is_proposer_next(job_id,
worker_id,
treatment,
max_judgments=max_judgments)
table_all = get_table(BASE, "all", schema=None)
con = get_db()
if table_exists(con, table_all):
with con:
res = con.execute(f"SELECT * from {table_all} WHERE worker_id=?",
(worker_id, )).fetchone()
# if res:
# flash(f"You already took part on this survey. Thank you for your participation")
# return render_template("error.html")
if previous_worker_code is None:
if is_proposer:
return redirect(url_for(f"{treatment}.prop.index", **request.args))
else:
return redirect(url_for(f"{treatment}.resp.index", **request.args))
else:
if prop_BASE in previous_worker_code:
return redirect(url_for(f"{treatment}.prop.index", **request.args))
else:
return redirect(url_for(f"{treatment}.resp.index", **request.args))
def create_resp_data_auto_prop_table(treatment, ref, use_ai_offer=None):
app.logger.debug(f"create_resp_data_auto_prop_table - {ref}, {treatment}")
con = get_db()
table = get_table(BASE, None, "data", treatment=treatment)
assert len(ref) == 4, "expected references of the form <txyz>"
required_columns = """ai_calls_acceptance_probability,ai_calls_best_offer_probability,ai_calls_count_repeated,ai_calls_offers,ai_calls_pauses,ai_nb_calls,ai_offer,feedback_accuracy,feedback_explanation,feedback_understanding,job_id,model_type,offer,offer_dss,offer_final,prop_time_spent,prop_worker_id,timestamp,worker_id""".split(
",")
columns_to_clear = """ai_calls_acceptance_probability,ai_calls_best_offer_probability,ai_calls_count_repeated,ai_calls_offers,ai_calls_pauses,ai_nb_calls,feedback_accuracy,feedback_explanation,feedback_understanding,job_id,prop_time_spent,timestamp,worker_id""".split(
",")
if not table_exists(con, table):
df = pd.read_csv(
os.path.join(CODE_DIR, 'data', ref, 'export',
f'result__{ref}_prop.csv'))
for col in required_columns:
if col not in df.columns:
df[col] = None
columns = [col for col in required_columns if col in df.columns]
df = df[columns]
if use_ai_offer:
df["offer_final"] = df["ai_offer"]
df["offer"] = df["ai_offer"]
df["offer_dss"] = df["ai_offer"]
df[[col for col in columns_to_clear]] = None
df["job_id"] = f"REFAUTO{ref.upper()}"
df[STATUS_KEY] = RowState.JUDGEABLE
df[WORKER_KEY] = None
df["updated"] = 0
with con:
df.to_sql(table, con, index=False)
app.logger.debug("create_table_data: table created")
def process_insert_row(job_id, resp_row, treatment, overwrite=False):
"""
Insert a new row for the proposer assuming no model is available
"""
app.logger.debug("process_insert_row")
resp_row = dict(resp_row)
_, features_dict = get_features(job_id,
resp_worker_id=resp_row[WORKER_KEY],
treatment=treatment)
resp_row = dict(resp_row)
resp_row.update(features_dict)
df = pd.DataFrame(data=[resp_row])
df[STATUS_KEY] = RowState.JUDGEABLE
df[LAST_MODIFIED_KEY] = time.time()
df[WORKER_KEY] = None
df["resp_worker_id"] = resp_row[WORKER_KEY]
table = get_table(BASE, job_id=job_id, schema="data", treatment=treatment)
con = get_db("DATA")
insert(df,
table,
con=con,
overwrite=overwrite,
unique_fields=["resp_worker_id"])
app.logger.debug("process_insert_row - done")
def process_insert_row_dss(job_id, resp_row, treatment, overwrite=False):
"""
Insert a new row for the proposer with the ai predicted min_offer
"""
app.logger.debug("process_insert_row_dss")
ENABLED_FAKE_MODEL_KEY = f"{treatment.upper()}_FAKE_MODEL"
MODEL_KEY = f"{TREATMENTS_MODEL_REFS[treatment.upper()]}_MODEL"
FAKE_MODEL_TYPES = [WEAK_FAKE_MODEL, STRONG_FAKE_MODEL]
model_type = None
ai_offer = 0
features, features_dict = get_features(job_id,
resp_worker_id=resp_row[WORKER_KEY],
treatment=treatment)
resp_row = dict(resp_row)
resp_row.update(features_dict)
df = pd.DataFrame(data=[resp_row])
df[STATUS_KEY] = RowState.JUDGEABLE
df[LAST_MODIFIED_KEY] = time.time()
df[WORKER_KEY] = None
df["resp_worker_id"] = resp_row[WORKER_KEY]
df["ai_offer"] = ai_offer
df["model_type"] = model_type
table = get_table(BASE, job_id=job_id, schema="data", treatment=treatment)
con = get_db("DATA")
insert(df,
table,
con=con,
overwrite=overwrite,
unique_fields=["resp_worker_id"])
with con:
rowid = con.execute(
f"SELECT {PK_KEY} FROM {table} where resp_worker_id=?",
(resp_row[WORKER_KEY], )).fetchone()[PK_KEY]
# if true, fake models should be used for this treatment
if app.config[ENABLED_FAKE_MODEL_KEY]:
if rowid is not None:
model_type = FAKE_MODEL_TYPES[rowid % len(FAKE_MODEL_TYPES)]
else:
model_type = REAL_MODEL
else:
model_type = REAL_MODEL
if model_type == WEAK_FAKE_MODEL:
ai_offer = predict_weak(resp_row["min_offer"])
elif model_type == STRONG_FAKE_MODEL:
ai_offer = predict_strong(resp_row["min_offer"])
else:
# Models predict a vector
ai_offer = app.config[MODEL_KEY].predict(features)[0]
ai_offer = int(ai_offer)
with con:
update(
sql=f"UPDATE {table} SET ai_offer=?, model_type=? where rowid=?",
args=(ai_offer, model_type, rowid),
con=con)
app.logger.debug("process_insert_row_dss - done MODEL_TYPE: " +
str(rowid % len(FAKE_MODEL_TYPES)) + " " + str(rowid))
def test_webhook(client, treatment, resp_feedback_fields=None):
# In real conditions, the tasks/webhook are delayed with +500 ms
job_id = "test"
resp_worker_id = generate_worker_id("webhook_resp")
prop_worker_id = generate_worker_id("webhook_prop")
process_tasks(client,
job_id=job_id,
worker_id=resp_worker_id,
bonus_mode="random")
_process_resp(client,
treatment,
job_id=job_id,
worker_id=resp_worker_id,
min_offer=get_min_offer(),
resp_feedback_fields=resp_feedback_fields)
emit_webhook(client,
url=f"/{treatment}/webhook/",
treatment=treatment,
worker_id=resp_worker_id,
by_get=True)
time.sleep(WEBHOOK_DELAY)
_process_prop(client,
treatment,
job_id=job_id,
worker_id=prop_worker_id,
offer=get_offer(),
response_available=True)
emit_webhook(client,
url=f"/{treatment}/webhook/",
treatment=treatment,
worker_id=prop_worker_id,
by_get=True)
time.sleep(WEBHOOK_DELAY)
with app.app_context():
assert is_worker_available(
resp_worker_id,
get_table("resp",
job_id=job_id,
schema="result",
treatment=treatment))
assert is_worker_available(
prop_worker_id,
get_table("prop",
job_id=job_id,
schema="result",
treatment=treatment))
def test_eff(client):
with app.app_context():
job_id = "test"
worker_id = generate_worker_id("task")
res = _process_eff(client, job_id=job_id, worker_id=worker_id)
assert b"eff:" in res
assert is_worker_available(worker_id,
get_table("eff", job_id, "result"))
assert has_worker_and_features("eff", eff, job_id, worker_id)
def create_prop_data_table(treatment, ref):
app.logger.debug(f"create_table_data - {ref}, {treatment}")
con = get_db()
table_data = get_table(BASE, None, "data", treatment=treatment)
table_resp = get_table("resp", None, "result", treatment=treatment)
assert len(ref) == 4, "expected references of the form <txyz>"
job_id = f"REF{ref.upper()}"
if not table_exists(con, table_data):
df_resp = None
try:
# normaly, we expect an exported data-table to be available with all required features
df = pd.read_csv(
os.path.join(CODE_DIR, 'data', ref, 'export',
f'data__{ref}_prop.csv'))
df[STATUS_KEY] = RowState.JUDGEABLE
df[WORKER_KEY] = None
df["updated"] = 0
df["job_id"] = f"REF{ref.upper()}"
with con:
df.to_sql(table_data, con, index=False)
app.logger.debug("create_table_data: table created")
df_resp = pd.read_csv(
os.path.join(CODE_DIR, 'data', ref, 'export',
f'result__{ref}_resp.csv'))
df["job_id"] = f"REF{ref.upper()}"
except Exception as err:
app.logger.warn(f"silenced-error: {err}")
# otherwise, we work with the resp. table, which means the model does/should not expect any features.
df = pd.read_csv(
os.path.join(CODE_DIR, 'data', ref, 'export',
f'result__{ref}_resp.csv'))
df["job_id"] = f"REF{ref.upper()}"
for idx in range(len(df)):
resp_row = df.iloc[idx]
resp_row = dict(**resp_row)
insert_row(job_id, resp_row, treatment)
app.logger.debug("create_table_data: table created")
df_resp = df
df_resp.to_sql(table_resp, con, index=False, if_exists='replace')
app.logger.debug("resp-table-cloned: table created")
def save_prop_result2db(con,
proposal_result,
job_id,
overwrite=False,
treatment=None):
table = get_table(BASE,
job_id=job_id,
schema="result",
treatment=treatment)
df = pd.DataFrame(data=[proposal_result])
insert(df, table=table, con=con, overwrite=overwrite)
def test_auto_finalize(client, treatment, resp_feedback_fields=None):
# Test automatic webhook triggering to finalize tasks
job_id = "test"
resp_worker_id = generate_worker_id("auto_resp")
prop_worker_id = generate_worker_id("auto_prop")
_process_resp(client,
treatment,
job_id=job_id,
worker_id=resp_worker_id,
min_offer=get_min_offer(),
resp_feedback_fields=resp_feedback_fields)
process_tasks(client,
job_id=job_id,
worker_id=resp_worker_id,
bonus_mode="random",
url_kwargs={
"auto_finalize": 1,
"treatment": treatment
})
time.sleep(WEBHOOK_DELAY)
res = _process_prop(client,
treatment,
job_id=job_id,
worker_id=prop_worker_id,
offer=get_offer(),
response_available=True,
auto_finalize=True)
time.sleep(WEBHOOK_DELAY)
with app.app_context():
assert is_worker_available(
resp_worker_id,
get_table("resp",
job_id=job_id,
schema="result",
treatment=treatment))
assert is_worker_available(
prop_worker_id,
get_table("prop",
job_id=job_id,
schema="result",
treatment=treatment))
def close_row(con, job_id, row_id, treatment):
app.logger.debug("close_row")
table = get_table(BASE, job_id=job_id, schema="data", treatment=treatment)
if not table_exists(con, table):
app.logger.warning(f"table missing: <{table}>")
else:
with con:
update(
f'UPDATE {table} set {LAST_MODIFIED_KEY}=?, {STATUS_KEY}=? where {PK_KEY}=? and {STATUS_KEY}=?',
(time.time(), RowState.JUDGED, row_id, RowState.JUDGING),
con=con)
app.logger.debug("close_row - done")
def is_resp_in_prop_result(resp_worker_id, job_id, treatment):
con = get_db()
table = get_table("prop", job_id, "result", treatment=treatment)
sql = f"SELECT * FROM {table} WHERE job_id=? and resp_worker_id=?"
print("SQL: ", sql)
try:
with con:
res = con.execute(sql, [job_id, resp_worker_id]).fetchone()
if res is None:
return False
else:
return True
except Exception:
pass
return False
def test_auto_finalize(client, treatment, resp_feedback_fields=None):
# Test automatic webhook triggering to finalize tasks
job_id = "test"
resp_worker_id = generate_worker_id("auto_resp")
_process_resp(client, treatment, job_id=job_id, worker_id=resp_worker_id, min_offer=get_min_offer(), resp_feedback_fields=resp_feedback_fields)
process_tasks(client, job_id=job_id, worker_id=resp_worker_id, bonus_mode="random", url_kwargs={"auto_finalize": 1, "treatment": treatment})
time.sleep(WEBHOOK_DELAY)
with app.app_context():
# let the auto-responder kick-in
for _ in range(NB_MAX_AUTO_PLAY_RETRIES):
if is_resp_in_prop_result(resp_worker_id, job_id, treatment):
break
time.sleep(AUTO_PROP_DELAY)
assert is_worker_available(resp_worker_id, get_table("resp", job_id=job_id, schema="result", treatment=treatment))
assert is_resp_in_prop_result(resp_worker_id, job_id, treatment)
def test_webhook(client, treatment, resp_feedback_fields=None):
# In real conditions, the tasks/webhook are delayed with +500 ms
job_id = "test"
resp_worker_id = generate_worker_id("webhook_resp")
process_tasks(client, job_id=job_id, worker_id=resp_worker_id, bonus_mode="random")
_process_resp(client, treatment, job_id=job_id, worker_id=resp_worker_id, min_offer=get_min_offer(), resp_feedback_fields=resp_feedback_fields)
emit_webhook(client, url=f"/{treatment}/webhook/", job_id=job_id, treatment=treatment, worker_id=resp_worker_id, by_get=True)
time.sleep(WEBHOOK_DELAY)
with app.app_context():
# let the auto-responder kick-in
for _ in range(NB_MAX_AUTO_PLAY_RETRIES):
if is_resp_in_prop_result(resp_worker_id, job_id, treatment):
break
time.sleep(AUTO_PROP_DELAY)
assert is_worker_available(resp_worker_id, get_table("resp", job_id=job_id, schema="result", treatment=treatment))
assert is_resp_in_prop_result(resp_worker_id, job_id, treatment)
def handle_index(treatment, resp_only=None, prop_only=None):
job_id = request.args.get("job_id", "na")
worker_id = request.args.get("worker_id", "na")
max_judgments = None
try:
max_judgments = int(request.args.get("max_judgments", "0"))
except ValueError:
pass
previous_worker_code = get_previous_worker_code(job_id, worker_id,
treatment)
app.logger.debug(f"handle_index: job_id: {job_id}, worker_id: {worker_id}")
is_proposer = check_is_proposer_next(job_id,
worker_id,
treatment,
max_judgments=max_judgments,
resp_only=resp_only,
prop_only=prop_only)
table_all = get_table(BASE, "all", schema=None)
con = get_db()
if previous_worker_code is None:
if resp_only:
return redirect(url_for(f"{treatment}.resp.index", **request.args))
elif prop_only:
return redirect(url_for(f"{treatment}.prop.index", **request.args))
else:
if is_proposer:
return redirect(
url_for(f"{treatment}.prop.index", **request.args))
else:
return redirect(
url_for(f"{treatment}.resp.index", **request.args))
else:
flash("You already completed the main task!")
if resp_BASE in previous_worker_code:
return render_template("txx/resp.done.html",
worker_code=previous_worker_code)
else:
return render_template("txx/prop.done.html",
worker_code=previous_worker_code)
def finalize_round(job_id, prop_worker_id, treatment):
app.logger.debug("finalize_round")
# TODO: At this point, all features have been gathered
resp_worker_id = "na"
con = get_db("RESULT")
resp_bonus = 0
prop_bonus = 0
offer, min_offer = 0, 0
with con:
table = get_table(BASE, job_id, schema="result", treatment=treatment)
try:
# min_offer_final was introduce late, so it may be missing on some tables...
# in case of absence, a fallback to min_offer is perfectly fine.
res = con.execute(
f"SELECT offer_final as offer, min_offer_final as min_offer, resp_worker_id from {table} WHERE prop_worker_id=?",
(prop_worker_id, )).fetchone()
except Exception as err:
app.logger.warn(f"{err}")
res = con.execute(
f"SELECT offer_final as offer, min_offer, resp_worker_id from {table} WHERE prop_worker_id=?",
(prop_worker_id, )).fetchone()
offer, min_offer = res["offer"], res["min_offer"]
resp_worker_id = res["resp_worker_id"]
offer = max(min(offer, MAX_GAIN), 0)
min_offer = max(min(min_offer, MAX_GAIN), 0)
if offer >= min_offer:
resp_bonus += offer
prop_bonus += (MAX_GAIN - offer)
assert (resp_bonus + prop_bonus) == 0 or (resp_bonus +
prop_bonus) == MAX_GAIN
con2 = get_db("DB")
with con2:
if not is_fake_worker(resp_worker_id):
increase_worker_bonus(job_id, resp_worker_id, resp_bonus, con2)
if not is_fake_worker(prop_worker_id):
increase_worker_bonus(job_id, prop_worker_id, prop_bonus, con2)
def handle_start(treatment=None,
template=None,
code_prefixes=None,
form_class=None,
overview_url=None,
resp_only=None,
prop_only=None):
app.logger.info("handle_start")
if template is None:
template = "txx/start.html"
if overview_url is None:
overview_url = url_for("overview")
cookie_obj = get_cookie_obj(BASE)
adapter = select_adapter()
app.logger.debug(f"adapter: {adapter.to_dict()}")
arg_job_id = adapter.get_job_id()
arg_worker_id = adapter.get_worker_id()
assignment_id = adapter.get_assignment_id()
job_id = arg_job_id or f"na"
worker_id = arg_worker_id
if worker_id in {"", "na", None}:
worker_id = str(uuid.uuid4())
adapter.worker_id = worker_id
table_all = get_table("txx", "all", schema=None)
# The task was already completed, so we skip to the completion code display
# if cookie_obj.get(BASE) and cookie_obj.get(worker_code_key):
# req_response = make_response(redirect(url_for(f"{treatment}.survey.survey_done", **request.args)))
# set_cookie_obj(req_response, BASE, cookie_obj)
# return req_response
if treatment is None:
treatment = get_latest_treatment()
cookie_obj["job_id"] = job_id
cookie_obj["worker_id"] = worker_id
cookie_obj["treatment"] = treatment
cookie_obj["adapter"] = adapter.to_dict()
cookie_obj[BASE] = True
con = get_db()
if table_exists(con, table_all):
with con:
res = con.execute(f"SELECT * from {table_all} WHERE worker_id=?",
(worker_id, )).fetchone()
if res:
flash(f"You already took part on this survey.")
flash(
"You can just ignore this HIT for the assignment to be RETURNED later to another worker."
)
req_response = make_response(
render_template("error.html",
worker_code=WORKER_CODE_DROPPED))
return req_response
req_response = make_response(
render_template(template,
job_id=job_id,
worker_id=worker_id,
assignment_id=assignment_id,
treatment=treatment,
submit_to_url=adapter.get_submit_to_URL()))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
def handle_survey_feedback_done(template=None):
""" Only used for treatment t10_feedback"""
if template is None:
template = "txx/survey.done.html"
cookie_obj = get_cookie_obj(BASE)
if not cookie_obj.get(BASE):
flash("Sorry, you are not allowed to use this service. ^_^")
return render_template("error.html")
worker_code_key = f"{BASE}_worker_code"
worker_code = WORKER_CODE_DROPPED
if True or not cookie_obj.get(worker_code_key):
job_id = cookie_obj["job_id"]
worker_id = cookie_obj["worker_id"]
treatment = cookie_obj["treatment"]
assignment_id = cookie_obj.get("assignment_id")
worker_code = generate_completion_code(base=treatment, job_id=job_id)
response = cookie_obj["response"]
app.logger.debug("DONE_response: {response}")
dropped = response.get("drop")
if dropped == "1":
worker_code = WORKER_CODE_DROPPED
flash(
f"You have been disqualified as you made more than {MAXIMUM_CONTROL_MISTAKES} mistakes on the control questions. Close this window and don't submit any completion on MTurk to avoid getting a REJECTION."
)
response["worker_id"] = worker_id
response["job_id"] = job_id
response["assignment_id"] = assignment_id
response["completion_code"] = worker_code
result = {
k: v
for k, v in response.items() if k not in {'csrf_token', 'drop'}
}
try:
save_result2file(
get_output_filename(base=BASE,
job_id=job_id,
treatment=treatment), result)
except Exception as err:
app.log_exception(err)
try:
save_result2db(table=get_table(base=BASE,
job_id=job_id,
schema="result",
treatment=treatment),
response_result=result,
unique_fields=["worker_id"])
except Exception as err:
app.log_exception(err)
adapter = get_adapter_from_dict(cookie_obj["adapter"])
#adapter = get_adapter()
app.logger.debug(f"adapter: {cookie_obj['adapter']}")
cookie_obj.clear()
#save_worker_id(job_id=job_id, worker_id=worker_id, worker_code=worker_code, assignment_id=adapter.assignment_id)
app.logger.debug(
f"request-args: {request.args}, adapter: {adapter.to_dict()} ")
try:
expected_max_judgments = request.args.get("expected_max_judgments")
if expected_max_judgments is not None:
expected_max_judgments = int(expected_max_judgments)
api = adapter.get_api()
max_judgments = api.get_max_judgments()
app.logger.debug(
f"survey.done: max_judgments: {max_judgments} - {type(max_judgments)}, expected_max_judgments: {expected_max_judgments} - {type(expected_max_judgments)}"
)
if max_judgments < expected_max_judgments:
app.logger.debug(f"try create new assignments")
create_res = api.create_additional_assignments(1)
app.logger.debug(
f"post assignment creation: new-max: {api.get_max_judgments()} , mturk-api-res: {create_res}"
)
except Exception as err:
app.log_exception(err)
app.logger.info(
f"handle_survey_feedback_done: saved new survey - job_id: {job_id}, worker_id: {worker_id}"
)
req_response = make_response(
render_template(template, worker_code=worker_code, dropped=True))
set_cookie_obj(req_response, BASE, cookie_obj)
for cookie in session.get(ALL_COOKIES_KEY, []):
req_response.set_cookie(cookie, expires=0)
session[ALL_COOKIES_KEY] = []
return req_response
def handle_survey_feedback(treatment=None,
template=None,
code_prefixes=None,
form_class=None,
overview_url=None):
app.logger.info("handle_survey_feedback")
if code_prefixes is None:
code_prefixes = {
"code_cpc": "cpc:",
"code_exp": "exp:",
"code_risk": "risk:",
"code_cc": "cc:",
"code_ras": "ras:"
}
if form_class is None:
form_class = MainFormFeeback
if template is None:
template = "txx/survey_feedback.html"
if overview_url is None:
if treatment and treatment.upper() in TREATMENTS_AUTO_DSS:
overview_url = url_for("overview_auto_prop_feedback")
else:
overview_url = url_for("overview_feedback")
cookie_obj = get_cookie_obj(BASE)
adapter_cookie = get_adapter_from_dict(cookie_obj.get("adapter", {}))
adapter_args = get_adapter_from_dict(request.args)
adapter_referrer = get_adapter()
if adapter_referrer.job_id not in {"", "na", None}:
adapter = adapter_referrer
elif adapter_cookie.job_id not in {"", "na", None}:
adapter = adapter_cookie
else:
adapter = adapter_args
app.logger.debug(f"adapter: {adapter.to_dict()}")
worker_code_key = f"{BASE}_worker_code"
arg_job_id = adapter.get_job_id()
arg_worker_id = adapter.get_worker_id()
job_id = arg_job_id or f"na"
worker_id = arg_worker_id
if worker_id in {"", "na", None}:
worker_id = str(uuid.uuid4())
adapter.worker_id = worker_id
max_judgments = 0 #not set
if adapter.has_api():
api = adapter.get_api()
max_judgments = api.get_max_judgments()
else:
try:
max_judgments = int(
request.args.get("max_judgments", max_judgments))
except:
pass
job_config = get_job_config(get_db(), job_id)
max_judgments = max(max_judgments, job_config.expected_judgments)
next_player = check_is_proposer_next(job_id,
worker_id,
treatment,
max_judgments=max_judgments)
table_all = get_table("txx", "all", schema=None)
# The task was already completed, so we skip to the completion code display
if cookie_obj.get(BASE) and cookie_obj.get(worker_code_key):
req_response = make_response(
redirect(url_for(f"{treatment}.survey.survey_done",
**request.args)))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
if treatment is None:
treatment = get_latest_treatment()
cookie_obj["job_id"] = job_id
cookie_obj["worker_id"] = worker_id
cookie_obj["assignment_id"] = adapter.get_assignment_id()
cookie_obj["treatment"] = treatment
cookie_obj["adapter"] = adapter.to_dict()
cookie_obj[BASE] = True
form = form_class(request.form)
drop = request.form.get("drop")
con = get_db()
# if table_exists(con, table_all):
# with con:
# res = con.execute(f"SELECT * from {table_all} WHERE worker_id=?", (worker_id,)).fetchone()
# if res:
# flash(f"You already took part on this survey. You can just ignore this HIT for the assignment to be RETURNED later to another worker or you can submit right now for a REJECTION using the survey code provided.")
# req_response = make_response(render_template("error.html", worker_code=WORKER_CODE_DROPPED))
# return req_response
#The next player should be a proposer but some responders may still be processing data
## Should not matter in feedback surveys
if next_player == NEXT_IS_WAITING:
resp_table = get_table(base="resp",
job_id=job_id,
schema="result",
treatment=treatment)
prop_table = get_table(base="prop",
job_id=job_id,
schema="result",
treatment=treatment)
# We make sure the user didn't accidentaly refreshed the page after processing the main task
if (not is_worker_available(worker_id, resp_table)
and not is_worker_available(worker_id, prop_table)):
flash(
"Unfortunately there is no task available right now. Please check again in 15 minutes. Otherwise you can just ignore this HIT for the assignment to be RETURNED later to another worker or you can submit right now for a REJECTION using the survey code provided."
)
return render_template("error.html",
worker_code=WORKER_CODE_DROPPED)
if adapter.is_preview() or job_id == "na":
flash(
"Please do note that you are currently in the preview mode of the survey. You SHOULD NOT FILL NEITHER SUBMIT the survey in this mode. Please go back to Mturk and read the instructions about how to correctly start this survey."
)
if request.method == "POST" and (drop == "1" or form.validate_on_submit()):
form = form_class(request.form)
response = request.form.to_dict()
response["timestamp"] = str(datetime.datetime.now())
cookie_obj["response"] = response
is_codes_valid = True
# Responders have to fill and submit tasks
if is_codes_valid and job_id != "na" and worker_id != "na":
cookie_obj["response"] = response
#NOTE: the url should be pointing to handle_survey_feedback_done
req_response = make_response(
redirect(
url_for(f"{treatment}.survey.survey_done",
**request.args)))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
elif is_codes_valid:
flash(
"Your data was submitted and validated but not save as you are currently in the preview mode of the survey. Please go back to Mturk and read the instructions about how to correctly start this survey."
)
req_response = make_response(
render_template(template,
job_id=job_id,
worker_id=worker_id,
treatment=treatment,
form=form,
max_judgments=max_judgments,
max_gain=MAX_GAIN,
maximum_control_mistakes=MAXIMUM_CONTROL_MISTAKES,
overview_url=overview_url,
tasks=app.config["TASKS"]))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
def check_is_proposer_next(job_id,
worker_id,
treatment,
max_judgments=None,
resp_only=None,
prop_only=None):
app.logger.debug("check_is_proposer_next")
resp_table = get_table(resp_BASE,
job_id=job_id,
schema="result",
treatment=treatment)
prop_table = get_table(prop_BASE,
job_id=job_id,
schema="result",
treatment=treatment)
prop_table_data = get_table(prop_BASE,
job_id=job_id,
schema="data",
treatment=treatment)
job_config = get_job_config(get_db("DB"), job_id)
con = get_db("DATA")
nb_resp = 0
nb_prop = 0
nb_prop_open = 0
if table_exists(con, resp_table):
with con:
tmp = con.execute(
f"SELECT COUNT(*) as count from {resp_table} where job_id=?",
(job_id, )).fetchone()
if tmp:
nb_resp = tmp["count"]
if table_exists(con, prop_table_data):
with con:
judging_timeout = time.time() - JUDGING_TIMEOUT_SEC
tmp = con.execute(
f"SELECT COUNT(*) as count from {prop_table_data} where (job_id=? OR job_id like 'REF%') and ({STATUS_KEY}=? OR ({STATUS_KEY}=? and {LAST_MODIFIED_KEY}<?) OR ({WORKER_KEY}=?))",
(job_id, RowState.JUDGEABLE, RowState.JUDGING, judging_timeout,
worker_id)).fetchone()
if tmp:
nb_prop_open = tmp["count"]
if table_exists(con, prop_table):
with con:
tmp = con.execute(
f"SELECT COUNT(*) as count from {prop_table} where (job_id=? OR job_id like 'REF%')",
(job_id, )).fetchone()
if tmp:
nb_prop = tmp["count"]
#TODO: if nb_resp >= expected row/2, should only take props
if max_judgments is None or max_judgments == 0:
max_judgments = job_config["expected_judgments"]
max_resp = (max_judgments // 2)
max_prop = (max_judgments // 2)
if resp_only:
max_resp = max_judgments
elif prop_only:
max_prop = max_judgments
if max_judgments > 0:
#if (max_judgments // 2) <= nb_resp and (max_judgments // 2) > nb_prop:
if max_resp <= nb_resp and max_prop > nb_prop:
if nb_prop_open > 0:
is_proposer = NEXT_IS_PROPOSER
else:
is_proposer = NEXT_IS_WAITING
elif nb_prop_open > 0:
is_proposer = NEXT_IS_PROPOSER
else:
if resp_only or prop_only:
is_proposer = NEXT_IS_WAITING
else:
is_proposer = NEXT_IS_RESPONDER
if resp_only:
if max_judgments > nb_resp:
is_proposer = NEXT_IS_RESPONDER
else:
is_proposer = NEXT_IS_WAITING
elif prop_only:
if max_judgments > nb_prop:
is_proposer = NEXT_IS_PROPOSER
else:
is_proposer = NEXT_IS_WAITING
elif nb_prop_open > 0:
is_proposer = NEXT_IS_PROPOSER
else:
is_proposer = NEXT_IS_RESPONDER
app.logger.debug(
f"max_judgments: {max_judgments}, nb_prop: {nb_prop}, nb_resp: {nb_resp}, nb_prop_open: {nb_prop_open}, is_proposer: {is_proposer}"
)
return is_proposer
def _process_judgments(signal,
payload,
job_id,
job_config,
treatment,
auto_finalize=False):
"""
:param signal: (str)
:param payload: (dict)
:param job_id: (int|str)
:param job_config: (JobConfig)
:param auto_finalize (bool)
"""
error_happened = False
app.logger.debug(
f"_process_judgments: {signal}, job_id: {job_id}, auto_finalize: {auto_finalize}"
)
with app.app_context():
try:
if signal == "new_judgments":
judgments_count = payload['judgments_count']
fig8 = FigureEight(job_id, job_config["api_key"])
for idx in range(judgments_count):
if auto_finalize == True:
try:
con = get_db("RESULT")
worker_judgment = payload['results']['judgments'][
idx]
worker_id = worker_judgment["worker_id"]
app.logger.debug(
f"_process_judgments: {signal}, job_id: {job_id}, worker_id: {worker_id}"
)
is_responder = False
is_proposer = False
table_resp = get_table(resp_BASE,
job_id=job_id,
schema="result",
treatment=treatment)
table_prop = get_table(prop_BASE,
job_id=job_id,
schema="result",
treatment=treatment)
with con:
if table_exists(con, table_resp):
res = con.execute(
f"SELECT * from {table_resp} WHERE job_id=? and worker_id=?",
(job_id, worker_id)).fetchone()
if res:
is_responder = True
if not is_responder and table_exists(
con, table_prop):
res = con.execute(
f"SELECT * from {table_prop} WHERE job_id=? and worker_id=?",
(job_id, worker_id)).fetchone()
if res:
is_proposer = True
if is_responder:
finalize_resp(job_id=job_id,
worker_id=worker_id,
treatment=treatment)
elif is_proposer:
finalize_round(job_id=job_id,
prop_worker_id=worker_id,
treatment=treatment)
else:
app.logger.error(
f"Error: unknown worker_id: {worker_id} for job_id: {job_id}"
)
except Exception as err:
if not error_happened:
app.log_exception(err)
error_happened = True
else:
worker_judgment = payload['results']['judgments'][idx]
worker_id = worker_judgment["worker_id"]
pay_worker_bonus(job_id, worker_id, fig8)
elif signal == "unit_complete":
judgments_count = payload['judgments_count']
fig8 = FigureEight(job_id, job_config["api_key"])
for idx in range(judgments_count):
if auto_finalize == False:
worker_judgment = payload['results']['judgments'][idx]
worker_id = worker_judgment["worker_id"]
# PAY_WORKER won't pay someone twice.
pay_worker_bonus(job_id, worker_id, fig8)
#TODO: may process the whole unit here
pass
except Exception as err:
app.log_exception(err)
app.logger.debug(f"_process_judgments: {signal}, job_id: {job_id} - done")
def handle_done_no_prop(treatment, template=None, no_features=None):
app.logger.debug("handle_done")
if template is None:
template = f"txx/resp.done.html"
if no_features is None:
completion_code_base = BASE
else:
# survey should not require tasks
completion_code_base = BASE + "NF"
cookie_obj = get_cookie_obj(BASE)
worker_code_key = f"{BASE}_worker_code"
if not cookie_obj.get(BASE, None):
flash("Sorry, you are not allowed to use this service. ^_^")
return render_template("error.html")
if not cookie_obj.get(worker_code_key):
job_id = cookie_obj["job_id"]
worker_code = generate_completion_code(completion_code_base, job_id)
response = cookie_obj["response"]
response["completion_code"] = worker_code
worker_id = cookie_obj["worker_id"]
resp_result = resp_to_resp_result(response,
job_id=job_id,
worker_id=worker_id)
try:
#save_resp_result(TUBE_RES_FILENAME, resp_result)
save_result2file(
get_output_filename(BASE, job_id, treatment=treatment),
resp_result)
except Exception as err:
app.log_exception(err)
bonus_cents = 0
row_id = None
prop_row = {}
try:
prop_row = get_row_ignore_job(get_db(), job_id, worker_id,
treatment)
offer = prop_row.get("offer_final", prop_row["offer"])
row_id = prop_row.get(PK_KEY)
if offer >= response.get("min_offer_final", response["min_offer"]):
bonus_cents = MAX_GAIN - offer
else:
bonus_cents = 0
except Exception as err:
app.log_exception(err)
try:
#save_resp_result2db(get_db("RESULT"), resp_result, job_id)
save_result2db(table=get_table(base=BASE,
job_id=job_id,
schema="result",
treatment=treatment),
response_result=resp_result,
unique_fields=["worker_id"])
increase_worker_bonus(job_id=job_id,
worker_id=worker_id,
bonus_cents=bonus_cents)
close_row(get_db(), job_id, row_id, treatment)
prop_result = {
k: v
for k, v in resp_result.items() if k not in SKIP_RESP_KEYS
}
prop_result = {
k: (v if "feedback" not in k and "time" not in k else None)
for k, v in prop_result.items()
}
prop_result["resp_worker_id"] = worker_id
prop_result["worker_id"] = prop_row["prop_worker_id"]
prop_result.update(prop_row)
save_result2db(table=get_table(base="prop",
job_id=job_id,
schema="result",
treatment=treatment),
response_result=prop_result,
unique_fields=["worker_id"])
except Exception as err:
app.log_exception(err)
cookie_obj.clear()
cookie_obj[BASE] = True
cookie_obj["worker_id"] = worker_id
cookie_obj[worker_code_key] = worker_code
req_response = make_response(
render_template(template, worker_code=cookie_obj[worker_code_key]))
set_cookie_obj(req_response, BASE, cookie_obj)
return req_response
