def qualify(qualification, value, worker):
"""Assign a qualification to a worker."""
# create connection to AWS
from boto.mturk.connection import MTurkConnection
config = PsiturkConfig()
config.load_config()
aws_access_key_id = config.get('AWS Access', 'aws_access_key_id')
aws_secret_access_key = config.get('AWS Access', 'aws_secret_access_key')
conn = MTurkConnection(aws_access_key_id, aws_secret_access_key)
def get_workers_with_qualification(qualification):
"""Get workers with the given qualification."""
results = []
continue_flag = True
page = 1
while(continue_flag):
new_results = conn.get_qualifications_for_qualification_type(
qualification,
page_size=100,
page_number=page)
if(len(new_results) == 0):
continue_flag = False
else:
results.extend(new_results)
page = page + 1
return results
results = get_workers_with_qualification(qualification)
workers = [x.SubjectId for x in results]
# assign the qualification
click.echo(
"Assigning qualification {} with value {} to worker {}".format(
qualification,
value,
worker))
if worker in workers:
result = conn.update_qualification_score(qualification, worker, value)
else:
result = conn.assign_qualification(qualification, worker, value)
if result:
click.echo(result)
# print out the current set of workers with the qualification
results = get_workers_with_qualification(qualification)
click.echo("{} workers with qualification {}:".format(
len(results),
qualification))
values = [r.IntegerValue for r in results]
unique_values = list(set([r.IntegerValue for r in results]))
for v in unique_values:
click.echo("{} with value {}".format(
len([val for val in values if val == v]),
v))
def qualify(qualification, value, worker):
"""Assign a qualification to a worker."""
# create connection to AWS
from boto.mturk.connection import MTurkConnection
config = PsiturkConfig()
config.load_config()
aws_access_key_id = config.get('AWS Access', 'aws_access_key_id')
aws_secret_access_key = config.get('AWS Access', 'aws_secret_access_key')
conn = MTurkConnection(aws_access_key_id, aws_secret_access_key)
def get_workers_with_qualification(qualification):
"""Get workers with the given qualification."""
results = []
continue_flag = True
page = 1
while(continue_flag):
new_results = conn.get_qualifications_for_qualification_type(
qualification,
page_size=100,
page_number=page)
if(len(new_results) == 0):
continue_flag = False
else:
results.extend(new_results)
page = page + 1
return results
results = get_workers_with_qualification(qualification)
workers = [x.SubjectId for x in results]
# assign the qualification
click.echo(
"Assigning qualification {} with value {} to worker {}".format(
qualification,
value,
worker))
if worker in workers:
result = conn.update_qualification_score(qualification, worker, value)
else:
result = conn.assign_qualification(qualification, worker, value)
if result:
click.echo(result)
# print out the current set of workers with the qualification
results = get_workers_with_qualification(qualification)
click.echo("{} workers with qualification {}:".format(
len(results),
qualification))
values = [r.IntegerValue for r in results]
unique_values = list(set([r.IntegerValue for r in results]))
for v in unique_values:
click.echo("{} with value {}".format(
len([val for val in values if val == v]),
v))
def awaken(app, databaseurl):
"""Restore the database from a given url."""
config = PsiturkConfig()
config.load_config()
database_size = config.get('Database Parameters', 'database_size')
subprocess.call(
"heroku addons:create heroku-postgresql:{} --app {}".format(
database_size, app),
shell=True)
subprocess.call("heroku pg:wait --app {}".format(app), shell=True)
conn = boto.connect_s3(
config.get('AWS Access', 'aws_access_key_id'),
config.get('AWS Access', 'aws_secret_access_key'),
)
bucket = conn.get_bucket(app)
key = bucket.lookup('database.dump')
url = key.generate_url(expires_in=300)
cmd = "heroku pg:backups restore"
subprocess.call("{} '{}' DATABASE_URL --app {} --confirm {}".format(
cmd, url, app, app),
shell=True)
subprocess.call("heroku addons:create rediscloud:250 --app {}".format(app),
shell=True)
# Scale up the dynos.
scale_up_dynos(app)
def ad_address(mode, hit_id):
if mode == "debug":
address = '/complete'
elif mode in ["sandbox", "live"]:
CONFIG = PsiturkConfig()
CONFIG.load_config()
username = os.getenv('psiturk_access_key_id', CONFIG.get("psiTurk Access", "psiturk_access_key_id"))
password = os.getenv('psiturk_secret_access_id', CONFIG.get("psiTurk Access", "psiturk_secret_access_id"))
try:
req = requests.get('https://api.psiturk.org/api/ad/lookup/' + hit_id,
auth=(username, password))
except:
raise ValueError('api_server_not_reachable')
else:
if req.status_code == 200:
hit_address = req.json()['ad_id']
else:
raise ValueError("something here")
if mode == "sandbox":
address = 'https://sandbox.ad.psiturk.org/complete/' + str(hit_address)
elif mode == "live":
address = 'https://ad.psiturk.org/complete/' + str(hit_address)
else:
raise ValueError("Unknown mode: {}".format(mode))
return Response(dumps({"address": address}), status=200)
def __init__(self):
config = PsiturkConfig()
config.load_config()
self.access_key = config.get("psiTurk Access", "psiturk_access_key_id")
self.secret_key = config.get("psiTurk Access", "psiturk_secret_access_id")
self.local_port = config.getint("Server Parameters", "port")
self.is_open = False
self.tunnel_port = 8000 # Set by tunnel server
self.tunnel_host = "tunnel.psiturk.org"
self.tunnel_server = os.path.join(os.path.dirname(__file__), "tunnel/ngrok")
self.tunnel_config = os.path.join(os.path.dirname(__file__), "tunnel/ngrok-config")
def __init__(self, config=None):
if not config:
config = PsiturkConfig()
config.load_config()
self.access_key = config.get('psiTurk Access', 'psiturk_access_key_id')
self.secret_key = config.get('psiTurk Access', 'psiturk_secret_access_id')
self.local_port = config.getint('Server Parameters', 'port')
self.is_open = False
self.tunnel_port = 8000 # Set by tunnel server
self.tunnel_host = 'tunnel.psiturk.org'
self.tunnel_server = os.path.join(os.path.dirname(__file__),
"tunnel/ngrok")
self.tunnel_config = os.path.join(os.path.dirname(__file__),
"tunnel/ngrok-config")
def run(script=None, execute=None, testfile=None, quiet=False):
try:
using_libedit = 'libedit' in readline.__doc__
if using_libedit:
print('\n'.join([
'libedit version of readline detected.',
'readline will not be well behaved, which may cause all sorts',
'of problems for the psiTurk shell. We highly recommend installing',
'the gnu version of readline by running "sudo pip install gnureadline".',
'Note: "pip install readline" will NOT work because of how the OSX',
'pythonpath is structured.'
]))
except TypeError:
pass # pyreadline doesn't have anything for __doc__
# Drop arguments which were already processed in command_line.py
sys.argv = [sys.argv[0]]
config = PsiturkConfig()
config.load_config()
server = control.ExperimentServerController(config)
shell = PsiturkNetworkShell(config,
server,
mode=config.get('Shell Parameters',
'launch_in_mode'),
quiet=quiet)
if script:
shell.runcmds_plus_hooks([f'load {script}'])
elif execute:
shell.runcmds_plus_hooks([execute])
else:
shell.cmdloop()
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=quals)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
questions=None,
response_groups=[
'Minimal', 'HITDetail', 'HITQuestion',
'HITAssignmentSummary'
])
def backup(app):
"""Dump the database."""
dump_path = dump_database(app)
config = PsiturkConfig()
config.load_config()
conn = boto.connect_s3(
config.get('AWS Access', 'aws_access_key_id'),
config.get('AWS Access', 'aws_secret_access_key'),
)
bucket = conn.create_bucket(app,
location=boto.s3.connection.Location.DEFAULT)
k = boto.s3.key.Key(bucket)
k.key = 'database.dump'
k.set_contents_from_filename(dump_path)
url = k.generate_url(expires_in=0, query_auth=False)
log("The database backup URL is...")
print(url)
def awaken(app, databaseurl):
"""Restore the database from a given url."""
config = PsiturkConfig()
config.load_config()
database_size = config.get('Database Parameters', 'database_size')
subprocess.call(
"heroku addons:create heroku-postgresql:{} --app {}".format(
database_size,
app),
shell=True)
subprocess.call("heroku pg:wait --app {}".format(app), shell=True)
conn = boto.connect_s3(
config.get('AWS Access', 'aws_access_key_id'),
config.get('AWS Access', 'aws_secret_access_key'),
)
bucket = conn.get_bucket(app)
key = bucket.lookup('database.dump')
url = key.generate_url(expires_in=300)
cmd = "heroku pg:backups restore"
subprocess.call(
"{} '{}' DATABASE_URL --app {} --confirm {}".format(
cmd,
url,
app,
app),
shell=True)
subprocess.call(
"heroku addons:create rediscloud:250 --app {}".format(app),
shell=True)
# Scale up the dynos.
scale_up_dynos(app)
def scale_up_dynos(id):
"""Scale up the Heroku dynos."""
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
dyno_type = config.get('Server Parameters', 'dyno_type')
num_dynos_web = config.get('Server Parameters', 'num_dynos_web')
num_dynos_worker = config.get('Server Parameters', 'num_dynos_worker')
log("Scaling up the dynos...")
subprocess.call(
"heroku ps:scale web=" + str(num_dynos_web) + ":" +
str(dyno_type) + " --app " + id, shell=True)
subprocess.call(
"heroku ps:scale worker=" + str(num_dynos_worker) + ":" +
str(dyno_type) + " --app " + id, shell=True)
clock_on = config.getboolean('Server Parameters', 'clock_on')
if clock_on:
subprocess.call(
"heroku ps:scale clock=1:" + dyno_type + " --app " + id,
shell=True)
def backup(app):
"""Dump the database."""
dump_path = dump_database(app)
config = PsiturkConfig()
config.load_config()
conn = boto.connect_s3(
config.get('AWS Access', 'aws_access_key_id'),
config.get('AWS Access', 'aws_secret_access_key'),
)
bucket = conn.create_bucket(
app,
location=boto.s3.connection.Location.DEFAULT
)
k = boto.s3.key.Key(bucket)
k.key = 'database.dump'
k.set_contents_from_filename(dump_path)
url = k.generate_url(expires_in=0, query_auth=False)
log("The database backup URL is...")
print(url)
def scale_up_dynos(id):
"""Scale up the Heroku dynos."""
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
dyno_type = config.get('Server Parameters', 'dyno_type')
num_dynos_web = config.get('Server Parameters', 'num_dynos_web')
num_dynos_worker = config.get('Server Parameters', 'num_dynos_worker')
log("Scaling up the dynos...")
subprocess.call("heroku ps:scale web=" + str(num_dynos_web) + ":" +
str(dyno_type) + " --app " + id,
shell=True)
subprocess.call("heroku ps:scale worker=" + str(num_dynos_worker) + ":" +
str(dyno_type) + " --app " + id,
shell=True)
clock_on = config.getboolean('Server Parameters', 'clock_on')
if clock_on:
subprocess.call("heroku ps:scale clock=1:" + dyno_type + " --app " +
id,
shell=True)
class PsiTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk via PsiTurk."""
def __init__(self):
"""Set up the connection to MTurk and psiTurk web services."""
# load the configuration options
self.config = PsiturkConfig()
self.config.load_config()
class FakeExperimentServerController(object):
def is_server_running(self):
return 'yes'
self.server = FakeExperimentServerController()
# Get keys from environment variables or config file.
self.aws_access_key_id = os.getenv(
"aws_access_key_id",
self.config.get("AWS Access", "aws_access_key_id"))
self.aws_secret_access_key = os.getenv(
"aws_secret_access_key",
self.config.get("AWS Access", "aws_secret_access_key"))
self.aws_region = os.getenv(
"aws_region", self.config.get("AWS Access", "aws_region"))
def open_recruitment(self, n=1):
"""Open recruitment for the first HIT, unless it's already open."""
from psiturk.amt_services import MTurkServices, RDSServices
from psiturk.psiturk_shell import PsiturkNetworkShell
from psiturk.psiturk_org_services import PsiturkOrgServices
psiturk_access_key_id = os.getenv(
"psiturk_access_key_id",
self.config.get("psiTurk Access", "psiturk_access_key_id"))
psiturk_secret_access_id = os.getenv(
"psiturk_secret_access_id",
self.config.get("psiTurk Access", "psiturk_secret_access_id"))
web_services = PsiturkOrgServices(psiturk_access_key_id,
psiturk_secret_access_id)
aws_rds_services = RDSServices(self.aws_access_key_id,
self.aws_secret_access_key,
self.aws_region)
self.amt_services = MTurkServices(
self.aws_access_key_id, self.aws_secret_access_key,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
self.shell = PsiturkNetworkShell(
self.config, self.amt_services, aws_rds_services, web_services,
self.server,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
try:
participants = Participant.query.all()
assert (participants)
except Exception:
# Create the first HIT.
self.shell.hit_create(
n, self.config.get('HIT Configuration', 'base_payment'),
self.config.get('HIT Configuration', 'duration'))
else:
# HIT was already created, no need to recreate it.
print "Reject recruitment reopening: experiment has started."
def recruit_participants(self, n=1):
"""Recruit n participants."""
auto_recruit = os.environ['auto_recruit'] == 'true'
if auto_recruit:
print "Starting Dallinger's recruit_participants."
hit_id = str(
Participant.query.with_entities(
Participant.hitid).first().hitid)
print "hit_id is {}.".format(hit_id)
is_sandbox = self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode')
if is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
self.mtc.extend_hit(hit_id, assignments_increment=int(n or 0))
expiration_increment = self.config.get('HIT Configuration',
'duration')
self.mtc.extend_hit(hit_id,
expiration_increment=int(
float(expiration_increment or 0) * 3600))
else:
print(">>>> auto_recruit set to {}: recruitment suppressed".format(
auto_recruit))
def approve_hit(self, assignment_id):
"""Approve the HIT."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id, self.aws_secret_access_key,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
return self.amt_services.approve_worker(assignment_id)
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker with a bonus."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id, self.aws_secret_access_key,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
return self.amt_services.bonus_worker(assignment_id, amount, reason)
def close_recruitment(self):
"""Close recruitment."""
pass
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(
hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=None)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(
hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(
hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
title=hit_config['title'],
description=hit_config['description'],
keywords=hit_config['keywords'],
reward=hit_config['reward'],
duration=hit_config['duration'],
approval_delay=None,
questions=None,
qualifications=quals,
response_groups=[
'Minimal',
'HITDetail',
'HITQuestion',
'HITAssignmentSummary'
])
config = PsiturkConfig()
config.load_config()
myauth = PsiTurkAuthorization(config) # if you want to add a password protect route use this
##added imports for file upload support
#some libraries
from flask import send_from_directory
from os import makedirs
import os.path
from werkzeug.utils import secure_filename
from sqlalchemy.orm.exc import NoResultFound
from werkzeug.exceptions import NotFound
#status codes
from psiturk.experiment import STARTED
#upload folder
UPLOADS = config.get('Server Parameters', 'upload_folder')
# explore the Blueprint
custom_code = Blueprint('custom_code', __name__, template_folder='templates', static_folder='static')
###########################################################
# serving warm, fresh, & sweet custom, user-provided routes
# add them here
###########################################################
@custom_code.route('/_wav_upload/<uid>/<fname>', methods = ['PUT', 'GET'])
def wav_upload(uid = None, fname = None):
'''
Accepts a PUT'ed wav file, saves it and returns the file url.
Accepts a GET for a file and returns it (if it exists).
from psiturk.psiturk_statuses import *
from flask_mail import Mail, Message
# load the configuration options
config = PsiturkConfig()
config.load_config()
# if you want to add a password protect route use this
myauth = PsiTurkAuthorization(config)
# explore the Blueprint
custom_code = Blueprint('custom_code',
__name__,
template_folder='templates',
static_folder='static')
app.config.update(MAIL_SERVER=config.get('Mail Parameters', 'mail_server'),
MAIL_PORT=config.get('Mail Parameters', 'mail_port'),
MAIL_USE_SSL=config.get('Mail Parameters', 'mail_use_ssl'),
MAIL_USE_TLS=config.get('Mail Parameters', 'mail_use_tls'),
MAIL_USERNAME=config.get('Mail Parameters', 'mail_username'),
MAIL_PASSWORD=config.get('Mail Parameters', 'mail_password'))
mail = Mail(app)
###########################################################
# serving warm, fresh, & sweet custom, user-provided routes
# add them here
###########################################################
leading_4_spaces = re.compile('^ ')
myauth = PsiTurkAuthorization(config)
# Import the blueprint
dashboard = Blueprint('dashboard',
__name__,
template_folder='templates',
static_folder='static',
url_prefix='/dashboard')
# ---------------------------------------------------------------------------- #
# CONSTANTS #
# ---------------------------------------------------------------------------- #
# Get the advanced qualifications for info dumping
advanced_quals_path = config.get('HIT Configuration',
'advanced_quals_path',
fallback=None)
advanced_qualifications = []
if advanced_quals_path:
with open(advanced_quals_path) as f:
advanced_qualifications = json.load(f)
if not isinstance(advanced_qualifications, list):
raise PsiturkException(
message=
f'JSON file "{advanced_quals_path}" must be a list of dicts')
else:
for el in advanced_qualifications:
if not isinstance(el, dict):
raise PsiturkException(
message=
f'JSON file "{advanced_quals_path}" must be a list of dicts'
def configure_hit(self, hit_config):
''' Configure HIT '''
# Qualification:
quals = []
quals.append(dict(
QualificationTypeId=PERCENT_ASSIGNMENTS_APPROVED_QUAL_ID,
Comparator='GreaterThanOrEqualTo',
IntegerValues=[int(hit_config['approve_requirement'])]
))
quals.append(dict(
QualificationTypeId=NUMBER_HITS_APPROVED_QUAL_ID,
Comparator='GreaterThanOrEqualTo',
IntegerValues=[int(hit_config['number_hits_approved'])]
))
if hit_config['require_master_workers']:
master_qualId = MASTERS_SANDBOX_QUAL_ID if self.is_sandbox else MASTERS_QUAL_ID
quals.append(dict(
QualificationTypeId=master_qualId,
Comparator='Exists'
))
if hit_config['us_only']:
quals.append(dict(
QualificationTypeId=LOCALE_QUAL_ID,
Comparator='EqualTo',
LocaleValues=[{'Country': 'US'}]
))
# Create a HIT type for this HIT.
hit_type = self.mtc.create_hit_type(
Title=hit_config['title'],
Description=hit_config['description'],
Reward=str(hit_config['reward']),
AssignmentDurationInSeconds=int(hit_config['duration'].total_seconds()),
Keywords=hit_config['keywords'],
QualificationRequirements=quals)
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
# TODO: not sure if this works. Can't find documentation in PsiTurk or MTurk
self.mtc.update_notification_settings(
HitTypeId=hit_type['HITTypeId'],
Notification=dict(
Destination=url,
Transport='REST',
Version=NOTIFICATION_VERSION,
EventTypes=all_event_types,
),
)
except Exception as e:
pass
schema_url = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"
template = '<ExternalQuestion xmlns="%(schema_url)s"><ExternalURL>%%(external_url)s</ExternalURL><FrameHeight>%%(frame_height)s</FrameHeight></ExternalQuestion>' % vars()
question = template % dict(
external_url=hit_config['ad_location'],
frame_height=600,
)
# Specify all the HIT parameters
self.param_dict = dict(
HITTypeId=hit_type['HITTypeId'],
Question=question,
LifetimeInSeconds=int(hit_config['lifetime'].total_seconds()),
MaxAssignments=hit_config['max_assignments'],
# TODO
# ResponseGroups=[
# 'Minimal',
# 'HITDetail',
# 'HITQuestion',
# 'HITAssignmentSummary'
# ]
)
def deploy_sandbox_shared_setup(verbose=True, app=None, web_procs=1):
"""Set up Git, push to Heroku, and launch the app."""
out = None if verbose else open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
ensure_heroku_logged_in()
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
cmds = [
"git init", "git add --all", 'git commit -m "Experiment ' + id + '"'
]
for cmd in cmds:
subprocess.call(cmd, stdout=out, shell=True)
time.sleep(0.5)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
subprocess.call(
"heroku apps:create " + id +
" --buildpack https://github.com/thenovices/heroku-buildpack-scipy",
stdout=out,
shell=True)
database_size = config.get('Database Parameters', 'database_size')
try:
if config.getboolean('Easter eggs', 'whimsical'):
whimsical = "true"
else:
whimsical = "false"
except:
whimsical = "false"
# Set up postgres database and AWS/psiTurk environment variables.
cmds = [
"heroku addons:create heroku-postgresql:{}".format(database_size),
"heroku pg:wait",
"heroku addons:create rediscloud:250",
"heroku addons:create papertrail",
"heroku config:set HOST=" + id + ".herokuapp.com",
"heroku config:set aws_access_key_id=" +
config.get('AWS Access', 'aws_access_key_id'),
"heroku config:set aws_secret_access_key=" +
config.get('AWS Access', 'aws_secret_access_key'),
"heroku config:set aws_region=" +
config.get('AWS Access', 'aws_region'),
"heroku config:set psiturk_access_key_id=" +
config.get('psiTurk Access', 'psiturk_access_key_id'),
"heroku config:set psiturk_secret_access_id=" +
config.get('psiTurk Access', 'psiturk_secret_access_id'),
"heroku config:set auto_recruit=" +
config.get('Experiment Configuration', 'auto_recruit'),
"heroku config:set wallace_email_username="******"heroku config:set wallace_email_key=" +
config.get('Email Access', 'wallace_email_password'),
"heroku config:set heroku_email_address=" +
config.get('Heroku Access', 'heroku_email_address'),
"heroku config:set heroku_password="******"heroku config:set whimsical=" + whimsical,
]
for cmd in cmds:
subprocess.call(cmd + " --app " + id, stdout=out, shell=True)
# Set the notification URL in the cofig file to the notifications URL.
config.set("Server Parameters", "notification_url",
"http://" + id + ".herokuapp.com/notifications")
# Set the database URL in the config file to the newly generated one.
log("Saving the URL of the postgres database...")
db_url = subprocess.check_output("heroku config:get DATABASE_URL --app " +
id,
shell=True)
config.set("Database Parameters", "database_url", db_url.rstrip())
subprocess.call("git add config.txt", stdout=out, shell=True),
time.sleep(0.25)
subprocess.call('git commit -m "Save URLs for database and notifications"',
stdout=out,
shell=True)
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
subprocess.call("git push heroku HEAD:master",
stdout=out,
stderr=out,
shell=True)
scale_up_dynos(id)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on MTurk...")
subprocess.call('curl --data "" http://{}.herokuapp.com/launch'.format(id),
shell=True)
time.sleep(8)
url = subprocess.check_output("heroku logs --app " + id + " | sort | " +
"sed -n 's|.*URL:||p'",
shell=True)
log("URLs:")
click.echo(url)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup_experiment(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Set the mode to debug.
config.set("Experiment Configuration", "mode", "debug")
config.set("Shell Parameters", "launch_in_sandbox_mode", "true")
config.set("Server Parameters", "logfile",
os.path.join(cwd, config.get("Server Parameters", "logfile")))
# Swap in the HotAirRecruiter
os.rename("wallace_experiment.py", "wallace_experiment_tmp.py")
with open("wallace_experiment_tmp.py", "r+") as f:
with open("wallace_experiment.py", "w+") as f2:
f2.write("from wallace.recruiters import HotAirRecruiter\n")
for idx, line in enumerate(f):
if re.search("\s*self.recruiter = (.*)", line):
p = line.partition("self.recruiter =")
f2.write(p[0] + p[1] + ' HotAirRecruiter\n')
else:
f2.write(line)
os.remove("wallace_experiment_tmp.py")
# Set environment variables.
aws_vars = ['aws_access_key_id', 'aws_secret_access_key', 'aws_region']
for var in aws_vars:
if var not in os.environ:
os.environ[var] = config.get('AWS Access', var)
pt_vars = ['psiturk_access_key_id', 'psiturk_secret_access_id']
for var in pt_vars:
if var not in os.environ:
os.environ[var] = config.get('psiTurk Access', var)
if "HOST" not in os.environ:
os.environ["HOST"] = config.get('Server Parameters', 'host')
# Start up the local server
log("Starting up the server...")
# Try opening the psiTurk shell.
try:
p = pexpect.spawn("psiturk")
p.expect_exact("]$")
p.sendline("server on")
p.expect_exact("Experiment server launching...")
# Launche the experiment.
time.sleep(4)
host = config.get("Server Parameters", "host")
port = config.get("Server Parameters", "port")
subprocess.call('curl --data "" http://{}:{}/launch'.format(
host, port),
shell=True)
log("Here's the psiTurk shell...")
p.interact()
except Exception:
click.echo("\nCouldn't open psiTurk shell. Internet connection okay?")
log("Completed debugging of experiment " + id + ".")
os.chdir(cwd)
"Fix the errors and then try running 'wallace verify'.")
# Verify that the Postgres server is running.
try:
psycopg2.connect(database="x", user="******", password="******")
except psycopg2.OperationalError, e:
if "could not connect to server" in str(e):
raise RuntimeError("The Postgres server isn't running.")
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Check that the version of Wallace specified in the config file is the one
# that we are currently running.
wallace_version = config.get('Experiment Configuration', 'wallace_version')
this_version = pkg_resources.require("wallace")[0].version
if wallace_version != this_version:
raise AssertionError(
"You are using Wallace v" + this_version + ", "
"but the experiment requires v" + wallace_version)
# Generate a unique id for this experiment.
id = "w" + str(uuid.uuid4())[0:28]
log("Running as experiment " + id + "...")
# Copy this directory into a temporary folder, ignoring .git
dst = os.path.join(tempfile.mkdtemp(), id)
to_ignore = shutil.ignore_patterns(
".git/*",
"*.db",
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Set the mode to debug.
config.set("Experiment Configuration", "mode", "debug")
config.set("Shell Parameters", "launch_in_sandbox_mode", "true")
config.set(
"Server Parameters",
"logfile",
os.path.join(cwd, config.get("Server Parameters", "logfile")))
# Swap in the HotAirRecruiter
os.rename("wallace_experiment.py", "wallace_experiment_tmp.py")
with open("wallace_experiment_tmp.py", "r+") as f:
with open("wallace_experiment.py", "w+") as f2:
f2.write("from wallace.recruiters import HotAirRecruiter\n")
for idx, line in enumerate(f):
if re.search("\s*self.recruiter = (.*)", line):
p = line.partition("self.recruiter =")
f2.write(p[0] + p[1] + ' HotAirRecruiter\n')
else:
f2.write(line)
os.remove("wallace_experiment_tmp.py")
# Set environment variables.
aws_vars = ['aws_access_key_id', 'aws_secret_access_key', 'aws_region']
for var in aws_vars:
if var not in os.environ:
os.environ[var] = config.get('AWS Access', var)
pt_vars = ['psiturk_access_key_id', 'psiturk_secret_access_id']
for var in pt_vars:
if var not in os.environ:
os.environ[var] = config.get('psiTurk Access', var)
if "HOST" not in os.environ:
os.environ["HOST"] = config.get('Server Parameters', 'host')
# Start up the local server
log("Starting up the server...")
# Try opening the psiTurk shell.
try:
p = pexpect.spawn("psiturk")
p.expect_exact("]$")
p.sendline("server on")
p.expect_exact("Experiment server launching...")
# Launche the experiment.
time.sleep(4)
host = config.get("Server Parameters", "host")
port = config.get("Server Parameters", "port")
subprocess.call(
'curl --data "" http://{}:{}/launch'.format(host, port),
shell=True)
log("Here's the psiTurk shell...")
p.interact()
except Exception:
print "\nCouldn't open the psiTurk shell. Internet connection okay?"
log("Completed debugging of experiment " + id + ".")
os.chdir(cwd)
from psiturk.models import Participant
from json import dumps, loads
# load the configuration options
config = PsiturkConfig()
config.load_config()
myauth = PsiTurkAuthorization(config) # if you want to add a password protect route use this
# explore the Blueprint
custom_code = Blueprint('custom_code', __name__, template_folder='templates', static_folder='static')
recaptcha_secret = <<<SECRET RECAPTCHA KEY GOES HERE>>>
# for bonuses
amt_services = MTurkServices(
config.get('AWS Access', 'aws_access_key_id'), \
config.get('AWS Access', 'aws_secret_access_key'),
config.getboolean('Shell Parameters', 'launch_in_sandbox_mode'))
# for bonuses
class WorkerAlreadyCredited(Exception):
pass
class CreditTransactionError(Exception):
pass
class WorkerNotRegistered(Exception):
pass
class NoAMTConnection(Exception):
pass
def configure_hit(self, hit_config):
''' Configure HIT '''
# Qualification:
quals = []
quals.append(dict(
QualificationTypeId=PERCENT_ASSIGNMENTS_APPROVED_QUAL_ID,
Comparator='GreaterThanOrEqualTo',
IntegerValues=[int(hit_config['approve_requirement'])]
))
quals.append(dict(
QualificationTypeId=NUMBER_HITS_APPROVED_QUAL_ID,
Comparator='GreaterThanOrEqualTo',
IntegerValues=[int(hit_config['number_hits_approved'])]
))
if hit_config['require_master_workers']:
master_qualId = MASTERS_SANDBOX_QUAL_ID if self.is_sandbox else MASTERS_QUAL_ID
quals.append(dict(
QualificationTypeId=master_qualId,
Comparator='Exists'
))
if hit_config['us_only']:
quals.append(dict(
QualificationTypeId=LOCALE_QUAL_ID,
Comparator='EqualTo',
LocaleValues=[{'Country': 'US'}]
))
# Create a HIT type for this HIT.
hit_type = self.mtc.create_hit_type(
Title=hit_config['title'],
Description=hit_config['description'],
Reward=str(hit_config['reward']),
AssignmentDurationInSeconds=int(
hit_config['duration'].total_seconds()),
Keywords=hit_config['keywords'],
QualificationRequirements=quals)
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
# TODO: not sure if this works. Can't find documentation in PsiTurk or MTurk
self.mtc.update_notification_settings(
HitTypeId=hit_type['HITTypeId'],
Notification=dict(
Destination=url,
Transport='REST',
Version=NOTIFICATION_VERSION,
EventTypes=all_event_types,
),
)
except Exception as e:
pass
schema_url = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"
template = '<ExternalQuestion xmlns="%(schema_url)s"><ExternalURL>%%(external_url)s</ExternalURL><FrameHeight>%%(frame_height)s</FrameHeight></ExternalQuestion>' % vars()
question = template % dict(
external_url=hit_config['ad_location'],
frame_height=600,
)
# Specify all the HIT parameters
self.param_dict = dict(
HITTypeId=hit_type['HITTypeId'],
Question=question,
LifetimeInSeconds=int(hit_config['lifetime'].total_seconds()),
MaxAssignments=hit_config['max_assignments'],
# TODO
# ResponseGroups=[
# 'Minimal',
# 'HITDetail',
# 'HITQuestion',
# 'HITAssignmentSummary'
# ]
)
def deploy_sandbox_shared_setup(verbose=True, web_procs=1):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
OUT = None
else:
OUT = open(os.devnull, 'w')
(id, tmp) = setup(debug=False, verbose=verbose)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
ensure_heroku_logged_in()
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
cmds = ["git init",
"git add --all",
'git commit -m "Experiment ' + id + '"']
for cmd in cmds:
subprocess.call(cmd, stdout=OUT, shell=True)
time.sleep(0.5)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
subprocess.call(
"heroku apps:create " + id +
" --buildpack https://github.com/thenovices/heroku-buildpack-scipy",
stdout=OUT,
shell=True)
database_size = config.get('Database Parameters', 'database_size')
# Set up postgres database and AWS/psiTurk environment variables.
cmds = [
"heroku addons:create heroku-postgresql:{}".format(database_size),
"heroku pg:wait",
"heroku addons:create redistogo:small",
"heroku addons:create papertrail",
"heroku config:set HOST=" +
id + ".herokuapp.com",
"heroku config:set aws_access_key_id=" +
config.get('AWS Access', 'aws_access_key_id'),
"heroku config:set aws_secret_access_key=" +
config.get('AWS Access', 'aws_secret_access_key'),
"heroku config:set aws_region=" +
config.get('AWS Access', 'aws_region'),
"heroku config:set psiturk_access_key_id=" +
config.get('psiTurk Access', 'psiturk_access_key_id'),
"heroku config:set psiturk_secret_access_id=" +
config.get('psiTurk Access', 'psiturk_secret_access_id'),
"heroku config:set auto_recruit=" +
config.get('Experiment Configuration', 'auto_recruit'),
]
for cmd in cmds:
subprocess.call(cmd + " --app " + id, stdout=OUT, shell=True)
# Set the notification URL in the cofig file to the notifications URL.
config.set(
"Server Parameters",
"notification_url",
"http://" + id + ".herokuapp.com/notifications")
# Set the database URL in the config file to the newly generated one.
log("Saving the URL of the postgres database...")
db_url = subprocess.check_output(
"heroku config:get DATABASE_URL --app " + id, shell=True)
config.set("Database Parameters", "database_url", db_url.rstrip())
subprocess.call("git add config.txt", stdout=OUT, shell=True),
time.sleep(0.25)
subprocess.call(
'git commit -m "Save URLs for database and notifications"',
stdout=OUT,
shell=True)
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
subprocess.call("git push heroku HEAD:master", stdout=OUT,
stderr=OUT, shell=True)
dyno_type = config.get('Server Parameters', 'dyno_type')
num_dynos_web = config.get('Server Parameters', 'num_dynos_web')
num_dynos_worker = config.get('Server Parameters', 'num_dynos_worker')
log("Starting up the web server...")
subprocess.call("heroku ps:scale web=" + str(num_dynos_web) + ":" +
str(dyno_type) + " --app " + id, stdout=OUT, shell=True)
subprocess.call("heroku ps:scale worker=" + str(num_dynos_worker) + ":" +
str(dyno_type) + " --app " + id, stdout=OUT, shell=True)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on MTurk...")
subprocess.call(
'curl --data "" http://{}.herokuapp.com/launch'.format(id),
shell=True)
time.sleep(8)
url = subprocess.check_output("heroku logs --app " + id + " | sort | " +
"sed -n 's|.*URL:||p'", shell=True)
log("URLs:")
print url
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
class PsiTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk."""
def __init__(self):
"""Set up the connection to MTurk and psiTurk web services."""
# load the configuration options
self.config = PsiturkConfig()
self.config.load_config()
class FakeExperimentServerController(object):
def is_server_running(self):
return "yes"
self.server = FakeExperimentServerController()
# Get keys from environment variables or config file.
self.aws_access_key_id = os.getenv("aws_access_key_id", self.config.get("AWS Access", "aws_access_key_id"))
self.aws_secret_access_key = os.getenv(
"aws_secret_access_key", self.config.get("AWS Access", "aws_secret_access_key")
)
self.aws_region = os.getenv("aws_region", self.config.get("AWS Access", "aws_region"))
def open_recruitment(self, n=1):
"""Open recruitment for the first HIT, unless it's already open."""
from psiturk.amt_services import MTurkServices, RDSServices
from psiturk.psiturk_shell import PsiturkNetworkShell
from psiturk.psiturk_org_services import PsiturkOrgServices
psiturk_access_key_id = os.getenv(
"psiturk_access_key_id", self.config.get("psiTurk Access", "psiturk_access_key_id")
)
psiturk_secret_access_id = os.getenv(
"psiturk_secret_access_id", self.config.get("psiTurk Access", "psiturk_secret_access_id")
)
web_services = PsiturkOrgServices(psiturk_access_key_id, psiturk_secret_access_id)
aws_rds_services = RDSServices(self.aws_access_key_id, self.aws_secret_access_key, self.aws_region)
self.amt_services = MTurkServices(
self.aws_access_key_id,
self.aws_secret_access_key,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
self.shell = PsiturkNetworkShell(
self.config,
self.amt_services,
aws_rds_services,
web_services,
self.server,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
try:
participants = Participant.query.all()
assert participants
except Exception:
# Create the first HIT.
self.shell.hit_create(
n,
self.config.get("HIT Configuration", "base_payment"),
self.config.get("HIT Configuration", "duration"),
)
else:
# HIT was already created, no need to recreate it.
print "Reject recruitment reopening: experiment has started."
def recruit_participants(self, n=1):
"""Extend the HIT to recruit more people."""
auto_recruit = os.environ["auto_recruit"] == "true"
if auto_recruit:
print "Starting Wallace's recruit_participants."
hit_id = str(Participant.query.with_entities(Participant.hitid).first().hitid)
print "hit_id is {}.".format(hit_id)
is_sandbox = self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode")
if is_sandbox:
host = "mechanicalturk.sandbox.amazonaws.com"
else:
host = "mechanicalturk.amazonaws.com"
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host
)
self.mtc = MTurkConnection(**mturkparams)
self.mtc.extend_hit(hit_id, assignments_increment=int(n or 0))
expiration_increment = self.config.get("HIT Configuration", "duration")
self.mtc.extend_hit(hit_id, expiration_increment=int(float(expiration_increment or 0) * 3600))
else:
print (">>>> auto_recruit set to {}: recruitment suppressed".format(auto_recruit))
def approve_hit(self, assignment_id):
"""Approve the HIT."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id,
self.aws_secret_access_key,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
return self.amt_services.approve_worker(assignment_id)
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker with a bonus."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id,
self.aws_secret_access_key,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
return self.amt_services.bonus_worker(assignment_id, amount, reason)
def close_recruitment(self):
"""Close recruitment."""
pass
from psiturk.psiturk_statuses import *
# load the configuration options
config = PsiturkConfig()
config.load_config()
# if you want to add a password protect route use this
myauth = PsiTurkAuthorization(config)
# explore the Blueprint
custom_code = Blueprint('custom_code',
__name__,
template_folder='templates',
static_folder='static')
try:
if config.get('Mail Parameters', 'mail_enabled'):
from flask_mail import Mail, Message
app.config.update(MAIL_SERVER=config.get('Mail Parameters',
'mail_server'),
MAIL_PORT=config.get('Mail Parameters', 'mail_port'),
MAIL_USE_SSL=config.get('Mail Parameters',
'mail_use_ssl'),
MAIL_USE_TLS=config.get('Mail Parameters',
'mail_use_tls'),
MAIL_USERNAME=config.get('Mail Parameters',
'mail_username'),
MAIL_PASSWORD=config.get('Mail Parameters',
'mail_password'))
mail = Mail(app)
except Exception as error:
app.logger.error('[flask_mail] %s' % error)
