def hibernate(app):
"""Pause an experiment and remove costly resources."""
verify_id(app)
log("The database backup URL is...")
backup_url = data.backup(app)
log(backup_url)
log("Scaling down the web servers...")
for process in ["web", "worker"]:
subprocess.check_call([
"heroku", "ps:scale", "{}=0".format(process), "--app",
app_name(app)
])
subprocess.call(["heroku", "ps:scale", "clock=0", "--app", app_name(app)])
log("Removing addons...")
addons = [
"heroku-postgresql",
# "papertrail",
"heroku-redis",
]
for addon in addons:
subprocess.check_call([
"heroku", "addons:destroy", addon, "--app",
app_name(app), "--confirm",
app_name(app)
])
def dump_database(id):
"""Dump the database to a temporary directory."""
tmp_dir = tempfile.mkdtemp()
current_dir = os.getcwd()
os.chdir(tmp_dir)
FNULL = open(os.devnull, 'w')
subprocess.call(
["heroku", "pg:backups:capture", "--app",
heroku.app_name(id)],
stdout=FNULL,
stderr=FNULL)
subprocess.call(
["heroku", "pg:backups:download", "--app",
heroku.app_name(id)],
stdout=FNULL,
stderr=FNULL)
for filename in os.listdir(tmp_dir):
if filename.startswith("latest.dump"):
os.rename(filename, "database.dump")
os.chdir(current_dir)
return os.path.join(tmp_dir, "database.dump")
def dump_database(id):
"""Backup the Postgres database locally."""
log("Generating a backup of the database on Heroku...")
dump_filename = "data.dump"
data_directory = "data"
dump_dir = os.path.join(data_directory, id)
if not os.path.exists(dump_dir):
os.makedirs(dump_dir)
subprocess.call(
"heroku pg:backups capture --app " + app_name(id), shell=True)
backup_url = subprocess.check_output(
"heroku pg:backups public-url --app " + app_name(id), shell=True)
backup_url = backup_url.replace('"', '').rstrip()
backup_url = re.search("https:.*", backup_url).group(0)
print(backup_url)
log("Downloading the backup...")
dump_path = os.path.join(dump_dir, dump_filename)
with open(dump_path, 'wb') as file:
subprocess.call(['curl', '-o', dump_path, backup_url], stdout=file)
return dump_path
def destroy(app):
"""Tear down an experiment server."""
subprocess.call(
"heroku destroy --app {} --confirm {}".format(
app_name(app),
app_name(app)
),
shell=True,
)
def destroy_server(app):
"""Tear down an experiment server."""
subprocess.check_call([
"heroku",
"destroy",
"--app",
app_name(app),
"--confirm",
app_name(app),
])
def logs(app):
"""Show the logs."""
if app is None:
raise TypeError("Select an experiment using the --app flag.")
else:
subprocess.call(
"heroku addons:open papertrail --app " + app_name(app),
shell=True)
def logs(app):
"""Show the logs."""
verify_id(app)
try:
subprocess.check_call(
["heroku", "addons:open", "papertrail", "--app",
app_name(app)])
except subprocess.CalledProcessError as e:
print("Invalid experiment ID. {}".format(e))
def copy_heroku_to_local(id):
"""Copy a Heroku database locally."""
try:
subprocess.call([
"dropdb",
heroku.app_name(id),
])
except Exception:
pass
subprocess.call([
"heroku",
"pg:pull",
"DATABASE_URL",
heroku.app_name(id),
"--app",
heroku.app_name(id),
])
def summary(app):
"""Print a summary of a deployed app's status."""
r = requests.get('https://{}.herokuapp.com/summary'.format(app_name(app)))
summary = r.json()['summary']
click.echo("\nstatus \t| count")
click.echo("----------------")
for s in summary:
click.echo("{}\t| {}".format(s[0], s[1]))
num_101s = sum([s[1] for s in summary if s[0] == 101])
num_10xs = sum([s[1] for s in summary if s[0] >= 100])
if num_10xs > 0:
click.echo("\nYield: {:.2%}".format(1.0 * num_101s / num_10xs))
def awaken(app, databaseurl):
"""Restore the database from a given url."""
id = app
config = get_config()
config.load_config()
database_size = config.get('database_size')
subprocess.check_call(
"heroku addons:create heroku-postgresql:{} --app {}".format(
database_size, app_name(id)),
shell=True)
subprocess.check_call("heroku pg:wait --app {}".format(app_name(id)),
shell=True)
bucket = data.user_s3_bucket()
key = bucket.lookup('{}.dump'.format(id))
url = key.generate_url(expires_in=300)
cmd = "heroku pg:backups restore"
subprocess.check_call("{} '{}' DATABASE_URL --app {} --confirm {}".format(
cmd, url, app_name(id), app_name(id)),
shell=True)
subprocess.check_call(
"heroku addons:create heroku-redis:premium-0 --app {}".format(
app_name(id)),
shell=True)
# Scale up the dynos.
log("Scaling up the dynos...")
scale_up_dynos(app_name(id))
def export(id, local=False, scrub_pii=False):
"""Export data from an experiment."""
print("Preparing to export the data...")
copy_heroku_to_local(id)
# Create the data package if it doesn't already exist.
subdata_path = os.path.join("data", id, "data")
try:
os.makedirs(subdata_path)
except OSError as e:
if e.errno != errno.EEXIST or not os.path.isdir(subdata_path):
raise
# Copy in the data.
copy_local_to_csv(heroku.app_name(id), subdata_path, scrub_pii=scrub_pii)
# Copy the experiment code into a code/ subdirectory.
try:
shutil.copyfile(os.path.join("snapshots", id + "-code.zip"),
os.path.join("data", id, id + "-code.zip"))
except Exception:
pass
# Copy in the DATA readme.
# open(os.path.join(id, "README.txt"), "a").close()
# Save the experiment id.
with open(os.path.join("data", id, "experiment_id.md"), "a+") as file:
file.write(id)
print("Zipping up the package...")
shutil.make_archive(os.path.join("data", id + "-data"), "zip",
os.path.join("data", id))
shutil.rmtree(os.path.join("data", id))
print("Done. Data available in {}-data.zip".format(id))
cwd = os.getcwd()
data_filename = '{}-data.zip'.format(id)
path_to_data = os.path.join(cwd, "data", data_filename)
# Backup data on S3.
k = Key(user_s3_bucket())
k.key = data_filename
k.set_contents_from_filename(path_to_data)
return path_to_data
def experiment_completed(self):
"""Checks the current state of the experiment to see whether it has
completed"""
status_url = 'https://{}.herokuapp.com/summary'.format(
app_name(self.app_id))
data = {}
try:
resp = requests.get(status_url)
data = resp.json()
except (ValueError, requests.exceptions.RequestException):
logger.exception('Error fetching experiment status.')
logger.debug('Current application state: {}'.format(data))
return data.get('completed', False)
def summary(app):
"""Print a summary of a deployed app's status."""
verify_id(app)
r = requests.get('https://{}.herokuapp.com/summary'.format(app_name(app)))
summary = r.json()['summary']
click.echo("\nstatus | count")
click.echo("-----------------")
for s in summary:
click.echo("{:<10}| {}".format(s[0], s[1]))
num_approved = sum([s[1] for s in summary if s[0] == u"approved"])
num_not_working = sum([s[1] for s in summary if s[0] != u"working"])
if num_not_working > 0:
the_yield = 1.0 * num_approved / num_not_working
click.echo("\nYield: {:.2%}".format(the_yield))
def awaken(app, databaseurl):
"""Restore the database from a given url."""
id = app
config = PsiturkConfig()
config.load_config()
database_size = config.get('Database Parameters', 'database_size')
subprocess.call(
"heroku addons:create heroku-postgresql:{} --app {}".format(
database_size,
app_name(id)),
shell=True)
subprocess.call(
"heroku pg:wait --app {}".format(app_name(id)),
shell=True)
conn = boto.connect_s3(
config.get('AWS Access', 'aws_access_key_id'),
config.get('AWS Access', 'aws_secret_access_key'),
)
bucket = conn.get_bucket(id)
key = bucket.lookup('database.dump')
url = key.generate_url(expires_in=300)
cmd = "heroku pg:backups restore"
subprocess.call(
"{} '{}' DATABASE_URL --app {} --confirm {}".format(
cmd,
url,
app_name(id),
app_name(id)),
shell=True)
subprocess.call(
"heroku addons:create heroku-redis:premium-0 --app {}".format(app_name(id)),
shell=True)
# Scale up the dynos.
log("Scaling up the dynos...")
scale_up_dynos(app_name(id))
def awaken(app, databaseurl):
"""Restore the database from a given url."""
id = app
config = get_config()
config.load()
subprocess.check_call([
"heroku",
"addons:create",
"heroku-postgresql:{}".format(config.get('database_size')),
"--app",
app_name(id),
])
bucket = data.user_s3_bucket()
key = bucket.lookup('{}.dump'.format(id))
url = key.generate_url(expires_in=300)
time.sleep(60)
subprocess.check_call(["heroku", "pg:wait", "--app", app_name(id)])
time.sleep(10)
subprocess.check_call([
"heroku", "addons:create", "heroku-redis:premium-0", "--app",
app_name(id)
])
subprocess.check_call([
"heroku",
"pg:backups:restore",
"{}".format(url),
"DATABASE_URL",
"--app",
app_name(id),
"--confirm",
app_name(id),
])
# Scale up the dynos.
log("Scaling up the dynos...")
heroku.scale_up_dynos(app_name(id))
def test_heroku_app_name(self):
id = "8fbe62f5-2e33-4274-8aeb-40fc3dd621a0"
assert(len(app_name(id)) < 30)
def test_app_name(self, heroku):
dallinger_uid = "8fbe62f5-2e33-4274-8aeb-40fc3dd621a0"
assert heroku.app_name(dallinger_uid) == "dlgr-8fbe62f5-2e33-4274"
def deploy_sandbox_shared_setup(verbose=True, app=None, web_procs=1):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
cmds = ["git init",
"git add --all",
'git commit -m "Experiment ' + id + '"']
for cmd in cmds:
subprocess.call(cmd, stdout=out, shell=True)
time.sleep(0.5)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
subprocess.call(
"heroku apps:create " + app_name(id) +
" --buildpack https://github.com/thenovices/heroku-buildpack-scipy",
stdout=out,
shell=True)
database_size = config.get('Database Parameters', 'database_size')
try:
if config.getboolean('Easter eggs', 'whimsical'):
whimsical = "true"
else:
whimsical = "false"
except:
whimsical = "false"
# Set up postgres database and AWS/psiTurk environment variables.
cmds = [
"heroku addons:create heroku-postgresql:{}".format(database_size),
"heroku pg:wait",
"heroku addons:create heroku-redis:premium-0",
"heroku addons:create papertrail",
"heroku config:set HOST=" +
app_name(id) + ".herokuapp.com",
"heroku config:set aws_access_key_id=" +
config.get('AWS Access', 'aws_access_key_id'),
"heroku config:set aws_secret_access_key=" +
config.get('AWS Access', 'aws_secret_access_key'),
"heroku config:set aws_region=" +
config.get('AWS Access', 'aws_region'),
"heroku config:set psiturk_access_key_id=" +
config.get('psiTurk Access', 'psiturk_access_key_id'),
"heroku config:set psiturk_secret_access_id=" +
config.get('psiTurk Access', 'psiturk_secret_access_id'),
"heroku config:set auto_recruit=" +
config.get('Experiment Configuration', 'auto_recruit'),
"heroku config:set dallinger_email_username="******"heroku config:set dallinger_email_key=" +
config.get('Email Access', 'dallinger_email_password'),
"heroku config:set heroku_email_address=" +
config.get('Heroku Access', 'heroku_email_address'),
"heroku config:set heroku_password="******"heroku config:set whimsical=" + whimsical,
]
for cmd in cmds:
subprocess.call(
cmd + " --app " + app_name(id), stdout=out, shell=True)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
redis_URL = subprocess.check_output(
"heroku config:get REDIS_URL --app {}".format(app_name(id)),
shell=True
)
ready = False
while not ready:
r = redis.from_url(redis_URL)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
pass
# Set the notification URL in the cofig file to the notifications URL.
config.set(
"Server Parameters",
"notification_url",
"http://" + app_name(id) + ".herokuapp.com/notifications")
# Set the database URL in the config file to the newly generated one.
log("Saving the URL of the postgres database...")
db_url = subprocess.check_output(
"heroku config:get DATABASE_URL --app " + app_name(id), shell=True)
config.set("Database Parameters", "database_url", db_url.rstrip())
subprocess.call("git add config.txt", stdout=out, shell=True),
time.sleep(0.25)
subprocess.call(
'git commit -m "Save URLs for database and notifications"',
stdout=out,
shell=True)
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
subprocess.call("git push heroku HEAD:master", stdout=out,
stderr=out, shell=True)
log("Scaling up the dynos...")
scale_up_dynos(app_name(id))
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on MTurk...")
subprocess.call(
'curl --data "" http://{}.herokuapp.com/launch'.format(app_name(id)),
shell=True)
time.sleep(8)
url = subprocess.check_output(
"heroku logs --app " + app_name(id) + " | sort | " +
"sed -n 's|.*URL:||p'", shell=True)
log("URLs:")
click.echo(url)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
def export(app, local):
"""Export the data."""
print_header()
log("Preparing to export the data...")
id = str(app)
subdata_path = os.path.join("data", id, "data")
# Create the data package
os.makedirs(subdata_path)
# Copy the experiment code into a code/ subdirectory
try:
shutil.copyfile(
os.path.join("snapshots", id + "-code.zip"),
os.path.join("data", id, id + "-code.zip")
)
except:
pass
# Copy in the DATA readme.
# open(os.path.join(id, "README.txt"), "a").close()
# Save the experiment id.
with open(os.path.join("data", id, "experiment_id.md"), "a+") as file:
file.write(id)
if not local:
# Export the logs
subprocess.call(
"heroku logs " +
"-n 10000 > " + os.path.join("data", id, "server_logs.md") +
" --app " + app_name(id),
shell=True)
dump_path = dump_database(id)
subprocess.call(
"pg_restore --verbose --no-owner --clean -d dallinger " +
os.path.join("data", id, "data.dump"),
shell=True)
all_tables = [
"node",
"network",
"vector",
"info",
"transformation",
"transmission",
"participant",
"notification",
"question"
]
for table in all_tables:
subprocess.call(
"psql -d dallinger --command=\"\\copy " + table + " to \'" +
os.path.join(subdata_path, table) + ".csv\' csv header\"",
shell=True)
if not local:
os.remove(dump_path)
log("Zipping up the package...")
shutil.make_archive(
os.path.join("data", id + "-data"),
"zip",
os.path.join("data", id)
)
shutil.rmtree(os.path.join("data", id))
log("Done. Data available in " + str(id) + ".zip")
def deploy_sandbox_shared_setup(verbose=True,
app=None,
web_procs=1,
exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False,
verbose=verbose,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
config = get_config()
if config.get("mode") == u"live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
subprocess.check_call(["git", "init"], stdout=out)
subprocess.check_call(["git", "add", "--all"], stdout=out)
subprocess.check_call(
["git", "commit", "-m", '"Experiment {}"'.format(id)],
stdout=out,
)
# Load configuration.
config = get_config()
if not config.ready:
config.load()
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
create_cmd = [
"heroku",
"apps:create",
app_name(id),
"--buildpack",
"https://github.com/thenovices/heroku-buildpack-scipy",
]
# If a team is specified, assign the app to the team.
try:
team = config.get("heroku_team", None)
if team:
create_cmd.extend(["--org", team])
except Exception:
pass
subprocess.check_call(create_cmd, stdout=out)
database_size = config.get('database_size')
# Set up postgres database and AWS environment variables.
cmds = [
[
"heroku", "addons:create",
"heroku-postgresql:{}".format(quote(database_size))
],
["heroku", "addons:create", "heroku-redis:premium-0"],
["heroku", "addons:create", "papertrail"],
]
for cmd in cmds:
subprocess.check_call(cmd + ["--app", app_name(id)], stdout=out)
heroku_config = {
"HOST": "{}.herokuapp.com".format(app_name(id)),
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"dallinger_email_username": config["dallinger_email_address"],
"dallinger_email_key": config["dallinger_email_password"],
"whimsical": config["whimsical"],
}
for key in heroku_config:
subprocess.check_call([
"heroku", "config:set", "{}={}".format(
key, quote(str(heroku_config[key]))), "--app",
app_name(id)
],
stdout=out)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
redis_URL = subprocess.check_output([
"heroku",
"config:get",
"REDIS_URL",
"--app",
app_name(id),
])
r = redis.from_url(redis_URL)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
time.sleep(2)
log("Saving the URL of the postgres database...")
subprocess.check_call(["heroku", "pg:wait", "--app", app_name(id)])
db_url = subprocess.check_output(
["heroku", "config:get", "DATABASE_URL", "--app",
app_name(id)])
# Set the notification URL and database URL in the config file.
config.extend({
"notification_url":
u"http://" + app_name(id) + ".herokuapp.com/notifications",
"database_url": db_url.rstrip().decode('utf8'),
})
config.write()
subprocess.check_call(["git", "add", "config.txt"], stdout=out),
time.sleep(0.25)
subprocess.check_call(
["git", "commit", "-m", '"Save URLs for database and notifications"'],
stdout=out)
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
subprocess.check_call(["git", "push", "heroku", "HEAD:master"],
stdout=out,
stderr=out)
log("Scaling up the dynos...")
heroku.scale_up_dynos(app_name(id))
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on MTurk...")
launch_request = requests.post('https://{}.herokuapp.com/launch'.format(
app_name(id)))
launch_data = launch_request.json()
log("URLs:")
log("App home: https://{}.herokuapp.com/".format(app_name(id)),
chevrons=False)
log("Initial recruitment: {}".format(
launch_data.get('recruitment_url', None)),
chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
