def deploy_sandbox_shared_setup(verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False,
verbose=verbose,
app=app,
exp_config=exp_config)
config = get_config() # We know it's ready; setup_experiment() does this.
# Register the experiment using all configured registration services.
if config.get("mode") == u"live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", '').strip() or None
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get('database_size')
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:premium-0", "papertrail"
]
if config.get("sentry", False):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"dallinger_email_username": config["dallinger_email_address"],
"dallinger_email_key": config["dallinger_email_password"],
"whimsical": config["whimsical"],
}
for k, v in sorted(heroku_config.items()): # sorted for testablility
heroku_app.set(k, v)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
r = redis.from_url(heroku_app.redis_url)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
time.sleep(2)
log("Saving the URL of the postgres database...")
# Set the notification URL and database URL in the config file.
config.extend({
"notification_url": heroku_app.url + u"/notifications",
"database_url": heroku_app.db_url,
})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URLs for database and notifications")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data('{}/launch'.format(heroku_app.url))
result = {
'app_name': heroku_app.name,
'app_home': heroku_app.url,
'recruitment_msg': launch_data.get('recruitment_msg', None),
}
log("Experiment details:")
log("App home: {}".format(result['app_home']), chevrons=False)
log("Recruiter info:")
log(result['recruitment_msg'], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
}
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
try:
r = redis.from_url(heroku_app.redis_url)
r.set("foo", "bar")
ready = True
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URL for database")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data("{}/launch".format(heroku_app.url),
error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
def deploy_sandbox_shared_setup(verbose=True,
app=None,
web_procs=1,
exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False,
verbose=verbose,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
config = get_config()
if config.get("mode") == u"live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
subprocess.check_call(["git", "init"], stdout=out)
subprocess.check_call(["git", "add", "--all"], stdout=out)
subprocess.check_call(
["git", "commit", "-m", '"Experiment {}"'.format(id)],
stdout=out,
)
# Load configuration.
config = get_config()
if not config.ready:
config.load()
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
create_cmd = [
"heroku",
"apps:create",
app_name(id),
"--buildpack",
"https://github.com/thenovices/heroku-buildpack-scipy",
]
# If a team is specified, assign the app to the team.
try:
team = config.get("heroku_team", None)
if team:
create_cmd.extend(["--org", team])
except Exception:
pass
subprocess.check_call(create_cmd, stdout=out)
database_size = config.get('database_size')
# Set up postgres database and AWS environment variables.
cmds = [
[
"heroku", "addons:create",
"heroku-postgresql:{}".format(quote(database_size))
],
["heroku", "addons:create", "heroku-redis:premium-0"],
["heroku", "addons:create", "papertrail"],
]
for cmd in cmds:
subprocess.check_call(cmd + ["--app", app_name(id)], stdout=out)
heroku_config = {
"HOST": "{}.herokuapp.com".format(app_name(id)),
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"dallinger_email_username": config["dallinger_email_address"],
"dallinger_email_key": config["dallinger_email_password"],
"whimsical": config["whimsical"],
}
for key in heroku_config:
subprocess.check_call([
"heroku", "config:set", "{}={}".format(
key, quote(str(heroku_config[key]))), "--app",
app_name(id)
],
stdout=out)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
redis_URL = subprocess.check_output([
"heroku",
"config:get",
"REDIS_URL",
"--app",
app_name(id),
])
r = redis.from_url(redis_URL)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
time.sleep(2)
log("Saving the URL of the postgres database...")
subprocess.check_call(["heroku", "pg:wait", "--app", app_name(id)])
db_url = subprocess.check_output(
["heroku", "config:get", "DATABASE_URL", "--app",
app_name(id)])
# Set the notification URL and database URL in the config file.
config.extend({
"notification_url":
u"http://" + app_name(id) + ".herokuapp.com/notifications",
"database_url": db_url.rstrip().decode('utf8'),
})
config.write()
subprocess.check_call(["git", "add", "config.txt"], stdout=out),
time.sleep(0.25)
subprocess.check_call(
["git", "commit", "-m", '"Save URLs for database and notifications"'],
stdout=out)
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
subprocess.check_call(["git", "push", "heroku", "HEAD:master"],
stdout=out,
stderr=out)
log("Scaling up the dynos...")
heroku.scale_up_dynos(app_name(id))
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on MTurk...")
launch_request = requests.post('https://{}.herokuapp.com/launch'.format(
app_name(id)))
launch_data = launch_request.json()
log("URLs:")
log("App home: https://{}.herokuapp.com/".format(app_name(id)),
chevrons=False)
log("Initial recruitment: {}".format(
launch_data.get('recruitment_url', None)),
chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
def deploy_sandbox_shared_setup(verbose=True, app=None, web_procs=1):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
cmds = ["git init",
"git add --all",
'git commit -m "Experiment ' + id + '"']
for cmd in cmds:
subprocess.call(cmd, stdout=out, shell=True)
time.sleep(0.5)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
subprocess.call(
"heroku apps:create " + app_name(id) +
" --buildpack https://github.com/thenovices/heroku-buildpack-scipy",
stdout=out,
shell=True)
database_size = config.get('Database Parameters', 'database_size')
try:
if config.getboolean('Easter eggs', 'whimsical'):
whimsical = "true"
else:
whimsical = "false"
except:
whimsical = "false"
# Set up postgres database and AWS/psiTurk environment variables.
cmds = [
"heroku addons:create heroku-postgresql:{}".format(database_size),
"heroku pg:wait",
"heroku addons:create heroku-redis:premium-0",
"heroku addons:create papertrail",
"heroku config:set HOST=" +
app_name(id) + ".herokuapp.com",
"heroku config:set aws_access_key_id=" +
config.get('AWS Access', 'aws_access_key_id'),
"heroku config:set aws_secret_access_key=" +
config.get('AWS Access', 'aws_secret_access_key'),
"heroku config:set aws_region=" +
config.get('AWS Access', 'aws_region'),
"heroku config:set psiturk_access_key_id=" +
config.get('psiTurk Access', 'psiturk_access_key_id'),
"heroku config:set psiturk_secret_access_id=" +
config.get('psiTurk Access', 'psiturk_secret_access_id'),
"heroku config:set auto_recruit=" +
config.get('Experiment Configuration', 'auto_recruit'),
"heroku config:set dallinger_email_username="******"heroku config:set dallinger_email_key=" +
config.get('Email Access', 'dallinger_email_password'),
"heroku config:set heroku_email_address=" +
config.get('Heroku Access', 'heroku_email_address'),
"heroku config:set heroku_password="******"heroku config:set whimsical=" + whimsical,
]
for cmd in cmds:
subprocess.call(
cmd + " --app " + app_name(id), stdout=out, shell=True)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
redis_URL = subprocess.check_output(
"heroku config:get REDIS_URL --app {}".format(app_name(id)),
shell=True
)
ready = False
while not ready:
r = redis.from_url(redis_URL)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
pass
# Set the notification URL in the cofig file to the notifications URL.
config.set(
"Server Parameters",
"notification_url",
"http://" + app_name(id) + ".herokuapp.com/notifications")
# Set the database URL in the config file to the newly generated one.
log("Saving the URL of the postgres database...")
db_url = subprocess.check_output(
"heroku config:get DATABASE_URL --app " + app_name(id), shell=True)
config.set("Database Parameters", "database_url", db_url.rstrip())
subprocess.call("git add config.txt", stdout=out, shell=True),
time.sleep(0.25)
subprocess.call(
'git commit -m "Save URLs for database and notifications"',
stdout=out,
shell=True)
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
subprocess.call("git push heroku HEAD:master", stdout=out,
stderr=out, shell=True)
log("Scaling up the dynos...")
scale_up_dynos(app_name(id))
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on MTurk...")
subprocess.call(
'curl --data "" http://{}.herokuapp.com/launch'.format(app_name(id)),
shell=True)
time.sleep(8)
url = subprocess.check_output(
"heroku logs --app " + app_name(id) + " | sort | " +
"sed -n 's|.*URL:||p'", shell=True)
log("URLs:")
click.echo(url)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
def test_log_in_fails(self, heroku, check_output):
check_output.side_effect = Exception("boom!")
with pytest.raises(Exception) as excinfo:
heroku.log_in()
assert excinfo.match("You are not logged into Heroku.")
def test_log_in_ok(self, heroku, check_output):
check_output.return_value = b"all good"
heroku.log_in()
def deploy_sandbox_shared_setup(log,
verbose=True,
app=None,
exp_config=None,
prelaunch_actions=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(heroku_app_id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(heroku_app_id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(heroku_app_id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
heroku_app.set("PYTHON_NO_SQLITE3", "true")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
"FLASK_SECRET_KEY": codecs.encode(os.urandom(16),
"hex").decode("ascii"),
}
# Set up the preferred class as an environment variable, if one is set
# This is needed before the config is parsed, but we also store it in the
# config to make things easier for recording into bundles.
preferred_class = config.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class:
heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...", nl=False)
ready = False
while not ready:
try:
r = connect_to_redis(url=heroku_app.redis_url)
r.set("foo", "bar")
ready = True
log("\n✓ connected at {}".format(heroku_app.redis_url),
chevrons=False)
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log(".", chevrons=False, nl=False)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
git.commit("Save URL for database")
log("Generating dashboard links...")
heroku_addons = heroku_app.addon_parameters()
heroku_addons = json.dumps(heroku_addons)
if six.PY2:
heroku_addons = heroku_addons.decode("utf-8")
config.extend({"infrastructure_debug_details": heroku_addons})
config.write()
git.add("config.txt")
git.commit("Save URLs for heroku addon management")
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
default_size = config.get("dyno_type")
for process in ["web", "worker"]:
size = config.get("dyno_type_" + process, default_size)
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
if prelaunch_actions is not None:
for task in prelaunch_actions:
task(heroku_app, config)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_url = "{}/launch".format(heroku_app.url)
log("Calling {}".format(launch_url), chevrons=False)
launch_data = _handle_launch_data(launch_url, error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"dashboard_url": "{}/dashboard/".format(heroku_app.url),
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False)
log("Dashboard user: {}".format(config.get("dashboard_user")),
chevrons=False)
log(
"Dashboard password: {}".format(config.get("dashboard_password")),
chevrons=False,
)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed Heroku deployment of experiment ID {} using app ID {}.".
format(config.get("id"), heroku_app_id))
return result
