def deploy_sandbox_shared_setup(log,
verbose=True,
app=None,
exp_config=None,
prelaunch_actions=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(heroku_app_id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(heroku_app_id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(heroku_app_id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
heroku_app.set("PYTHON_NO_SQLITE3", "true")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
"FLASK_SECRET_KEY": codecs.encode(os.urandom(16),
"hex").decode("ascii"),
}
# Set up the preferred class as an environment variable, if one is set
# This is needed before the config is parsed, but we also store it in the
# config to make things easier for recording into bundles.
preferred_class = config.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class:
heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...", nl=False)
ready = False
while not ready:
try:
r = connect_to_redis(url=heroku_app.redis_url)
r.set("foo", "bar")
ready = True
log("\n✓ connected at {}".format(heroku_app.redis_url),
chevrons=False)
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log(".", chevrons=False, nl=False)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
git.commit("Save URL for database")
log("Generating dashboard links...")
heroku_addons = heroku_app.addon_parameters()
heroku_addons = json.dumps(heroku_addons)
if six.PY2:
heroku_addons = heroku_addons.decode("utf-8")
config.extend({"infrastructure_debug_details": heroku_addons})
config.write()
git.add("config.txt")
git.commit("Save URLs for heroku addon management")
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
default_size = config.get("dyno_type")
for process in ["web", "worker"]:
size = config.get("dyno_type_" + process, default_size)
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
if prelaunch_actions is not None:
for task in prelaunch_actions:
task(heroku_app, config)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_url = "{}/launch".format(heroku_app.url)
log("Calling {}".format(launch_url), chevrons=False)
launch_data = _handle_launch_data(launch_url, error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"dashboard_url": "{}/dashboard/".format(heroku_app.url),
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False)
log("Dashboard user: {}".format(config.get("dashboard_user")),
chevrons=False)
log(
"Dashboard password: {}".format(config.get("dashboard_password")),
chevrons=False,
)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed Heroku deployment of experiment ID {} using app ID {}.".
format(config.get("id"), heroku_app_id))
return result
logger = logging.getLogger("dallinger.db")
db_url_default = "postgresql://*****:*****@localhost/dallinger"
db_url = os.environ.get("DATABASE_URL", db_url_default)
if db_url.startswith("postgres://"):
# The sqlalchemy dialect name needs to be `postgresql`, not `postgres`
db_url = f"postgresql://{db_url[11:]}"
logger.debug(f"Using database URL {db_url}")
engine = create_engine(db_url, pool_size=1000)
session_factory = sessionmaker(autocommit=False, autoflush=True, bind=engine)
session = scoped_session(session_factory)
Base = declarative_base()
Base.query = session.query_property()
redis_conn = connect_to_redis()
db_user_warning = """
*********************************************************
*********************************************************
Dallinger now requires a database user named "dallinger".
Run:
createuser -P dallinger --createdb
Consult the developer guide for more information.
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
heroku_app.set("PYTHON_NO_SQLITE3", "true")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
}
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
try:
r = connect_to_redis(url=heroku_app.redis_url)
r.set("foo", "bar")
ready = True
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URL for database")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data("{}/launch".format(heroku_app.url),
error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result