def copy_heroku_to_local(id):
"""Copy a Heroku database locally."""
heroku_app = HerokuApp(dallinger_uid=id)
try:
subprocess.call(["dropdb", heroku_app.name])
except Exception:
pass
heroku_app.pg_pull()
def herokuapp():
# Patch addon since we're using a free app which doesn't support them:
from dallinger.heroku.tools import HerokuApp
instance = HerokuApp('fake-uid', output=None, team=None)
instance.addon = mock.Mock()
with mock.patch('dallinger.command_line.HerokuApp') as mock_app_class:
mock_app_class.return_value = instance
yield instance
instance.destroy()
def monitor(app):
"""Set up application monitoring."""
heroku_app = HerokuApp(dallinger_uid=app)
webbrowser.open(heroku_app.dashboard_url)
webbrowser.open("https://requester.mturk.com/mturk/manageHITs")
heroku_app.open_logs()
check_call(["open", heroku_app.db_uri])
while _keep_running():
summary = get_summary(app)
click.clear()
click.echo(header)
click.echo("\nExperiment {}\n".format(app))
click.echo(summary)
time.sleep(10)
def app(self):
from dallinger.heroku.tools import HerokuApp
with mock.patch('dallinger.heroku.tools.subprocess'):
the_app = HerokuApp(dallinger_uid='fake-uid',
output=None,
team="fake team")
yield the_app
def heroku():
"""Assemble links from Heroku add-on info, stored in config, plus some
standard dashboard links.
"""
config = get_config()
if config.get("mode") == "debug":
flash(
"This experiment is running in debug mode and is not deployed to Heroku",
"warning",
)
return render_template("dashboard_heroku.html", links=[])
heroku_app = HerokuApp(config.get("heroku_app_id_root"))
links = [
{
"url": heroku_app.dashboard_url,
"title": "Heroku dashboard"
},
{
"url": heroku_app.dashboard_metrics_url,
"title": "Heroku metrics"
},
]
details = json.loads(
config.get("infrastructure_debug_details", six.text_type("{}")))
links.extend([{
"title": v["title"].title(),
"url": v["url"]
} for v in details.values()])
return render_template("dashboard_heroku.html", links=links)
def export(id, local=False, scrub_pii=False):
"""Export data from an experiment."""
print("Preparing to export the data...")
if local:
local_db = db.db_url
else:
local_db = HerokuApp(id).name
copy_heroku_to_local(id)
# Create the data package if it doesn't already exist.
subdata_path = os.path.join("data", id, "data")
try:
os.makedirs(subdata_path)
except OSError as e:
if e.errno != errno.EEXIST or not os.path.isdir(subdata_path):
raise
# Copy in the data.
copy_local_to_csv(local_db, subdata_path, scrub_pii=scrub_pii)
# Copy the experiment code into a code/ subdirectory.
try:
shutil.copyfile(os.path.join("snapshots", id + "-code.zip"),
os.path.join("data", id, id + "-code.zip"))
except Exception:
pass
# Copy in the DATA readme.
# open(os.path.join(id, "README.txt"), "a").close()
# Save the experiment id.
with open(os.path.join("data", id, "experiment_id.md"), "a+") as file:
file.write(id)
# Zip data
src = os.path.join("data", id)
dst = os.path.join("data", id + "-data.zip")
archive_data(id, src, dst)
cwd = os.getcwd()
data_filename = '{}-data.zip'.format(id)
path_to_data = os.path.join(cwd, "data", data_filename)
# Backup data on S3 unless run locally
if not local:
k = Key(user_s3_bucket())
k.key = data_filename
k.set_contents_from_filename(path_to_data)
url = k.generate_url(expires_in=0, query_auth=False)
# Register experiment UUID with dallinger
register(id, url)
return path_to_data
def end_experiment(self):
"""Terminates a running experiment"""
# Debug runs synchronously
if self.exp_config.get('mode') != 'debug':
self.log("Waiting for experiment to complete.", "")
while self.experiment_completed() is False:
time.sleep(30)
HerokuApp(self.app_id).destroy()
return True
def export(id, local=False, scrub_pii=False):
"""Export data from an experiment."""
print("Preparing to export the data...")
if local:
db_uri = db.db_url
else:
db_uri = HerokuApp(id).db_uri
return export_db_uri(id, db_uri=db_uri, local=local, scrub_pii=scrub_pii)
def test_renders_links_for_heroku_services(self, active_config, logged_in):
from dallinger.heroku.tools import HerokuApp
details = '{"REDIS": {"url": "https://redis-url", "title": "REDIS"}}'
active_config.set("infrastructure_debug_details", details)
active_config.set("mode", "sandbox")
heroku_app = HerokuApp(active_config.get("heroku_app_id_root"))
resp = logged_in.get("/dashboard/heroku")
assert '<a href="https://redis-url"' in resp.data.decode("utf8")
assert '<a href="{}"'.format(
heroku_app.dashboard_metrics_url) in resp.data.decode("utf8")
def experiment_completed(self):
"""Checks the current state of the experiment to see whether it has
completed"""
heroku_app = HerokuApp(self.app_id)
status_url = '{}/summary'.format(heroku_app.url)
data = {}
try:
resp = requests.get(status_url)
data = resp.json()
except (ValueError, requests.exceptions.RequestException):
logger.exception('Error fetching experiment status.')
logger.debug('Current application state: {}'.format(data))
return data.get('completed', False)
def get_summary(app):
heroku_app = HerokuApp(app)
r = requests.get('{}/summary'.format(heroku_app.url))
summary = r.json()['summary']
out = []
out.append("\nstatus | count")
out.append("----------------")
for s in summary:
out.append("{:<10}| {}".format(s[0], s[1]))
num_approved = sum([s[1] for s in summary if s[0] == u"approved"])
num_not_working = sum([s[1] for s in summary if s[0] != u"working"])
if num_not_working > 0:
the_yield = 1.0 * num_approved / num_not_working
out.append("\nYield: {:.2%}".format(the_yield))
return "\n".join(out)
def experiment_completed(self):
"""Checks the current state of the experiment to see whether it has
completed. This makes use of the experiment server `/summary` route,
which in turn uses :meth:`~Experiment.is_complete`.
"""
heroku_app = HerokuApp(self.app_id)
status_url = "{}/summary".format(heroku_app.url)
data = {}
try:
resp = requests.get(status_url)
data = resp.json()
except (ValueError, requests.exceptions.RequestException):
logger.exception("Error fetching experiment status.")
logger.debug("Current application state: {}".format(data))
return data.get("completed", False)
def bot(app, debug):
"""Run the experiment bot."""
if debug is None:
verify_id(None, None, app)
(id, tmp) = setup_experiment()
if debug:
url = debug
else:
heroku_app = HerokuApp(dallinger_uid=app)
worker = generate_random_id()
hit = generate_random_id()
assignment = generate_random_id()
ad_url = '{}/ad'.format(heroku_app.url)
ad_parameters = 'assignmentId={}&hitId={}&workerId={}&mode=sandbox'
ad_parameters = ad_parameters.format(assignment, hit, worker)
url = '{}?{}'.format(ad_url, ad_parameters)
bot = bot_factory(url)
bot.run_experiment()
def hibernate(app):
"""Pause an experiment and remove costly resources."""
log("The database backup URL is...")
backup_url = data.backup(app)
log(backup_url)
log("Scaling down the web servers...")
heroku_app = HerokuApp(app)
heroku_app.scale_down_dynos()
log("Removing addons...")
addons = [
"heroku-postgresql",
# "papertrail",
"heroku-redis",
]
for addon in addons:
heroku_app.addon_destroy(addon)
def dump_database(id):
"""Dump the database to a temporary directory."""
tmp_dir = tempfile.mkdtemp()
current_dir = os.getcwd()
os.chdir(tmp_dir)
FNULL = open(os.devnull, 'w')
heroku_app = HerokuApp(dallinger_uid=id, output=FNULL)
heroku_app.backup_capture()
heroku_app.backup_download()
for filename in os.listdir(tmp_dir):
if filename.startswith("latest.dump"):
os.rename(filename, "database.dump")
os.chdir(current_dir)
return os.path.join(tmp_dir, "database.dump")
def deploy_sandbox_shared_setup(log,
verbose=True,
app=None,
exp_config=None,
prelaunch_actions=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(heroku_app_id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(heroku_app_id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(heroku_app_id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
heroku_app.set("PYTHON_NO_SQLITE3", "true")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
"FLASK_SECRET_KEY": codecs.encode(os.urandom(16),
"hex").decode("ascii"),
}
# Set up the preferred class as an environment variable, if one is set
# This is needed before the config is parsed, but we also store it in the
# config to make things easier for recording into bundles.
preferred_class = config.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class:
heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...", nl=False)
ready = False
while not ready:
try:
r = connect_to_redis(url=heroku_app.redis_url)
r.set("foo", "bar")
ready = True
log("\n✓ connected at {}".format(heroku_app.redis_url),
chevrons=False)
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log(".", chevrons=False, nl=False)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
git.commit("Save URL for database")
log("Generating dashboard links...")
heroku_addons = heroku_app.addon_parameters()
heroku_addons = json.dumps(heroku_addons)
if six.PY2:
heroku_addons = heroku_addons.decode("utf-8")
config.extend({"infrastructure_debug_details": heroku_addons})
config.write()
git.add("config.txt")
git.commit("Save URLs for heroku addon management")
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
default_size = config.get("dyno_type")
for process in ["web", "worker"]:
size = config.get("dyno_type_" + process, default_size)
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
if prelaunch_actions is not None:
for task in prelaunch_actions:
task(heroku_app, config)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_url = "{}/launch".format(heroku_app.url)
log("Calling {}".format(launch_url), chevrons=False)
launch_data = _handle_launch_data(launch_url, error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"dashboard_url": "{}/dashboard/".format(heroku_app.url),
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False)
log("Dashboard user: {}".format(config.get("dashboard_user")),
chevrons=False)
log(
"Dashboard password: {}".format(config.get("dashboard_password")),
chevrons=False,
)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed Heroku deployment of experiment ID {} using app ID {}.".
format(config.get("id"), heroku_app_id))
return result
def awaken(app, databaseurl):
"""Restore the database from a given url."""
id = app
config = get_config()
config.load()
bucket = data.user_s3_bucket()
key = bucket.lookup('{}.dump'.format(id))
url = key.generate_url(expires_in=300)
heroku_app = HerokuApp(id, output=None, team=None)
heroku_app.addon("heroku-postgresql:{}".format(
config.get('database_size')))
time.sleep(60)
heroku_app.pg_wait()
time.sleep(10)
heroku_app.addon("heroku-redis:premium-0")
heroku_app.restore(url)
# Scale up the dynos.
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
def destroy(app):
"""Tear down an experiment server."""
HerokuApp(app).destroy()
def deploy_sandbox_shared_setup(verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, 'w')
(id, tmp) = setup_experiment(debug=False,
verbose=verbose,
app=app,
exp_config=exp_config)
config = get_config() # We know it's ready; setup_experiment() does this.
# Register the experiment using all configured registration services.
if config.get("mode") == u"live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
click.echo("")
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", '').strip() or None
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get('database_size')
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:premium-0", "papertrail"
]
if config.get("sentry", False):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"dallinger_email_username": config["dallinger_email_address"],
"dallinger_email_key": config["dallinger_email_password"],
"whimsical": config["whimsical"],
}
for k, v in sorted(heroku_config.items()): # sorted for testablility
heroku_app.set(k, v)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
r = redis.from_url(heroku_app.redis_url)
try:
r.set("foo", "bar")
ready = True
except redis.exceptions.ConnectionError:
time.sleep(2)
log("Saving the URL of the postgres database...")
# Set the notification URL and database URL in the config file.
config.extend({
"notification_url": heroku_app.url + u"/notifications",
"database_url": heroku_app.db_url,
})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URLs for database and notifications")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data('{}/launch'.format(heroku_app.url))
result = {
'app_name': heroku_app.name,
'app_home': heroku_app.url,
'recruitment_msg': launch_data.get('recruitment_msg', None),
}
log("Experiment details:")
log("App home: {}".format(result['app_home']), chevrons=False)
log("Recruiter info:")
log(result['recruitment_msg'], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
def export(id, local=False, scrub_pii=False):
"""Export data from an experiment."""
print("Preparing to export the data...")
if local:
db_uri = db.db_url
else:
db_uri = HerokuApp(id).db_uri
# Create the data package if it doesn't already exist.
subdata_path = os.path.join("data", id, "data")
try:
os.makedirs(subdata_path)
except OSError as e:
if e.errno != errno.EEXIST or not os.path.isdir(subdata_path):
raise
# Copy in the data.
copy_db_to_csv(db_uri, subdata_path, scrub_pii=scrub_pii)
# Copy the experiment code into a code/ subdirectory.
try:
shutil.copyfile(
os.path.join("snapshots", id + "-code.zip"),
os.path.join("data", id, id + "-code.zip"),
)
except Exception:
pass
# Copy in the DATA readme.
# open(os.path.join(id, "README.txt"), "a").close()
# Save the experiment id.
with open(os.path.join("data", id, "experiment_id.md"), "a+") as file:
file.write(id)
# Zip data
src = os.path.join("data", id)
dst = os.path.join("data", id + "-data.zip")
archive_data(id, src, dst)
cwd = os.getcwd()
data_filename = "{}-data.zip".format(id)
path_to_data = os.path.join(cwd, "data", data_filename)
# Backup data on S3 unless run locally
if not local:
bucket = user_s3_bucket()
config = get_config()
try:
bucket.upload_file(path_to_data, data_filename)
registration_url = _generate_s3_url(bucket, data_filename)
s3_console_url = (
f"https://s3.console.aws.amazon.com/s3/object/{bucket.name}"
f"?region={config.aws_region}&prefix={data_filename}")
# Register experiment UUID with dallinger
register(id, registration_url)
print("A copy of your export was saved also to Amazon S3:\n"
f" - bucket name: {bucket.name}\n"
f" - S3 console URL: {s3_console_url}")
except AttributeError:
raise S3BucketUnavailable("Could not find an S3 bucket!")
return path_to_data
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(id, tmp) = setup_experiment(log,
debug=False,
app=app,
exp_config=exp_config)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack(
"https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
}
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
try:
r = redis.from_url(heroku_app.redis_url)
r.set("foo", "bar")
ready = True
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URL for database")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
size = config.get("dyno_type")
for process in ["web", "worker"]:
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment..."
)
launch_data = _handle_launch_data("{}/launch".format(heroku_app.url),
error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log("Completed deployment of experiment " + id + ".")
return result
def destroy(ctx, app, expire_hit, sandbox):
"""Tear down an experiment server."""
if expire_hit:
ctx.invoke(expire, app=app, sandbox=sandbox, exit=False)
HerokuApp(app).destroy()
def end_experiment(self):
"""Terminates a running experiment"""
if self.exp_config.get("mode") != "debug":
HerokuApp(self.app_id).destroy()
return True
def full_app(self):
from dallinger.heroku.tools import HerokuApp
the_app = HerokuApp(dallinger_uid="fake-uid", output=None, team=None)
yield the_app
the_app.destroy()
def logs(app):
"""Show the logs."""
if app is None:
raise TypeError("Select an experiment using the --app flag.")
HerokuApp(dallinger_uid=app).open_logs()
