def app(self): from dallinger.heroku.tools import HerokuApp with mock.patch('dallinger.heroku.tools.subprocess'): the_app = HerokuApp(dallinger_uid='fake-uid', output=None, team="fake team") yield the_app
def heroku(): """Assemble links from Heroku add-on info, stored in config, plus some standard dashboard links. """ config = get_config() if config.get("mode") == "debug": flash( "This experiment is running in debug mode and is not deployed to Heroku", "warning", ) return render_template("dashboard_heroku.html", links=[]) heroku_app = HerokuApp(config.get("heroku_app_id_root")) links = [ { "url": heroku_app.dashboard_url, "title": "Heroku dashboard" }, { "url": heroku_app.dashboard_metrics_url, "title": "Heroku metrics" }, ] details = json.loads( config.get("infrastructure_debug_details", six.text_type("{}"))) links.extend([{ "title": v["title"].title(), "url": v["url"] } for v in details.values()]) return render_template("dashboard_heroku.html", links=links)
def awaken(app, databaseurl): """Restore the database from a given url.""" id = app config = get_config() config.load() bucket = data.user_s3_bucket() key = bucket.lookup('{}.dump'.format(id)) url = key.generate_url(expires_in=300) heroku_app = HerokuApp(id, output=None, team=None) heroku_app.addon("heroku-postgresql:{}".format( config.get('database_size'))) time.sleep(60) heroku_app.pg_wait() time.sleep(10) heroku_app.addon("heroku-redis:premium-0") heroku_app.restore(url) # Scale up the dynos. log("Scaling up the dynos...") size = config.get("dyno_type") for process in ["web", "worker"]: qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size)
def export(id, local=False, scrub_pii=False): """Export data from an experiment.""" print("Preparing to export the data...") if local: local_db = db.db_url else: local_db = HerokuApp(id).name copy_heroku_to_local(id) # Create the data package if it doesn't already exist. subdata_path = os.path.join("data", id, "data") try: os.makedirs(subdata_path) except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(subdata_path): raise # Copy in the data. copy_local_to_csv(local_db, subdata_path, scrub_pii=scrub_pii) # Copy the experiment code into a code/ subdirectory. try: shutil.copyfile(os.path.join("snapshots", id + "-code.zip"), os.path.join("data", id, id + "-code.zip")) except Exception: pass # Copy in the DATA readme. # open(os.path.join(id, "README.txt"), "a").close() # Save the experiment id. with open(os.path.join("data", id, "experiment_id.md"), "a+") as file: file.write(id) # Zip data src = os.path.join("data", id) dst = os.path.join("data", id + "-data.zip") archive_data(id, src, dst) cwd = os.getcwd() data_filename = '{}-data.zip'.format(id) path_to_data = os.path.join(cwd, "data", data_filename) # Backup data on S3 unless run locally if not local: k = Key(user_s3_bucket()) k.key = data_filename k.set_contents_from_filename(path_to_data) url = k.generate_url(expires_in=0, query_auth=False) # Register experiment UUID with dallinger register(id, url) return path_to_data
def end_experiment(self): """Terminates a running experiment""" # Debug runs synchronously if self.exp_config.get('mode') != 'debug': self.log("Waiting for experiment to complete.", "") while self.experiment_completed() is False: time.sleep(30) HerokuApp(self.app_id).destroy() return True
def copy_heroku_to_local(id): """Copy a Heroku database locally.""" heroku_app = HerokuApp(dallinger_uid=id) try: subprocess.call(["dropdb", heroku_app.name]) except Exception: pass heroku_app.pg_pull()
def herokuapp(): # Patch addon since we're using a free app which doesn't support them: from dallinger.heroku.tools import HerokuApp instance = HerokuApp('fake-uid', output=None, team=None) instance.addon = mock.Mock() with mock.patch('dallinger.command_line.HerokuApp') as mock_app_class: mock_app_class.return_value = instance yield instance instance.destroy()
def export(id, local=False, scrub_pii=False): """Export data from an experiment.""" print("Preparing to export the data...") if local: db_uri = db.db_url else: db_uri = HerokuApp(id).db_uri return export_db_uri(id, db_uri=db_uri, local=local, scrub_pii=scrub_pii)
def test_renders_links_for_heroku_services(self, active_config, logged_in): from dallinger.heroku.tools import HerokuApp details = '{"REDIS": {"url": "https://redis-url", "title": "REDIS"}}' active_config.set("infrastructure_debug_details", details) active_config.set("mode", "sandbox") heroku_app = HerokuApp(active_config.get("heroku_app_id_root")) resp = logged_in.get("/dashboard/heroku") assert '<a href="https://redis-url"' in resp.data.decode("utf8") assert '<a href="{}"'.format( heroku_app.dashboard_metrics_url) in resp.data.decode("utf8")
def experiment_completed(self): """Checks the current state of the experiment to see whether it has completed""" heroku_app = HerokuApp(self.app_id) status_url = '{}/summary'.format(heroku_app.url) data = {} try: resp = requests.get(status_url) data = resp.json() except (ValueError, requests.exceptions.RequestException): logger.exception('Error fetching experiment status.') logger.debug('Current application state: {}'.format(data)) return data.get('completed', False)
def monitor(app): """Set up application monitoring.""" heroku_app = HerokuApp(dallinger_uid=app) webbrowser.open(heroku_app.dashboard_url) webbrowser.open("https://requester.mturk.com/mturk/manageHITs") heroku_app.open_logs() check_call(["open", heroku_app.db_uri]) while _keep_running(): summary = get_summary(app) click.clear() click.echo(header) click.echo("\nExperiment {}\n".format(app)) click.echo(summary) time.sleep(10)
def experiment_completed(self): """Checks the current state of the experiment to see whether it has completed. This makes use of the experiment server `/summary` route, which in turn uses :meth:`~Experiment.is_complete`. """ heroku_app = HerokuApp(self.app_id) status_url = "{}/summary".format(heroku_app.url) data = {} try: resp = requests.get(status_url) data = resp.json() except (ValueError, requests.exceptions.RequestException): logger.exception("Error fetching experiment status.") logger.debug("Current application state: {}".format(data)) return data.get("completed", False)
def get_summary(app): heroku_app = HerokuApp(app) r = requests.get('{}/summary'.format(heroku_app.url)) summary = r.json()['summary'] out = [] out.append("\nstatus | count") out.append("----------------") for s in summary: out.append("{:<10}| {}".format(s[0], s[1])) num_approved = sum([s[1] for s in summary if s[0] == u"approved"]) num_not_working = sum([s[1] for s in summary if s[0] != u"working"]) if num_not_working > 0: the_yield = 1.0 * num_approved / num_not_working out.append("\nYield: {:.2%}".format(the_yield)) return "\n".join(out)
def dump_database(id): """Dump the database to a temporary directory.""" tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) FNULL = open(os.devnull, 'w') heroku_app = HerokuApp(dallinger_uid=id, output=FNULL) heroku_app.backup_capture() heroku_app.backup_download() for filename in os.listdir(tmp_dir): if filename.startswith("latest.dump"): os.rename(filename, "database.dump") os.chdir(current_dir) return os.path.join(tmp_dir, "database.dump")
def hibernate(app): """Pause an experiment and remove costly resources.""" log("The database backup URL is...") backup_url = data.backup(app) log(backup_url) log("Scaling down the web servers...") heroku_app = HerokuApp(app) heroku_app.scale_down_dynos() log("Removing addons...") addons = [ "heroku-postgresql", # "papertrail", "heroku-redis", ] for addon in addons: heroku_app.addon_destroy(addon)
def bot(app, debug): """Run the experiment bot.""" if debug is None: verify_id(None, None, app) (id, tmp) = setup_experiment() if debug: url = debug else: heroku_app = HerokuApp(dallinger_uid=app) worker = generate_random_id() hit = generate_random_id() assignment = generate_random_id() ad_url = '{}/ad'.format(heroku_app.url) ad_parameters = 'assignmentId={}&hitId={}&workerId={}&mode=sandbox' ad_parameters = ad_parameters.format(assignment, hit, worker) url = '{}?{}'.format(ad_url, ad_parameters) bot = bot_factory(url) bot.run_experiment()
def destroy(app): """Tear down an experiment server.""" HerokuApp(app).destroy()
def deploy_sandbox_shared_setup(verbose=True, app=None, exp_config=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, 'w') (id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app, exp_config=exp_config) config = get_config() # We know it's ready; setup_experiment() does this. # Register the experiment using all configured registration services. if config.get("mode") == u"live": log("Registering the experiment on configured services...") registration.register(id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) click.echo("") # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. git = GitClient(output=out) git.init() git.add("--all") git.commit('"Experiment {}"'.format(id)) # Initialize the app on Heroku. log("Initializing app on Heroku...") team = config.get("heroku_team", '').strip() or None heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team) heroku_app.bootstrap() heroku_app.buildpack( "https://github.com/stomita/heroku-buildpack-phantomjs") # Set up add-ons and AWS environment variables. database_size = config.get('database_size') addons = [ "heroku-postgresql:{}".format(quote(database_size)), "heroku-redis:premium-0", "papertrail" ] if config.get("sentry", False): addons.append("sentry") for name in addons: heroku_app.addon(name) heroku_config = { "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "dallinger_email_username": config["dallinger_email_address"], "dallinger_email_key": config["dallinger_email_password"], "whimsical": config["whimsical"], } for k, v in sorted(heroku_config.items()): # sorted for testablility heroku_app.set(k, v) # Wait for Redis database to be ready. log("Waiting for Redis...") ready = False while not ready: r = redis.from_url(heroku_app.redis_url) try: r.set("foo", "bar") ready = True except redis.exceptions.ConnectionError: time.sleep(2) log("Saving the URL of the postgres database...") # Set the notification URL and database URL in the config file. config.extend({ "notification_url": heroku_app.url + u"/notifications", "database_url": heroku_app.db_url, }) config.write() git.add("config.txt") time.sleep(0.25) git.commit("Save URLs for database and notifications") time.sleep(0.25) # Launch the Heroku app. log("Pushing code to Heroku...") git.push(remote="heroku", branch="HEAD:master") log("Scaling up the dynos...") size = config.get("dyno_type") for process in ["web", "worker"]: qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size) time.sleep(8) # Launch the experiment. log("Launching the experiment on the remote server and starting recruitment..." ) launch_data = _handle_launch_data('{}/launch'.format(heroku_app.url)) result = { 'app_name': heroku_app.name, 'app_home': heroku_app.url, 'recruitment_msg': launch_data.get('recruitment_msg', None), } log("Experiment details:") log("App home: {}".format(result['app_home']), chevrons=False) log("Recruiter info:") log(result['recruitment_msg'], chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed deployment of experiment " + id + ".") return result
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, "w") config = get_config() if not config.ready: config.load() heroku.sanity_check(config) (id, tmp) = setup_experiment(log, debug=False, app=app, exp_config=exp_config) # Register the experiment using all configured registration services. if config.get("mode") == "live": log("Registering the experiment on configured services...") registration.register(id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) log("", chevrons=False) # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. git = GitClient(output=out) git.init() git.add("--all") git.commit('"Experiment {}"'.format(id)) # Initialize the app on Heroku. log("Initializing app on Heroku...") team = config.get("heroku_team", None) heroku_app = HerokuApp(dallinger_uid=id, output=out, team=team) heroku_app.bootstrap() heroku_app.buildpack( "https://github.com/stomita/heroku-buildpack-phantomjs") # Set up add-ons and AWS environment variables. database_size = config.get("database_size") redis_size = config.get("redis_size") addons = [ "heroku-postgresql:{}".format(quote(database_size)), "heroku-redis:{}".format(quote(redis_size)), "papertrail", ] if config.get("sentry"): addons.append("sentry") for name in addons: heroku_app.addon(name) heroku_config = { "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "smtp_username": config["smtp_username"], "smtp_password": config["smtp_password"], "whimsical": config["whimsical"], } heroku_app.set_multiple(**heroku_config) # Wait for Redis database to be ready. log("Waiting for Redis...") ready = False while not ready: try: r = redis.from_url(heroku_app.redis_url) r.set("foo", "bar") ready = True except (ValueError, redis.exceptions.ConnectionError): time.sleep(2) log("Saving the URL of the postgres database...") config.extend({"database_url": heroku_app.db_url}) config.write() git.add("config.txt") time.sleep(0.25) git.commit("Save URL for database") time.sleep(0.25) # Launch the Heroku app. log("Pushing code to Heroku...") git.push(remote="heroku", branch="HEAD:master") log("Scaling up the dynos...") size = config.get("dyno_type") for process in ["web", "worker"]: qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size) time.sleep(8) # Launch the experiment. log("Launching the experiment on the remote server and starting recruitment..." ) launch_data = _handle_launch_data("{}/launch".format(heroku_app.url), error=log) result = { "app_name": heroku_app.name, "app_home": heroku_app.url, "recruitment_msg": launch_data.get("recruitment_msg", None), } log("Experiment details:") log("App home: {}".format(result["app_home"]), chevrons=False) log("Recruiter info:") log(result["recruitment_msg"], chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed deployment of experiment " + id + ".") return result
def end_experiment(self): """Terminates a running experiment""" if self.exp_config.get("mode") != "debug": HerokuApp(self.app_id).destroy() return True
def export(id, local=False, scrub_pii=False): """Export data from an experiment.""" print("Preparing to export the data...") if local: db_uri = db.db_url else: db_uri = HerokuApp(id).db_uri # Create the data package if it doesn't already exist. subdata_path = os.path.join("data", id, "data") try: os.makedirs(subdata_path) except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(subdata_path): raise # Copy in the data. copy_db_to_csv(db_uri, subdata_path, scrub_pii=scrub_pii) # Copy the experiment code into a code/ subdirectory. try: shutil.copyfile( os.path.join("snapshots", id + "-code.zip"), os.path.join("data", id, id + "-code.zip"), ) except Exception: pass # Copy in the DATA readme. # open(os.path.join(id, "README.txt"), "a").close() # Save the experiment id. with open(os.path.join("data", id, "experiment_id.md"), "a+") as file: file.write(id) # Zip data src = os.path.join("data", id) dst = os.path.join("data", id + "-data.zip") archive_data(id, src, dst) cwd = os.getcwd() data_filename = "{}-data.zip".format(id) path_to_data = os.path.join(cwd, "data", data_filename) # Backup data on S3 unless run locally if not local: bucket = user_s3_bucket() config = get_config() try: bucket.upload_file(path_to_data, data_filename) registration_url = _generate_s3_url(bucket, data_filename) s3_console_url = ( f"https://s3.console.aws.amazon.com/s3/object/{bucket.name}" f"?region={config.aws_region}&prefix={data_filename}") # Register experiment UUID with dallinger register(id, registration_url) print("A copy of your export was saved also to Amazon S3:\n" f" - bucket name: {bucket.name}\n" f" - S3 console URL: {s3_console_url}") except AttributeError: raise S3BucketUnavailable("Could not find an S3 bucket!") return path_to_data
def destroy(ctx, app, expire_hit, sandbox): """Tear down an experiment server.""" if expire_hit: ctx.invoke(expire, app=app, sandbox=sandbox, exit=False) HerokuApp(app).destroy()
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None, prelaunch_actions=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, "w") config = get_config() if not config.ready: config.load() heroku.sanity_check(config) (heroku_app_id, tmp) = setup_experiment(log, debug=False, app=app, exp_config=exp_config) # Register the experiment using all configured registration services. if config.get("mode") == "live": log("Registering the experiment on configured services...") registration.register(heroku_app_id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) log("", chevrons=False) # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. git = GitClient(output=out) git.init() git.add("--all") git.commit('"Experiment {}"'.format(heroku_app_id)) # Initialize the app on Heroku. log("Initializing app on Heroku...") team = config.get("heroku_team", None) heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team) heroku_app.bootstrap() heroku_app.buildpack( "https://github.com/stomita/heroku-buildpack-phantomjs") heroku_app.set("PYTHON_NO_SQLITE3", "true") # Set up add-ons and AWS environment variables. database_size = config.get("database_size") redis_size = config.get("redis_size") addons = [ "heroku-postgresql:{}".format(quote(database_size)), "heroku-redis:{}".format(quote(redis_size)), "papertrail", ] if config.get("sentry"): addons.append("sentry") for name in addons: heroku_app.addon(name) heroku_config = { "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "smtp_username": config["smtp_username"], "smtp_password": config["smtp_password"], "whimsical": config["whimsical"], "FLASK_SECRET_KEY": codecs.encode(os.urandom(16), "hex").decode("ascii"), } # Set up the preferred class as an environment variable, if one is set # This is needed before the config is parsed, but we also store it in the # config to make things easier for recording into bundles. preferred_class = config.get("EXPERIMENT_CLASS_NAME", None) if preferred_class: heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class heroku_app.set_multiple(**heroku_config) # Wait for Redis database to be ready. log("Waiting for Redis...", nl=False) ready = False while not ready: try: r = connect_to_redis(url=heroku_app.redis_url) r.set("foo", "bar") ready = True log("\n✓ connected at {}".format(heroku_app.redis_url), chevrons=False) except (ValueError, redis.exceptions.ConnectionError): time.sleep(2) log(".", chevrons=False, nl=False) log("Saving the URL of the postgres database...") config.extend({"database_url": heroku_app.db_url}) config.write() git.add("config.txt") git.commit("Save URL for database") log("Generating dashboard links...") heroku_addons = heroku_app.addon_parameters() heroku_addons = json.dumps(heroku_addons) if six.PY2: heroku_addons = heroku_addons.decode("utf-8") config.extend({"infrastructure_debug_details": heroku_addons}) config.write() git.add("config.txt") git.commit("Save URLs for heroku addon management") # Launch the Heroku app. log("Pushing code to Heroku...") git.push(remote="heroku", branch="HEAD:master") log("Scaling up the dynos...") default_size = config.get("dyno_type") for process in ["web", "worker"]: size = config.get("dyno_type_" + process, default_size) qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size) if prelaunch_actions is not None: for task in prelaunch_actions: task(heroku_app, config) # Launch the experiment. log("Launching the experiment on the remote server and starting recruitment..." ) launch_url = "{}/launch".format(heroku_app.url) log("Calling {}".format(launch_url), chevrons=False) launch_data = _handle_launch_data(launch_url, error=log) result = { "app_name": heroku_app.name, "app_home": heroku_app.url, "dashboard_url": "{}/dashboard/".format(heroku_app.url), "recruitment_msg": launch_data.get("recruitment_msg", None), } log("Experiment details:") log("App home: {}".format(result["app_home"]), chevrons=False) log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False) log("Dashboard user: {}".format(config.get("dashboard_user")), chevrons=False) log( "Dashboard password: {}".format(config.get("dashboard_password")), chevrons=False, ) log("Recruiter info:") log(result["recruitment_msg"], chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed Heroku deployment of experiment ID {} using app ID {}.". format(config.get("id"), heroku_app_id)) return result
def full_app(self): from dallinger.heroku.tools import HerokuApp the_app = HerokuApp(dallinger_uid="fake-uid", output=None, team=None) yield the_app the_app.destroy()
def logs(app): """Show the logs.""" if app is None: raise TypeError("Select an experiment using the --app flag.") HerokuApp(dallinger_uid=app).open_logs()