def hibernate(app): """Pause an experiment and remove costly resources.""" verify_id(app) log("The database backup URL is...") backup_url = data.backup(app) log(backup_url) log("Scaling down the web servers...") for process in ["web", "worker"]: subprocess.check_call([ "heroku", "ps:scale", "{}=0".format(process), "--app", app_name(app) ]) subprocess.call(["heroku", "ps:scale", "clock=0", "--app", app_name(app)]) log("Removing addons...") addons = [ "heroku-postgresql", # "papertrail", "heroku-redis", ] for addon in addons: subprocess.check_call([ "heroku", "addons:destroy", addon, "--app", app_name(app), "--confirm", app_name(app) ])
def dump_database(id): """Dump the database to a temporary directory.""" tmp_dir = tempfile.mkdtemp() current_dir = os.getcwd() os.chdir(tmp_dir) FNULL = open(os.devnull, 'w') subprocess.call( ["heroku", "pg:backups:capture", "--app", heroku.app_name(id)], stdout=FNULL, stderr=FNULL) subprocess.call( ["heroku", "pg:backups:download", "--app", heroku.app_name(id)], stdout=FNULL, stderr=FNULL) for filename in os.listdir(tmp_dir): if filename.startswith("latest.dump"): os.rename(filename, "database.dump") os.chdir(current_dir) return os.path.join(tmp_dir, "database.dump")
def dump_database(id): """Backup the Postgres database locally.""" log("Generating a backup of the database on Heroku...") dump_filename = "data.dump" data_directory = "data" dump_dir = os.path.join(data_directory, id) if not os.path.exists(dump_dir): os.makedirs(dump_dir) subprocess.call( "heroku pg:backups capture --app " + app_name(id), shell=True) backup_url = subprocess.check_output( "heroku pg:backups public-url --app " + app_name(id), shell=True) backup_url = backup_url.replace('"', '').rstrip() backup_url = re.search("https:.*", backup_url).group(0) print(backup_url) log("Downloading the backup...") dump_path = os.path.join(dump_dir, dump_filename) with open(dump_path, 'wb') as file: subprocess.call(['curl', '-o', dump_path, backup_url], stdout=file) return dump_path
def destroy(app): """Tear down an experiment server.""" subprocess.call( "heroku destroy --app {} --confirm {}".format( app_name(app), app_name(app) ), shell=True, )
def destroy_server(app): """Tear down an experiment server.""" subprocess.check_call([ "heroku", "destroy", "--app", app_name(app), "--confirm", app_name(app), ])
def logs(app): """Show the logs.""" if app is None: raise TypeError("Select an experiment using the --app flag.") else: subprocess.call( "heroku addons:open papertrail --app " + app_name(app), shell=True)
def logs(app): """Show the logs.""" verify_id(app) try: subprocess.check_call( ["heroku", "addons:open", "papertrail", "--app", app_name(app)]) except subprocess.CalledProcessError as e: print("Invalid experiment ID. {}".format(e))
def copy_heroku_to_local(id): """Copy a Heroku database locally.""" try: subprocess.call([ "dropdb", heroku.app_name(id), ]) except Exception: pass subprocess.call([ "heroku", "pg:pull", "DATABASE_URL", heroku.app_name(id), "--app", heroku.app_name(id), ])
def summary(app): """Print a summary of a deployed app's status.""" r = requests.get('https://{}.herokuapp.com/summary'.format(app_name(app))) summary = r.json()['summary'] click.echo("\nstatus \t| count") click.echo("----------------") for s in summary: click.echo("{}\t| {}".format(s[0], s[1])) num_101s = sum([s[1] for s in summary if s[0] == 101]) num_10xs = sum([s[1] for s in summary if s[0] >= 100]) if num_10xs > 0: click.echo("\nYield: {:.2%}".format(1.0 * num_101s / num_10xs))
def awaken(app, databaseurl): """Restore the database from a given url.""" id = app config = get_config() config.load_config() database_size = config.get('database_size') subprocess.check_call( "heroku addons:create heroku-postgresql:{} --app {}".format( database_size, app_name(id)), shell=True) subprocess.check_call("heroku pg:wait --app {}".format(app_name(id)), shell=True) bucket = data.user_s3_bucket() key = bucket.lookup('{}.dump'.format(id)) url = key.generate_url(expires_in=300) cmd = "heroku pg:backups restore" subprocess.check_call("{} '{}' DATABASE_URL --app {} --confirm {}".format( cmd, url, app_name(id), app_name(id)), shell=True) subprocess.check_call( "heroku addons:create heroku-redis:premium-0 --app {}".format( app_name(id)), shell=True) # Scale up the dynos. log("Scaling up the dynos...") scale_up_dynos(app_name(id))
def export(id, local=False, scrub_pii=False): """Export data from an experiment.""" print("Preparing to export the data...") copy_heroku_to_local(id) # Create the data package if it doesn't already exist. subdata_path = os.path.join("data", id, "data") try: os.makedirs(subdata_path) except OSError as e: if e.errno != errno.EEXIST or not os.path.isdir(subdata_path): raise # Copy in the data. copy_local_to_csv(heroku.app_name(id), subdata_path, scrub_pii=scrub_pii) # Copy the experiment code into a code/ subdirectory. try: shutil.copyfile(os.path.join("snapshots", id + "-code.zip"), os.path.join("data", id, id + "-code.zip")) except Exception: pass # Copy in the DATA readme. # open(os.path.join(id, "README.txt"), "a").close() # Save the experiment id. with open(os.path.join("data", id, "experiment_id.md"), "a+") as file: file.write(id) print("Zipping up the package...") shutil.make_archive(os.path.join("data", id + "-data"), "zip", os.path.join("data", id)) shutil.rmtree(os.path.join("data", id)) print("Done. Data available in {}-data.zip".format(id)) cwd = os.getcwd() data_filename = '{}-data.zip'.format(id) path_to_data = os.path.join(cwd, "data", data_filename) # Backup data on S3. k = Key(user_s3_bucket()) k.key = data_filename k.set_contents_from_filename(path_to_data) return path_to_data
def experiment_completed(self): """Checks the current state of the experiment to see whether it has completed""" status_url = 'https://{}.herokuapp.com/summary'.format( app_name(self.app_id)) data = {} try: resp = requests.get(status_url) data = resp.json() except (ValueError, requests.exceptions.RequestException): logger.exception('Error fetching experiment status.') logger.debug('Current application state: {}'.format(data)) return data.get('completed', False)
def summary(app): """Print a summary of a deployed app's status.""" verify_id(app) r = requests.get('https://{}.herokuapp.com/summary'.format(app_name(app))) summary = r.json()['summary'] click.echo("\nstatus | count") click.echo("-----------------") for s in summary: click.echo("{:<10}| {}".format(s[0], s[1])) num_approved = sum([s[1] for s in summary if s[0] == u"approved"]) num_not_working = sum([s[1] for s in summary if s[0] != u"working"]) if num_not_working > 0: the_yield = 1.0 * num_approved / num_not_working click.echo("\nYield: {:.2%}".format(the_yield))
def awaken(app, databaseurl): """Restore the database from a given url.""" id = app config = PsiturkConfig() config.load_config() database_size = config.get('Database Parameters', 'database_size') subprocess.call( "heroku addons:create heroku-postgresql:{} --app {}".format( database_size, app_name(id)), shell=True) subprocess.call( "heroku pg:wait --app {}".format(app_name(id)), shell=True) conn = boto.connect_s3( config.get('AWS Access', 'aws_access_key_id'), config.get('AWS Access', 'aws_secret_access_key'), ) bucket = conn.get_bucket(id) key = bucket.lookup('database.dump') url = key.generate_url(expires_in=300) cmd = "heroku pg:backups restore" subprocess.call( "{} '{}' DATABASE_URL --app {} --confirm {}".format( cmd, url, app_name(id), app_name(id)), shell=True) subprocess.call( "heroku addons:create heroku-redis:premium-0 --app {}".format(app_name(id)), shell=True) # Scale up the dynos. log("Scaling up the dynos...") scale_up_dynos(app_name(id))
def awaken(app, databaseurl): """Restore the database from a given url.""" id = app config = get_config() config.load() subprocess.check_call([ "heroku", "addons:create", "heroku-postgresql:{}".format(config.get('database_size')), "--app", app_name(id), ]) bucket = data.user_s3_bucket() key = bucket.lookup('{}.dump'.format(id)) url = key.generate_url(expires_in=300) time.sleep(60) subprocess.check_call(["heroku", "pg:wait", "--app", app_name(id)]) time.sleep(10) subprocess.check_call([ "heroku", "addons:create", "heroku-redis:premium-0", "--app", app_name(id) ]) subprocess.check_call([ "heroku", "pg:backups:restore", "{}".format(url), "DATABASE_URL", "--app", app_name(id), "--confirm", app_name(id), ]) # Scale up the dynos. log("Scaling up the dynos...") heroku.scale_up_dynos(app_name(id))
def test_heroku_app_name(self): id = "8fbe62f5-2e33-4274-8aeb-40fc3dd621a0" assert(len(app_name(id)) < 30)
def test_app_name(self, heroku): dallinger_uid = "8fbe62f5-2e33-4274-8aeb-40fc3dd621a0" assert heroku.app_name(dallinger_uid) == "dlgr-8fbe62f5-2e33-4274"
def deploy_sandbox_shared_setup(verbose=True, app=None, web_procs=1): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, 'w') (id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() click.echo("") # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. cmds = ["git init", "git add --all", 'git commit -m "Experiment ' + id + '"'] for cmd in cmds: subprocess.call(cmd, stdout=out, shell=True) time.sleep(0.5) # Load psiTurk configuration. config = PsiturkConfig() config.load_config() # Initialize the app on Heroku. log("Initializing app on Heroku...") subprocess.call( "heroku apps:create " + app_name(id) + " --buildpack https://github.com/thenovices/heroku-buildpack-scipy", stdout=out, shell=True) database_size = config.get('Database Parameters', 'database_size') try: if config.getboolean('Easter eggs', 'whimsical'): whimsical = "true" else: whimsical = "false" except: whimsical = "false" # Set up postgres database and AWS/psiTurk environment variables. cmds = [ "heroku addons:create heroku-postgresql:{}".format(database_size), "heroku pg:wait", "heroku addons:create heroku-redis:premium-0", "heroku addons:create papertrail", "heroku config:set HOST=" + app_name(id) + ".herokuapp.com", "heroku config:set aws_access_key_id=" + config.get('AWS Access', 'aws_access_key_id'), "heroku config:set aws_secret_access_key=" + config.get('AWS Access', 'aws_secret_access_key'), "heroku config:set aws_region=" + config.get('AWS Access', 'aws_region'), "heroku config:set psiturk_access_key_id=" + config.get('psiTurk Access', 'psiturk_access_key_id'), "heroku config:set psiturk_secret_access_id=" + config.get('psiTurk Access', 'psiturk_secret_access_id'), "heroku config:set auto_recruit=" + config.get('Experiment Configuration', 'auto_recruit'), "heroku config:set dallinger_email_username="******"heroku config:set dallinger_email_key=" + config.get('Email Access', 'dallinger_email_password'), "heroku config:set heroku_email_address=" + config.get('Heroku Access', 'heroku_email_address'), "heroku config:set heroku_password="******"heroku config:set whimsical=" + whimsical, ] for cmd in cmds: subprocess.call( cmd + " --app " + app_name(id), stdout=out, shell=True) # Wait for Redis database to be ready. log("Waiting for Redis...") redis_URL = subprocess.check_output( "heroku config:get REDIS_URL --app {}".format(app_name(id)), shell=True ) ready = False while not ready: r = redis.from_url(redis_URL) try: r.set("foo", "bar") ready = True except redis.exceptions.ConnectionError: pass # Set the notification URL in the cofig file to the notifications URL. config.set( "Server Parameters", "notification_url", "http://" + app_name(id) + ".herokuapp.com/notifications") # Set the database URL in the config file to the newly generated one. log("Saving the URL of the postgres database...") db_url = subprocess.check_output( "heroku config:get DATABASE_URL --app " + app_name(id), shell=True) config.set("Database Parameters", "database_url", db_url.rstrip()) subprocess.call("git add config.txt", stdout=out, shell=True), time.sleep(0.25) subprocess.call( 'git commit -m "Save URLs for database and notifications"', stdout=out, shell=True) time.sleep(0.25) # Launch the Heroku app. log("Pushing code to Heroku...") subprocess.call("git push heroku HEAD:master", stdout=out, stderr=out, shell=True) log("Scaling up the dynos...") scale_up_dynos(app_name(id)) time.sleep(8) # Launch the experiment. log("Launching the experiment on MTurk...") subprocess.call( 'curl --data "" http://{}.herokuapp.com/launch'.format(app_name(id)), shell=True) time.sleep(8) url = subprocess.check_output( "heroku logs --app " + app_name(id) + " | sort | " + "sed -n 's|.*URL:||p'", shell=True) log("URLs:") click.echo(url) # Return to the branch whence we came. os.chdir(cwd) log("Completed deployment of experiment " + id + ".")
def export(app, local): """Export the data.""" print_header() log("Preparing to export the data...") id = str(app) subdata_path = os.path.join("data", id, "data") # Create the data package os.makedirs(subdata_path) # Copy the experiment code into a code/ subdirectory try: shutil.copyfile( os.path.join("snapshots", id + "-code.zip"), os.path.join("data", id, id + "-code.zip") ) except: pass # Copy in the DATA readme. # open(os.path.join(id, "README.txt"), "a").close() # Save the experiment id. with open(os.path.join("data", id, "experiment_id.md"), "a+") as file: file.write(id) if not local: # Export the logs subprocess.call( "heroku logs " + "-n 10000 > " + os.path.join("data", id, "server_logs.md") + " --app " + app_name(id), shell=True) dump_path = dump_database(id) subprocess.call( "pg_restore --verbose --no-owner --clean -d dallinger " + os.path.join("data", id, "data.dump"), shell=True) all_tables = [ "node", "network", "vector", "info", "transformation", "transmission", "participant", "notification", "question" ] for table in all_tables: subprocess.call( "psql -d dallinger --command=\"\\copy " + table + " to \'" + os.path.join(subdata_path, table) + ".csv\' csv header\"", shell=True) if not local: os.remove(dump_path) log("Zipping up the package...") shutil.make_archive( os.path.join("data", id + "-data"), "zip", os.path.join("data", id) ) shutil.rmtree(os.path.join("data", id)) log("Done. Data available in " + str(id) + ".zip")
def deploy_sandbox_shared_setup(verbose=True, app=None, web_procs=1, exp_config=None): """Set up Git, push to Heroku, and launch the app.""" if verbose: out = None else: out = open(os.devnull, 'w') (id, tmp) = setup_experiment(debug=False, verbose=verbose, app=app, exp_config=exp_config) # Register the experiment using all configured registration services. config = get_config() if config.get("mode") == u"live": log("Registering the experiment on configured services...") registration.register(id, snapshot=None) # Log in to Heroku if we aren't already. log("Making sure that you are logged in to Heroku.") heroku.log_in() config.set("heroku_auth_token", heroku.auth_token()) click.echo("") # Change to temporary directory. cwd = os.getcwd() os.chdir(tmp) # Commit Heroku-specific files to tmp folder's git repo. subprocess.check_call(["git", "init"], stdout=out) subprocess.check_call(["git", "add", "--all"], stdout=out) subprocess.check_call( ["git", "commit", "-m", '"Experiment {}"'.format(id)], stdout=out, ) # Load configuration. config = get_config() if not config.ready: config.load() # Initialize the app on Heroku. log("Initializing app on Heroku...") create_cmd = [ "heroku", "apps:create", app_name(id), "--buildpack", "https://github.com/thenovices/heroku-buildpack-scipy", ] # If a team is specified, assign the app to the team. try: team = config.get("heroku_team", None) if team: create_cmd.extend(["--org", team]) except Exception: pass subprocess.check_call(create_cmd, stdout=out) database_size = config.get('database_size') # Set up postgres database and AWS environment variables. cmds = [ [ "heroku", "addons:create", "heroku-postgresql:{}".format(quote(database_size)) ], ["heroku", "addons:create", "heroku-redis:premium-0"], ["heroku", "addons:create", "papertrail"], ] for cmd in cmds: subprocess.check_call(cmd + ["--app", app_name(id)], stdout=out) heroku_config = { "HOST": "{}.herokuapp.com".format(app_name(id)), "aws_access_key_id": config["aws_access_key_id"], "aws_secret_access_key": config["aws_secret_access_key"], "aws_region": config["aws_region"], "auto_recruit": config["auto_recruit"], "dallinger_email_username": config["dallinger_email_address"], "dallinger_email_key": config["dallinger_email_password"], "whimsical": config["whimsical"], } for key in heroku_config: subprocess.check_call([ "heroku", "config:set", "{}={}".format( key, quote(str(heroku_config[key]))), "--app", app_name(id) ], stdout=out) # Wait for Redis database to be ready. log("Waiting for Redis...") ready = False while not ready: redis_URL = subprocess.check_output([ "heroku", "config:get", "REDIS_URL", "--app", app_name(id), ]) r = redis.from_url(redis_URL) try: r.set("foo", "bar") ready = True except redis.exceptions.ConnectionError: time.sleep(2) log("Saving the URL of the postgres database...") subprocess.check_call(["heroku", "pg:wait", "--app", app_name(id)]) db_url = subprocess.check_output( ["heroku", "config:get", "DATABASE_URL", "--app", app_name(id)]) # Set the notification URL and database URL in the config file. config.extend({ "notification_url": u"http://" + app_name(id) + ".herokuapp.com/notifications", "database_url": db_url.rstrip().decode('utf8'), }) config.write() subprocess.check_call(["git", "add", "config.txt"], stdout=out), time.sleep(0.25) subprocess.check_call( ["git", "commit", "-m", '"Save URLs for database and notifications"'], stdout=out) time.sleep(0.25) # Launch the Heroku app. log("Pushing code to Heroku...") subprocess.check_call(["git", "push", "heroku", "HEAD:master"], stdout=out, stderr=out) log("Scaling up the dynos...") heroku.scale_up_dynos(app_name(id)) time.sleep(8) # Launch the experiment. log("Launching the experiment on MTurk...") launch_request = requests.post('https://{}.herokuapp.com/launch'.format( app_name(id))) launch_data = launch_request.json() log("URLs:") log("App home: https://{}.herokuapp.com/".format(app_name(id)), chevrons=False) log("Initial recruitment: {}".format( launch_data.get('recruitment_url', None)), chevrons=False) # Return to the branch whence we came. os.chdir(cwd) log("Completed deployment of experiment " + id + ".")