def awaken(app, databaseurl): """Restore the database from a given url.""" id = app config = get_config() config.load() bucket = data.user_s3_bucket() key = bucket.lookup("{}.dump".format(id)) url = key.generate_url(expires_in=300) heroku_app = HerokuApp(id, output=None, team=None) heroku_app.addon("heroku-postgresql:{}".format( config.get("database_size"))) time.sleep(60) heroku_app.pg_wait() time.sleep(10) heroku_app.addon("heroku-redis:{}".format(config.get("redis_size"))) heroku_app.restore(url) # Scale up the dynos. log("Scaling up the dynos...") size = config.get("dyno_type") for process in ["web", "worker"]: qty = config.get("num_dynos_" + process) heroku_app.scale_up_dyno(process, qty, size) if config.get("clock_on"): heroku_app.scale_up_dyno("clock", 1, size)
def debug(verbose, bot, proxy, no_browsers=False, exp_config=None): """Run the experiment locally using docker compose.""" from dallinger.docker.deployment import DockerDebugDeployment debugger = DockerDebugDeployment(Output(), verbose, bot, proxy, exp_config, no_browsers) log(header, chevrons=False) debugger.run()
def load(app, verbose, replay, exp_config=None): """Import database state from an exported zip file and leave the server running until stopping the process with <control>-c. """ if replay: exp_config = exp_config or {} exp_config["replay"] = True log(header, chevrons=False) loader = LoaderDeployment(app, Output(), verbose, exp_config) loader.run()
def export(app, local, no_scrub): """Export the experiment data to a zip archive on your local computer, and by default, to Amazon S3.""" log(header, chevrons=False) try: data.export(str(app), local=local, scrub_pii=(not no_scrub)) except data.S3BucketUnavailable: log("Your local export completed normally, but you don't have an " "Amazon S3 bucket accessible for a remote export. " "Either add an S3 bucket, or run with the --local option to " 'avoid this warning. Run "dallinger export -h" for more details.')
def _handle_timeout(signum, frame): config = get_config() if not config.ready: config.load() message = { "subject": "Idle Experiment.", "body": idle_template.format( app_id=config.get("id"), minutes_so_far=round(seconds / 60) ), } log("Reporting problem with idle experiment...") admin_notifier(config).send(**message)
def setup(): """Walk the user though the Dallinger setup.""" # Create the Dallinger config file if it does not already exist. config_name = ".dallingerconfig" config_path = os.path.join(os.path.expanduser("~"), config_name) if os.path.isfile(config_path): log("Dallinger config file already exists.", chevrons=False) else: log("Creating Dallinger config file at ~/.dallingerconfig...", chevrons=False) src = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", "default_configs", config_name, ) shutil.copyfile(src, config_path)
def _deploy_in_mode(mode, verbose, log, app=None, archive=None): if app: verify_id(None, None, app) log(header, chevrons=False) prelaunch = [] if archive: archive_path = os.path.abspath(archive) if not os.path.exists(archive_path): raise click.BadParameter( 'Experiment archive "{}" does not exist.'.format(archive_path)) prelaunch.append(prelaunch_db_bootstrapper(archive_path, log)) config = get_config() config.load() config.extend({"mode": mode, "logfile": "-"}) deploy_sandbox_shared_setup(log=log, verbose=verbose, app=app, prelaunch_actions=prelaunch)
def verify(): """Verify that app is compatible with Dallinger.""" verbose = True log( "Verifying current directory as a Dallinger experiment...", verbose=verbose, ) ok = verify_package(verbose=verbose) if ok: log("✓ Everything looks good!", verbose=verbose) else: log("☹ Some problems were found.", verbose=verbose)
def hibernate(app): """Pause an experiment and remove costly resources.""" log("The database backup URL is...") backup_url = data.backup(app) log(backup_url) log("Scaling down the web servers...") heroku_app = HerokuApp(app) heroku_app.scale_down_dynos() log("Removing addons...") addons = [ "heroku-postgresql", # "papertrail", "heroku-redis", ] for addon in addons: heroku_app.addon_destroy(addon)
def _bootstrap(exp_config=None): """Creates a directory called 'develop' which will be used to host the development version of the experiment.""" bootstrapper = DevelopmentDeployment(Output(), exp_config) log(header, chevrons=False) bootstrapper.run()
def bootstrap_db(heroku_app, config): # Pre-populate the database if an archive was given log("Ingesting dataset from {}...".format(os.path.basename(zip_path))) engine = db.create_db_engine(heroku_app.db_url) data.bootstrap_db_from_zip(zip_path, engine)
def debug(verbose, bot, proxy, no_browsers=False, exp_config=None): """Run the experiment locally.""" debugger = DebugDeployment(Output(), verbose, bot, proxy, exp_config, no_browsers) log(header, chevrons=False) debugger.run()