def run(self): """Set up the environment, get a wrapper instance, and pass it to the concrete class's execute() method. """ self.configure() self.setup() self.update_dir() if self.DO_INIT_DB: db.init_db(drop_all=True) config = get_config() environ = None if self.environ: environ = os.environ.copy() environ.update(self.environ) self.out.log(f"Starting up the {self.DEPLOY_NAME} Local server...") with self.WRAPPER_CLASS( config, self.out, self.original_dir, self.tmp_dir, verbose=self.verbose, env=environ, needs_chrome=self.bot, ) as wrapper: try: self.execute(wrapper) except KeyboardInterrupt: pass finally: os.chdir(self.original_dir) self.cleanup()
def debug(verbose): """Run the experiment locally.""" (id, tmp) = setup_experiment(debug=True, verbose=verbose) # Drop all the tables from the database. db.init_db(drop_all=True) # Switch to the temporary directory. cwd = os.getcwd() os.chdir(tmp) # Set the mode to debug. config = get_config() logfile = config.get('logfile') if logfile != '-': logfile = os.path.join(cwd, logfile) config.extend({"mode": u"debug", "loglevel": 0, "logfile": logfile}) config.write_config() # Start up the local server log("Starting up the server...") path = os.path.realpath(os.path.join(__file__, '..', 'heroku', 'launch.py')) p = subprocess.Popen( [sys.executable, '-u', path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) # Wait for server to start ready = False for line in iter(p.stdout.readline, ''): if re.match('^Ready.$', line): ready = True break sys.stdout.write(line) if ready: host = config.get('host') port = config.get('port') public_interface = "{}:{}".format(host, port) log("Server is running on {}. Press Ctrl+C to exit.".format( public_interface)) # Call endpoint to launch the experiment log("Launching the experiment...") requests.post('http://{}/launch'.format(public_interface)) # Monitor output from server process for line in iter(p.stdout.readline, ''): sys.stdout.write(line) # Open browser for new participants match = re.search('New participant requested: (.*)$', line) if match: url = match.group(1) webbrowser.open(url, new=1, autoraise=True) log("Completed debugging of experiment with id " + id) os.chdir(cwd)
def run(self): """Set up the environment, get a HerokuLocalWrapper instance, and pass it to the concrete class's execute() method. """ self.configure() self.setup() self.update_dir() db.init_db(drop_all=True) config = get_config() environ = None if self.environ: environ = os.environ.copy() environ.update(self.environ) self.out.log("Starting up the Heroku Local server...") with HerokuLocalWrapper(config, self.out, verbose=self.verbose, env=environ) as wrapper: try: self.execute(wrapper) except KeyboardInterrupt: pass finally: os.chdir(self.original_dir) self.cleanup()
def bootstrap_db_from_zip(zip_path, engine): """Given a path to a zip archive created with `export()`, first empty the database, then recreate it based on the data stored in the included .csv files. """ db.init_db(drop_all=True, bind=engine) ingest_zip(zip_path, engine=engine)
def run(self): """Bootstrap the environment and reset the database.""" self.out.log("Preparing your pristine development environment...") experiment_uid, dst = bootstrap_development_session( self.exp_config, os.getcwd(), self.out.log) self.out.log("Re-initializing database...") db.init_db(drop_all=True) self.out.log(f"Files symlinked in {dst}.\n" "Run './run.sh' in that directory to start Flask, " "plus the worker and clock processes.")
def launch(): """Launch the experiment.""" exp = Experiment(db.init_db(drop_all=False)) exp.log("Launching experiment...", "-----") url_info = exp.recruiter().open_recruitment(n=exp.initial_recruitment_size) session.commit() return success_response("recruitment_url", url_info, request_type="launch")
def execute(self, heroku): """Start the server, load the zip file into the database, then loop until terminated with <control>-c. """ db.init_db(drop_all=True) self.out.log( "Ingesting dataset from {}...".format(os.path.basename(self.zip_path)) ) data.ingest_zip(self.zip_path) base_url = get_base_url() self.out.log("Server is running on {}. Press Ctrl+C to exit.".format(base_url)) if self.exp_config.get("replay"): self.out.log("Launching the experiment...") time.sleep(4) _handle_launch_data("{}/launch".format(base_url), error=self.out.error) heroku.monitor(listener=self.notify) # Just run until interrupted: while self.keep_running(): time.sleep(1)
def setup(self): self._db = db.init_db(drop_all=True) # This is only needed for config, which loads on import os.chdir('demos/bartlett1932')
def setup(self): self.db = db.init_db(drop_all=True)
def restore_state_from_replay(self, app_id, session, zip_path=None, **configuration_options): # We need to fake dallinger_experiment to point at the current experiment module = sys.modules[type(self).__module__] if sys.modules.get('dallinger_experiment', module) != module: raise RuntimeError('dallinger_experiment is already set') sys.modules['dallinger_experiment'] = module # Load the configuration system and globals config = get_config() config.register_extra_parameters() config.load() self.app_id = self.original_app_id = app_id self.session = session self.exp_config = config # The replay index is initialised to 1970 as that is guaranteed # to be before any experiment Info objects self._replay_time_index = datetime.datetime(1970, 1, 1, 1, 1, 1) # Create a second database session so we can load the full history # of the experiment to be replayed and selectively import events # into the main database import_engine = create_engine( "postgresql://*****:*****@localhost/dallinger-import") import_session = scoped_session( sessionmaker(autocommit=False, autoflush=True, bind=import_engine)) # Find the real data for this experiment if zip_path is None: zip_path = find_experiment_export(app_id) if zip_path is None: msg = u'Dataset export for app id "{}" could not be found.' raise IOError(msg.format(app_id)) # Clear the temporary storage and import it init_db(drop_all=True, bind=import_engine) print("Ingesting dataset from {}...".format( os.path.basename(zip_path))) ingest_zip(zip_path, engine=import_engine) def go_to(time): """Scrub to a point in the experiment replay, given by time which is a datetime object.""" if self._replay_time_index > time: # We do not support going back in time raise NotImplementedError events = self.events_for_replay(session=import_session) for event in events: if event.creation_time <= self._replay_time_index: # Skip events we've already handled continue if event.creation_time > time: # Stop once we get future events break self.replay_event(event) self._replay_time_index = time # Override app_id to allow exports to be created that don't # overwrite the original dataset self.app_id = "{}_{}".format(self.original_app_id, time.isoformat()) # We apply the configuration options we were given and yield # the scrubber function into the context manager, so within the # with experiment.restore_state_from_replay(...): block the configuration # options are correctly set with config.override(configuration_options, strict=True): yield go_to # Clear up global state import_session.close() config._reset(register_defaults=True) del sys.modules['dallinger_experiment']
def debug(verbose): """Run the experiment locally.""" (id, tmp) = setup_experiment(debug=True, verbose=verbose) # Drop all the tables from the database. db.init_db(drop_all=True) # Switch to the temporary directory. cwd = os.getcwd() os.chdir(tmp) # Set the mode to debug. config = get_config() logfile = config.get('logfile') if logfile != '-': logfile = os.path.join(cwd, logfile) config.extend({"mode": u"debug", "loglevel": 0, "logfile": logfile}) config.write() # Start up the local server log("Starting up the server...") try: p = subprocess.Popen( ['heroku', 'local'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) except OSError: error("Couldn't start Heroku for local debugging.") raise # Wait for server to start ready = False for line in iter(p.stdout.readline, ''): if verbose: sys.stdout.write(line) line = line.strip() if re.match('^.*? worker.1 .*? Connection refused.$', line): error( 'Could not connect to redis instance, experiment may not behave correctly.' ) if not verbose and re.match('^.*? web.1 .*? \[ERROR\] (.*?)$', line): error(line) if re.match('^.*? web.1 .*? Ready.$', line): ready = True break if ready: host = config.get('host') port = config.get('port') public_interface = "{}:{}".format(host, port) log("Server is running on {}. Press Ctrl+C to exit.".format( public_interface)) # Call endpoint to launch the experiment log("Launching the experiment...") requests.post('http://{}/launch'.format(public_interface)) # Monitor output from server process for line in iter(p.stdout.readline, ''): sys.stdout.write(line) # Open browser for new participants match = re.search('New participant requested: (.*)$', line) if match: url = match.group(1) webbrowser.open(url, new=1, autoraise=True) log("Completed debugging of experiment with id " + id) os.chdir(cwd)
def setup(self): from dallinger.config import get_config config = get_config() config.load() self.db = db.init_db(drop_all=True)
def debug(verbose): """Run the experiment locally.""" (id, tmp) = setup_experiment(debug=True, verbose=verbose) # Drop all the tables from the database. db.init_db(drop_all=True) # Switch to the temporary directory. cwd = os.getcwd() os.chdir(tmp) # Load psiTurk configuration. config = PsiturkConfig() config.load_config() # Set the mode to debug. config.set("Experiment Configuration", "mode", "debug") config.set("Shell Parameters", "launch_in_sandbox_mode", "true") config.set( "Server Parameters", "logfile", os.path.join(cwd, config.get("Server Parameters", "logfile"))) # Swap in the HotAirRecruiter os.rename("dallinger_experiment.py", "dallinger_experiment_tmp.py") with open("dallinger_experiment_tmp.py", "r+") as f: with open("dallinger_experiment.py", "w+") as f2: f2.write("from dallinger.recruiters import HotAirRecruiter\n") for idx, line in enumerate(f): if re.search("\s*self.recruiter = (.*)", line): p = line.partition("self.recruiter =") f2.write(p[0] + p[1] + ' HotAirRecruiter\n') else: f2.write(line) os.remove("dallinger_experiment_tmp.py") # Set environment variables. vars = [ ("AWS Access", "aws_access_key_id"), ("AWS Access", "aws_secret_access_key"), ("AWS Access", "aws_region"), ("psiTurk Access", "psiturk_access_key_id"), ("psiTurk Access", "psiturk_secret_access_id"), ] for var in vars: if var[0] not in os.environ: os.environ[var[1]] = config.get(var[0], var[1]) if "HOST" not in os.environ: os.environ["HOST"] = config.get('Server Parameters', 'host') # Start up the local server log("Starting up the server...") # Try opening the psiTurk shell. try: p = pexpect.spawn("psiturk") p.expect_exact("]$") p.sendline("server on") p.expect_exact("Experiment server launching...") # Launche the experiment. time.sleep(4) host = config.get("Server Parameters", "host") port = config.get("Server Parameters", "port") subprocess.call( 'curl --data "" http://{}:{}/launch'.format(host, port), shell=True) log("Here's the psiTurk shell...") p.interact() except Exception: click.echo("\nCouldn't open psiTurk shell. Internet connection okay?") log("Completed debugging of experiment with id " + id) os.chdir(cwd)
def setup(self): self.db = db.init_db(drop_all=True) os.chdir(os.path.join("demos", "bartlett1932"))
def initdb(no_drop): from dallinger.db import init_db drop_all = not no_drop init_db(drop_all=drop_all)
def setup(self): """Set up the environment by resetting the tables.""" self.db = db.init_db(drop_all=True)
def restore_state_from_replay( self, app_id, session, zip_path=None, **configuration_options ): # We need to fake dallinger_experiment to point at the current experiment module = sys.modules[type(self).__module__] if sys.modules.get("dallinger_experiment", module) != module: logger.warning("dallinger_experiment is already set, updating") sys.modules["dallinger_experiment"] = module # Load the configuration system and globals config = get_config() # Manually load extra parameters and ignore errors try: from dallinger_experiment.experiment import extra_parameters try: extra_parameters() extra_parameters.loaded = True except KeyError: pass except ImportError: pass config.load() self.app_id = self.original_app_id = app_id self.session = session self.exp_config = config # The replay index is initialised to 1970 as that is guaranteed # to be before any experiment Info objects self._replay_time_index = datetime.datetime(1970, 1, 1, 1, 1, 1) # Create a second database session so we can load the full history # of the experiment to be replayed and selectively import events # into the main database specific_db_url = db_url + "-import-" + app_id import_engine = create_engine(specific_db_url) try: # Clear the temporary storage and import it init_db(drop_all=True, bind=import_engine) except Exception: create_db_engine = create_engine(db_url) conn = create_db_engine.connect() conn.execute("COMMIT;") conn.execute( 'CREATE DATABASE "{}"'.format(specific_db_url.rsplit("/", 1)[1]) ) conn.close() import_engine = create_engine(specific_db_url) init_db(drop_all=True, bind=import_engine) self.import_session = scoped_session( sessionmaker(autocommit=False, autoflush=True, bind=import_engine) ) # Find the real data for this experiment if zip_path is None: zip_path = find_experiment_export(app_id) if zip_path is None: msg = 'Dataset export for app id "{}" could not be found.' raise IOError(msg.format(app_id)) print("Ingesting dataset from {}...".format(os.path.basename(zip_path))) ingest_zip(zip_path, engine=import_engine) self._replay_range = tuple( self.import_session.query( func.min(Info.creation_time), func.max(Info.creation_time) ) )[0] # We apply the configuration options we were given and yield # the scrubber function into the context manager, so within the # with experiment.restore_state_from_replay(...): block the configuration # options are correctly set with config.override(configuration_options, strict=True): self.replay_start() yield Scrubber(self, session=self.import_session) self.replay_finish() # Clear up global state self.import_session.rollback() self.import_session.close() session.rollback() session.close() # Remove marker preventing experiment config variables being reloaded try: del module.extra_parameters.loaded except AttributeError: pass config._reset(register_defaults=True) del sys.modules["dallinger_experiment"]
def setup(self): self._db = db.init_db(drop_all=True) # This is only needed for config, which loads on import os.chdir(os.path.join("demos", "dlgr", "demos", "bartlett1932"))