def run(self):
"""Set up the environment, get a wrapper instance, and pass
it to the concrete class's execute() method.
"""
self.configure()
self.setup()
self.update_dir()
if self.DO_INIT_DB:
db.init_db(drop_all=True)
config = get_config()
environ = None
if self.environ:
environ = os.environ.copy()
environ.update(self.environ)
self.out.log(f"Starting up the {self.DEPLOY_NAME} Local server...")
with self.WRAPPER_CLASS(
config,
self.out,
self.original_dir,
self.tmp_dir,
verbose=self.verbose,
env=environ,
needs_chrome=self.bot,
) as wrapper:
try:
self.execute(wrapper)
except KeyboardInterrupt:
pass
finally:
os.chdir(self.original_dir)
self.cleanup()
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup_experiment(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Set the mode to debug.
config = get_config()
logfile = config.get('logfile')
if logfile != '-':
logfile = os.path.join(cwd, logfile)
config.extend({"mode": u"debug", "loglevel": 0, "logfile": logfile})
config.write_config()
# Start up the local server
log("Starting up the server...")
path = os.path.realpath(os.path.join(__file__, '..', 'heroku',
'launch.py'))
p = subprocess.Popen(
[sys.executable, '-u', path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
# Wait for server to start
ready = False
for line in iter(p.stdout.readline, ''):
if re.match('^Ready.$', line):
ready = True
break
sys.stdout.write(line)
if ready:
host = config.get('host')
port = config.get('port')
public_interface = "{}:{}".format(host, port)
log("Server is running on {}. Press Ctrl+C to exit.".format(
public_interface))
# Call endpoint to launch the experiment
log("Launching the experiment...")
requests.post('http://{}/launch'.format(public_interface))
# Monitor output from server process
for line in iter(p.stdout.readline, ''):
sys.stdout.write(line)
# Open browser for new participants
match = re.search('New participant requested: (.*)$', line)
if match:
url = match.group(1)
webbrowser.open(url, new=1, autoraise=True)
log("Completed debugging of experiment with id " + id)
os.chdir(cwd)
def run(self):
"""Set up the environment, get a HerokuLocalWrapper instance, and pass
it to the concrete class's execute() method.
"""
self.configure()
self.setup()
self.update_dir()
db.init_db(drop_all=True)
config = get_config()
environ = None
if self.environ:
environ = os.environ.copy()
environ.update(self.environ)
self.out.log("Starting up the Heroku Local server...")
with HerokuLocalWrapper(config,
self.out,
verbose=self.verbose,
env=environ) as wrapper:
try:
self.execute(wrapper)
except KeyboardInterrupt:
pass
finally:
os.chdir(self.original_dir)
self.cleanup()
def bootstrap_db_from_zip(zip_path, engine):
"""Given a path to a zip archive created with `export()`, first empty the
database, then recreate it based on the data stored in the included .csv
files.
"""
db.init_db(drop_all=True, bind=engine)
ingest_zip(zip_path, engine=engine)
def run(self):
"""Bootstrap the environment and reset the database."""
self.out.log("Preparing your pristine development environment...")
experiment_uid, dst = bootstrap_development_session(
self.exp_config, os.getcwd(), self.out.log)
self.out.log("Re-initializing database...")
db.init_db(drop_all=True)
self.out.log(f"Files symlinked in {dst}.\n"
"Run './run.sh' in that directory to start Flask, "
"plus the worker and clock processes.")
def launch():
"""Launch the experiment."""
exp = Experiment(db.init_db(drop_all=False))
exp.log("Launching experiment...", "-----")
url_info = exp.recruiter().open_recruitment(n=exp.initial_recruitment_size)
session.commit()
return success_response("recruitment_url", url_info, request_type="launch")
def execute(self, heroku):
"""Start the server, load the zip file into the database, then loop
until terminated with <control>-c.
"""
db.init_db(drop_all=True)
self.out.log(
"Ingesting dataset from {}...".format(os.path.basename(self.zip_path))
)
data.ingest_zip(self.zip_path)
base_url = get_base_url()
self.out.log("Server is running on {}. Press Ctrl+C to exit.".format(base_url))
if self.exp_config.get("replay"):
self.out.log("Launching the experiment...")
time.sleep(4)
_handle_launch_data("{}/launch".format(base_url), error=self.out.error)
heroku.monitor(listener=self.notify)
# Just run until interrupted:
while self.keep_running():
time.sleep(1)
def setup(self):
self._db = db.init_db(drop_all=True)
# This is only needed for config, which loads on import
os.chdir('demos/bartlett1932')
def setup(self):
self.db = db.init_db(drop_all=True)
def restore_state_from_replay(self,
app_id,
session,
zip_path=None,
**configuration_options):
# We need to fake dallinger_experiment to point at the current experiment
module = sys.modules[type(self).__module__]
if sys.modules.get('dallinger_experiment', module) != module:
raise RuntimeError('dallinger_experiment is already set')
sys.modules['dallinger_experiment'] = module
# Load the configuration system and globals
config = get_config()
config.register_extra_parameters()
config.load()
self.app_id = self.original_app_id = app_id
self.session = session
self.exp_config = config
# The replay index is initialised to 1970 as that is guaranteed
# to be before any experiment Info objects
self._replay_time_index = datetime.datetime(1970, 1, 1, 1, 1, 1)
# Create a second database session so we can load the full history
# of the experiment to be replayed and selectively import events
# into the main database
import_engine = create_engine(
"postgresql://*****:*****@localhost/dallinger-import")
import_session = scoped_session(
sessionmaker(autocommit=False, autoflush=True, bind=import_engine))
# Find the real data for this experiment
if zip_path is None:
zip_path = find_experiment_export(app_id)
if zip_path is None:
msg = u'Dataset export for app id "{}" could not be found.'
raise IOError(msg.format(app_id))
# Clear the temporary storage and import it
init_db(drop_all=True, bind=import_engine)
print("Ingesting dataset from {}...".format(
os.path.basename(zip_path)))
ingest_zip(zip_path, engine=import_engine)
def go_to(time):
"""Scrub to a point in the experiment replay, given by time
which is a datetime object."""
if self._replay_time_index > time:
# We do not support going back in time
raise NotImplementedError
events = self.events_for_replay(session=import_session)
for event in events:
if event.creation_time <= self._replay_time_index:
# Skip events we've already handled
continue
if event.creation_time > time:
# Stop once we get future events
break
self.replay_event(event)
self._replay_time_index = time
# Override app_id to allow exports to be created that don't
# overwrite the original dataset
self.app_id = "{}_{}".format(self.original_app_id,
time.isoformat())
# We apply the configuration options we were given and yield
# the scrubber function into the context manager, so within the
# with experiment.restore_state_from_replay(...): block the configuration
# options are correctly set
with config.override(configuration_options, strict=True):
yield go_to
# Clear up global state
import_session.close()
config._reset(register_defaults=True)
del sys.modules['dallinger_experiment']
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup_experiment(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Set the mode to debug.
config = get_config()
logfile = config.get('logfile')
if logfile != '-':
logfile = os.path.join(cwd, logfile)
config.extend({"mode": u"debug", "loglevel": 0, "logfile": logfile})
config.write()
# Start up the local server
log("Starting up the server...")
try:
p = subprocess.Popen(
['heroku', 'local'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
except OSError:
error("Couldn't start Heroku for local debugging.")
raise
# Wait for server to start
ready = False
for line in iter(p.stdout.readline, ''):
if verbose:
sys.stdout.write(line)
line = line.strip()
if re.match('^.*? worker.1 .*? Connection refused.$', line):
error(
'Could not connect to redis instance, experiment may not behave correctly.'
)
if not verbose and re.match('^.*? web.1 .*? \[ERROR\] (.*?)$', line):
error(line)
if re.match('^.*? web.1 .*? Ready.$', line):
ready = True
break
if ready:
host = config.get('host')
port = config.get('port')
public_interface = "{}:{}".format(host, port)
log("Server is running on {}. Press Ctrl+C to exit.".format(
public_interface))
# Call endpoint to launch the experiment
log("Launching the experiment...")
requests.post('http://{}/launch'.format(public_interface))
# Monitor output from server process
for line in iter(p.stdout.readline, ''):
sys.stdout.write(line)
# Open browser for new participants
match = re.search('New participant requested: (.*)$', line)
if match:
url = match.group(1)
webbrowser.open(url, new=1, autoraise=True)
log("Completed debugging of experiment with id " + id)
os.chdir(cwd)
def setup(self):
from dallinger.config import get_config
config = get_config()
config.load()
self.db = db.init_db(drop_all=True)
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup_experiment(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Set the mode to debug.
config.set("Experiment Configuration", "mode", "debug")
config.set("Shell Parameters", "launch_in_sandbox_mode", "true")
config.set(
"Server Parameters",
"logfile",
os.path.join(cwd, config.get("Server Parameters", "logfile")))
# Swap in the HotAirRecruiter
os.rename("dallinger_experiment.py", "dallinger_experiment_tmp.py")
with open("dallinger_experiment_tmp.py", "r+") as f:
with open("dallinger_experiment.py", "w+") as f2:
f2.write("from dallinger.recruiters import HotAirRecruiter\n")
for idx, line in enumerate(f):
if re.search("\s*self.recruiter = (.*)", line):
p = line.partition("self.recruiter =")
f2.write(p[0] + p[1] + ' HotAirRecruiter\n')
else:
f2.write(line)
os.remove("dallinger_experiment_tmp.py")
# Set environment variables.
vars = [
("AWS Access", "aws_access_key_id"),
("AWS Access", "aws_secret_access_key"),
("AWS Access", "aws_region"),
("psiTurk Access", "psiturk_access_key_id"),
("psiTurk Access", "psiturk_secret_access_id"),
]
for var in vars:
if var[0] not in os.environ:
os.environ[var[1]] = config.get(var[0], var[1])
if "HOST" not in os.environ:
os.environ["HOST"] = config.get('Server Parameters', 'host')
# Start up the local server
log("Starting up the server...")
# Try opening the psiTurk shell.
try:
p = pexpect.spawn("psiturk")
p.expect_exact("]$")
p.sendline("server on")
p.expect_exact("Experiment server launching...")
# Launche the experiment.
time.sleep(4)
host = config.get("Server Parameters", "host")
port = config.get("Server Parameters", "port")
subprocess.call(
'curl --data "" http://{}:{}/launch'.format(host, port),
shell=True)
log("Here's the psiTurk shell...")
p.interact()
except Exception:
click.echo("\nCouldn't open psiTurk shell. Internet connection okay?")
log("Completed debugging of experiment with id " + id)
os.chdir(cwd)
def setup(self):
self.db = db.init_db(drop_all=True)
os.chdir(os.path.join("demos", "bartlett1932"))
def initdb(no_drop):
from dallinger.db import init_db
drop_all = not no_drop
init_db(drop_all=drop_all)
def setup(self):
"""Set up the environment by resetting the tables."""
self.db = db.init_db(drop_all=True)
def restore_state_from_replay(
self, app_id, session, zip_path=None, **configuration_options
):
# We need to fake dallinger_experiment to point at the current experiment
module = sys.modules[type(self).__module__]
if sys.modules.get("dallinger_experiment", module) != module:
logger.warning("dallinger_experiment is already set, updating")
sys.modules["dallinger_experiment"] = module
# Load the configuration system and globals
config = get_config()
# Manually load extra parameters and ignore errors
try:
from dallinger_experiment.experiment import extra_parameters
try:
extra_parameters()
extra_parameters.loaded = True
except KeyError:
pass
except ImportError:
pass
config.load()
self.app_id = self.original_app_id = app_id
self.session = session
self.exp_config = config
# The replay index is initialised to 1970 as that is guaranteed
# to be before any experiment Info objects
self._replay_time_index = datetime.datetime(1970, 1, 1, 1, 1, 1)
# Create a second database session so we can load the full history
# of the experiment to be replayed and selectively import events
# into the main database
specific_db_url = db_url + "-import-" + app_id
import_engine = create_engine(specific_db_url)
try:
# Clear the temporary storage and import it
init_db(drop_all=True, bind=import_engine)
except Exception:
create_db_engine = create_engine(db_url)
conn = create_db_engine.connect()
conn.execute("COMMIT;")
conn.execute(
'CREATE DATABASE "{}"'.format(specific_db_url.rsplit("/", 1)[1])
)
conn.close()
import_engine = create_engine(specific_db_url)
init_db(drop_all=True, bind=import_engine)
self.import_session = scoped_session(
sessionmaker(autocommit=False, autoflush=True, bind=import_engine)
)
# Find the real data for this experiment
if zip_path is None:
zip_path = find_experiment_export(app_id)
if zip_path is None:
msg = 'Dataset export for app id "{}" could not be found.'
raise IOError(msg.format(app_id))
print("Ingesting dataset from {}...".format(os.path.basename(zip_path)))
ingest_zip(zip_path, engine=import_engine)
self._replay_range = tuple(
self.import_session.query(
func.min(Info.creation_time), func.max(Info.creation_time)
)
)[0]
# We apply the configuration options we were given and yield
# the scrubber function into the context manager, so within the
# with experiment.restore_state_from_replay(...): block the configuration
# options are correctly set
with config.override(configuration_options, strict=True):
self.replay_start()
yield Scrubber(self, session=self.import_session)
self.replay_finish()
# Clear up global state
self.import_session.rollback()
self.import_session.close()
session.rollback()
session.close()
# Remove marker preventing experiment config variables being reloaded
try:
del module.extra_parameters.loaded
except AttributeError:
pass
config._reset(register_defaults=True)
del sys.modules["dallinger_experiment"]
def setup(self):
self._db = db.init_db(drop_all=True)
# This is only needed for config, which loads on import
os.chdir(os.path.join("demos", "dlgr", "demos", "bartlett1932"))
