def execute(self, heroku):
"""Start the server, load the zip file into the database, then loop
until terminated with <control>-c.
"""
db.init_db(drop_all=True)
self.out.log(
"Ingesting dataset from {}...".format(os.path.basename(self.zip_path))
)
data.ingest_zip(self.zip_path)
base_url = get_base_url()
self.out.log("Server is running on {}. Press Ctrl+C to exit.".format(base_url))
if self.exp_config.get("replay"):
self.out.log("Launching the experiment...")
time.sleep(4)
_handle_launch_data("{}/launch".format(base_url), error=self.out.error)
heroku.monitor(listener=self.notify)
# Just run until interrupted:
while self.keep_running():
time.sleep(1)
def restore_state_from_replay(
self, app_id, session, zip_path=None, **configuration_options
):
# We need to fake dallinger_experiment to point at the current experiment
module = sys.modules[type(self).__module__]
if sys.modules.get("dallinger_experiment", module) != module:
logger.warning("dallinger_experiment is already set, updating")
sys.modules["dallinger_experiment"] = module
# Load the configuration system and globals
config = get_config()
# Manually load extra parameters and ignore errors
try:
from dallinger_experiment.experiment import extra_parameters
try:
extra_parameters()
extra_parameters.loaded = True
except KeyError:
pass
except ImportError:
pass
config.load()
self.app_id = self.original_app_id = app_id
self.session = session
self.exp_config = config
# The replay index is initialised to 1970 as that is guaranteed
# to be before any experiment Info objects
self._replay_time_index = datetime.datetime(1970, 1, 1, 1, 1, 1)
# Create a second database session so we can load the full history
# of the experiment to be replayed and selectively import events
# into the main database
specific_db_url = db_url + "-import-" + app_id
import_engine = create_engine(specific_db_url)
try:
# Clear the temporary storage and import it
init_db(drop_all=True, bind=import_engine)
except Exception:
create_db_engine = create_engine(db_url)
conn = create_db_engine.connect()
conn.execute("COMMIT;")
conn.execute(
'CREATE DATABASE "{}"'.format(specific_db_url.rsplit("/", 1)[1])
)
conn.close()
import_engine = create_engine(specific_db_url)
init_db(drop_all=True, bind=import_engine)
self.import_session = scoped_session(
sessionmaker(autocommit=False, autoflush=True, bind=import_engine)
)
# Find the real data for this experiment
if zip_path is None:
zip_path = find_experiment_export(app_id)
if zip_path is None:
msg = 'Dataset export for app id "{}" could not be found.'
raise IOError(msg.format(app_id))
print("Ingesting dataset from {}...".format(os.path.basename(zip_path)))
ingest_zip(zip_path, engine=import_engine)
self._replay_range = tuple(
self.import_session.query(
func.min(Info.creation_time), func.max(Info.creation_time)
)
)[0]
# We apply the configuration options we were given and yield
# the scrubber function into the context manager, so within the
# with experiment.restore_state_from_replay(...): block the configuration
# options are correctly set
with config.override(configuration_options, strict=True):
self.replay_start()
yield Scrubber(self, session=self.import_session)
self.replay_finish()
# Clear up global state
self.import_session.rollback()
self.import_session.close()
session.rollback()
session.close()
# Remove marker preventing experiment config variables being reloaded
try:
del module.extra_parameters.loaded
except AttributeError:
pass
config._reset(register_defaults=True)
del sys.modules["dallinger_experiment"]
def restore_state_from_replay(self,
app_id,
session,
zip_path=None,
**configuration_options):
# We need to fake dallinger_experiment to point at the current experiment
module = sys.modules[type(self).__module__]
if sys.modules.get('dallinger_experiment', module) != module:
raise RuntimeError('dallinger_experiment is already set')
sys.modules['dallinger_experiment'] = module
# Load the configuration system and globals
config = get_config()
config.register_extra_parameters()
config.load()
self.app_id = self.original_app_id = app_id
self.session = session
self.exp_config = config
# The replay index is initialised to 1970 as that is guaranteed
# to be before any experiment Info objects
self._replay_time_index = datetime.datetime(1970, 1, 1, 1, 1, 1)
# Create a second database session so we can load the full history
# of the experiment to be replayed and selectively import events
# into the main database
import_engine = create_engine(
"postgresql://*****:*****@localhost/dallinger-import")
import_session = scoped_session(
sessionmaker(autocommit=False, autoflush=True, bind=import_engine))
# Find the real data for this experiment
if zip_path is None:
zip_path = find_experiment_export(app_id)
if zip_path is None:
msg = u'Dataset export for app id "{}" could not be found.'
raise IOError(msg.format(app_id))
# Clear the temporary storage and import it
init_db(drop_all=True, bind=import_engine)
print("Ingesting dataset from {}...".format(
os.path.basename(zip_path)))
ingest_zip(zip_path, engine=import_engine)
def go_to(time):
"""Scrub to a point in the experiment replay, given by time
which is a datetime object."""
if self._replay_time_index > time:
# We do not support going back in time
raise NotImplementedError
events = self.events_for_replay(session=import_session)
for event in events:
if event.creation_time <= self._replay_time_index:
# Skip events we've already handled
continue
if event.creation_time > time:
# Stop once we get future events
break
self.replay_event(event)
self._replay_time_index = time
# Override app_id to allow exports to be created that don't
# overwrite the original dataset
self.app_id = "{}_{}".format(self.original_app_id,
time.isoformat())
# We apply the configuration options we were given and yield
# the scrubber function into the context manager, so within the
# with experiment.restore_state_from_replay(...): block the configuration
# options are correctly set
with config.override(configuration_options, strict=True):
yield go_to
# Clear up global state
import_session.close()
config._reset(register_defaults=True)
del sys.modules['dallinger_experiment']
