def add_checkpoint(checkpoint_name):
"""
Create a new checkpoint with specified name, write all data required to restore the simulation
to its current state.
Detect any changed tables , re-wrap them and write the current version to the pipeline store.
Write the current state of the random number generator.
Parameters
----------
checkpoint_name : str
"""
timestamp = dt.datetime.now()
logger.debug("set_checkpoint %s timestamp %s" %
(checkpoint_name, timestamp))
for table_name in orca_dataframe_tables():
# if we have not already checkpointed it or it has changed
# FIXME - this won't detect if the orca table was modified
if (table_name not in _LAST_CHECKPOINT
or len(orca.list_columns_for_table(table_name))):
# rewrap the changed orca table as a unitary DataFrame-backed DataFrameWrapper table
df = rewrap(table_name)
logger.debug("set_checkpoint %s writing %s to store" % (
checkpoint_name,
table_name,
))
# write it to store
write_df(df, table_name, checkpoint_name)
# remember which checkpoint it was last written
_LAST_CHECKPOINT[table_name] = checkpoint_name
_LAST_CHECKPOINT[_CHECKPOINT_NAME] = checkpoint_name
_LAST_CHECKPOINT[_TIMESTAMP] = timestamp
# current state of the random number generator
_LAST_CHECKPOINT[_PRNG_CHANNELS] = cPickle.dumps(_PRNG.get_channels())
# append to the array of checkpoint history
_CHECKPOINTS.append(_LAST_CHECKPOINT.copy())
# create a pandas dataframe of the checkpoint history, one row per checkpoint
checkpoints = pd.DataFrame(_CHECKPOINTS)
# convert empty values to str so PyTables doesn't pickle object types
for c in checkpoints.columns:
checkpoints[c] = checkpoints[c].fillna('')
# write it to the store, overwriting any previous version (no way to simply extend)
write_df(checkpoints, _CHECKPOINT_TABLE_NAME)
for channel_state in _PRNG.get_channels():
logger.debug("channel_name '%s', step_name '%s', offset: %s" %
channel_state)
def drop_table(table_name):
if orca.is_table(table_name):
logger.debug("drop_table dropping orca table '%s'" % table_name)
# don't trigger function call of TableFuncWrapper
t = orca.get_raw_table(table_name)
t.clear_cached()
for column_name in orca.list_columns_for_table(table_name):
# logger.debug("pop %s.%s: %s" % (table_name, column_name, t.column_type(column_name)))
orca.orca._COLUMNS.pop((table_name, column_name), None)
# remove from orca's table list
orca.orca._TABLES.pop(table_name, None)
if table_name in _PIPELINE.replaced_tables:
logger.debug("drop_table forgetting replaced_tables '%s'" % table_name)
del _PIPELINE.replaced_tables[table_name]
if table_name in _PIPELINE.last_checkpoint:
logger.debug("drop_table removing table %s from last_checkpoint" % table_name)
_PIPELINE.last_checkpoint[table_name] = ''
def rewrap(table_name, df=None):
"""
Add or replace an orca registered table as a unitary DataFrame-backed DataFrameWrapper table
if df is None, then get the dataframe from orca (table_name should be registered, or
an error will be thrown) which may involve evaluating added columns, etc.
If the orca table already exists, deregister it along with any associated columns before
re-registering it.
The net result is that the dataframe is a registered orca DataFrameWrapper table with no
computed or added columns.
Parameters
----------
table_name
df
Returns
-------
the underlying df of the rewrapped table
"""
logger.debug("rewrap table %s inplace=%s" % (table_name, (df is None)))
if orca.is_table(table_name):
if df is None:
logger.debug("rewrap - orca.get_table(%s)" % (table_name, ))
t = orca.get_table(table_name)
df = t.to_frame()
else:
logger.debug("rewrap - orca.get_raw_table(%s)" % (table_name, ))
# don't trigger function call of TableFuncWrapper
t = orca.get_raw_table(table_name)
t.clear_cached()
for column_name in orca.list_columns_for_table(table_name):
# logger.debug("pop %s.%s: %s" % (table_name, column_name, t.column_type(column_name)))
orca.orca._COLUMNS.pop((table_name, column_name), None)
# remove from orca's table list
orca.orca._TABLES.pop(table_name, None)
assert df is not None
logger.debug("rewrap - orca.add_table(%s)" % (table_name, ))
orca.add_table(table_name, df)
return df