def add_checkpoint(checkpoint_name): """ Create a new checkpoint with specified name, write all data required to restore the simulation to its current state. Detect any changed tables , re-wrap them and write the current version to the pipeline store. Write the current state of the random number generator. Parameters ---------- checkpoint_name : str """ timestamp = dt.datetime.now() logger.debug("set_checkpoint %s timestamp %s" % (checkpoint_name, timestamp)) for table_name in orca_dataframe_tables(): # if we have not already checkpointed it or it has changed # FIXME - this won't detect if the orca table was modified if (table_name not in _LAST_CHECKPOINT or len(orca.list_columns_for_table(table_name))): # rewrap the changed orca table as a unitary DataFrame-backed DataFrameWrapper table df = rewrap(table_name) logger.debug("set_checkpoint %s writing %s to store" % ( checkpoint_name, table_name, )) # write it to store write_df(df, table_name, checkpoint_name) # remember which checkpoint it was last written _LAST_CHECKPOINT[table_name] = checkpoint_name _LAST_CHECKPOINT[_CHECKPOINT_NAME] = checkpoint_name _LAST_CHECKPOINT[_TIMESTAMP] = timestamp # current state of the random number generator _LAST_CHECKPOINT[_PRNG_CHANNELS] = cPickle.dumps(_PRNG.get_channels()) # append to the array of checkpoint history _CHECKPOINTS.append(_LAST_CHECKPOINT.copy()) # create a pandas dataframe of the checkpoint history, one row per checkpoint checkpoints = pd.DataFrame(_CHECKPOINTS) # convert empty values to str so PyTables doesn't pickle object types for c in checkpoints.columns: checkpoints[c] = checkpoints[c].fillna('') # write it to the store, overwriting any previous version (no way to simply extend) write_df(checkpoints, _CHECKPOINT_TABLE_NAME) for channel_state in _PRNG.get_channels(): logger.debug("channel_name '%s', step_name '%s', offset: %s" % channel_state)
def drop_table(table_name): if orca.is_table(table_name): logger.debug("drop_table dropping orca table '%s'" % table_name) # don't trigger function call of TableFuncWrapper t = orca.get_raw_table(table_name) t.clear_cached() for column_name in orca.list_columns_for_table(table_name): # logger.debug("pop %s.%s: %s" % (table_name, column_name, t.column_type(column_name))) orca.orca._COLUMNS.pop((table_name, column_name), None) # remove from orca's table list orca.orca._TABLES.pop(table_name, None) if table_name in _PIPELINE.replaced_tables: logger.debug("drop_table forgetting replaced_tables '%s'" % table_name) del _PIPELINE.replaced_tables[table_name] if table_name in _PIPELINE.last_checkpoint: logger.debug("drop_table removing table %s from last_checkpoint" % table_name) _PIPELINE.last_checkpoint[table_name] = ''
def rewrap(table_name, df=None): """ Add or replace an orca registered table as a unitary DataFrame-backed DataFrameWrapper table if df is None, then get the dataframe from orca (table_name should be registered, or an error will be thrown) which may involve evaluating added columns, etc. If the orca table already exists, deregister it along with any associated columns before re-registering it. The net result is that the dataframe is a registered orca DataFrameWrapper table with no computed or added columns. Parameters ---------- table_name df Returns ------- the underlying df of the rewrapped table """ logger.debug("rewrap table %s inplace=%s" % (table_name, (df is None))) if orca.is_table(table_name): if df is None: logger.debug("rewrap - orca.get_table(%s)" % (table_name, )) t = orca.get_table(table_name) df = t.to_frame() else: logger.debug("rewrap - orca.get_raw_table(%s)" % (table_name, )) # don't trigger function call of TableFuncWrapper t = orca.get_raw_table(table_name) t.clear_cached() for column_name in orca.list_columns_for_table(table_name): # logger.debug("pop %s.%s: %s" % (table_name, column_name, t.column_type(column_name))) orca.orca._COLUMNS.pop((table_name, column_name), None) # remove from orca's table list orca.orca._TABLES.pop(table_name, None) assert df is not None logger.debug("rewrap - orca.add_table(%s)" % (table_name, )) orca.add_table(table_name, df) return df