def main(settings, rescue_running=[]):
"""
Perform the primary loop of building, submitting, monitoring, and analyzing jobs.
This function works via a loop of calls to thread.process and thread.interpret for each thread that hasn't
terminated, until either the global termination criterion is met or all the individual threads have completed.
Parameters
----------
settings : argparse.Namespace
Settings namespace object
rescue_running : list
List of threads passed in from handle_loop_exception, containing running threads. If given, setup is skipped and
the function proceeds directly to the main loop.
Returns
-------
exit_message : str
A message indicating the status of ATESA at the end of main
"""
if not rescue_running:
# Implement resample
if settings.job_type in ['aimless_shooting', 'committor_analysis'
] and settings.resample:
# Store settings object in the working directory for compatibility with analysis/utility scripts
if not settings.dont_dump:
temp_settings = copy.deepcopy(
settings
) # initialize temporary copy of settings to modify
temp_settings.__dict__.pop(
'env') # env attribute is not picklable
pickle.dump(temp_settings,
open(settings.working_directory + '/settings.pkl',
'wb'),
protocol=2)
# Run resampling
if settings.job_type == 'aimless_shooting':
utilities.resample(settings,
partial=False,
full_cvs=settings.full_cvs)
if settings.information_error_checking: # update info_err.out if called for by settings
information_error.main()
elif settings.job_type == 'committor_analysis':
resample_committor_analysis.resample_committor_analysis(
settings)
return 'Resampling complete'
# Make working directory if it does not exist, handling overwrite and restart as needed
if os.path.exists(settings.working_directory):
if settings.overwrite and not settings.restart:
if os.path.exists(
settings.working_directory +
'/cvs.txt'): # a kludge to avoid removing cvs.txt
if os.path.exists('ATESA_TEMP_CVS.txt'):
raise RuntimeError(
'tried to create temporary file ATESA_TEMP_CVS.txt in directory: '
+ os.getcwd() +
', but it already exists. Please move, delete, or rename it.'
)
shutil.move(settings.working_directory + '/cvs.txt',
'ATESA_TEMP_CVS.txt')
shutil.rmtree(settings.working_directory)
os.mkdir(settings.working_directory)
if os.path.exists('ATESA_TEMP_CVS.txt'
): # continuation of aforementioned kludge
shutil.move('ATESA_TEMP_CVS.txt',
settings.working_directory + '/cvs.txt')
elif not settings.restart and glob.glob(
settings.working_directory +
'/*') == [settings.working_directory + '/cvs.txt']:
# Occurs when restart = False, overwrite = False, and auto_cvs is used
pass
elif not settings.restart:
raise RuntimeError(
'Working directory ' + settings.working_directory +
' already exists, but overwrite '
'= False and restart = False. Either change one of these two settings or choose a '
'different working directory.')
else:
if not settings.restart:
os.mkdir(settings.working_directory)
else:
raise RuntimeError('Working directory ' +
settings.working_directory +
' does not yet exist, but '
'restart = True.')
# Store settings object in the working directory for compatibility with analysis/utility scripts
if os.path.exists(
settings.working_directory +
'/settings.pkl'): # for checking for need for resample later
previous_settings = pickle.load(
open(settings.working_directory + '/settings.pkl', 'rb'))
settings.previous_cvs = previous_settings.cvs
try:
settings.previous_information_error_max_dims = previous_settings.information_error_max_dims
except AttributeError:
pass
try:
settings.previous_information_error_lmax_string = previous_settings.information_error_lmax_string
except AttributeError:
pass
if not settings.dont_dump:
temp_settings = copy.deepcopy(
settings) # initialize temporary copy of settings to modify
temp_settings.__dict__.pop(
'env'
) # env attribute is not picklable (update: maybe no longer true, but doesn't matter)
pickle.dump(temp_settings,
open(settings.working_directory + '/settings.pkl',
'wb'),
protocol=2)
# Build or load threads
allthreads = init_threads(settings)
# Move runtime to working directory
os.chdir(settings.working_directory)
running = allthreads.copy() # to be pruned later by thread.process()
attempted_rescue = False # to keep track of general error handling below
else:
allthreads = pickle.load(
open(settings.working_directory + '/restart.pkl', 'rb'))
running = rescue_running
attempted_rescue = True
# Initialize threads with first process step
try:
if not rescue_running: # if rescue_running, this step has already finished and we just want the while loop
for thread in allthreads:
running = thread.process(running, settings)
except Exception as e:
if settings.restart:
print(
'The following error occurred while attempting to initialize threads from restart.pkl. It may be '
'corrupted.')
#'If you haven\'t already done so, consider running verify_threads.py to remove corrupted threads from this file.'
raise e
try:
if settings.job_type == 'aimless_shooting' and len(
os.sched_getaffinity(0)) > 1:
# Initialize Manager for shared data across processes; this is necessary because multiprocessing is being
# retrofitted to code designed for serial processing, but it works!
manager = Manager()
# Setup Managed allthreads list
managed_allthreads = []
for thread in allthreads:
thread_dict = thread.__dict__
thread_history_dict = thread.history.__dict__
managed_thread = Thread()
managed_thread.history = manager.Namespace()
managed_thread.__dict__.update(thread_dict)
managed_thread.history.__dict__.update(thread_history_dict)
managed_allthreads.append(managed_thread)
allthreads = manager.list(managed_allthreads)
# Setup Managed settings Namespace
settings_dict = settings.__dict__
managed_settings = manager.Namespace()
# Need to explicitly update every key because of how the Managed Namespace works.
# Calling exec is the best way to do this I could find. Updating managed_settings.__dict__ doesn't work.
for key in settings_dict.keys():
exec('managed_settings.' + key + ' = settings_dict[key]')
# Distribute processes among available core Pool
with get_context("spawn").Pool(len(os.sched_getaffinity(0))) as p:
p.starmap(
main_loop,
zip(itertools.repeat(managed_settings),
itertools.repeat(allthreads),
[[thread] for thread in allthreads]))
else:
main_loop(settings, allthreads, running)
except AttributeError: # os.sched_getaffinity raises AttributeError on non-UNIX systems.
main_loop(settings, allthreads, running)
## Deprecated thread pool
# pool = ThreadPool(len(allthreads))
# func = partial(main_loop, settings)
# results = pool.map(func, [[thread] for thread in allthreads])
jobtype = factory.jobtype_factory(settings.job_type)
jobtype.cleanup(settings)
return 'ATESA run exiting normally'
def __init__(self, inputShape, n_reservoir,
filterSize=1, stride=1, borderMode="mirror", nWorkers="auto",
spectralRadius=1.0, noiseLevel=0.0, inputScaling=None,
leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True,
out_activation=lambda x: x, out_inverse_activation=lambda x: x,
weightGeneration='naive', bias=1.0, outputBias=1.0,
outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh,
activationDerivation=lambda x: 1.0 / B.cosh(x) ** 2):
self._averageOutputWeights = averageOutputWeights
if averageOutputWeights and solver != "lsqr":
raise ValueError(
"`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)")
self._borderMode = borderMode
if not borderMode in ["mirror", "padding", "edge", "wrap"]:
raise ValueError(
"`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`.")
self._regressionParameters = regressionParameters
self._solver = solver
n_inputDimensions = len(inputShape)
if filterSize % 2 == 0:
raise ValueError("filterSize has to be an odd number (1, 3, 5, ...).")
self._filterSize = filterSize
self._filterWidth = int(np.floor(filterSize / 2))
self._stride = stride
self._n_input = int(np.power(np.ceil(filterSize / stride), n_inputDimensions))
self.n_inputDimensions = n_inputDimensions
self.inputShape = inputShape
if not self._averageOutputWeights:
self._WOuts = B.empty((np.prod(inputShape), 1, self._n_input + n_reservoir + 1))
self._WOut = None
else:
self._WOuts = None
self._WOut = B.zeros((1, self._n_input + n_reservoir + 1))
self._xs = B.empty((np.prod(inputShape), n_reservoir, 1))
if nWorkers == "auto":
self._nWorkers = np.max((cpu_count() - 1, 1))
else:
self._nWorkers = nWorkers
manager = Manager()
self.sharedNamespace = manager.Namespace()
if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None:
self.parallelWorkerIDs = manager.Queue()
for i in range(self._nWorkers):
self.parallelWorkerIDs.put((i))
super(SpatioTemporalESN, self).__init__(n_input=self._n_input, n_reservoir=n_reservoir, n_output=1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel, inputScaling=inputScaling,
leakingRate=leakingRate, reservoirDensity=reservoirDensity,
randomSeed=randomSeed, out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration, bias=bias, outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity, activation=activation,
activationDerivation=activationDerivation)
"""
def __init__(
self,
inputShape,
n_reservoir,
filterSize=1,
stride=1,
borderMode="mirror",
nWorkers="auto",
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
reservoirDensity=0.2,
randomSeed=None,
averageOutputWeights=True,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration="naive",
bias=1.0,
outputBias=1.0,
outputInputScaling=1.0,
inputDensity=1.0,
solver="pinv",
regressionParameters={},
activation=B.tanh,
activationDerivative=lambda x: 1.0 / B.cosh(x)**2,
chunkSize=16,
):
""" ESN that predicts (steps of) a spatio-temporal time series based on a time series.
Args:
inputShape : Shape of the input w/o the time axis, e.g. (W, H) for a 2D input.
n_reservoir : Number of units in the reservoir.
filterSize : Size of patches used to predict a single output element.
stride : Stride between different patches.
borderMode : How to handle border values. Choices: mirror, padding, edge, wrap.
nWorkers : Number of CPU threads executed in parallel to solve the problem.
spectralRadius : Spectral radius of the reservoir's connection/weight matrix.
noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting.
inputScaling : Scaling factor of the input.
leakingRate : Convex combination factor between 0 and 1 that weights current and new state value.
reservoirDensity : Percentage of non-zero weight connections in the reservoir.
randomSeed : Seed for random processes, e.g. weight initialization.
averageOutputWeights : Average output matrices after fitting across all pixels or use a distinct matrix
per pixel. The former assumes homogeneity of the problem across all pixels.
out_activation : Final activation function (i.e. activation function of the output).
out_inverse_activation : Inverse of the final activation function
weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom
bias : Size of the bias added for the internal update process.
outputBias : Size of the bias added for the final linear regression of the output.
outputInputScaling : Rescaling factor for the input of the ESN for the regression.
inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix.
solver : Algorithm to find output matrix. Choices: pinv, lsqr.
regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization.
activation : (Non-linear) Activation function.
activationDerivative : Derivative of the activation function.
chunkSize : Internal parameter for the multi-threading. For long time series this should be reduced to
avoid OOM errors/getting stuck and to reduce memory consumption.
"""
self._averageOutputWeights = averageOutputWeights
if averageOutputWeights and solver != "lsqr":
raise ValueError(
"`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)"
)
self._borderMode = borderMode
if not borderMode in ["mirror", "padding", "edge", "wrap"]:
raise ValueError(
"`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`."
)
self._regressionParameters = regressionParameters
self._solver = solver
n_inputDimensions = len(inputShape)
if filterSize % 2 == 0:
raise ValueError(
"filterSize has to be an odd number (1, 3, 5, ...).")
self._filterSize = filterSize
self._filterWidth = int(np.floor(filterSize / 2))
self._stride = stride
self._n_input = int(
np.power(np.ceil(filterSize / stride), n_inputDimensions))
self.n_inputDimensions = n_inputDimensions
self.inputShape = inputShape
if not self._averageOutputWeights:
self._WOuts = B.empty(
(np.prod(inputShape), 1, self._n_input + n_reservoir + 1))
self._WOut = None
else:
self._WOuts = None
self._WOut = B.zeros((1, self._n_input + n_reservoir + 1))
self._xs = B.empty((np.prod(inputShape), n_reservoir, 1))
if nWorkers == "auto":
self._nWorkers = np.max((cpu_count() - 1, 1))
else:
self._nWorkers = nWorkers
manager = Manager()
self.sharedNamespace = manager.Namespace()
if hasattr(self,
"fitWorkerID") == False or self.parallelWorkerIDs is None:
self.parallelWorkerIDs = manager.Queue()
for i in range(self._nWorkers):
self.parallelWorkerIDs.put((i))
self._chunkSize = chunkSize
super(SpatioTemporalESN, self).__init__(
n_input=self._n_input,
n_reservoir=n_reservoir,
n_output=1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivative=activationDerivative,
)
"""