def main(settings, rescue_running=[]): """ Perform the primary loop of building, submitting, monitoring, and analyzing jobs. This function works via a loop of calls to thread.process and thread.interpret for each thread that hasn't terminated, until either the global termination criterion is met or all the individual threads have completed. Parameters ---------- settings : argparse.Namespace Settings namespace object rescue_running : list List of threads passed in from handle_loop_exception, containing running threads. If given, setup is skipped and the function proceeds directly to the main loop. Returns ------- exit_message : str A message indicating the status of ATESA at the end of main """ if not rescue_running: # Implement resample if settings.job_type in ['aimless_shooting', 'committor_analysis' ] and settings.resample: # Store settings object in the working directory for compatibility with analysis/utility scripts if not settings.dont_dump: temp_settings = copy.deepcopy( settings ) # initialize temporary copy of settings to modify temp_settings.__dict__.pop( 'env') # env attribute is not picklable pickle.dump(temp_settings, open(settings.working_directory + '/settings.pkl', 'wb'), protocol=2) # Run resampling if settings.job_type == 'aimless_shooting': utilities.resample(settings, partial=False, full_cvs=settings.full_cvs) if settings.information_error_checking: # update info_err.out if called for by settings information_error.main() elif settings.job_type == 'committor_analysis': resample_committor_analysis.resample_committor_analysis( settings) return 'Resampling complete' # Make working directory if it does not exist, handling overwrite and restart as needed if os.path.exists(settings.working_directory): if settings.overwrite and not settings.restart: if os.path.exists( settings.working_directory + '/cvs.txt'): # a kludge to avoid removing cvs.txt if os.path.exists('ATESA_TEMP_CVS.txt'): raise RuntimeError( 'tried to create temporary file ATESA_TEMP_CVS.txt in directory: ' + os.getcwd() + ', but it already exists. Please move, delete, or rename it.' ) shutil.move(settings.working_directory + '/cvs.txt', 'ATESA_TEMP_CVS.txt') shutil.rmtree(settings.working_directory) os.mkdir(settings.working_directory) if os.path.exists('ATESA_TEMP_CVS.txt' ): # continuation of aforementioned kludge shutil.move('ATESA_TEMP_CVS.txt', settings.working_directory + '/cvs.txt') elif not settings.restart and glob.glob( settings.working_directory + '/*') == [settings.working_directory + '/cvs.txt']: # Occurs when restart = False, overwrite = False, and auto_cvs is used pass elif not settings.restart: raise RuntimeError( 'Working directory ' + settings.working_directory + ' already exists, but overwrite ' '= False and restart = False. Either change one of these two settings or choose a ' 'different working directory.') else: if not settings.restart: os.mkdir(settings.working_directory) else: raise RuntimeError('Working directory ' + settings.working_directory + ' does not yet exist, but ' 'restart = True.') # Store settings object in the working directory for compatibility with analysis/utility scripts if os.path.exists( settings.working_directory + '/settings.pkl'): # for checking for need for resample later previous_settings = pickle.load( open(settings.working_directory + '/settings.pkl', 'rb')) settings.previous_cvs = previous_settings.cvs try: settings.previous_information_error_max_dims = previous_settings.information_error_max_dims except AttributeError: pass try: settings.previous_information_error_lmax_string = previous_settings.information_error_lmax_string except AttributeError: pass if not settings.dont_dump: temp_settings = copy.deepcopy( settings) # initialize temporary copy of settings to modify temp_settings.__dict__.pop( 'env' ) # env attribute is not picklable (update: maybe no longer true, but doesn't matter) pickle.dump(temp_settings, open(settings.working_directory + '/settings.pkl', 'wb'), protocol=2) # Build or load threads allthreads = init_threads(settings) # Move runtime to working directory os.chdir(settings.working_directory) running = allthreads.copy() # to be pruned later by thread.process() attempted_rescue = False # to keep track of general error handling below else: allthreads = pickle.load( open(settings.working_directory + '/restart.pkl', 'rb')) running = rescue_running attempted_rescue = True # Initialize threads with first process step try: if not rescue_running: # if rescue_running, this step has already finished and we just want the while loop for thread in allthreads: running = thread.process(running, settings) except Exception as e: if settings.restart: print( 'The following error occurred while attempting to initialize threads from restart.pkl. It may be ' 'corrupted.') #'If you haven\'t already done so, consider running verify_threads.py to remove corrupted threads from this file.' raise e try: if settings.job_type == 'aimless_shooting' and len( os.sched_getaffinity(0)) > 1: # Initialize Manager for shared data across processes; this is necessary because multiprocessing is being # retrofitted to code designed for serial processing, but it works! manager = Manager() # Setup Managed allthreads list managed_allthreads = [] for thread in allthreads: thread_dict = thread.__dict__ thread_history_dict = thread.history.__dict__ managed_thread = Thread() managed_thread.history = manager.Namespace() managed_thread.__dict__.update(thread_dict) managed_thread.history.__dict__.update(thread_history_dict) managed_allthreads.append(managed_thread) allthreads = manager.list(managed_allthreads) # Setup Managed settings Namespace settings_dict = settings.__dict__ managed_settings = manager.Namespace() # Need to explicitly update every key because of how the Managed Namespace works. # Calling exec is the best way to do this I could find. Updating managed_settings.__dict__ doesn't work. for key in settings_dict.keys(): exec('managed_settings.' + key + ' = settings_dict[key]') # Distribute processes among available core Pool with get_context("spawn").Pool(len(os.sched_getaffinity(0))) as p: p.starmap( main_loop, zip(itertools.repeat(managed_settings), itertools.repeat(allthreads), [[thread] for thread in allthreads])) else: main_loop(settings, allthreads, running) except AttributeError: # os.sched_getaffinity raises AttributeError on non-UNIX systems. main_loop(settings, allthreads, running) ## Deprecated thread pool # pool = ThreadPool(len(allthreads)) # func = partial(main_loop, settings) # results = pool.map(func, [[thread] for thread in allthreads]) jobtype = factory.jobtype_factory(settings.job_type) jobtype.cleanup(settings) return 'ATESA run exiting normally'
def __init__(self, inputShape, n_reservoir, filterSize=1, stride=1, borderMode="mirror", nWorkers="auto", spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration='naive', bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh, activationDerivation=lambda x: 1.0 / B.cosh(x) ** 2): self._averageOutputWeights = averageOutputWeights if averageOutputWeights and solver != "lsqr": raise ValueError( "`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)") self._borderMode = borderMode if not borderMode in ["mirror", "padding", "edge", "wrap"]: raise ValueError( "`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`.") self._regressionParameters = regressionParameters self._solver = solver n_inputDimensions = len(inputShape) if filterSize % 2 == 0: raise ValueError("filterSize has to be an odd number (1, 3, 5, ...).") self._filterSize = filterSize self._filterWidth = int(np.floor(filterSize / 2)) self._stride = stride self._n_input = int(np.power(np.ceil(filterSize / stride), n_inputDimensions)) self.n_inputDimensions = n_inputDimensions self.inputShape = inputShape if not self._averageOutputWeights: self._WOuts = B.empty((np.prod(inputShape), 1, self._n_input + n_reservoir + 1)) self._WOut = None else: self._WOuts = None self._WOut = B.zeros((1, self._n_input + n_reservoir + 1)) self._xs = B.empty((np.prod(inputShape), n_reservoir, 1)) if nWorkers == "auto": self._nWorkers = np.max((cpu_count() - 1, 1)) else: self._nWorkers = nWorkers manager = Manager() self.sharedNamespace = manager.Namespace() if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None: self.parallelWorkerIDs = manager.Queue() for i in range(self._nWorkers): self.parallelWorkerIDs.put((i)) super(SpatioTemporalESN, self).__init__(n_input=self._n_input, n_reservoir=n_reservoir, n_output=1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, reservoirDensity=reservoirDensity, randomSeed=randomSeed, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) """
def __init__( self, inputShape, n_reservoir, filterSize=1, stride=1, borderMode="mirror", nWorkers="auto", spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration="naive", bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver="pinv", regressionParameters={}, activation=B.tanh, activationDerivative=lambda x: 1.0 / B.cosh(x)**2, chunkSize=16, ): """ ESN that predicts (steps of) a spatio-temporal time series based on a time series. Args: inputShape : Shape of the input w/o the time axis, e.g. (W, H) for a 2D input. n_reservoir : Number of units in the reservoir. filterSize : Size of patches used to predict a single output element. stride : Stride between different patches. borderMode : How to handle border values. Choices: mirror, padding, edge, wrap. nWorkers : Number of CPU threads executed in parallel to solve the problem. spectralRadius : Spectral radius of the reservoir's connection/weight matrix. noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting. inputScaling : Scaling factor of the input. leakingRate : Convex combination factor between 0 and 1 that weights current and new state value. reservoirDensity : Percentage of non-zero weight connections in the reservoir. randomSeed : Seed for random processes, e.g. weight initialization. averageOutputWeights : Average output matrices after fitting across all pixels or use a distinct matrix per pixel. The former assumes homogeneity of the problem across all pixels. out_activation : Final activation function (i.e. activation function of the output). out_inverse_activation : Inverse of the final activation function weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom bias : Size of the bias added for the internal update process. outputBias : Size of the bias added for the final linear regression of the output. outputInputScaling : Rescaling factor for the input of the ESN for the regression. inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix. solver : Algorithm to find output matrix. Choices: pinv, lsqr. regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization. activation : (Non-linear) Activation function. activationDerivative : Derivative of the activation function. chunkSize : Internal parameter for the multi-threading. For long time series this should be reduced to avoid OOM errors/getting stuck and to reduce memory consumption. """ self._averageOutputWeights = averageOutputWeights if averageOutputWeights and solver != "lsqr": raise ValueError( "`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)" ) self._borderMode = borderMode if not borderMode in ["mirror", "padding", "edge", "wrap"]: raise ValueError( "`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`." ) self._regressionParameters = regressionParameters self._solver = solver n_inputDimensions = len(inputShape) if filterSize % 2 == 0: raise ValueError( "filterSize has to be an odd number (1, 3, 5, ...).") self._filterSize = filterSize self._filterWidth = int(np.floor(filterSize / 2)) self._stride = stride self._n_input = int( np.power(np.ceil(filterSize / stride), n_inputDimensions)) self.n_inputDimensions = n_inputDimensions self.inputShape = inputShape if not self._averageOutputWeights: self._WOuts = B.empty( (np.prod(inputShape), 1, self._n_input + n_reservoir + 1)) self._WOut = None else: self._WOuts = None self._WOut = B.zeros((1, self._n_input + n_reservoir + 1)) self._xs = B.empty((np.prod(inputShape), n_reservoir, 1)) if nWorkers == "auto": self._nWorkers = np.max((cpu_count() - 1, 1)) else: self._nWorkers = nWorkers manager = Manager() self.sharedNamespace = manager.Namespace() if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None: self.parallelWorkerIDs = manager.Queue() for i in range(self._nWorkers): self.parallelWorkerIDs.put((i)) self._chunkSize = chunkSize super(SpatioTemporalESN, self).__init__( n_input=self._n_input, n_reservoir=n_reservoir, n_output=1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, reservoirDensity=reservoirDensity, randomSeed=randomSeed, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivative=activationDerivative, ) """