def setup_output(self, output_file, force=False, injection_file=None): """Sets up the sampler's checkpoint and output files. The checkpoint file has the same name as the output file, but with ``.checkpoint`` appended to the name. A backup file will also be created. If the output file already exists, an ``OSError`` will be raised. This can be overridden by setting ``force`` to ``True``. Parameters ---------- sampler : sampler instance Sampler output_file : str Name of the output file. force : bool, optional If the output file already exists, overwrite it. injection_file : str, optional If an injection was added to the data, write its information. """ # check that the output file doesn't already exist if os.path.exists(output_file): if force: os.remove(output_file) else: raise OSError("output-file already exists; use force if you " "wish to overwrite it.") # check for backup file(s) checkpoint_file = output_file + '.checkpoint' backup_file = output_file + '.bkup' # check if we have a good checkpoint and/or backup file logging.info("Looking for checkpoint file") checkpoint_valid = validate_checkpoint_files(checkpoint_file, backup_file) # Create a new file if the checkpoint doesn't exist, or if it is # corrupted self.new_checkpoint = False # keeps track if this is a new file or not if not checkpoint_valid: logging.info("Checkpoint not found or not valid") create_new_output_file(self, checkpoint_file, injection_file=injection_file) # now the checkpoint is valid self.new_checkpoint = True # copy to backup shutil.copy(checkpoint_file, backup_file) # write the command line, startup for fn in [checkpoint_file, backup_file]: with self.io(fn, "a") as fp: fp.write_command_line() fp.write_resume_point() # store self.checkpoint_file = checkpoint_file self.backup_file = backup_file self.checkpoint_valid = checkpoint_valid
def finalize(self): """Finalze and write it to the results file """ logz = self._sampler.results.logz[-1:][0] dlogz = self._sampler.results.logzerr[-1:][0] logging.info("log Z, dlog Z: {}, {}".format(logz, dlogz)) self.checkpoint() logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files( self.checkpoint_file, self.backup_file, check_nsamples=False) if not checkpoint_valid: raise IOError("error writing to checkpoint file")
def checkpoint(self): """Dumps current samples to the checkpoint file.""" logging.info("Writing samples to files") for f_n in [self.checkpoint_file, self.backup_file]: self.write_results(f_n) with self.io(f_n, "a") as f_p: f_p.write_niterations(self.niterations) logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files( self.checkpoint_file, self.backup_file) if not checkpoint_valid: raise IOError("error writing to checkpoint file")
def finalize(self): logz = self._sampler.results.logz[-1:][0] dlogz = self._sampler.results.logzerr[-1:][0] logging.info("log Z, dlog Z: {}, {}".format(logz, dlogz)) for fn in [self.checkpoint_file]: with self.io(fn, "a") as fp: fp.write_logevidence(logz, dlogz) logging.info("Writing samples to files") for fn in [self.checkpoint_file, self.backup_file]: self.write_results(fn) logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files( self.checkpoint_file, self.backup_file, check_nsamples=False) if not checkpoint_valid: raise IOError("error writing to checkpoint file")
def setup_output(self, output_file, force=False, injection_file=None): """Sets up the sampler's checkpoint and output files. The checkpoint file has the same name as the output file, but with ``.checkpoint`` appended to the name. A backup file will also be created. If the output file already exists, an ``OSError`` will be raised. This can be overridden by setting ``force`` to ``True``. Parameters ---------- sampler : sampler instance Sampler output_file : str Name of the output file. force : bool, optional If the output file already exists, overwrite it. injection_file : str, optional If an injection was added to the data, write its information. """ # check for backup file(s) checkpoint_file = output_file + '.checkpoint' backup_file = output_file + '.bkup' # check if we have a good checkpoint and/or backup file logging.info("Looking for checkpoint file") checkpoint_valid = validate_checkpoint_files(checkpoint_file, backup_file) # Create a new file if the checkpoint doesn't exist, or if it is # corrupted self.new_checkpoint = False # keeps track if this is a new file or not if not checkpoint_valid: logging.info("Checkpoint not found or not valid") create_new_output_file(self, checkpoint_file, force=force, injection_file=injection_file) # now the checkpoint is valid self.new_checkpoint = True # copy to backup shutil.copy(checkpoint_file, backup_file) # write the command line, startup for fn in [checkpoint_file, backup_file]: with self.io(fn, "a") as fp: fp.write_command_line() fp.write_resume_point() # store self.checkpoint_file = checkpoint_file self.backup_file = backup_file self.checkpoint_valid = checkpoint_valid
def checkpoint(self): """Dumps current samples to the checkpoint file.""" # write new samples logging.info("Writing samples to files") for fn in [self.checkpoint_file, self.backup_file]: self.write_results(fn) with self.io(fn, "a") as fp: # write the current number of iterations fp.write_niterations(self.niterations) # check for burn in, compute the acls self.acls = None if self.burn_in is not None: logging.info("Updating burn in") self.burn_in.evaluate(self.checkpoint_file) burn_in_iter = self.burn_in.burn_in_iteration logging.info("Is burned in: {}".format(self.burn_in.is_burned_in)) if self.burn_in.is_burned_in: logging.info("Burn-in iteration: {}".format( self.burn_in.burn_in_iteration)) else: burn_in_iter = 0 # Compute acls; the burn_in test may have calculated an acl and saved # it, in which case we don't need to do it again. if self.acls is None: logging.info("Computing acls") self.acls = self.compute_acl(self.checkpoint_file, start_index=burn_in_iter) logging.info("ACL: {}".format(numpy.array(self.acls.values()).max())) # write for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: if self.burn_in is not None: fp.write_burn_in(self.burn_in) if self.acls is not None: fp.write_acls(self.acls) # write effective number of samples fp.write_effective_nsamples(self.effective_nsamples) # check validity logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files( self.checkpoint_file, self.backup_file) if not checkpoint_valid: raise IOError("error writing to checkpoint file") # clear the in-memory chain to save memory logging.info("Clearing samples from memory") self.clear_samples()
def setup_output(sampler, output_file, check_nsamples=True): r"""Sets up the sampler's checkpoint and output files. The checkpoint file has the same name as the output file, but with ``.checkpoint`` appended to the name. A backup file will also be created. Parameters ---------- sampler : sampler instance Sampler output_file : str Name of the output file. """ # check for backup file(s) checkpoint_file = output_file + '.checkpoint' backup_file = output_file + '.bkup' # check if we have a good checkpoint and/or backup file logging.info("Looking for checkpoint file") checkpoint_valid = validate_checkpoint_files(checkpoint_file, backup_file, check_nsamples) # Create a new file if the checkpoint doesn't exist, or if it is # corrupted sampler.new_checkpoint = False # keeps track if this is a new file or not if not checkpoint_valid: logging.info("Checkpoint not found or not valid") create_new_output_file(sampler, checkpoint_file) # now the checkpoint is valid sampler.new_checkpoint = True # copy to backup shutil.copy(checkpoint_file, backup_file) # write the command line, startup for fn in [checkpoint_file, backup_file]: with sampler.io(fn, "a") as fp: fp.write_command_line() fp.write_resume_point() fp.write_run_start_time() # store sampler.checkpoint_file = checkpoint_file sampler.backup_file = backup_file
def checkpoint(self): """Dumps current samples to the checkpoint file.""" # thin and write new samples # get the updated thin interval to use thin_interval = self.get_thin_interval() for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: # write the current number of iterations fp.write_niterations(self.niterations) # thin samples on disk if it changed if thin_interval > 1: # if this is the first time writing, set the file's # thinned_by if fp.last_iteration() == 0: fp.thinned_by = thin_interval elif thin_interval < fp.thinned_by: # whatever was done previously resulted in a larger # thin interval, so we'll set it to the file's thin_interval = fp.thinned_by elif thin_interval > fp.thinned_by: # we need to thin the samples on disk logging.info( "Thinning samples in %s by a factor " "of %i", fn, int(thin_interval)) fp.thin(thin_interval) fp_lastiter = fp.last_iteration() logging.info("Writing samples to %s with thin interval %i", fn, thin_interval) self.write_results(fn) # update the running thin interval self.thin_interval = thin_interval # see if we had anything to write after thinning; if not, don't try # to compute anything with self.io(self.checkpoint_file, "r") as fp: nsamples_written = fp.last_iteration() - fp_lastiter if nsamples_written == 0: logging.info("No samples written due to thinning") else: # check for burn in, compute the acls self.raw_acls = None if self.burn_in is not None: logging.info("Updating burn in") self.burn_in.evaluate(self.checkpoint_file) # write for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: self.burn_in.write(fp) # Compute acls; the burn_in test may have calculated an acl and # saved it, in which case we don't need to do it again. if self.raw_acls is None: logging.info("Computing autocorrelation time") self.raw_acls = self.compute_acl(self.checkpoint_file) # write acts, effective number of samples for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: if self.raw_acls is not None: fp.raw_acls = self.raw_acls fp.acl = self.acl # write effective number of samples fp.write_effective_nsamples(self.effective_nsamples) # check validity logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files(self.checkpoint_file, self.backup_file) if not checkpoint_valid: raise IOError("error writing to checkpoint file") elif self.checkpoint_signal: # kill myself with the specified signal logging.info("Exiting with SIG{}".format(self.checkpoint_signal)) kill_cmd = "os.kill(os.getpid(), signal.SIG{})".format( self.checkpoint_signal) exec(kill_cmd) # clear the in-memory chain to save memory logging.info("Clearing samples from memory") self.clear_samples()
def checkpoint(self): """Dumps current samples to the checkpoint file.""" # thin and write new samples for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: # write the current number of iterations fp.write_niterations(self.niterations) thin_interval = self.get_thin_interval() # thin samples on disk if it changed if thin_interval > 1: # if this is the first time writing, set the file's # thinned_by if fp.last_iteration() == 0: fp.thinned_by = thin_interval else: # check if we need to thin the current samples on disk thin_by = thin_interval // fp.thinned_by if thin_by > 1: logging.info("Thinning samples in %s by a factor " "of %i", fn, int(thin_by)) fp.thin(thin_by) fp_lastiter = fp.last_iteration() logging.info("Writing samples to %s with thin interval %i", fn, thin_interval) self.write_results(fn) # see if we had anything to write after thinning; if not, don't try # to compute anything with self.io(self.checkpoint_file, "r") as fp: nsamples_written = fp.last_iteration() - fp_lastiter if nsamples_written == 0: logging.info("No samples written due to thinning") else: # check for burn in, compute the acls self.acls = None if self.burn_in is not None: logging.info("Updating burn in") self.burn_in.evaluate(self.checkpoint_file) burn_in_index = self.burn_in.burn_in_index logging.info("Is burned in: %r", self.burn_in.is_burned_in) if self.burn_in.is_burned_in: logging.info("Burn-in iteration: %i", int(self.burn_in.burn_in_iteration)) else: burn_in_index = 0 # Compute acls; the burn_in test may have calculated an acl and # saved it, in which case we don't need to do it again. if self.acls is None: logging.info("Computing acls") self.acls = self.compute_acl(self.checkpoint_file, start_index=burn_in_index) logging.info("ACT: %s", str(numpy.array(self.acts.values()).max())) # write for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: if self.burn_in is not None: fp.write_burn_in(self.burn_in) if self.acls is not None: fp.write_acls(self.acls) # write effective number of samples fp.write_effective_nsamples(self.effective_nsamples) # check validity logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files( self.checkpoint_file, self.backup_file) if not checkpoint_valid: raise IOError("error writing to checkpoint file") # clear the in-memory chain to save memory logging.info("Clearing samples from memory") self.clear_samples()
def checkpoint(self): """Dumps current samples to the checkpoint file.""" # thin and write new samples for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: # write the current number of iterations fp.write_niterations(self.niterations) thin_interval = self.get_thin_interval() # thin samples on disk if it changed if thin_interval > 1: # if this is the first time writing, set the file's # thinned_by if fp.last_iteration() == 0: fp.thinned_by = thin_interval else: # check if we need to thin the current samples on disk thin_by = thin_interval // fp.thinned_by if thin_by > 1: logging.info( "Thinning samples in %s by a factor " "of %i", fn, int(thin_by)) fp.thin(thin_by) fp_lastiter = fp.last_iteration() logging.info("Writing samples to %s with thin interval %i", fn, thin_interval) self.write_results(fn) # see if we had anything to write after thinning; if not, don't try # to compute anything with self.io(self.checkpoint_file, "r") as fp: nsamples_written = fp.last_iteration() - fp_lastiter if nsamples_written == 0: logging.info("No samples written due to thinning") else: # check for burn in, compute the acls self.acls = None if self.burn_in is not None: logging.info("Updating burn in") self.burn_in.evaluate(self.checkpoint_file) burn_in_index = self.burn_in.burn_in_index logging.info("Is burned in: %r", self.burn_in.is_burned_in) if self.burn_in.is_burned_in: logging.info("Burn-in iteration: %i", int(self.burn_in.burn_in_iteration)) else: burn_in_index = 0 # Compute acls; the burn_in test may have calculated an acl and # saved it, in which case we don't need to do it again. if self.acls is None: logging.info("Computing acls") self.acls = self.compute_acl(self.checkpoint_file, start_index=burn_in_index) logging.info("ACT: %s", str(numpy.array(self.acts.values()).max())) # write for fn in [self.checkpoint_file, self.backup_file]: with self.io(fn, "a") as fp: if self.burn_in is not None: fp.write_burn_in(self.burn_in) if self.acls is not None: fp.write_acls(self.acls) # write effective number of samples fp.write_effective_nsamples(self.effective_nsamples) # check validity logging.info("Validating checkpoint and backup files") checkpoint_valid = validate_checkpoint_files(self.checkpoint_file, self.backup_file) if not checkpoint_valid: raise IOError("error writing to checkpoint file") elif self.checkpoint_signal: # kill myself with the specified signal logging.info("Exiting with SIG{}".format(self.checkpoint_signal)) kill_cmd = "os.kill(os.getpid(), signal.SIG{})".format( self.checkpoint_signal) exec(kill_cmd) # clear the in-memory chain to save memory logging.info("Clearing samples from memory") self.clear_samples()