def setup_output(self, output_file, force=False, injection_file=None):
"""Sets up the sampler's checkpoint and output files.
The checkpoint file has the same name as the output file, but with
``.checkpoint`` appended to the name. A backup file will also be
created.
If the output file already exists, an ``OSError`` will be raised.
This can be overridden by setting ``force`` to ``True``.
Parameters
----------
sampler : sampler instance
Sampler
output_file : str
Name of the output file.
force : bool, optional
If the output file already exists, overwrite it.
injection_file : str, optional
If an injection was added to the data, write its information.
"""
# check that the output file doesn't already exist
if os.path.exists(output_file):
if force:
os.remove(output_file)
else:
raise OSError("output-file already exists; use force if you "
"wish to overwrite it.")
# check for backup file(s)
checkpoint_file = output_file + '.checkpoint'
backup_file = output_file + '.bkup'
# check if we have a good checkpoint and/or backup file
logging.info("Looking for checkpoint file")
checkpoint_valid = validate_checkpoint_files(checkpoint_file,
backup_file)
# Create a new file if the checkpoint doesn't exist, or if it is
# corrupted
self.new_checkpoint = False # keeps track if this is a new file or not
if not checkpoint_valid:
logging.info("Checkpoint not found or not valid")
create_new_output_file(self,
checkpoint_file,
injection_file=injection_file)
# now the checkpoint is valid
self.new_checkpoint = True
# copy to backup
shutil.copy(checkpoint_file, backup_file)
# write the command line, startup
for fn in [checkpoint_file, backup_file]:
with self.io(fn, "a") as fp:
fp.write_command_line()
fp.write_resume_point()
# store
self.checkpoint_file = checkpoint_file
self.backup_file = backup_file
self.checkpoint_valid = checkpoint_valid
def finalize(self):
"""Finalze and write it to the results file
"""
logz = self._sampler.results.logz[-1:][0]
dlogz = self._sampler.results.logzerr[-1:][0]
logging.info("log Z, dlog Z: {}, {}".format(logz, dlogz))
self.checkpoint()
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(
self.checkpoint_file, self.backup_file, check_nsamples=False)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
def checkpoint(self):
"""Dumps current samples to the checkpoint file."""
logging.info("Writing samples to files")
for f_n in [self.checkpoint_file, self.backup_file]:
self.write_results(f_n)
with self.io(f_n, "a") as f_p:
f_p.write_niterations(self.niterations)
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(
self.checkpoint_file, self.backup_file)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
def finalize(self):
logz = self._sampler.results.logz[-1:][0]
dlogz = self._sampler.results.logzerr[-1:][0]
logging.info("log Z, dlog Z: {}, {}".format(logz, dlogz))
for fn in [self.checkpoint_file]:
with self.io(fn, "a") as fp:
fp.write_logevidence(logz, dlogz)
logging.info("Writing samples to files")
for fn in [self.checkpoint_file, self.backup_file]:
self.write_results(fn)
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(
self.checkpoint_file, self.backup_file, check_nsamples=False)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
def setup_output(self, output_file, force=False, injection_file=None):
"""Sets up the sampler's checkpoint and output files.
The checkpoint file has the same name as the output file, but with
``.checkpoint`` appended to the name. A backup file will also be
created.
If the output file already exists, an ``OSError`` will be raised.
This can be overridden by setting ``force`` to ``True``.
Parameters
----------
sampler : sampler instance
Sampler
output_file : str
Name of the output file.
force : bool, optional
If the output file already exists, overwrite it.
injection_file : str, optional
If an injection was added to the data, write its information.
"""
# check for backup file(s)
checkpoint_file = output_file + '.checkpoint'
backup_file = output_file + '.bkup'
# check if we have a good checkpoint and/or backup file
logging.info("Looking for checkpoint file")
checkpoint_valid = validate_checkpoint_files(checkpoint_file,
backup_file)
# Create a new file if the checkpoint doesn't exist, or if it is
# corrupted
self.new_checkpoint = False # keeps track if this is a new file or not
if not checkpoint_valid:
logging.info("Checkpoint not found or not valid")
create_new_output_file(self, checkpoint_file, force=force,
injection_file=injection_file)
# now the checkpoint is valid
self.new_checkpoint = True
# copy to backup
shutil.copy(checkpoint_file, backup_file)
# write the command line, startup
for fn in [checkpoint_file, backup_file]:
with self.io(fn, "a") as fp:
fp.write_command_line()
fp.write_resume_point()
# store
self.checkpoint_file = checkpoint_file
self.backup_file = backup_file
self.checkpoint_valid = checkpoint_valid
def checkpoint(self):
"""Dumps current samples to the checkpoint file."""
# write new samples
logging.info("Writing samples to files")
for fn in [self.checkpoint_file, self.backup_file]:
self.write_results(fn)
with self.io(fn, "a") as fp:
# write the current number of iterations
fp.write_niterations(self.niterations)
# check for burn in, compute the acls
self.acls = None
if self.burn_in is not None:
logging.info("Updating burn in")
self.burn_in.evaluate(self.checkpoint_file)
burn_in_iter = self.burn_in.burn_in_iteration
logging.info("Is burned in: {}".format(self.burn_in.is_burned_in))
if self.burn_in.is_burned_in:
logging.info("Burn-in iteration: {}".format(
self.burn_in.burn_in_iteration))
else:
burn_in_iter = 0
# Compute acls; the burn_in test may have calculated an acl and saved
# it, in which case we don't need to do it again.
if self.acls is None:
logging.info("Computing acls")
self.acls = self.compute_acl(self.checkpoint_file,
start_index=burn_in_iter)
logging.info("ACL: {}".format(numpy.array(self.acls.values()).max()))
# write
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
if self.burn_in is not None:
fp.write_burn_in(self.burn_in)
if self.acls is not None:
fp.write_acls(self.acls)
# write effective number of samples
fp.write_effective_nsamples(self.effective_nsamples)
# check validity
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(
self.checkpoint_file, self.backup_file)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
# clear the in-memory chain to save memory
logging.info("Clearing samples from memory")
self.clear_samples()
def setup_output(sampler, output_file, check_nsamples=True):
r"""Sets up the sampler's checkpoint and output files.
The checkpoint file has the same name as the output file, but with
``.checkpoint`` appended to the name. A backup file will also be
created.
Parameters
----------
sampler : sampler instance
Sampler
output_file : str
Name of the output file.
"""
# check for backup file(s)
checkpoint_file = output_file + '.checkpoint'
backup_file = output_file + '.bkup'
# check if we have a good checkpoint and/or backup file
logging.info("Looking for checkpoint file")
checkpoint_valid = validate_checkpoint_files(checkpoint_file, backup_file,
check_nsamples)
# Create a new file if the checkpoint doesn't exist, or if it is
# corrupted
sampler.new_checkpoint = False # keeps track if this is a new file or not
if not checkpoint_valid:
logging.info("Checkpoint not found or not valid")
create_new_output_file(sampler, checkpoint_file)
# now the checkpoint is valid
sampler.new_checkpoint = True
# copy to backup
shutil.copy(checkpoint_file, backup_file)
# write the command line, startup
for fn in [checkpoint_file, backup_file]:
with sampler.io(fn, "a") as fp:
fp.write_command_line()
fp.write_resume_point()
fp.write_run_start_time()
# store
sampler.checkpoint_file = checkpoint_file
sampler.backup_file = backup_file
def checkpoint(self):
"""Dumps current samples to the checkpoint file."""
# thin and write new samples
# get the updated thin interval to use
thin_interval = self.get_thin_interval()
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
# write the current number of iterations
fp.write_niterations(self.niterations)
# thin samples on disk if it changed
if thin_interval > 1:
# if this is the first time writing, set the file's
# thinned_by
if fp.last_iteration() == 0:
fp.thinned_by = thin_interval
elif thin_interval < fp.thinned_by:
# whatever was done previously resulted in a larger
# thin interval, so we'll set it to the file's
thin_interval = fp.thinned_by
elif thin_interval > fp.thinned_by:
# we need to thin the samples on disk
logging.info(
"Thinning samples in %s by a factor "
"of %i", fn, int(thin_interval))
fp.thin(thin_interval)
fp_lastiter = fp.last_iteration()
logging.info("Writing samples to %s with thin interval %i", fn,
thin_interval)
self.write_results(fn)
# update the running thin interval
self.thin_interval = thin_interval
# see if we had anything to write after thinning; if not, don't try
# to compute anything
with self.io(self.checkpoint_file, "r") as fp:
nsamples_written = fp.last_iteration() - fp_lastiter
if nsamples_written == 0:
logging.info("No samples written due to thinning")
else:
# check for burn in, compute the acls
self.raw_acls = None
if self.burn_in is not None:
logging.info("Updating burn in")
self.burn_in.evaluate(self.checkpoint_file)
# write
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
self.burn_in.write(fp)
# Compute acls; the burn_in test may have calculated an acl and
# saved it, in which case we don't need to do it again.
if self.raw_acls is None:
logging.info("Computing autocorrelation time")
self.raw_acls = self.compute_acl(self.checkpoint_file)
# write acts, effective number of samples
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
if self.raw_acls is not None:
fp.raw_acls = self.raw_acls
fp.acl = self.acl
# write effective number of samples
fp.write_effective_nsamples(self.effective_nsamples)
# check validity
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(self.checkpoint_file,
self.backup_file)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
elif self.checkpoint_signal:
# kill myself with the specified signal
logging.info("Exiting with SIG{}".format(self.checkpoint_signal))
kill_cmd = "os.kill(os.getpid(), signal.SIG{})".format(
self.checkpoint_signal)
exec(kill_cmd)
# clear the in-memory chain to save memory
logging.info("Clearing samples from memory")
self.clear_samples()
def checkpoint(self):
"""Dumps current samples to the checkpoint file."""
# thin and write new samples
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
# write the current number of iterations
fp.write_niterations(self.niterations)
thin_interval = self.get_thin_interval()
# thin samples on disk if it changed
if thin_interval > 1:
# if this is the first time writing, set the file's
# thinned_by
if fp.last_iteration() == 0:
fp.thinned_by = thin_interval
else:
# check if we need to thin the current samples on disk
thin_by = thin_interval // fp.thinned_by
if thin_by > 1:
logging.info("Thinning samples in %s by a factor "
"of %i", fn, int(thin_by))
fp.thin(thin_by)
fp_lastiter = fp.last_iteration()
logging.info("Writing samples to %s with thin interval %i", fn,
thin_interval)
self.write_results(fn)
# see if we had anything to write after thinning; if not, don't try
# to compute anything
with self.io(self.checkpoint_file, "r") as fp:
nsamples_written = fp.last_iteration() - fp_lastiter
if nsamples_written == 0:
logging.info("No samples written due to thinning")
else:
# check for burn in, compute the acls
self.acls = None
if self.burn_in is not None:
logging.info("Updating burn in")
self.burn_in.evaluate(self.checkpoint_file)
burn_in_index = self.burn_in.burn_in_index
logging.info("Is burned in: %r", self.burn_in.is_burned_in)
if self.burn_in.is_burned_in:
logging.info("Burn-in iteration: %i",
int(self.burn_in.burn_in_iteration))
else:
burn_in_index = 0
# Compute acls; the burn_in test may have calculated an acl and
# saved it, in which case we don't need to do it again.
if self.acls is None:
logging.info("Computing acls")
self.acls = self.compute_acl(self.checkpoint_file,
start_index=burn_in_index)
logging.info("ACT: %s", str(numpy.array(self.acts.values()).max()))
# write
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
if self.burn_in is not None:
fp.write_burn_in(self.burn_in)
if self.acls is not None:
fp.write_acls(self.acls)
# write effective number of samples
fp.write_effective_nsamples(self.effective_nsamples)
# check validity
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(
self.checkpoint_file, self.backup_file)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
# clear the in-memory chain to save memory
logging.info("Clearing samples from memory")
self.clear_samples()
def checkpoint(self):
"""Dumps current samples to the checkpoint file."""
# thin and write new samples
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
# write the current number of iterations
fp.write_niterations(self.niterations)
thin_interval = self.get_thin_interval()
# thin samples on disk if it changed
if thin_interval > 1:
# if this is the first time writing, set the file's
# thinned_by
if fp.last_iteration() == 0:
fp.thinned_by = thin_interval
else:
# check if we need to thin the current samples on disk
thin_by = thin_interval // fp.thinned_by
if thin_by > 1:
logging.info(
"Thinning samples in %s by a factor "
"of %i", fn, int(thin_by))
fp.thin(thin_by)
fp_lastiter = fp.last_iteration()
logging.info("Writing samples to %s with thin interval %i", fn,
thin_interval)
self.write_results(fn)
# see if we had anything to write after thinning; if not, don't try
# to compute anything
with self.io(self.checkpoint_file, "r") as fp:
nsamples_written = fp.last_iteration() - fp_lastiter
if nsamples_written == 0:
logging.info("No samples written due to thinning")
else:
# check for burn in, compute the acls
self.acls = None
if self.burn_in is not None:
logging.info("Updating burn in")
self.burn_in.evaluate(self.checkpoint_file)
burn_in_index = self.burn_in.burn_in_index
logging.info("Is burned in: %r", self.burn_in.is_burned_in)
if self.burn_in.is_burned_in:
logging.info("Burn-in iteration: %i",
int(self.burn_in.burn_in_iteration))
else:
burn_in_index = 0
# Compute acls; the burn_in test may have calculated an acl and
# saved it, in which case we don't need to do it again.
if self.acls is None:
logging.info("Computing acls")
self.acls = self.compute_acl(self.checkpoint_file,
start_index=burn_in_index)
logging.info("ACT: %s", str(numpy.array(self.acts.values()).max()))
# write
for fn in [self.checkpoint_file, self.backup_file]:
with self.io(fn, "a") as fp:
if self.burn_in is not None:
fp.write_burn_in(self.burn_in)
if self.acls is not None:
fp.write_acls(self.acls)
# write effective number of samples
fp.write_effective_nsamples(self.effective_nsamples)
# check validity
logging.info("Validating checkpoint and backup files")
checkpoint_valid = validate_checkpoint_files(self.checkpoint_file,
self.backup_file)
if not checkpoint_valid:
raise IOError("error writing to checkpoint file")
elif self.checkpoint_signal:
# kill myself with the specified signal
logging.info("Exiting with SIG{}".format(self.checkpoint_signal))
kill_cmd = "os.kill(os.getpid(), signal.SIG{})".format(
self.checkpoint_signal)
exec(kill_cmd)
# clear the in-memory chain to save memory
logging.info("Clearing samples from memory")
self.clear_samples()
