def anneal(self, Ti=10.1, Tf=0.1, dT=-0.1, Nreps=3, mcs=1,
nEqlib=1000, quiet=False):
if mcs == None:
mcs = self.N*self.N
if nEqlib == None:
nEqlib = mcs * 100
self.annealSched = np.arange(Ti,Tf+dT,dT)
self.resetRecord()
self.annealEHist = []
self.anneal_mcsHist = []
self.annealedTHist = []
if not quiet:
wstdout("Running SA, N="
+str((mcs*len(self.annealSched)+nEqlib)*Nreps)
+"...\n")
stepNum = 0
for rep in range(Nreps):
for T in self.annealSched:
self.setTemp(T)
self.mhStep(N=mcs, record=True)
stepNum += mcs
self.anneal_mcsHist.append(stepNum)
self.annealEHist.append(self.energy)
self.annealedTHist.append(T)
TFinal = T
for n in xrange(nEqlib):
#wstdout(".")
self.setTemp(TFinal)
self.mhStep(N=mcs, record=True)
stepNum += mcs
self.anneal_mcsHist.append(stepNum)
self.annealEHist.append(self.energy)
self.annealedTHist.append(T)
def anneal(self, Ti=1000.1, Tf=0.1, dT=-10, mcs=None, nEqlib=None,
quiet=False):
if mcs == None:
mcs = 1 #self.N*self.N
if nEqlib == None:
nEqlib = self.N*self.N #mcs * 100
self.annealSched = np.arange(Ti,Tf+dT,dT)
self.resetRecord()
self.annealEHist = []
self.anneal_mcsHist = []
self.annealedTHist = []
if not quiet:
wstdout("Running SA, N = "+str(mcs*(len(self.annealSched)+
nEqlib))+
"...\n")
stepNum = 0
for T in self.annealSched:
self.setTemp(T)
self.mhStep(N=mcs, record=True)
stepNum += mcs
self.anneal_mcsHist.append(stepNum)
self.annealEHist.append(self.energy)
self.annealedTHist.append(T)
for n in xrange(nEqlib):
self.mhStep(N=mcs)
stepNum += mcs
self.anneal_mcsHist.append(stepNum)
self.annealEHist.append(self.energy)
self.annealedTHist.append(T)
def process_event(self, *args, **kwargs):
"""Pass this method as an IceTray module."""
if 'frame' in kwargs.keys():
frame = kwargs['frame']
elif not args:
frame = None
elif len(args) == 1:
frame = args[0]
elif len(args) > 1:
raise ValueError('Got %d frames, can only handle 1.'
% len(args))
if frame is None:
return False
if frame.Has(self.srt_pulse_name):
self.event_number += 1
else:
return False
if (self.event_number < self.skip
and not (self.skip == 0 and self.event_number == -1)):
return False
if self.n_events > 0 and self.event_number >= self.skip + self.n_events:
return False
wstdout('> Processing an event; total events processed will be %5d\n'
% (self.event_number + 1))
self.events_run.append(self.event_number)
return True
def process_event(self, *args, **kwargs):
"""Pass this method as an IceTray module."""
if 'frame' in kwargs.keys():
frame = kwargs['frame']
elif not args:
frame = None
elif len(args) == 1:
frame = args[0]
elif len(args) > 1:
raise ValueError('Got %d frames, can only handle 1.' % len(args))
if frame is None:
return False
if frame.Has(self.srt_pulse_name):
self.event_number += 1
else:
return False
if (self.event_number < self.skip
and not (self.skip == 0 and self.event_number == -1)):
return False
if self.n_events > 0 and self.event_number >= self.skip + self.n_events:
return False
wstdout('> Processing an event; total events processed will be %5d\n' %
(self.event_number + 1))
self.events_run.append(self.event_number)
return True
def cleanup_lockfiles(self):
"""Remove all expired and obsolete-format lockfiles from the
directory.
"""
start_time = time.time()
wstdout('> Cleaning up stale and old-format lock files...\n')
for lockfilepath in deepcopy(self._allfilepaths):
if not lockfilepath.endswith(LOCK_SUFFIX):
continue
# 1. if source file doesn't exist, remove the lock file
sourcefilepath = lockfilepath[:-len(LOCK_SUFFIX)]
if sourcefilepath not in self._allfilepaths:
self._remove(lockfilepath)
continue
# NOTE: temporarily removed this section since for a time the lock
# files were being overwritten, so they're 0-length... but still
# valid. Once the set of runs done on 2017-02-22 completes, I think
# this issue is fixed and so this sectoun should be re-introduced.
## 2. If lock file has outdated format, remove it
## * Remove locked file if...?
#if 'type' not in lock_info.keys():
# self._remove(lockfilepath)
# continue
## 3. If lock is expired, remove the lock file
#if lock_info['expires_at'] > time.time():
# # Remove locked file if lock type is outfile_lock, as this
# # means the output file was not fully written before the
# # process died
# if lock_info['type'] == 'outfile_lock':
# self._remove(lock_info['outfile'])
# self._remove(lockfilepath)
# TODO: the following might require some ssh magic, and/or qstat
# magic... whatever it is, it'll be a lot of magic to get it to
# work. So do this another day, since there's no magic today.
# TODO: SSH via subprocess fails when run under Parrot on ACI!
# "PRIV_END: seteuid: Operation not permitted"
# 4. If lock is not expired but process is dead, remove lock
# * Remove locked file if lock type is outfile_lock
# 5. If lock can be acquired on the file, then the lock has expired
# and therefore should be deleted.
try:
lock_f = acquire_lock(lockfilepath)
except IOError, err:
# Could not acquire lock on file, since it is locked: OK
if err.errno == 11:
continue
continue
except:
def __init__(self, chainsQueue, nSteps, changesInDir=(0,-1,+1), nChains=1):
Process.__init__(self)
wstdout("0")
self.chainsQueue = chainsQueue
self.nSteps = nSteps
self.changesInDir = changesInDir
self.nChains = nChains
self.nChangesInDir = len(changesInDir)
wstdout("1\n")
def acquire_lock(lock_path, lock_info=None):
"""Acquire a lock on the file at `lock_path` and record `lock_info` to
that file.
Parameters
----------
lock_path : string
lock_info : None or Mapping
Returns
-------
lock_f : file object
This holds an exlcusive lock; close the file or use fcntl.flock to
release the lock.
Raises
------
IOError: [Errno 11] Resource temporarily unavailable
The lock is held by a different process on the file. Note that the
same process can re-acquire a lock infinitely many times (but there
is no lock counter, so the first file descriptor to be closed or
explicitly release the lock also releases the lock for all other
instaces within the process).
ValueError: I/O operation on closed file
This might be the case if the file has disappeared between opening it
and actually acquiring the exclusive lock.
Notes
-----
See
https://loonytek.com/2015/01/15/advisory-file-locking-differences-between-posix-and-bsd-locks
for more info about locks. Note that this function uses flock, i.e.
POSIX--not BSD--locking. This means that it should work even with an NFS
filesystem, although there are other tradeoffs as well. And locking is
"cooperative," so another process can simply ignore the `flock` locking
protocol altogether and read/write/delete the file.
"""
lock_acq_timeout_time = time.time() + LOCK_ACQ_TIMEOUT
lock_f = file(lock_path, 'a')
lock_acquired = False
while time.time() <= lock_acq_timeout_time:
try:
flock(lock_f, LOCK_EX | LOCK_NB)
except IOError, err:
if err.errno == 11:
wstdout('.')
time.sleep(random.random() * LOCK_ACQ_TIMEOUT / 100)
continue
else:
raise
else:
lock_acquired = True
def acquire_lock(lock_path, lock_info=None):
"""Acquire a lock on the file at `lock_path` and record `lock_info` to
that file.
Parameters
----------
lock_path : string
lock_info : None or Mapping
Returns
-------
lock_f : file object
This holds an exlcusive lock; close the file or use fcntl.flock to
release the lock.
Raises
------
IOError: [Errno 11] Resource temporarily unavailable
The lock is held by a different process on the file. Note that the
same process can re-acquire a lock infinitely many times (but there
is no lock counter, so the first file descriptor to be closed or
explicitly release the lock also releases the lock for all other
instaces within the process).
ValueError: I/O operation on closed file
This might be the case if the file has disappeared between opening it
and actually acquiring the exclusive lock.
Notes
-----
See
https://loonytek.com/2015/01/15/advisory-file-locking-differences-between-posix-and-bsd-locks
for more info about locks. Note that this function uses flock, i.e.
POSIX--not BSD--locking. This means that it should work even with an NFS
filesystem, although there are other tradeoffs as well. And locking is
"cooperative," so another process can simply ignore the `flock` locking
protocol altogether and read/write/delete the file.
"""
lock_acq_timeout_time = time.time() + LOCK_ACQ_TIMEOUT
lock_f = file(lock_path, 'a')
lock_acquired = False
while time.time() <= lock_acq_timeout_time:
try:
flock(lock_f, LOCK_EX | LOCK_NB)
except IOError, err:
if err.errno == 11:
wstdout('.')
time.sleep(random.random()*LOCK_ACQ_TIMEOUT/100)
continue
else:
raise
else:
lock_acquired = True
def get_recos(frame):
"""Get reconstruction #'s from an I3 file frame.
Looks for all recos specified in `RECO`, where the reconstruction's
name appended with `FIT_FIELD_SUFFIX` (e.g. "_FitParams"; defined in
repeated_reco.py) is expected to be in the frame for that
reconstruction to be deemed present.
Parameters
----------
frame : I3Frame
Frame to inspect
Returns
-------
recos : list of integers
Reconstruction numbers found in the frame, where numbers are
references to the `RECO` constant defined in `repeated_reco.py`.
"""
from icecube import dataio, multinest_icetray
if isinstance(frame, basestring):
filepath = expand(frame)
try:
i3file = dataio.I3File(filepath)
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not open file "%s"\n%s\n'
% (filepath, exc_str))
return []
try:
frame = i3file.pop_physics()
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not find P frame in file "%s"\n%s\n'
% (filepath, exc_str))
return []
else:
i3file.close()
keys = frame.keys()
recos = []
for reco_num, reco_info in enumerate(RECOS):
field_name = reco_info['name'] + FIT_FIELD_SUFFIX
#if field_name in keys and not frame[field_name].has_reached_time_limit:
if field_name in keys:
recos.append(reco_num)
return sorted(recos)
def remove_empty_files(self):
"""Remove all empty (0-length) files in the directory, regardless of
the type of file (but skip directories).
"""
start_time = time.time()
wstdout('> Removing empty files from dir "%s"\n' % self.dirpath)
for filepath in deepcopy(self._allfilepaths):
try:
if getsize(filepath) == 0:
self._remove(filepath)
except (IOError, OSError):
self._done_with(filepath)
self.report()
wstdout('> Time to remove empty files: %s\n'
% timediffstamp(time.time() - start_time))
def testAnnealing(self, plotLat=True, plotHist=True, plotEnergy=True,
quiet=False, **kwargs):
self.e_min = -self.N*self.N*(2+self.extH)
self.anneal(quiet=quiet, **kwargs)
self.steps = np.cumsum(self.anneal_mcsHist)
self.finalEnergy = (self.energy-self.e_min)/self.N2
self.minEnergy = (self.record['hist'].min - self.e_min)/linSA.N2
if not quiet:
wstdout("Final energy = "+str(self.finalEnergy)+"\n")
wstdout("Minimum energy = "+str(self.minEnergy)+"\n")
if plotLat:
self.plotLattice()
if plotHist:
self.plotHist(linewidth=0.0, color=(0.1,0.5,0.6))
if plotEnergy:
fig3 = figure(3)
fig3.clf()
ax1 = fig3.add_subplot(111)
ax1.plot(self.anneal_mcsHist, self.annealedTHist,
'k--', lw=3, label=r"$T$")
ax1.set_xlabel(r"Step number")
ax1.set_ylabel(r"$T$", color='k')
ax1.autoscale(tight=True)
eps = 1/(self.N2*100)
ax2 = ax1.twinx()
ax2.plot(self.anneal_mcsHist,
((np.array(self.annealEHist)-self.e_min)
/self.N2),
'g-', lw=3, label=r"$E$")
ax2.set_xlabel(r"Step number")
ax2.set_ylabel(r"$\left(E-E_\mathrm{min}\right)/N$",
color='g')
#ax2.set_ylabel(r"$\log_{10}\left[\
#left(E-E_\mathrm{min}\right)"+
# r"/E_\mathrm{min}\right]$",
# color='g')
ax2.autoscale(tight=True)
yl = ax2.get_ylim()
ax2.set_ylim((0,yl[1]))
[tl.set_color('g') for tl in ax2.get_yticklabels()]
def _rename_or_remove(self, filepath):
"""Rename or remove an I3 file based on reconstructions found within
the file. If there is a name conflict with an existing file,
recursively resolve the conflict by checking the conflicting file, and
so forth.
Parameters
----------
filepath : string
Path to file to be checked.
Returns
-------
status : string, one of ('kept', 'removed', 'renamed', 'failed')
"""
from icecube import dataio, icetray
if not filepath.endswith(EXTENSION):
self._done_with(filepath)
return 'kept'
lockfilepath = filepath + LOCK_SUFFIX
lock_f = None
#
# Acquire lock file
#
lock_info = {'type': 'rename_or_remove'}
try:
lock_f = acquire_lock(lockfilepath, lock_info)
except IOError, err:
if err.errno == 11:
lock_info = read_lockfile(lockfilepath)
wstdout('File is locked, skipping: "%s"\n' % filepath)
self._done_with(lockfilepath)
return 'kept'
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Trying to acquire lock on lockfile "%s"\n%s\n'
% (lockfilepath, exc_str))
return 'kept'
def deep_clean(self, n_procs=None):
"""Run the cleaning process
Parameters
----------
ignore_locks : bool
Whether to ignore (valid) locks. Note that invalid locks are
ignored regardless of the value of `ignore_locks`.
"""
start_time = time.time()
wstdout('> Deep-cleaning the I3 files in the directory...\n')
if n_procs is None:
n_procs = min(len(self._allfilepaths), 8*cpu_count())
# Create a manager for objects synchronized across workers
mgr = Manager()
groups = [mgr.list(g) for g in self.group_by_event().values()]
pool = Pool(processes=n_procs)
ret = pool.map(DeepCleaner(), groups)
removed = []
renamed = []
failed_to_remove = []
failed_to_rename = []
for d in ret:
removed.extend(d['removed'])
renamed.extend(d['renamed'])
failed_to_remove.extend(d['failed_to_remove'])
failed_to_rename.extend(d['failed_to_rename'])
wstderr(' '.join([str(len(g)) for g in groups]) + '\n')
# TODO: remove files that -- while not redundant -- have fewer recos
# (or, stretch goal: merge the non-redundant info together)
self.report()
wstdout('> Time to run deep cleaning: %s\n'
% timediffstamp(time.time() - start_time))
def testStrategy(lattice, strategy, nRuns=10, plotVals=True, quiet=True):
finalEnergies = []
minEnergies = []
for runNum in xrange(nRuns):
lattice.resetLattice()
lattice.testAnnealing(plotLat=False,plotHist=False,
plotEnergy=False, quiet=quiet,
**strategy)
finalEnergies.append(lattice.finalEnergy)
minEnergies.append(lattice.minEnergy)
wstdout("strat: " + dictPrint(strategy) +"\n")
ST = [5,5]
SF = 3
wstdout(" E_min: mean = " +
simpleFormat(np.mean(minEnergies),sigFigs=SF,sciThresh=ST) +
", std = " +
simpleFormat(np.std(minEnergies),sigFigs=SF,sciThresh=ST)
+ "\n")
wstdout(" E_fin: mean = " +
simpleFormat(np.mean(finalEnergies),sigFigs=SF,sciThresh=ST)
+ ", std = " +
simpleFormat(np.std(finalEnergies),sigFigs=SF,sciThresh=ST)
+ "\n")
#print "mean(E_fin):", simpleFormat(np.mean(finalEnergies))
if plotVals:
fig = figure()
ax = fig.add_subplot(111)
#plot()
ax.plot(minEnergies, 'o', fillstyle='none', markeredgecolor='k',
label=r"$E_\mathrm{min}/N$")
ax.plot(minEnergies, 'k+', label=r"$E_\mathrm{fin}/N$")
ax.set_ylabel(r"$E/N$")
ax.set_xlabel("Run number")
legend(loc='best')
return finalEnergies, minEnergies
def main():
"""Main"""
start_time_sec = time.time()
args = parse_args()
cleaner = CleanupRecoFiles(args.dir)
if args.remove_empty:
cleaner.remove_empty_files()
cleaner.cleanup_lockfiles()
if args.deep_clean:
cleaner.merge_and_rename()
#lock_count = 0
#i3_count = 0
#other_count = 0
#wstdout(' -> %3d I3 files removed\n' % i3_count)
#wstdout(' -> %3d lock files removed\n' % lock_count)
#wstdout(' -> %3d other files removed\n' % other_count)
wstdout('Script run time: %s\n'
% timediffstamp(time.time() - start_time_sec))
def report(self, kept=None, removed=None, renamed=None,
failed_to_remove=None, failed_to_rename=None):
"""Report what was done to stdout"""
if removed is None:
removed = self.removed
if renamed is None:
renamed = self.renamed
if failed_to_remove is None:
failed_to_remove = self.failed_to_remove
if failed_to_rename is None:
failed_to_rename = self.failed_to_rename
if kept is None:
kept = self.kept
wstdout('> Total files removed : %5d\n'
% len(removed))
wstdout('> Total files failed to be removed: %5d\n'
% len(failed_to_remove))
wstdout('> Total files renamed : %5d\n'
% len(renamed))
wstdout('> Total files failed to be renamed: %5d\n'
% len(failed_to_rename))
def merge_and_rename(self, n_procs=None):
"""Merge and/or rename"""
if n_procs is None:
n_procs = min(len(self._allfilepaths), 8*cpu_count())
start_time = time.time()
wstdout('> Merging and/or renaming I3 files in the directory...\n')
# Create a manager for objects synchronized across workers
mgr = Manager()
groups = [mgr.list(g) for g in self.group_by_event().values()]
pool = Pool(processes=n_procs)
ret = pool.map(merge_and_rename, groups)
successes = []
failures = 0
for r in ret:
if r is None:
failures += 1
else:
successes.append(r)
wstdout('> Failures: %5d; successes: %5d\n'
% (failures, len(successes)))
wstdout('> Time to run merge_and_rename: %s\n'
% timediffstamp(time.time() - start_time, hms_always=True))
return failures, successes
def runSimulation(self, targetSuccesses=10, stepsRange=(4,50),
plotting=False):
#-- Reset state variables for a new simulation run
self.sd.simulationCompleted = False
self.sd.postprocCompleted = False
timeLastSaved = time.time()
self.sd.targetSuccesses = targetSuccesses
self.sd.stepsInChains = range(stepsRange[0],stepsRange[1])
self.sd.allChainFinalCoords = []
self.sd.allMeanChainFinalCoords = []
self.sd.meanChainFinalCoords = []
self.sd.chainSquareLengthAvg = []
self.sd.successRatio = []
self.sd.timingAvg = []
if plotting:
self.fig1 = plt.figure(1)
self.fig1.clf()
self.ax1 = fig1.add_subplot(111)
line, = ax1.plot([], [], 'ko-', lw=2)
self.ax1.set_xlim(-20,20)
self.ax1.set_ylim(-20,20)
ax1.axis('image')
plt.draw()
for stepsThisChain in self.sd.stepsInChains:
startTime = time.time()
snake = Snake(stepsThisChain, validDirs=(-1,0,1))
#successfulChains = []
#chainSquareLengths = []
#chainFinalCoords = []
#meanChainFinalCoord = []
nSuccesses = 0
trialN = 0
while nSuccesses < self.sd.targetSuccesses:
trialN += 1
#-- Perform as many reptations as chain links, to
# help ensure an independent configuration
[ snake.reptate() for n in xrange(stepsThisChain) ]
nSuccesses += 1
if plotting:
snake.plot()
plt.draw()
time.sleep(0.005)
self.sd.chainSquareLengthAvg.append(snake.meanR2())
self.sd.timingAvg.append( (time.time()-startTime)/nSuccesses )
wstdout("\nstepsThisChain = " + str(stepsThisChain) + "\n")
wstdout(" time/success = " + str(self.sd.timingAvg[-1]) + "\n")
if (time.time() - timeLastSaved) > 60*5:
self.saveState()
timeLastSaved = time.time()
self.sd.allMeanChainFinalCoords = \
np.array(self.sd.allMeanChainFinalCoords)
self.sd.simulationCompleted = True
self.saveState()
def main():
"""Main"""
start_time_sec = time.time()
args = parse_args()
def _sigint_handler(signal, frame): # pylint: disable=unused-argument, redefined-outer-name
wstderr('=' * 79 + '\n')
wstderr('*** CAUGHT CTL-C (sigint) *** ... attempting to cleanup!\n')
wstderr('=' * 79 + '\n')
raise KeyboardInterrupt
# Import IceCube things now
from I3Tray import I3Tray # pylint: disable=import-error
from icecube import dataclasses, dataio, icetray, multinest_icetray # pylint: disable=unused-variable, import-error
from cluster import get_spline_tables
lock_info = get_process_info()
wstdout('=' * 79 + '\n')
for d in [vars(args), lock_info]:
wstdout('\n')
wstdout('\n'.join([(('%20s' % k) + ' = %s' % d[k])
for k in sorted(d.keys())]))
wstdout('\n' * 2)
file_lister = FileLister(infile=args.infile, indir=args.indir)
event_counter = EventCounter(srt_pulse_name=args.srt_pulse_name,
skip=args.skip,
n_events=args.n_events)
expiration = time.time() + args.seconds_remaining
expiration_timestamp = timestamp(at=expiration, utc=True)
while True:
infile_path = file_lister.get_next_file()
if infile_path is None:
wstdout('> No more files that can be processed. Quitting.\n')
break
# NOTE: cannot run on a file that has _all_ recos already run, since
# output file cannot be same as input file (which it will have same
# name, since the name is derived from recos run / etc.)
already_run = recos_from_path(infile_path)
# NOTE: now skipping a reco is determined ONLY by the "If" kwarg, and
# not by the filename at all (swap the comment on the next line for the
# line below to change behavior back)
#recos_not_run_yet = sorted(set(args.requested) - set(already_run))
recos_not_run_yet = sorted(set(args.requested))
if not recos_not_run_yet:
wstdout('> Nothing more to be done on file. Moving on. ("%s")\n' %
infile_path)
continue
# See if file still exists
if not isfile(infile_path):
wstdout('> File no longer exists. Moving on. ("%s")\n' %
infile_path)
continue
# Skip if empty input files
if getsize(infile_path) == 0:
wstdout('> Input file is 0-length. Moving on. ("%s")\n' %
infile_path)
continue
# NOTE: commenting out the following and forcing an extremely long
# timeout to allow all recos to run (of which many won't have to,
# becuase they've already been run). Uncomment the following three
# lines and comment out the "time_remaining =" line below to change the
# behavior back when most or all recos have to be run
#time_remaining = np.ceil(
# args.seconds_remaining - (time.time() - start_time_sec)
#)
time_remaining = 3600 * 24 * 10000
# See if any reco at all fits in the remaining time
if time_remaining <= MIN_RECO_TIME:
wstdout('Not enough time to run *any* reco. Quitting.\n')
break
# See if any of the recos needing to be run on *this* file fit in the
# remaining time; register all `reco_num`s that can be run
recos_to_run = []
after_proc_time_remaining = time_remaining
for reco_num in recos_not_run_yet:
time_limit = RECOS[reco_num]['time_limit']
if time_limit > after_proc_time_remaining:
continue
recos_to_run.append(reco_num)
after_proc_time_remaining -= time_limit
time_to_run_processing = time_remaining - after_proc_time_remaining
# Give the lock an extra minute beyond the strict time to run
expiration = time.time() + time_to_run_processing + 60
expiration_timestamp = timestamp(at=expiration, utc=True)
if not recos_to_run:
wstdout('Not enough time to run any remaining reco on file. Moving'
' on. ("%s")\n' % infile_path)
continue
infile_lock_f, outfile_lock_f = None, None
infile_lock_path = infile_path + LOCK_SUFFIX
outfile_lock_path = None
allrecos = set(recos_to_run).union(already_run)
outfile_name = basename(
path_from_recos(orig_path=infile_path, recos=allrecos))
outfile_path = abspath(expand(join(args.outdir, outfile_name)))
#print('args.outdir: "%s", outfile_name: "%s", outfile_path: "%s"'
# % (args.outdir, outfile_name, outfile_path))
#break # debug
outfile_lock_path = outfile_path + LOCK_SUFFIX
if outfile_name == infile_path or outfile_path == infile_path:
wstdout('Outfile is same as infile, which will lead to removal of'
' infile. Path = "%s" ; Moving on to next input file.\n' %
infile_path)
continue
lock_info['acquired_at'] = timestamp(utc=True)
lock_info['expires_at'] = expiration_timestamp
lock_info['infile'] = infile_path
lock_info['outfile'] = outfile_path
if isfile(outfile_path):
wstdout('> Outfile path exists; will overwrite if both infile and'
' outfile locks can be obtained! ...\n'
'> "%s"\n' % outfile_path)
# NOTE:
# Create lockfiles (if they don't exist) for each of the infile and
# outfile, and try to acquire exclusive locks on these before
# working with either the infile or outfile.
#
# Also: write info to the lockfiles to know when it's okay to clean
# each up manually. Note that the `flock` will be removed by the OS
# as soon as the lock file is closed or when this process dies.
lock_info['type'] = 'infile_lock'
try:
if args.use_locks:
infile_lock_f = acquire_lock(infile_lock_path, lock_info)
except IOError:
wstdout('> infile lock failed to be obtained.'
' "%s"\n' % infile_lock_path)
infile_lock_f = None
continue
lock_info['type'] = 'outfile_lock'
try:
if args.use_locks:
outfile_lock_f = acquire_lock(outfile_lock_path, lock_info)
except IOError:
wstdout('ERROR: outfile lock failed to be obtained.'
' Cleaning up infile lock and moving on.\n'
' "%s" (infile lock)\n'
' "%s" (outfile lock)\n' %
(infile_lock_path, outfile_lock_path))
cleanup_lock_f(infile_lock_f)
infile_lock_f = None
continue
try:
remove(outfile_path)
except OSError, err:
# OSError.errno == 2 => "No such file or directory", which is OK
# since the point of `remove` is to make sure the path doesn't
# exist; otherwise, we can't go on since the output file exists but
# apparently cannot be overwritten
if err.errno != 2:
wstdout(
'> ERROR: obtained locks but outfile path exists and'
' cannot be removed. Cleaning up locks and moving on.\n'
'> "%s" (outfile path)\n'
'> "%s" (infile_lock_path)\n'
'> "%s" (outfile_lock_path)\n' %
(outfile_path, infile_lock_path, outfile_lock_path))
cleanup_lock_f(infile_lock_f)
infile_lock_f = None
cleanup_lock_f(outfile_lock_f)
outfile_lock_f = None
continue
except Exception:
cleanup_lock_f(infile_lock_f)
infile_lock_f = None
cleanup_lock_f(outfile_lock_f)
outfile_lock_f = None
raise
self.fig2.savefig("2014-01-14_problem7x28_plots.png", dpi=120)
plt.show()
if __name__ == "__main__":
startTime = time.time()
#-- Instantiate the Simulation object
#sim = Simulation()
##-- Try to load the sim data from any previous run; if no data saved
## to disk in the default location, run a new simulation
##try:
## sim.loadState()
##except Exception as e:
## print "Error({0}: {1}".format(e.errno, e.strerror)
## #sim.runSimulation(targetSuccesses=10, stepsRange=(4,101))
#sim.runSimulation(targetSuccesses=10, stepsRange=(5,30))
##-- *Always* perform post-processing and plotting (allows easy modification
## of the postprocessing (curve fitting) and plotting routines
## without needing to re-run the simulation, which can take hours)
#sim.postproc()
#sim.plotResults()
##print simpleCreateChain(nSteps=20)
chains = createChainParallel(nSteps=60, nProcs=1, nChains=1)
print time.time()-startTime
[wstdout(str(len(chain)) + " ") for chain in chains]
wstdout("\n")
def main():
"""Main"""
start_time_sec = time.time()
args = parse_args()
def _sigint_handler(signal, frame): # pylint: disable=unused-argument, redefined-outer-name
wstderr('='*79 + '\n')
wstderr('*** CAUGHT CTL-C (sigint) *** ... attempting to cleanup!\n')
wstderr('='*79 + '\n')
raise KeyboardInterrupt
# Import IceCube things now
from I3Tray import I3Tray # pylint: disable=import-error
from icecube import dataclasses, dataio, icetray, multinest_icetray # pylint: disable=unused-variable, import-error
from cluster import get_spline_tables
lock_info = getProcessInfo()
wstdout('='*79 + '\n')
for d in [vars(args), lock_info]:
wstdout('\n')
wstdout(
'\n'.join([(('%20s'%k) + ' = %s'%d[k]) for k in sorted(d.keys())])
)
wstdout('\n'*2)
file_lister = FileLister(infile=args.infile, indir=args.indir)
event_counter = EventCounter(srt_pulse_name=args.srt_pulse_name,
skip=args.skip, n_events=args.n_events)
expiration = time.time() + args.seconds_remaining
expiration_timestamp = timestamp(at=expiration, utc=True)
while True:
infile_path = file_lister.get_next_file()
if infile_path is None:
wstdout('> No more files that can be processed. Quitting.\n')
break
# NOTE: cannot run on a file that has _all_ recos already run, since
# output file cannot be same as input file (which it will have same
# name, since the name is derived from recos run / etc.)
already_run = recosFromPath(infile_path)
# NOTE: now skipping a reco is determined ONLY by the "If" kwarg, and
# not by the filename at all (swap the comment on the next line for the
# line below to change behavior back)
#recos_not_run_yet = sorted(set(args.requested) - set(already_run))
recos_not_run_yet = sorted(set(args.requested))
if not recos_not_run_yet:
wstdout('> Nothing more to be done on file. Moving on. ("%s")\n'
% infile_path)
continue
# See if file still exists
if not isfile(infile_path):
wstdout('> File no longer exists. Moving on. ("%s")\n'
% infile_path)
continue
# Skip if empty input files
if getsize(infile_path) == 0:
wstdout('> Input file is 0-length. Moving on. ("%s")\n'
% infile_path)
continue
# NOTE: commenting out the following and forcing an extremely long
# timeout to allow all recos to run (of which many won't have to,
# becuase they've already been run). Uncomment the following three
# lines and comment out the "time_remaining =" line below to change the
# behavior back when most or all recos have to be run
#time_remaining = np.ceil(
# args.seconds_remaining - (time.time() - start_time_sec)
#)
time_remaining = 3600 * 24 * 10000
# See if any reco at all fits in the remaining time
if time_remaining <= MIN_RECO_TIME:
wstdout('Not enough time to run *any* reco. Quitting.\n')
break
# See if any of the recos needing to be run on *this* file fit in the
# remaining time; register all `reco_num`s that can be run
recos_to_run = []
after_proc_time_remaining = time_remaining
for reco_num in recos_not_run_yet:
time_limit = RECOS[reco_num]['time_limit']
if time_limit > after_proc_time_remaining:
continue
recos_to_run.append(reco_num)
after_proc_time_remaining -= time_limit
time_to_run_processing = time_remaining - after_proc_time_remaining
# Give the lock an extra minute beyond the strict time to run
expiration = time.time() + time_to_run_processing + 60
expiration_timestamp = timestamp(at=expiration, utc=True)
if not recos_to_run:
wstdout('Not enough time to run any remaining reco on file. Moving'
' on. ("%s")\n' % infile_path)
continue
infile_lock_f, outfile_lock_f = None, None
infile_lock_path = infile_path + LOCK_SUFFIX
outfile_lock_path = None
allrecos = set(recos_to_run).union(already_run)
outfile_name = basename(pathFromRecos(orig_path=infile_path, recos=allrecos))
outfile_path = abspath(expand(join(args.outdir, outfile_name)))
#print('args.outdir: "%s", outfile_name: "%s", outfile_path: "%s"'
# % (args.outdir, outfile_name, outfile_path))
#break # debug
outfile_lock_path = outfile_path + LOCK_SUFFIX
if outfile_name == infile_path or outfile_path == infile_path:
wstdout(
'Outfile is same as infile, which will lead to removal of'
' infile. Path = "%s" ; Moving on to next input file.\n'
% infile_path
)
continue
lock_info['acquired_at'] = timestamp(utc=True)
lock_info['expires_at'] = expiration_timestamp
lock_info['infile'] = infile_path
lock_info['outfile'] = outfile_path
if isfile(outfile_path):
wstdout('> Outfile path exists; will overwrite if both infile and'
' outfile locks can be obtained! ...\n'
'> "%s"\n' % outfile_path)
# NOTE:
# Create lockfiles (if they don't exist) for each of the infile and
# outfile, and try to acquire exclusive locks on these before
# working with either the infile or outfile.
#
# Also: write info to the lockfiles to know when it's okay to clean
# each up manually. Note that the `flock` will be removed by the OS
# as soon as the lock file is closed or when this process dies.
lock_info['type'] = 'infile_lock'
try:
if args.use_locks:
infile_lock_f = acquire_lock(infile_lock_path, lock_info)
except IOError:
wstdout(
'> infile lock failed to be obtained.'
' "%s"\n'
% infile_lock_path
)
infile_lock_f = None
continue
lock_info['type'] = 'outfile_lock'
try:
if args.use_locks:
outfile_lock_f = acquire_lock(outfile_lock_path, lock_info)
except IOError:
wstdout(
'ERROR: outfile lock failed to be obtained.'
' Cleaning up infile lock and moving on.\n'
' "%s" (infile lock)\n'
' "%s" (outfile lock)\n'
% (infile_lock_path, outfile_lock_path)
)
cleanup_lock_f(infile_lock_f)
infile_lock_f = None
continue
try:
remove(outfile_path)
except OSError, err:
# OSError.errno == 2 => "No such file or directory", which is OK
# since the point of `remove` is to make sure the path doesn't
# exist; otherwise, we can't go on since the output file exists but
# apparently cannot be overwritten
if err.errno != 2:
wstdout(
'> ERROR: obtained locks but outfile path exists and'
' cannot be removed. Cleaning up locks and moving on.\n'
'> "%s" (outfile path)\n'
'> "%s" (infile_lock_path)\n'
'> "%s" (outfile_lock_path)\n'
% (outfile_path, infile_lock_path, outfile_lock_path)
)
cleanup_lock_f(infile_lock_f)
infile_lock_f = None
cleanup_lock_f(outfile_lock_f)
outfile_lock_f = None
continue
except Exception:
cleanup_lock_f(infile_lock_f)
infile_lock_f = None
cleanup_lock_f(outfile_lock_f)
outfile_lock_f = None
raise
try:
lock_f = acquire_lock(lockfilepath, lock_info)
except IOError, err:
if err.errno == 11:
lock_info = read_lockfile(lockfilepath)
wstdout('File is locked, skipping: "%s"\n' % filepath)
self._done_with(lockfilepath)
return 'kept'
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Trying to acquire lock on lockfile "%s"\n%s\n'
% (lockfilepath, exc_str))
return 'kept'
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Trying to acquire lock on lockfile "%s"\n%s\n'
% (lockfilepath, exc_str))
return 'kept'
#
# With lock file acquired, ...
#
try:
purportedly_run = set(recosFromPath(filepath))
unique_recos_in_file = []
#
# Open I3 file
#
try:
i3file = dataio.I3File(filepath)
except:
track_spline_abs=spline_tables['trk_amplitude'],
track_spline_prob=spline_tables['trk_timing'],
bad_doms_name='BadDomsList',
input_pulses=args.srt_pulse_name,
wave_form_range='WaveformRange',
use_ball_speedup=True,
ball_radius=150.0,
)
for reco_num in recos_to_run:
reco_name = RECOS[reco_num]['name']
time_limit = RECOS[reco_num]['time_limit']
kwargs = RECOS[reco_num]['kwargs']
wstdout(
'> Setting up tray to run reco #%3d, %s (time limit %s)\n'
% (reco_num, reco_name, timediffstamp(time_limit)))
tray.AddSegment(multinest_icetray.MultiNestReco,
reco_name,
input_pulses=args.srt_pulse_name,
base_geometry=args.geometry,
**kwargs)
tray.AddModule(
'I3Writer',
'EventWriter',
Filename=outfile_path,
Streams=[icetray.I3Frame.Physics, icetray.I3Frame.DAQ],
DropOrphanStreams=[icetray.I3Frame.DAQ])
def merge_and_rename(files):
"""Merge and/or rename all I3 files passed.
Parameters
----------
files : sequence or iterable
Returns
-------
None if failure, or -- if success -- string containing path of generated
file
"""
from icecube import dataio, icetray
if len(files) == 1:
filepath = files[0]
recos_in_file = get_recos(filepath)
new_path = pathFromRecos(orig_path=filepath, recos=recos_in_file)
if new_path == filepath:
return [filepath]
if isfile(new_path):
wstdout('Warning! "%s" exists and will be overwritten when current'
' file "%s" is renamed.\n' % (filepath, new_path))
try:
rename(filepath, new_path)
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not move file from "%s" to "%s".\n%s\n'
% (filepath, new_path, exc_str))
return None
return filepath
# TODO: make sure recos are the same for all frames...
frames = merge(files)
for frame in frames:
if frame.Stop == icetray.I3Frame.Physics:
recos = get_recos(frame)
new_path = pathFromRecos(orig_path=expand(files[0]), recos=recos)
if isfile(new_path):
recos_in_existing_file = get_recos(new_path)
if set(recos_in_existing_file) != set(recos):
wstdout('Warning! "%s" exists, has different recos from than "%s",'
' and will be overwritten by the latter.\n'
% (filepath, new_path))
try:
remove(new_path)
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not remove file "%s".\n%s\n'
% (new_path, exc_str))
return None
i3file = None
try:
i3file = dataio.I3File(new_path, 'w')
for frame in frames:
i3file.push(frame)
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not write to file "%s".\n%s\n'
% (new_path, exc_str))
return None
finally:
if i3file is not None:
i3file.close()
for filepath in files:
if expand(filepath) == new_path:
continue
try:
remove(filepath)
except:
exc_str = ''.join(format_exception(*sys.exc_info()))
wstdout('ERROR! Could not remove file "%s".\n%s\n'
% (filepath, exc_str))
return new_path
track_spline_abs=spline_tables['trk_amplitude'],
track_spline_prob=spline_tables['trk_timing'],
bad_doms_name='BadDomsList',
input_pulses=args.srt_pulse_name,
wave_form_range='WaveformRange',
use_ball_speedup=True,
ball_radius=150.0,
)
for reco_num in recos_to_run:
reco_name = RECOS[reco_num]['name']
time_limit = RECOS[reco_num]['time_limit']
kwargs = RECOS[reco_num]['kwargs']
wstdout(
'> Setting up tray to run reco #%3d, %s (time limit %s)\n'
% (reco_num, reco_name, timediffstamp(time_limit))
)
tray.AddSegment(
multinest_icetray.MultiNestReco, reco_name,
input_pulses=args.srt_pulse_name,
base_geometry=args.geometry,
**kwargs
)
tray.AddModule(
'I3Writer', 'EventWriter',
Filename=outfile_path,
Streams=[icetray.I3Frame.Physics, icetray.I3Frame.DAQ],
DropOrphanStreams=[icetray.I3Frame.DAQ]
)
