def __nolock_update(self, done_file, inprogress_file):
done_file.seek(0, os.SEEK_END)
new_done_filepos = done_file.tell()
if new_done_filepos >= self.__done_filepos:
if new_done_filepos > self.__done_filepos:
done_file.seek(self.__done_filepos, os.SEEK_SET)
new_done = list()
while True:
try:
new_done.append(pickle.load(done_file))
self.__done_filepos = done_file.tell()
except:
done_file.truncate(self.__done_filepos)
break
self.__done.update(new_done)
self.__remaining.difference_update(new_done)
self.__filtered_done.update(
set(new_done).intersection(self.__sweeps))
inprogress_file.seek(0, os.SEEK_SET)
try:
self.__inprogress = pickle.load(inprogress_file)
except:
inprogress_file.truncate(0)
self.__inprogress.clear()
self.__remaining.difference_update(self.__inprogress)
self.__filtered_inprogress.intersection_update(self.__inprogress)
else:
self.__nolock_full_update(done_file, inprogress_file)
def igeom(range_min, range_max, num_steps):
"""Return an integer geometric progression from range_min to range_max with num_steps"""
if num_steps == 0: return []
if num_steps == 1: return [int(range_min)]
return sorted(
set([int(range_min)] +
[int(round(x)) for x in geom(range_min, range_max, num_steps)] +
[int(range_max)]))
def cancel_batch(self, combinations):
"""cancel processing of the given combination(s), but don't mark it/them as skipped, they comes back in the *todo* queue."""
with self.__lock:
with _openlock(os.path.join(self.__persistence_dir,
"done")) as done_file:
with _openlock(
os.path.join(self.__persistence_dir,
"inprogress")) as inprogress_file:
self.__nolock_update(done_file, inprogress_file)
filtered_combinations = set(combinations)
filtered_combinations.intersection_update(self.__sweeps)
self.__remaining.update(filtered_combinations)
self.__inprogress.difference_update(combinations)
self.__filtered_inprogress.difference_update(combinations)
inprogress_file.truncate(0)
pickle.dump(self.__inprogress, inprogress_file)
logger.trace("%s combinations cancelled: %s", self.__name,
combinations)
logger.trace(self)
def skip_batch(self, combinations):
"""mark the given element(s) *skipped*"""
with self.__lock:
with _openlock(os.path.join(self.__persistence_dir,
"done")) as done_file:
with _openlock(
os.path.join(self.__persistence_dir,
"inprogress")) as inprogress_file:
self.__nolock_update(done_file, inprogress_file)
self.__skipped.update(combinations)
filtered_combinations = set(combinations)
filtered_combinations.intersection_update(self.__sweeps)
self.__filtered_skipped.update(filtered_combinations)
self.__inprogress.difference_update(combinations)
self.__filtered_inprogress.difference_update(combinations)
inprogress_file.truncate(0)
pickle.dump(self.__inprogress, inprogress_file)
logger.trace("%s combinations skipped: %s", self.__name,
combinations)
logger.trace(self)
def done_batch(self, combinations):
"""mark the given element(s) *done*"""
with self.__lock:
with _openlock(os.path.join(self.__persistence_dir,
"done")) as done_file:
with _openlock(
os.path.join(self.__persistence_dir,
"inprogress")) as inprogress_file:
self.__nolock_update(done_file, inprogress_file)
self.__remaining.difference_update(combinations)
self.__inprogress.difference_update(combinations)
self.__filtered_inprogress.difference_update(combinations)
self.__done.update(combinations)
filtered_combinations = set(combinations)
filtered_combinations.intersection_update(self.__sweeps)
self.__filtered_done.update(filtered_combinations)
done_file.seek(0, os.SEEK_END)
for combination in combinations:
pickle.dump(combination, done_file)
inprogress_file.truncate(0)
pickle.dump(self.__inprogress, inprogress_file)
logger.trace("%s combinations done: %s", self.__name, combinations)
logger.trace(self)
def __nolock_full_update(self, done_file, inprogress_file):
self.__done.clear()
self.__done_filepos = 0
done_file.seek(0, os.SEEK_SET)
while True:
try:
self.__done.add(pickle.load(done_file))
self.__done_filepos = done_file.tell()
except:
done_file.truncate(self.__done_filepos)
break
inprogress_file.seek(0, os.SEEK_SET)
try:
self.__inprogress = pickle.load(inprogress_file)
except:
inprogress_file.truncate(0)
self.__inprogress.clear()
self.__remaining = set(self.__sweeps).difference(
self.__done, self.__skipped, self.__inprogress)
self.__filtered_done = self.__done.intersection(self.__sweeps)
self.__filtered_inprogress = self.__inprogress.intersection(
self.__sweeps)
self.__filtered_skipped = self.__skipped.intersection(self.__sweeps)
def set_sweeps(self, sweeps=None, save_sweeps=False):
"""Change the list of what to iterate on.
:param sweeps: iterable
:param save_sweeps: boolean. default False. If True, the
sweeps are written to disk.
"""
with self.__lock:
if sweeps:
self.__sweeps = set(sweeps)
if save_sweeps:
with _openlock(
os.path.join(self.__persistence_dir,
"sweeps")) as sweeps_file:
sweeps_file.truncate(0)
pickle.dump(self.__sweeps, sweeps_file)
else:
with _openlock(os.path.join(self.__persistence_dir,
"sweeps")) as sweeps_file:
sweeps_file.seek(0, os.SEEK_SET)
self.__sweeps = pickle.load(sweeps_file)
self.full_update()
def __init__(self,
persistence_dir,
sweeps=None,
save_sweeps=False,
name=None):
"""
:param persistence_dir: path to persistence directory. In this
directory will be created to python pickle files: ``done``
and ``inprogress`` This files can be erased if needed.
:param sweeps: An iterable, what to iterate on. If None
(default), try to load it from ``persistence_dir``
:param save_sweeps: boolean. default False. If True, the
sweeps are written to disk during initialization (this may
take some time but occurs only once)
:param name: a convenient name to identify an instance in
logs. If None, compute one from persistence_dir.
"""
self.__lock = threading.RLock()
self.__persistence_dir = persistence_dir
try:
os.makedirs(self.__persistence_dir)
except os.error:
pass
self.__name = name
if not self.__name:
self.__name = os.path.basename(self.__persistence_dir)
self.__done = set()
self.__inprogress = set()
self.__skipped = set()
self.__filtered_done = set()
self.__filtered_inprogress = set()
self.__filtered_skipped = set()
# __filtered_done, __filtered_inprogress, __filtered_skipped
# are the intersections of __sweeps and __done, __inprogress,
# __skipped. They exist because:
#
# - client may call set_sweeps with different sweeps, still we
# want to remember the complete list of __done,
# __inprogress, __skipped
#
# - different ParamSweeper instances may share the same
# storage though having different individual
# __sweeps. __inprogress and __done on storage will be the
# union of all inprogress and done, so the ParamSweeper must
# be prepared to deal correctly with __inprogress and __done
# containing elements not in *its* __sweeps (and it must not
# discard them)
#
# - on the other hand, when displaying with __str__(),
# stats(), or when retrieving with get_done(),
# get_inprogress(), get_skipped(), client expects to get
# only lists of done, inprogress, skipped relative to the
# current __sweeps.
#
# - we could filter (do the intersection with __sweeps) only
# when displaying or returning the lists, but it's a costly
# operation to do the full intersection, whereas doing it
# incrementaly is fast.
self.__remaining = set()
self.__done_filepos = None
self.set_sweeps(sweeps, save_sweeps)
