def run(self, runstate=None):
print("run...")
if runstate is None:
runstate = RunState(str(self))
# Display and interact with the Window using an Event Loop
window = self.window
print("window =", window)
with runstate:
while not runstate.cancelled:
print("event?")
event, values = window.read()
print("event =", repr(event), repr(values))
# See if user wants to quit or window was closed
if event == sg.WINDOW_CLOSED or event == 'Quit':
runstate.cancel()
elif event == self.tree.key:
record_key, = values[event]
print("record_key =", record_key)
try:
record = self.tree[record_key]
except KeyError as e:
warning("no self.tree[%r]: %s", record_key, e)
else:
print("record =", record)
self.fspath = record.fullpath
else:
warning("unexpected event %r: %r", event, values)
def __init__(self,
tmpdir=None,
suffix='.dat',
max_files=None,
max_file_size=None):
if max_files is None:
max_files = self.MAX_FILES
if max_file_size is None:
max_file_size = self.MAX_FILE_SIZE
self.tmpdir = tmpdir
self.suffix = suffix
self.max_files = max_files
self.max_file_size = max_file_size
self.blockmaps = {} # hashcode -> BlockMapping
self._tempfiles = [] # in play BlockTempfiles
self._lock = Lock()
self.runstate = RunState()
self.runstate.start()
def run(self, runstate=None):
''' Run the GUI.
'''
print("run...")
if runstate is None:
runstate = RunState(str(self))
with runstate:
print("before mainloop")
self.app.mainloop()
print("after mainloop")
def __init__(self,
*a,
lock=None,
cancel_on_exception=False,
cancel_on_result=None,
func,
func_args=(),
func_kwargs=None,
**kw):
if lock is None:
lock = RLock()
if func_kwargs is None:
func_kwargs = {}
super().__init__(*a, lock=lock, **kw)
self._required = set()
self.cancel_on_exception = cancel_on_exception
self.cancel_on_result = cancel_on_result
self.func = func
self.func_args = func_args
self.func_kwargs = func_kwargs
self.runstate = RunState(self.name)
def run(self, **kw_options):
''' Run a command.
Returns the exit status of the command.
May raise `GetoptError` from subcommands.
Any keyword arguments are used to override `self.options` attributes
for the duration of the run,
for example to presupply a shared `RunState` from an outer context.
If the first command line argument *foo*
has a corresponding method `cmd_`*foo*
then that argument is removed from the start of `argv`
and `self.cmd_`*foo*`(cmd=`*foo*`)` is called
and its value returned.
Otherwise `self.main(argv)` is called
and its value returned.
If the command implementation requires some setup or teardown
then this may be provided by the `run_context`
context manager method,
called with `cmd=`*subcmd* for subcommands
and with `cmd=None` for `main`.
'''
# short circuit if we've already complaints about bad invocation
if self._printed_usage:
return 2
options = self.options
try:
runstate = getattr(options, 'runstate', RunState(self.cmd))
upd = getattr(options, 'upd', self.loginfo.upd)
upd_context = nullcontext() if upd is None else upd
with runstate:
with upd_context:
with stackattrs(
options,
runstate=runstate,
upd=upd,
):
with stackattrs(options, **kw_options):
with self.run_context():
return self._run(self._subcmd, self, self._argv)
except GetoptError as e:
if self.getopt_error_handler(
self.cmd,
options,
e,
(None if self._printed_usage else self.usage_text(
cmd=self.cmd, subcmd=self._subcmd)),
subcmd=self._subcmd,
):
self._printed_usage = True
return 2
raise
def __init__(self, name, capacity=None, hashclass=None, runstate=None):
''' Initialise the Store.
Parameters:
* `name`: a name for this Store;
if None, a sequential name based on the Store class name
is generated
* `capacity`: a capacity for the internal `Later` queue, default 4
* `hashclass`: the hash class to use for this Store,
default: `DEFAULT_HASHCLASS` (`{DEFAULT_HASHCLASS.__name__}`)
* `runstate`: a `cs.resources.RunState` for external control;
if not supplied one is allocated
'''
with Pfx("_BasicStoreCommon.__init__(%s,..)", name):
if not isinstance(name, str):
raise TypeError(
"initial `name` argument must be a str, got %s" %
(type(name), ))
if name is None:
name = "%s%d" % (type(self).__name__, next(self._seq()))
if hashclass is None:
hashclass = DEFAULT_HASHCLASS
elif isinstance(hashclass, str):
hashclass = HASHCLASS_BY_NAME[hashclass]
assert issubclass(hashclass, HashCode)
if capacity is None:
capacity = 4
if runstate is None:
runstate = RunState(name)
RunStateMixin.__init__(self, runstate=runstate)
self._str_attrs = {}
self.name = name
self._capacity = capacity
self.__funcQ = None
self.hashclass = hashclass
self._config = None
self.logfp = None
self.mountdir = None
self.readonly = False
self.writeonly = False
self._archives = {}
self._blockmapdir = None
self.block_cache = None
class BlockCache:
''' A temporary file based cache for whole Blocks.
This is to support filesystems' and files' direct read/write
actions by passing them straight through to this cache is
there's a mapping.
We accrue complete Block contents in unlinked files.
'''
# default cace size
MAX_FILES = 32
MAX_FILE_SIZE = 1024 * 1024 * 1024
def __init__(self,
tmpdir=None,
suffix='.dat',
max_files=None,
max_file_size=None):
if max_files is None:
max_files = self.MAX_FILES
if max_file_size is None:
max_file_size = self.MAX_FILE_SIZE
self.tmpdir = tmpdir
self.suffix = suffix
self.max_files = max_files
self.max_file_size = max_file_size
self.blockmaps = {} # hashcode -> BlockMapping
self._tempfiles = [] # in play BlockTempfiles
self._lock = Lock()
self.runstate = RunState()
self.runstate.start()
def close(self):
''' Release all the blockmaps.
'''
self.runstate.cancel()
with self._lock:
self.blockmaps = {}
for tempf in self._tempfiles:
tempf.close()
self._tempfiles = []
def __getitem__(self, hashcode):
''' Fetch BlockMapping associated with `hashcode`, raise KeyError if missing.
'''
return self.blockmaps[hashcode]
def get_blockmap(self, block):
''' Add the specified Block to the cache, return the BlockMapping.
'''
blockmaps = self.blockmaps
h = block.hashcode
with self._lock:
bm = blockmaps.get(h)
if bm is None:
tempfiles = self._tempfiles
if tempfiles:
tempf = tempfiles[-1]
if tempf.size >= self.max_file_size:
tempf = None
else:
tempf = None
if tempf is None:
while len(tempfiles) >= self.max_files:
otempf = tempfiles.pop(0)
otempf.close()
tempf = BlockTempfile(self, self.tmpdir, self.suffix)
tempfiles.append(tempf)
bm = tempf.append_block(block, self.runstate)
blockmaps[h] = bm
return bm
class Task(Result):
''' A task which may require the completion of other tasks.
This is a subclass of `Result`.
Keyword parameters:
* `cancel_on_exception`: if true, cancel this `Task` if `.call`
raises an exception; the default is `False`, allowing repair
and retry
* `cancel_on_result`: optional callable to test the `Task.result`
after `.call`; if it returns `True` the `Task` is marked
as cancelled
* `func`: the function to call to complete the `Task`;
it will be called as `func(*func_args,**func_kwargs)`
* `func_args`: optional positional arguments, default `()`
* `func_kwargs`: optional keyword arguments, default `{}`
* `lock`: optional lock, default an `RLock`
Other arguments are passed to the `Result` initialiser.
Example:
t1 = Task(name="task1")
t1.bg(time.sleep, 10)
t2 = Task("name="task2")
# prevent t2 from running until t1 completes
t2.require(t1)
# try to run sleep(5) for t2 immediately after t1 completes
t1.notify(t2.call, sleep, 5)
The model here may not be quite as expected; it is aimed at
tasks which can be repaired and rerun.
As such, if `self.call(func,...)` raises an exception from
`func` then this `Task` will still block dependent `Task`s.
Dually, a `Task` which completes without an exception is
considered complete and does not block dependent `Task`s.
To cancel dependent `Tasks` the function should raise a
`CancellationError`.
Users wanting more immediate semantics can supply `cancel_on_exception`
and/or `cancel_on_result` to control these behaviours.
Example:
t1 = Task(name="task1")
t1.bg(time.sleep, 2)
t2 = Task("name="task2")
# prevent t2 from running until t1 completes
t2.require(t1)
# try to run sleep(5) for t2 immediately after t1 completes
t1.notify(t2.call, sleep, 5)
>>>
'''
_seq = Seq()
_state = ThreadState(current_task=None)
def __init__(self,
*a,
lock=None,
cancel_on_exception=False,
cancel_on_result=None,
func,
func_args=(),
func_kwargs=None,
**kw):
if lock is None:
lock = RLock()
if func_kwargs is None:
func_kwargs = {}
super().__init__(*a, lock=lock, **kw)
self._required = set()
self.cancel_on_exception = cancel_on_exception
self.cancel_on_result = cancel_on_result
self.func = func
self.func_args = func_args
self.func_kwargs = func_kwargs
self.runstate = RunState(self.name)
def __hash__(self):
return id(self)
def __eq__(self, otask):
return self is otask
@classmethod
def current_task(cls):
''' The current `Task`, valid during `Task.call()`.
This allows the function called by the `Task` to access the
task, typically to poll its `.runstate` attribute.
'''
return cls._state.current_task # pylint: disable=no-member
def abort(self):
''' Calling `abort()` calls `self.runstate.cancel()` to indicate
to the running function that it should cease operation.
'''
self.runstate.cancel()
def required(self):
''' Return a `set` containing any required tasks.
'''
with self._lock:
return set(self._required)
def require(self, otask):
''' Add a requirement that `otask` be complete before we proceed.
'''
assert otask is not self
assert self.state == ResultState.pending
with self._lock:
self._required.add(otask)
def block(self, otask):
''' Block another task until we are complete.
'''
otask.require(self)
def then(self, func, *a, **kw):
''' Queue a call to `func(*a,**kw)` to run after the completion of
this task.
This supports a chain of actions:
>>> t = Task(func=lambda: 1)
>>> final_t = t.then(print,1).then(print,2)
>>> final_t.ready # the final task has not yet run
False
>>> # finalise t, wait for final_t (which runs immediately)
>>> t.call(); print(final_t.join())
1
2
(None, None)
>>> final_t.ready
True
'''
post_task = type(self)(func=func, func_args=a, func_kwargs=kw)
post_task.require(self)
self.notify(lambda _: post_task.bg())
return post_task
def blockers(self):
''' A generator yielding tasks from `self.required()`
which should block this task.
Cancelled tasks are not blockers
but if we encounter one we do cancel the current task.
'''
for otask in self.required():
if otask.cancelled:
warning("%s cancelled because %s is also cancelled" %
(self, otask))
self.cancel()
continue
if not otask.ready:
yield otask
continue
if otask.exc_info:
yield otask
continue
# pylint: disable=arguments-differ
def bg(self):
''' Submit a function to complete the `Task` in a separate `Thread`,
returning the `Thread`.
This dispatches a `Thread` to run `self.call()`
and as such the `Task` must be in "pending" state,
and transitions to "running".
'''
return bg_thread(self.call, name=self.name)
# pylint: disable=arguments-differ
def call(self):
''' Attempt to perform the `Task` by calling `func(*func_args,**func_kwargs)`.
If we are cancelled, raise `CancellationError`.
If there are blocking required tasks, raise `BlockedError`.
Otherwise run `r=func(self,*self.func_args,**self.func_kwargsw)`
with the following effects:
* if `func()` raises a `CancellationError`, cancel the `Task`
* otherwise, if an exception is raised and `self.cancel_on_exception`
is true, cancel the `Task`;
store the exception information from `sys.exc_info()` as `self.exc_info`
regardless
* otherwise, if `self.cancel_on_result` is not `None`
and `self.cancel_on_result(r)` is true, cancel the `Task`;
store `r` as `self.result` regardless
If we were cancelled, raise `CancellationError`.
During the duration of the call the property `Task.current_task`
is set to `self` allowing access to the `Task`.
A typical use is to access the current `Task`'s `.runstate`
attribute which can be polled by long running tasks to
honour calls to `Task.abort()`.
'''
if not self.cancelled:
for otask in self.blockers():
raise BlockedError("%s blocked by %s" % (self, otask))
if not self.cancelled:
state = type(self)._state
with self._lock:
with state(current_task=self):
try:
with self.runstate:
r = self.func(*self.func_args,
**self.func_kwargs)
except CancellationError:
self.cancel()
except BaseException:
if self.cancel_on_exception:
self.cancel()
# store the exception regardless
self.exc_info = sys.exc_info()
else:
if self.cancel_on_result and self.cancel_on_result(
r):
self.cancel()
# store the result regardless
self.result = r
if self.cancelled:
raise CancellationError()
def callif(self):
''' Trigger a call to `func(self,*self.func_args,**self.func_kwargsw)`
if we're pending and not blocked or cancelled.
'''
with self._lock:
if not self.ready:
try:
self.call()
except (BlockedError, CancellationError) as e:
debug("%s.callif: %s", self, e)
# Default OS level file high water mark.
# This is used for rollover levels for DataDir files and cache files.
MAX_FILE_SIZE = 1024 * 1024 * 1024
# path separator, hardwired
PATHSEP = '/'
_progress = Progress(name="cs.vt.common.progress"),
_over_progress = OverProgress(name="cs.vt.common.over_progress")
# some shared default state, Thread independent
common = NS(
progress=_progress,
over_progress=_over_progress,
runstate=RunState("cs.vt.common.runstate"),
config=None,
S=None,
)
del _progress
del _over_progress
class _Defaults(ThreadState):
''' Per-thread default context stack.
A Store's __enter__/__exit__ methods push/pop that store
from the `.S` attribute.
'''