def noexc_gen_wrapper(*args, **kwargs): try: it = iter(func(*args, **kwargs)) except Exception as e0: try: exception("exception calling %s(*%s, **(%s)): %s", func.__name__, args, kwargs, e0) except Exception as e2: try: X("exception calling %s(*%s, **(%s)): %s", func.__name__, args, kwargs, e2) except Exception: pass return while True: try: item = next(it) except StopIteration: raise except Exception as e: try: exception("exception calling next(%s(*%s, **(%s))): %s", func.__name__, args, kwargs, e) except Exception: try: X("exception calling next(%s(*%s, **(%s))): %s", func.__name__, args, kwargs, e) except Exception: pass return else: yield item
def reap(Q): for LF in Q: if handler: try: handler(LF) except Exception as e: # pylint: disable=broad-except exception("%s: reap %s: %s", self, LF, e)
def _updated(self): datum = self.latest for notify in list(self.notify_update): try: notify(self, datum) except Exception as e: # pylint: disable=broad-except exception("%s: notify_update %s: %s", self, notify, e)
def _run_request(self, channel, tag, handler, rq_type, flags, payload): ''' Run a request and queue a response packet. ''' with Pfx( "_run_request[channel=%d,tag=%d,rq_type=%d,flags=0x%02x,payload=%s", channel, tag, rq_type, flags, repr(payload) if len(payload) <= 32 else repr(payload[:32]) + '...'): result_flags = 0 result_payload = b'' try: result = handler(rq_type, flags, payload) if result is not None: if isinstance(result, int): result_flags = result elif isinstance(result, bytes): result_payload = result elif isinstance(result, str): result_payload = result.encode( encoding='utf-8', errors='xmlcharrefreplace') else: result_flags, result_payload = result except Exception as e: # pylint: disable=broad-except exception("exception: %s", e) self._reject(channel, tag, "exception during handler") else: self._respond(channel, tag, result_flags, result_payload) self._channel_request_tags[channel].remove(tag)
def iterate_once(): ''' Call `iterate`. Place the result on outQ. Close the queue at end of iteration or other exception. Otherwise, requeue ourself to collect the next iteration value. ''' if test_ready is not None and not test_ready(): raise RetryError("iterate_once: not ready yet") try: item = iterate() except StopIteration: outQ.close() R.result = iterationss[0] except Exception as e: # pylint: disable=broad-except exception( "defer_iterable: iterate_once: exception during iteration: %s", e ) outQ.close() R.exc_info = sys.exc_info() else: iterationss[0] += 1 # put the item onto the output queue # this may itself defer various tasks (eg in a pipeline) debug("L.defer_iterable: iterate_once: %s.put(%r)", outQ, item) outQ.put(item) # now queue another iteration to run after those defered tasks self._defer(iterate_once)
def __getitem__(self, hashcode): ''' Return the decompressed data associated with the supplied `hashcode`. ''' unindexed = self._unindexed try: return unindexed[hashcode] except KeyError: index = self.index try: with self._lock: entry_bs = index[hashcode] except KeyError: raise KeyError("%s[%s]: hash not in index" % (self, hashcode)) entry = FileDataIndexEntry.from_bytes(entry_bs) filenum = entry.filenum try: try: rfd = self._rfds[filenum] except KeyError: # TODO: shove this sideways to self.open_datafile # which releases an existing datafile if too many are open DFstate = self._filemap[filenum] rfd = self._rfds[filenum] = openfd_read(DFstate.pathname) return entry.fetch_fd(rfd) except Exception as e: exception("%s[%s]:%s not available: %s", self, hashcode, entry, e) raise KeyError(str(hashcode)) from e
def _updated(self): with self._lock: notifiers = list(self.notify_update) for notify in notifiers: try: notify(self, None) except Exception as e: # pylint: disable=broad-except exception("%s: notify_update %s: %s", self, notify, e)
def thread_body(): with Pfx("parser-thread"): try: run_parser(bfr) except Exception as e: exception("exception: %s", e) raise finally: offsetQ.close()
def handle(self, *a, **kw): ''' Wrapper for FUSE handler methods. ''' syscall = method.__name__ if syscall == 'write': fh, offset, bs = a arg_desc = [ str(a[0]), str(a[1]), "%d bytes:%r..." % (len(bs), bytes(bs[:16])) ] else: arg_desc = [(("<%s>" % (type(arg).__name__, )) if isinstance( arg, llfuse.RequestContext) else repr(arg)) for arg in a] arg_desc.extend("%s=%r" % (kw_name, kw_value) for kw_name, kw_value in kw.items()) arg_desc = ','.join(arg_desc) with Pfx("%s.%s(%s)", type(self).__name__, syscall, arg_desc): trace = syscall in ( ##'getxattr', ##'setxattr', ##'statfs', ) if trace: X("CALL %s(%s)", syscall, arg_desc) fs = self._vtfs try: with stackattrs(defaults, fs=fs): with fs.S: with LogTime("SLOW SYSCALL", threshold=5.0): result = method(self, *a, **kw) if trace: if isinstance(result, bytes): X("CALL %s result => %d bytes, %r...", syscall, len(result), result[:16]) else: X("CALL %s result => %s", syscall, result) return result ##except FuseOSError as e: ## warning("=> FuseOSError %s", e, exc_info=False) ## raise except OSError as e: ##warning("=> OSError %s => FuseOSError", e, exc_info=False) raise FuseOSError(e.errno) from e except MissingHashcodeError as e: error("raising IOError from missing hashcode: %s", e) raise FuseOSError(errno.EIO) from e except Exception as e: exception("unexpected exception, raising EINVAL %s:%s", type(e), e) raise FuseOSError(errno.EINVAL) from e except BaseException as e: error("UNCAUGHT EXCEPTION: %s", e) raise RuntimeError("UNCAUGHT EXCEPTION") from e except: error("=> EXCEPTION %r", sys.exc_info())
def doit(self): # pull off our pending task and untick it Tfunc = None with self._lock: if self.pending: T, Twhen, Tfunc = self.pending self.pending = None # run it if we haven't been told not to if Tfunc: try: retval = Tfunc() except Exception as e: # pylint: disable=broad-except exception("func %s threw exception: %s", Tfunc, e) else: debug("func %s returns %s", Tfunc, retval)
def noexc_wrapper(*args, **kwargs): from cs.logutils import exception from cs.x import X try: return func(*args, **kwargs) except Exception: try: exception("exception calling %s(%s, **(%s))", func.__name__, args, kwargs) except Exception as e: try: X("exception calling %s(%s, **(%s)): %s", func.__name__, args, kwargs, e) except Exception: pass
def docmd_wrapper(self, *a, **kw): ''' Run a `Cmd` "do" method with some context and handling. ''' if not funcname.startswith('do_'): raise ValueError("function does not start with 'do_': %s" % (funcname,)) argv0 = funcname[3:] with Pfx(argv0): try: return dofunc(self, *a, **kw) except GetoptError as e: warning("%s", e) self.do_help(argv0) return None except Exception as e: # pylint: disable=broad-except exception("%s", e) return None
def parsed(self): ''' The URL content parsed as HTML by BeautifulSoup. ''' content = self.content if self.content_type == 'text/html': parser_names = ('html5lib', 'html.parser', 'lxml', 'xml') else: parser_names = ('lxml', 'xml') try: P = BeautifulSoup(content.decode('utf-8', 'replace'), 'html5lib') ##P = BeautifulSoup(content.decode('utf-8', 'replace'), list(parser_names)) except Exception as e: exception("%s: .parsed: BeautifulSoup(unicode(content)) fails: %s", self, e) with open("cs.urlutils-unparsed.html", "wb") as bs: bs.write(self.content) raise return P
def control(self, E, argv): ''' Perform a control action `argv` on the Dirent `E`. ''' with Pfx("%s.control(E=%s,argv=%r)", self, E, argv): if not argv: raise ValueError('empty argv') op = argv.pop() with Pfx(op): try: action = getattr(self, 'cmd_' + op) except AttributeError as e: exception("unknown operation %r: %s", op, e) raise ValueError("unknown operation %r: %s" % (op, e)) try: return action(E, argv) except GetoptError as e: exception("%s: %s", type(e), e) raise ValueError("%s: %s" % (type(e), e))
def call(self, func, *a, **kw): ''' Have the `Result` call `func(*a,**kw)` and store its return value as `self.result`. If `func` raises an exception, store it as `self.exc_info`. ''' with self._lock: if self.state != ResultState.pending: raise RuntimeError("%s: state should be pending but is %s" % (self, self.state)) self.state = ResultState.running try: r = func(*a, **kw) except BaseException: self.exc_info = sys.exc_info() except: # pylint: disable=bare-except exception("%s: unexpected exception: %r", func, sys.exc_info()) self.exc_info = sys.exc_info() else: self.result = r
def save_filepath(self, filepath, key=None, nolink=False, flags=''): ''' Save the file specified by `filepath` into the Maildir. By default a hardlink is attempted unless `nolink` is supplied true. The optional `flags` is a string consisting of flag letters listed at: http://cr.yp.to/proto/maildir.html Return the key for the saved message. ''' with Pfx("save_filepath(%s)", filepath): if key is None: key = self.newkey() debug("new key = %s", key) elif not self.validkey(key): raise ValueError("invalid key: %s" % (key, )) elif key in self.msgmap: raise ValueError("key already in Maildir: %s" % (key, )) tmppath = os.path.join(self.path, 'tmp', key) if os.path.exists(tmppath): raise ValueError("temp file already in Maildir: %s" % (tmppath, )) if not nolink: try: debug("hardlink %s => %s", filepath, tmppath) os.link(filepath, tmppath) except OSError: debug("copyfile %s => %s", filepath, tmppath) shutil.copyfile(filepath, tmppath) else: debug("copyfile %s => %s", filepath, tmppath) shutil.copyfile(filepath, tmppath) newbase = key if flags: newbase += ':2,' + ''.join(sorted(flags)) newpath = os.path.join(self.path, 'new', newbase) try: debug("rename %s => %s", tmppath, newpath) os.rename(tmppath, newpath) except Exception as e: exception("%s: unlink %s", e, tmppath) os.unlink(tmppath) raise self.msgmap[key] = ('new', newbase) return key
def run_parser(): ''' Thread body to run the supplied scanner against the input data. ''' bfr = CornuCopyBuffer(chunk_iter) # pylint: disable=broad-except try: for offset in scanner(bfr): # the scanner should yield only offsets, not chunks and offsets if not isinstance(offset, int): warning("discarding non-int from scanner %s: %s", scanner, offset) else: parseQ.put(offset) except Exception as e: exception("exception from scanner %s: %s", scanner, e) # Consume the remainder of chunk_iter; the tee() will copy it to parseQ. for _ in chunk_iter: pass # end of offsets and chunks parseQ.close()
def _complete(self, result, exc_info): ''' Set the result. Alert people to completion. Expect to be called _inside_ `self._lock`. ''' if result is not None and exc_info is not None: raise ValueError( "one of (result, exc_info) must be None, got (%r, %r)" % (result, exc_info)) state = self.state if state in (ResultState.cancelled, ResultState.running, ResultState.pending): self._result = result # pylint: disable=attribute-defined-outside-init self._exc_info = exc_info # pylint: disable=attribute-defined-outside-init if state != ResultState.cancelled: self.state = ResultState.ready else: if state == ResultState.ready: warning( "<%s>.state is ResultState.ready, ignoring result=%r, exc_info=%r", self, result, exc_info) raise RuntimeError( "REPEATED _COMPLETE of %s: result=%r, exc_info=%r" % (self, result, exc_info)) raise RuntimeError( "<%s>.state is not one of (cancelled, running, pending, ready): %r" % (self, state)) self._get_lock.release() notifiers = self.notifiers del self.notifiers for notifier in notifiers: try: notifier(self) except Exception as e: # pylint: disable=broad-except exception("%s._complete: calling notifier %s: exc=%s", self, notifier, e) else: self.collected = True
def update(self, E, *, when=None, previous=None, force=False, source=None): ''' Save the supplied Dirent `E` with timestamp `when`. Return the Dirent transcription. Parameters: * `E`: the Dirent to save. * `when`: the POSIX timestamp for the save, default now. * `previous`: optional previous Dirent transcription; defaults to the latest Transcription from of the Archive * `force`: append an entry even if the previous entry has the same transcription as `previous`, default False * `source`: optional source indicator for the update, default None ''' assert isinstance(E, _Dirent), "expected E<%s> to be a _Dirent" % (type(E),) etc = E.name if not force: # see if we should discard this update if previous is None: previous = self._last_s if previous is not None: # do not save if the previous transcription is unchanged Es = str(E) if Es == previous: return Es if when is None: when = time.time() s = self.append(E, when, etc) self._last = when, E self._last_s = s for notify in self.notify_update: try: notify(E, when=when, source=source) except Exception as e: exception( "notify[%s](%s,when=%s,source=%s): %s", notify, E, when, source, e ) return s
def shutdown(self): ''' Quit and disconnect. ''' logmsg = debug logmsg("send client QUIT") try: quitR = self.client_quit_bg() logmsg("flush QUIT") self.flush() logmsg("join QUIT") quitR.join() except Exception as e: exception("client quit: %s", e) logmsg = warning if self._result_queue: logmsg("close result queue") self._result_queue.close() self._result_queue = None if self._client_worker: logmsg("join client worker") self._client_worker.join() self._client_worker = None logmsg("close sendf") self.sendf.close() self.sendf = None logmsg("check for uncollected server responses") bs = self.recvf.read() if bs: warning("received %d bytes from the server at shutdown", len(bs)) logmsg("close recvf") self.recvf.close() self.recvf = None logmsg("close socket") self._sock.close() self._sock = None logmsg("shutdown complete")
def monitor(): old_sig = None next_test_time = now() next_start_time = now() while True: # check for termination state if self.flag_stop: self.flag_stop = False break # check for process exit if self.subp is not None and not self.probe(): self.reap() if self.once: break next_start_time = now() + self.restart_delay if self.subp is None: # not running - see if it should start if now() >= max(next_test_time, next_start_time): if self.test(): # test passes, start service self.spawn() next_test_time = now() + self.test_rate else: # running - see if it should stop stop = False if self.flag_restart: self.flag_restart = False stop = True elif now() >= next_test_time: if not self.test(): stop = True next_test_time = now() + self.test_rate if not stop and self.sig_func is not None: try: new_sig = self.sig_func() except Exception as e: exception("sig_func: %s", e) new_sig = None if new_sig is not None: if old_sig is None: # initial signature probe old_sig = new_sig else: try: changed = new_sig != old_sig except TypeError as e: warning( "type error comparing old_sig %s with new_sig %s: %s", type(old_sig), type(new_sig), e, ) old_sig = new_sig else: if changed: old_sig = new_sig stop = True if stop: self._kill_subproc() sleep(self.restart_delay) sleep(1) if self.subp is not None: self._kill_subproc()
def parse(self, fp, parent_context=None, missing_ok=False): ''' Read a Mykefile and yield Macros and Targets. ''' from .make import Target, Action action_list = None # not in a target for context, line in readMakefileLines(self, fp, parent_context=parent_context, missing_ok=missing_ok): with Pfx(str(context)): if isinstance(line, OSError): e = line if e.errno == errno.ENOENT or e.errno == errno.EPERM: if missing_ok: continue e.context = context yield e break raise e try: if line.startswith(':'): # top level directive _, doffset = get_white(line, 1) word, offset = get_identifier(line, doffset) if not word: raise ParseError(context, doffset, "missing directive name") _, offset = get_white(line, offset) with Pfx(word): if word == 'append': if offset == len(line): raise ParseError(context, offset, "nothing to append") mexpr, offset = MacroExpression.parse( context, line, offset) assert offset == len(line) for include_file in mexpr( context, self.namespaces).split(): if include_file: if not os.path.isabs(include_file): include_file = os.path.join( realpath(dirname(fp.name)), include_file) self.add_appendfile(include_file) continue if word == 'import': if offset == len(line): raise ParseError(context, offset, "nothing to import") ok = True missing_envvars = [] for envvar in line[offset:].split(): if envvar: envvalue = os.environ.get(envvar) if envvalue is None: error("no $%s" % (envvar, )) ok = False missing_envvars.append(envvar) else: yield Macro( context, envvar, (), envvalue.replace('$', '$$')) if not ok: raise ValueError( "missing environment variables: %s" % (missing_envvars, )) continue if word == 'precious': if offset == len(line): raise ParseError( context, offset, "nothing to mark as precious") mexpr, offset = MacroExpression.parse( context, line, offset) self.precious.update(word for word in mexpr( context, self.namespaces).split() if word) continue raise ParseError(context, doffset, "unrecognised directive") if action_list is not None: # currently collating a Target if not line[0].isspace(): # new target or unindented assignment etc - fall through # action_list is already attached to targets, # so simply reset it to None to keep state action_list = None else: # action line _, offset = get_white(line) if offset >= len(line) or line[offset] != ':': # ordinary shell action action_silent = False if offset < len(line) and line[offset] == '@': action_silent = True offset += 1 A = Action(context, 'shell', line[offset:], silent=action_silent) self.debug_parse("add action: %s", A) action_list.append(A) continue # in-target directive like ":make" _, offset = get_white(line, offset + 1) directive, offset = get_identifier(line, offset) if not directive: raise ParseError( context, offset, "missing in-target directive after leading colon" ) A = Action(context, directive, line[offset:].lstrip()) self.debug_parse("add action: %s", A) action_list.append(A) continue try: macro = Macro.from_assignment(context, line) except ValueError: pass else: yield macro continue # presumably a target definition # gather up the target as a macro expression target_mexpr, offset = MacroExpression.parse(context, stopchars=':') if not context.text.startswith(':', offset): raise ParseError(context, offset, "no colon in target definition") prereqs_mexpr, offset = MacroExpression.parse( context, offset=offset + 1, stopchars=':') if offset < len( context.text) and context.text[offset] == ':': postprereqs_mexpr, offset = MacroExpression.parse( context, offset=offset + 1) else: postprereqs_mexpr = [] action_list = [] for target in target_mexpr(context, self.namespaces).split(): yield Target(self, target, context, prereqs=prereqs_mexpr, postprereqs=postprereqs_mexpr, actions=action_list) continue raise ParseError(context, 0, 'unparsed line') except ParseError as e: exception("%s", e) self.debug_parse("finish parse")
def __init__( self, E, *, S=None, archive=None, subpath=None, readonly=None, append_only=False, show_prev_dirent=False, thread_max=None, ): ''' Initialise a new mountpoint. Parameters: * `E`: the root directory reference * `S`: the backing Store * `archive`: if not None, an Archive or similar, with a `.update(Dirent[,when])` method * `subpath`: relative path to mount Dir * `readonly`: forbid data modification * `append_only`: append only mode: files may only grow, filenames may not be changed or deleted * `show_prev_dirent`: show previous Dir revision as the '...' entry ''' if not E.isdir: raise ValueError("not dir Dir: %s" % (E,)) if S is None: S = defaults.S self._old_S_block_cache = S.block_cache self.block_cache = S.block_cache or defaults.block_cache or BlockCache() S.block_cache = self.block_cache S.open() if readonly is None: readonly = S.readonly if thread_max is None: thread_max = DEFAULT_FS_THREAD_MAX self.E = E self.S = S self.archive = archive if archive is None: self._last_sync_state = None else: self._last_sync_state = bytes(E) self.subpath = subpath self.readonly = readonly self.append_only = append_only self.show_prev_dirent = show_prev_dirent if subpath: # locate subdirectory to display at mountpoint mntE, _, tail_path = resolve(E, subpath) if tail_path: raise ValueError("subpath %r does not resolve" % (subpath,)) if not mntE.isdir: raise ValueError("subpath %r is not a directory" % (subpath,)) self.mntE = mntE else: mntE = E self.mntE = mntE self.is_darwin = os.uname().sysname == 'Darwin' self.device_id = -1 self._fs_uid = os.geteuid() self._fs_gid = os.getegid() self._lock = RLock() self._later = Later(DEFAULT_FS_THREAD_MAX) self._later.open() self._path_files = {} self._file_handles = [] inodes = self._inodes = Inodes(self) self[1] = mntE try: with Pfx("fs_inode_dirents"): fs_inode_dirents = E.meta.get("fs_inode_dirents") X("FS INIT: fs_inode_dirents=%s", fs_inode_dirents) if fs_inode_dirents: inode_dir, offset = _Dirent.from_str(fs_inode_dirents) if offset < len(fs_inode_dirents): warning( "unparsed text after Dirent: %r", fs_inode_dirents[offset:] ) X("IMPORT INODES:") dump_Dirent(inode_dir) inodes.load_fs_inode_dirents(inode_dir) else: X("NO INODE IMPORT") X("FileSystem mntE:") with self.S: with stackattrs(defaults, fs=self): dump_Dirent(mntE) except Exception as e: exception("exception during initial report: %s", e)
def _main(self): ''' The main loop. Pull requests off the queue; they will come off in time order, so we always get the most urgent item. If we're already delayed waiting for a previous request, halt that request's timer and compare it with the new job; push the later request back onto the queue and proceed with the more urgent one. If it should run now, run it. Otherwise start a `Timer` to run it later. The loop continues processing items until the `TimerQueue` is closed. ''' with Pfx("TimerQueue._main()"): assert not self.mainRunning, "main loop already active" self.mainRunning = True while not self.closed: when, n, func = self.Q.get() debug("got when=%s, n=%s, func=%s", when, n, func) if when is None: # it should be the dummy item assert self.closed assert self.Q.empty() break with self._lock: if self.pending: # Cancel the pending Timer # and choose between the new job and the job the Timer served. # Requeue the lesser job and do or delay-via-Timer the more # urgent one. T, Twhen, Tfunc = self.pending self.pending[2] = None # prevent the function from running if racy T.cancel() self.pending = None # nothing pending now T = None # let go of the cancelled timer if when < Twhen: # push the pending function back onto the queue, but ahead of # later-queued funcs with the same timestamp requeue = (Twhen, 0, Tfunc) else: # push the dequeued function back - we prefer the pending one requeue = (when, n, func) when = Twhen func = Tfunc self.Q.put(requeue) # post: self.pending is None and the Timer is cancelled assert self.pending is None now = time.time() delay = when - now if delay <= 0: # function due now - run it try: retval = func() except Exception as e: # pylint: disable=broad-except exception("func %s threw exception: %s", func, e) else: debug("func %s returns %s", func, retval) else: # function due later - run it from a Timer def doit(self): # pull off our pending task and untick it Tfunc = None with self._lock: if self.pending: T, Twhen, Tfunc = self.pending self.pending = None # run it if we haven't been told not to if Tfunc: try: retval = Tfunc() except Exception as e: # pylint: disable=broad-except exception("func %s threw exception: %s", Tfunc, e) else: debug("func %s returns %s", Tfunc, retval) with self._lock: T = Timer(delay, partial(doit, self)) self.pending = [T, when, func] T.start() self.mainRunning = False
def entries(): ''' Generator to yield directory entries. ''' o = off D = FH.D fs = FH.fs S = self._vtfs.S names = FH.names while True: try: E = None EA = None if o == 0: name = '.' with S: E = D[name] elif o == 1: name = '..' if D is self._vtfs.mntE: try: st = os.stat(dirname(self._vtfs.mnt_path)) except OSError as e: warning("os.stat(%r): %s", dirname(self._vtfs.mnt_path), e) else: EA = self._stat_EntryAttributes(st) else: with S: E = D[name] else: o2 = o - 2 if o2 == len(names) and fs.show_prev_dirent: name = PREV_DIRENT_NAME try: E = D.prev_dirent except MissingHashcodeError as e: warning("prev_dirent unavailable: %s", e) elif o2 >= len(names): break else: name = names[o2] if name == '.' or name == '..': # already special cased E = None elif name == PREV_DIRENT_NAME and fs.show_prev_dirent: warning( "%s: readdir: suppressing entry %r because fs.show_prev_dirent is true", D, PREV_DIRENT_NAME) E = None else: with S: E = D.get(name) if EA is None: if E is not None: # yield name, attributes and next offset with stackattrs(defaults, fs=fs): with S: try: EA = self._vt_EntryAttributes(E) except Exception as e: warning("%r: %s", name, e) EA = None if EA is not None: yield self._vt_bytes(name), EA, o + 1 o += 1 except Exception as e: exception("READDIR: %s", e) raise
def handler(exc_type, exc_value, exc_tb): exception("EXCEPTION: <%s> %s", exc_type, exc_value) return conceal