def Connect(self): self.error(1) self.warning(1) def en(x): return x if len(x) else None self.dbc = dicty.PIPAx(cache=self.buffer, username=en(self.username), password=self.password) # check password if en(self.username) != None: try: self.dbc.mappings(reload=True) except dicty.AuthenticationError: self.error(1, "Wrong username or password") self.dbc = None except Exception as ex: print("Error when contacting the PIPA database", ex) sys.excepthook(*sys.exc_info()) try: # maybe cached? self.dbc.mappings() self.warning(1, "Can not access database - using cached data.") except Exception as ex: self.dbc = None self.error(1, "Can not access database.")
def restricted(code, environment=None, layer='Unknown'): """ Runs code in environment and returns the output. If an exception occurs in code it raises a RestrictedError containing the traceback. Layer is passed to RestrictedError to identify where the error occurred. """ if environment is None: environment = {} environment['__file__'] = layer environment['__name__'] = '__restricted__' try: if isinstance(code, types.CodeType): ccode = code else: ccode = compile2(code, layer) exec ccode in environment except HTTP: raise except RestrictedError: # do not encapsulate (obfuscate) the original RestrictedError raise except Exception, error: # extract the exception type and value (used as output message) etype, evalue, tb = sys.exc_info() # XXX Show exception in Wing IDE if running in debugger if __debug__ and 'WINGDB_ACTIVE' in os.environ: sys.excepthook(etype, evalue, tb) output = "%s %s" % (etype, evalue) raise RestrictedError(layer, code, output, environment)
def main(): parser = argparse.ArgumentParser() parser.add_argument("source", nargs="+", help="The exercise json files to make") parser.add_argument("--target", default="./exercises/", help="Target path for exercises") parser.add_argument("--overwrite", action="store_true", help="Overwrite existing exercises") args = parser.parse_args() errors = 0 for src in args.source: tgt = "%s.html" % splitext(basename(src))[0] tgt = join(args.target, tgt) if exists(tgt) and not args.overwrite: print("Skipping existing exercise: %s" % tgt) continue try: print("\n") print(("=[ %s ]=" % src).center(80, "-")) make_exercise(src, tgt) print("\n") except Exception as e: sys.excepthook(*sys.exc_info()) errors += 1 sys.exit(errors)
def check_http_signature(self, path, query): try: qargs = dict([b.split("=") for b in query.split("&")]) ts = qargs['ts'] sig = qargs['sig'] keyid = qargs.get("k", "default") # not used yet if sig in self.replay: print("replayed signature " + sig + " in request for " + path) return False its = int(ts) / 1000 now = time.time() max = self.signature_max_age print("request timestamp: " + ts + ", min: " + str(now - max) + ", max: " + str(now + max)) if (its < (now - max)) or (its > (now + max)): return False message = (path + "\n" + ts).encode() check = siphash.SipHash_2_4(self.get_sign_key(keyid), message).hexdigest() print("request signature: " + sig + ", expecting: " + check.decode()) if check == sig.encode(): self.replay.insert(0, sig) self.replay = self.replay[:self.replay_memory] return True return False except Exception as e: sys.excepthook(*sys.exc_info()) return False
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): '''Run forkserver.''' if preload: if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: spawn.import_main_path(main_path) finally: del process.current_process()._inheriting for modname in preload: try: __import__(modname) except ImportError: pass util._close_stdin() # ignoring SIGCHLD means no need to reap zombie processes # letting SIGINT through avoids KeyboardInterrupt tracebacks handlers = { signal.SIGCHLD: signal.SIG_IGN, signal.SIGINT: signal.SIG_DFL, } old_handlers = {sig: signal.signal(sig, val) for (sig, val) in handlers.items()} with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() selector.register(listener, selectors.EVENT_READ) selector.register(alive_r, selectors.EVENT_READ) while True: try: while True: rfds = [key.fileobj for (key, events) in selector.select()] if rfds: break if alive_r in rfds: # EOF because no more client processes left assert os.read(alive_r, 1) == b'' raise SystemExit assert listener in rfds with listener.accept()[0] as s: code = 1 if os.fork() == 0: try: _serve_one(s, listener, alive_r, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: os._exit(code) except OSError as e: if e.errno != errno.ECONNABORTED: raise
def emit_stream(streams, stream, outdir): if streams: i = max(streams.keys()) + 1 # streams are unnumbered else: i = 0xffff try: if swf.sound.supported(stream): filename = 'soundstream-%d%s' % (i, swf.consts.AudioCodec.FileExtensions[stream[0].soundFormat]) with open(path.join(outdir, filename), 'wb') as sf: swf.sound.write_stream_to_file(stream, sf) streams[i] = dict(status = 'extracted', id = i, kind = 'soundstream', filesize = path.getsize(path.join(outdir, filename)), mimetype = swf.consts.AudioCodec.MimeTypes[stream[0].soundFormat], codec = swf.consts.AudioCodec.tostring(stream[0].soundFormat), filename = filename, **get_sound_meta(path.join(outdir, filename))) elif swf.sound.junk(stream): pass # discard junk else: streams[i] = dict(status = 'skipped', id = i, kind = 'soundstream', reason = swf.sound.reason_unsupported(stream), codec = swf.consts.AudioCodec.tostring(stream[0].soundFormat)) except Exception: print 'stream', i, 'extraction failed:' sys.excepthook(*sys.exc_info())
def emit_sound(sounds, sound, outdir): try: i = sound.soundId if swf.sound.supported(sound): filename = 'sound-%d%s' % (i, swf.consts.AudioCodec.FileExtensions[sound.soundFormat]) with open(path.join(outdir, filename), 'wb') as sf: swf.sound.write_sound_to_file(sound, sf) sounds[i] = dict(status = 'extracted', id = i, kind = 'sound', filesize = path.getsize(path.join(outdir, filename)), mimetype = swf.consts.AudioCodec.MimeTypes[sound.soundFormat], codec = swf.consts.AudioCodec.tostring(sound.soundFormat), filename = filename, **get_sound_meta(path.join(outdir, filename))) elif swf.sound.junk(sound): pass # discard junk else: sounds[i] = dict(status = 'skipped', id = i, kind = 'sound', reason = swf.sound.reason_unsupported(sound), codec = swf.consts.AudioCodec.tostring(sound.soundFormat)) except Exception: print 'sound', i, 'extraction failed:' sys.excepthook(*sys.exc_info())
def tst_ctrl_flush(self): try: s3ql.ctrl.main(['flushcache', self.mnt_dir]) except: sys.excepthook(*sys.exc_info()) pytest.fail("s3qlctrl raised exception")
def run_with_except_hook(*args, **kw): try: run_old(*args, **kw) except (KeyboardInterrupt, SystemExit): raise except: sys.excepthook(*sys.exc_info())
def main(): args = sys.argv[1:] if not args: args.append('run') try: try: # SZ: print(__greeter__ % ENV) _load_default_settings() fabfile = _pick_fabfile() load(fabfile, fail='warn') commands = _parse_args(args) _validate_commands(commands) _execute_commands(commands) finally: _disconnect() # SZ: print("Done.") except SystemExit: # a number of internal functions might raise this one. raise except KeyboardInterrupt: print("Stopped.") sys.exit(1) except: sys.excepthook(*sys.exc_info()) # we might leave stale threads if we don't explicitly exit() sys.exit(1) sys.exit(0)
def tst_lock_rm(self): # Extract tar tempdir = os.path.join(self.mnt_dir, 'lock_dir') filename = os.path.join(tempdir, 'myfile') os.mkdir(tempdir) with open(filename, 'w') as fh: fh.write('Hello, world') # copy try: s3ql.lock.main([tempdir]) except: sys.excepthook(*sys.exc_info()) pytest.fail("s3qllock raised exception") # Try to delete assert_raises(PermissionError, os.unlink, filename) # Try to write with pytest.raises(PermissionError): open(filename, 'w+').write('Hello') # delete properly try: s3ql.remove.main([tempdir]) except: sys.excepthook(*sys.exc_info()) pytest.fail("s3qlrm raised exception") assert 'lock_dir' not in llfuse.listdir(self.mnt_dir)
def run(self): global path, version, initVersion, forcedVersion global buildVersion ## Make sure build directory is clean buildPath = os.path.join(path, self.build_lib) if os.path.isdir(buildPath): distutils.dir_util.remove_tree(buildPath) ret = build.build.run(self) # If the version in __init__ is different from the automatically-generated # version string, then we will update __init__ in the build directory if initVersion == version: return ret try: initfile = os.path.join(buildPath, 'pyqtgraph', '__init__.py') data = open(initfile, 'r').read() open(initfile, 'w').write(re.sub(r"__version__ = .*", "__version__ = '%s'" % version, data)) buildVersion = version except: if forcedVersion: raise buildVersion = initVersion sys.stderr.write("Warning: Error occurred while setting version string in build path. " "Installation will use the original version string " "%s instead.\n" % (initVersion) ) sys.excepthook(*sys.exc_info()) return ret
def _exitfunc(cls): # At shutdown invoke finalizers for which atexit is true. # This is called once all other non-daemonic threads have been # joined. reenable_gc = False try: if cls._registry: import gc if gc.isenabled(): reenable_gc = True gc.disable() pending = None while True: if pending is None or finalize._dirty: pending = cls._select_for_exit() finalize._dirty = False if not pending: break f = pending.pop() try: # gc is disabled, so (assuming no daemonic # threads) the following is the only line in # this function which might trigger creation # of a new finalizer f() except Exception: sys.excepthook(*sys.exc_info()) assert f not in cls._registry finally: # prevent any more finalizers from executing during shutdown finalize._shutdown = True if reenable_gc: gc.enable()
def load(): try: f = open(fullfn) except IOError: # e.g. file-not-found. that's ok return baseType(*defaultArgs) # some common types g = {baseType.__name__: baseType} # the baseType itself if namespace is None: g.update(globals()) # all what we have here if baseType.__module__: # the module of the basetype import sys m = sys.modules[baseType.__module__] g.update([(varname,getattr(m,varname)) for varname in dir(m)]) else: g.update(namespace) try: obj = eval(f.read(), g) except Exception: import sys sys.excepthook(*sys.exc_info()) return baseType(*defaultArgs) # Try to convert. if not isinstance(obj, baseType): obj = baseType(obj) return obj
def threadMain(self): better_exchook.install() thread = currentThread() setCurThreadName("PyMod %s" % self.name) while True: if self.module: try: reload(self.module) except Exception: print "couldn't reload module", self.module sys.excepthook(*sys.exc_info()) # continue anyway, maybe it still works and maybe the mainFunc does sth good/important else: self.module = __import__(self.moduleName) mainFunc = getattr(self.module, self.mainFuncName) try: mainFunc() except KeyboardInterrupt: break except Exception: print "Exception in module", self.name sys.excepthook(*sys.exc_info()) if not thread.reload: break sys.stdout.write("reloading module %s\n" % self.name) thread.cancel = False thread.reload = False thread.waitQueue = None
def execute(self): """Execute the PEX. This function makes assumptions that it is the last function called by the interpreter. """ try: with self.patch_sys(): working_set = self._env.activate() if 'PEX_COVERAGE' in os.environ: self.start_coverage() TRACER.log('PYTHONPATH contains:') for element in sys.path: TRACER.log(' %c %s' % (' ' if os.path.exists(element) else '*', element)) TRACER.log(' * - paths that do not exist or will be imported via zipimport') with self.patch_pkg_resources(working_set): self._execute() except Exception: # Allow the current sys.excepthook to handle this app exception before we tear things down in # finally, then reraise so that the exit status is reflected correctly. sys.excepthook(*sys.exc_info()) raise finally: # squash all exceptions on interpreter teardown -- the primary type here are # atexit handlers failing to run because of things such as: # http://stackoverflow.com/questions/2572172/referencing-other-modules-in-atexit if 'PEX_TEARDOWN_VERBOSE' not in os.environ: sys.stderr.flush() sys.stderr = DevNull() sys.excepthook = lambda *a, **kw: None
def _do_polling(self): while True: try: self.poll() except: sys.excepthook(*sys.exc_info()) time.sleep(self.getProperty("interval")/1000.0 or 1)
def _cfgRun(self): self._cfgsock = socket.socket() s = self._cfgsock s.bind((self._ip, self._cfgport)) s.listen(100) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) while True: try: self._cfgsock = s.accept() rs,addr = self._cfgsock rs.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) print "== received CONFIG connection from %s:%d ==" % (addr) self.stdin = FileSocket(rs) self.stdout = self.stdin self.stderr = self.stdin while True: try: self.cmdloop() except KillCfgLoop: break except: sys.excepthook(*sys.exc_info()) except: sys.excepthook(*sys.exc_info())
def printException(): """ Prints the exception information in case exection is thrown in a try/catch block """ e = sys.exc_info () sys.excepthook ( e[0], e[1], e[2] )
def onTextChange(): try: control.subjectObject = control.attr.__get__(control.parent.subjectObject) newText = unicode(label.stringValue()) control.subjectObject(updateText = newText) except Exception: sys.excepthook(*sys.exc_info())
def onPerformDragOperation(self, sender): self.guiCursor.setDrawsBackground_(False) import __builtin__ try: filenames = __builtin__.list(sender.draggingPasteboard().propertyListForType_(AppKit.NSFilenamesPboardType)) filenames = map(convertToUnicode, filenames) index = self.index internalDragCallback = getattr(sender.draggingSource(), "onInternalDrag", None) def doDragHandler(): control.attr.dragHandler( control.parent.subjectObject, control.subjectObject, index, filenames) if internalDragCallback: do_in_mainthread(lambda: internalDragCallback( control, index, filenames), wait=False) utils.daemonThreadCall(doDragHandler, name="DragHandler") return True except: sys.excepthook(*sys.exc_info()) return False
def _enum_dir_cb(self, fileenum, result, results): try: files = fileenum.next_files_finish(result) if files is None or len(files) == 0: print("okay, done; got %d files" % len(results)) fileenum.close_async(GLib.PRIORITY_DEFAULT, None, self._close_enum_cb, None) self.finished(results) return for f in files: ct = f.get_attribute_string("standard::content-type") # assume readable unless told otherwise readable = True if f.has_attribute("access::can-read"): readable = f.get_attribute_boolean("access::can-read") if ct is not None and ct.startswith("image/") and readable: results.append(f.get_name()) fileenum.next_files_async(ITEMS_PER_NOTIFICATION, GLib.PRIORITY_DEFAULT, None, self._enum_dir_cb, results) except Exception as e: print("okay, probably done: %s" % e) import sys sys.excepthook(*sys.exc_info()) self.finished(results) fileenum.close_async(GLib.PRIORITY_DEFAULT, None, self._close_enum_cb, None)
def emit(self, eventDict): if 'failure' in eventDict: vf = eventDict['failure'] e_t, e_v, e_tb = vf.type, vf.value, vf.getTracebackObject() sys.excepthook(e_t, e_v, e_tb) text = twlog.textFromEventDict(eventDict) if text is None: return timeStr = self.formatTime(eventDict['time']) fmtDict = {'system': eventDict['system'], 'text': text.replace("\n", "\n\t")} msgStr = twlog._safeFormat("[%(system)s] %(text)s\n", fmtDict) if GLLogObserver.suppressed == GLLogObserver.limit_suppressed: GLLogObserver.suppressed = 0 GLLogObserver.limit_suppressed += 5 GLLogObserver.last_exception_msg = "" try: # in addition to escape sequence removal on logfiles we also quote html chars util.untilConcludes(self.write, timeStr + " " + log_encode_html(msgStr)) util.untilConcludes(self.flush) # Hoorj! except Exception as excep: GLLogObserver.suppressed += 1 GLLogObserver.last_exception_msg = str(excep)
def exception_msg_handler(signal, data): """ Handler for the ExceptionSignal signal. :param signal: event data :type signal: (event_type, message_data) :param data: additional data :type data: any """ global exception_processed if exception_processed: # get data from the event data structure exception_info = signal.exception_info stack_trace = "\n" + App.get_scheduler().dump_stack() log.error(stack_trace) # exception_info is a list sys.excepthook(*exception_info) else: # show only the first exception do not spam user with others exception_processed = True loop = App.get_event_loop() # start new loop for handling the exception # this will stop processing all the old signals and prevent raising new exceptions loop.execute_new_loop(signal)
def do(self, s): """Does an arbitrary operation from a valid Chef string The command must be the first word, case sensitive. If the command does not exist, None is returned. The rest of the string is passed to the desired function as a list. Author: Sean """ def isVerbStatement(li): return len(li) == 3 and li[1] == "the" and li[2] in [v + "." for v in self.ingredients] t = s.split() try: if s == "Ingredients.": self._do_ingredients() elif len(t) < 2: raise Kitchen.Error("Syntax Error", "Commands must have at least two tokens.") elif hasattr(self, t[0]): return getattr(self, t[0])(t[1:]) elif isVerbStatement(t): # the word may be a verb print "Verbing!" print "Verb:", t[0] else: print "No such attribute" except Kitchen.Error as (title, message): import sys sys.excepthook(*sys.exc_info()) print "" print "{t}: {m}\a".format(t=title, m=message) return False
def sendColormap(self): if DEBUG: print("sending colormap") #prevent unexpected behaviour because of bad limits if self.minValue > self.maxValue: vmax = self.minValue vmin = self.maxValue else: vmax = self.maxValue vmin = self.minValue try: #self.emit(qt.SIGNAL("ColormapChanged"), # self.colormapIndex, self.autoscale, # vmin, vmax, # self.dataMin, self.dataMax, # self.colormapType) cmap = [self.colormapIndex, self.autoscale, vmin, vmax, self.dataMin, self.dataMax, self.colormapType] self.sigColormapChanged.emit(cmap) except: sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
def execute(self): """Execute the PEX. This function makes assumptions that it is the last function called by the interpreter. """ teardown_verbosity = self._vars.PEX_TEARDOWN_VERBOSE try: with self.patch_sys(): working_set = self._activate() TRACER.log('PYTHONPATH contains:') for element in sys.path: TRACER.log(' %c %s' % (' ' if os.path.exists(element) else '*', element)) TRACER.log(' * - paths that do not exist or will be imported via zipimport') with self.patch_pkg_resources(working_set): self._wrap_coverage(self._wrap_profiling, self._execute) except Exception: # Allow the current sys.excepthook to handle this app exception before we tear things down in # finally, then reraise so that the exit status is reflected correctly. sys.excepthook(*sys.exc_info()) raise except SystemExit as se: # Print a SystemExit error message, avoiding a traceback in python3. # This must happen here, as sys.stderr is about to be torn down if not isinstance(se.code, int) and se.code is not None: print(se.code, file=sys.stderr) raise finally: # squash all exceptions on interpreter teardown -- the primary type here are # atexit handlers failing to run because of things such as: # http://stackoverflow.com/questions/2572172/referencing-other-modules-in-atexit if not teardown_verbosity: sys.stderr.flush() sys.stderr = DevNull() sys.excepthook = lambda *a, **kw: None
def emit(self, eventDict): if 'failure' in eventDict: vf = eventDict['failure'] e_t, e_v, e_tb = vf.type, vf.value, vf.getTracebackObject() sys.excepthook(e_t, e_v, e_tb) text = twlog.textFromEventDict(eventDict) if text is None: return timeStr = self.formatTime(eventDict['time']) fmtDict = {'system': eventDict['system'], 'text': text.replace("\n", "\n\t")} msgStr = twlog._safeFormat("[%(system)s] %(text)s\n", fmtDict) if GLLogObserver.suppressed == GLLogObserver.limit_suppressed: # This code path flush the status of the broken log, in the case a flood is happen # for few moment or in the case something goes wrong when logging below. log.info("!! has been suppressed %d log lines due to error flood (last error %s)" % (GLLogObserver.limit_suppressed, GLLogObserver.last_exception_msg) ) GLLogObserver.suppressed = 0 GLLogObserver.limit_suppressed += 5 GLLogObserver.last_exception_msg = "" try: # in addition to escape sequence removal on logfiles we also quote html chars util.untilConcludes(self.write, timeStr + " " + log_encode_html(msgStr)) util.untilConcludes(self.flush) # Hoorj! except Exception as excep: GLLogObserver.suppressed += 1 GLLogObserver.last_exception_msg = str(excep)
def _do_polling(self): while True: try: self.value_changed() except BaseException: sys.excepthook(*sys.exc_info()) gevent.sleep(self.interval)
def update(ev=None, args=None, kwargs=None): control.subjectObject = control.attr.__get__(control.parent.subjectObject) s = "???" try: labelContent = control.getTextObj() s = convertToUnicode(labelContent) except Exception: sys.excepthook(*sys.exc_info()) def do_update(): label.setStringValue_(s) if backgroundColor(control): label.setDrawsBackground_(True) label.setBackgroundColor_(backgroundColor(control)) label.setTextColor_(foregroundColor(control)) if control.attr.autosizeWidth: label.sizeToFit() control.layoutLine() if label.onMouseEntered or label.onMouseExited: if getattr(label, "trackingRect", None): label.removeTrackingRect_(label.trackingRect) label.trackingRect = label.addTrackingRect_owner_userData_assumeInside_(label.bounds(), label, None, False) do_in_mainthread(do_update, wait=False)
def __init__(self, URI, options=None): """ @param URI: real URI for the vocabulary file @param options: the error handler (option) object to send warnings to @type options: L{options.Options} """ # First see if this particular vocab has been handled before. If yes, it is extracted and everything # else can be forgotten. self.uri = URI (self.filename, self.creation_date, self.expiration_date) = ("", None, None) self.graph = Graph() try: CachedVocabIndex.__init__(self, options) vocab_reference = self.get_ref(URI) self.caching = True except Exception: # what this means is that the caching becomes impossible through some system error... (type, value, traceback) = sys.exc_info() if self.report: options.add_info( "Could not access the vocabulary cache area %s" % value, VocabCachingInfo, URI) vocab_reference = None self.caching = False if vocab_reference == None: # This has never been cached before if self.report: options.add_info( "No cache exists for %s, generating one" % URI, VocabCachingInfo) # Store all the cache data unless caching proves to be impossible if self._get_vocab_data(newCache=True) and self.caching: self.filename = create_file_name(self.uri) self._store_caches() if self.report: options.add_info( "Generated a cache for %s, with an expiration date of %s" % (URI, self.expiration_date), VocabCachingInfo, URI) else: (self.filename, self.creation_date, self.expiration_date) = vocab_reference if self.report: options.add_info( "Found a cache for %s, expiring on %s" % (URI, self.expiration_date), VocabCachingInfo) # Check if the expiration date is still away if options.refresh_vocab_cache == False and datetime.datetime.utcnow( ) <= self.expiration_date: # We are fine, we can just extract the data from the cache and we're done if self.report: options.add_info( "Cache for %s is still valid; extracting the data" % URI, VocabCachingInfo) fname = os.path.join(self.app_data_dir, self.filename) try: self.graph = _load(fname) except Exception: # what this means is that the caching becomes impossible VocabCachingInfo (type, value, traceback) = sys.exc_info() sys.excepthook(type, value, traceback) if self.report: options.add_info( "Could not access the vocab cache %s (%s)" % (value, fname), VocabCachingInfo, URI) else: if self.report: if options.refresh_vocab_cache == True: options.add_info( "Time check is bypassed; refreshing the cache for %s" % URI, VocabCachingInfo) else: options.add_info( "Cache timeout; refreshing the cache for %s" % URI, VocabCachingInfo) # we have to refresh the graph if self._get_vocab_data(newCache=False) == False: # bugger; the cache could not be refreshed, using the current one, and setting the cache artificially # to be valid for the coming hour, hoping that the access issues will be resolved by then... if self.report: options.add_info( "Could not refresh vocabulary cache for %s, using the old cache, extended its expiration time by an hour (network problems?)" % URI, VocabCachingInfo, URI) fname = os.path.join(self.app_data_dir, self.filename) try: self.graph = _load(fname) self.expiration_date = datetime.datetime.utcnow( ) + datetime.timedelta(hours=1) except Exception: # what this means is that the caching becomes impossible VocabCachingInfo (type, value, traceback) = sys.exc_info() sys.excepthook(type, value, traceback) if self.report: options.add_info( "Could not access the vocabulary cache %s (%s)" % (value, fname), VocabCachingInfo, URI) self.creation_date = datetime.datetime.utcnow() if self.report: options.add_info( "Generated a new cache for %s, with an expiration date of %s" % (URI, self.expiration_date), VocabCachingInfo, URI) self._store_caches()
def switch_mozilla_repo(): """If the mozilla/ repo matches SWITCH_MOZILLA_REPO_REGEXP then: 1) Backup (unused anymore) checkout of mozilla/. 2a) If SWITCH_MOZILLA_REPO_OLD_REPO_LOCATION exists, move that to mozilla/. 2b) Else clone the backup of mozilla/ up to the SWITCH_MOZILLA_BASE_REV revision and set the pull location to SWITCH_MOZILLA_REPO_REPLACE. It is expected that the normal pull/update functions in this script will update the new mozilla/ repo to the latest version. """ mozilla_path = os.path.join(topsrcdir, 'mozilla') # Do nothing if there is no Mozilla directory. if not os.path.exists(mozilla_path): return import ConfigParser import re config = ConfigParser.ConfigParser() config_path = os.path.join(mozilla_path, '.hg', 'hgrc') config.read([config_path]) if not config.has_option('paths', 'default'): # Abort, not to get into a possibly inconsistent state. sys.exit("Error: default path in %s is undefined!" % config_path) # Compile the Mozilla repository regex. moz_old_regex = re.compile(SWITCH_MOZILLA_REPO_REGEXP, re.I) match = moz_old_regex.match(config.get('paths', 'default')) # Do nothing if not pulling from the one we're trying to switch from. if not match: return config.set('paths', 'default', SWITCH_MOZILLA_REPO_REPLACE % match.group(1)) if config.has_option('paths', 'default-push'): match = moz_old_regex.match(config.get('paths', 'default-push')) # Do not update this property if not pushing to Mozilla trunk. if match: config.set('paths', 'default-push', SWITCH_MOZILLA_REPO_REPLACE % match.group(1)) hgcloneopts = [] if options.hgcloneopts: hgcloneopts = options.hgcloneopts.split() hgopts = [] if options.hgopts: hgopts = options.hgopts.split() backup_mozilla_path = os.path.join(topsrcdir, SWITCH_MOZILLA_REPO_BACKUP_LOCATION) print "Moving mozilla to " + SWITCH_MOZILLA_REPO_BACKUP_LOCATION + "..." try: os.rename(mozilla_path, backup_mozilla_path) except: # Print the exception without its traceback. sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], None) sys.exit("Error: Mozilla directory renaming failed!") # Does the user have a pre-existing backup repository? old_backup_repository = os.path.join( topsrcdir, SWITCH_MOZILLA_REPO_OLD_REPO_LOCATION) if SWITCH_MOZILLA_REPO_OLD_REPO_LOCATION != "" and \ os.path.exists(old_backup_repository): # Yes, so let's use that print "Moving " + old_backup_repository + " to " + mozilla_path + "..." try: os.rename(old_backup_repository, mozilla_path) except: # Print the exception without its traceback. sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], None) sys.exit( "Error: Renaming old backup directory failed! You must recover manually." ) # Let's leave the hgrc as it was, so any repo specific config is left # the same. return # Locally clone common repository history. check_call_noisy([options.hg, 'clone', '-r', SWITCH_MOZILLA_BASE_REV] + hgcloneopts + hgopts + [backup_mozilla_path, mozilla_path], retryMax=options.retries) # Rewrite hgrc for new local mozilla repo based on pre-existing hgrc # but with new values f = open(os.path.join(topsrcdir, 'mozilla', '.hg', 'hgrc'), 'w') try: config.write(f) finally: f.close()
def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: for line in f: try: cmd, name, rtype = line.strip().decode('ascii').split(':') cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( f'Cannot register {name} for automatic cleanup: ' f'unknown resource type {rtype}') if cmd == 'REGISTER': cache[rtype].add(name) elif cmd == 'UNREGISTER': cache[rtype].remove(name) elif cmd == 'PROBE': pass else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: # all processes have terminated; cleanup any remaining resources for rtype, rtype_cache in cache.items(): if rtype_cache: try: warnings.warn('resource_tracker: There appear to be %d ' 'leaked %s objects to clean up at shutdown' % (len(rtype_cache), rtype)) except Exception: pass for name in rtype_cache: # For some reason the process which created and registered this # resource has failed to unregister it. Presumably it has # died. We therefore unlink it. try: try: _CLEANUP_FUNCS[rtype](name) except Exception as e: warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass
def handle_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): sys.excepthook(exc_type, exc_value, exc_traceback) return # handle Breaking exceptions if exc_type in [ exceptions.ConfigException, exceptions.RouteNotFound, exceptions.InvalidRoutes, exceptions.CandleNotFoundInDatabase ]: click.clear() print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color('{}: {}'.format(exc_type.__name__, exc_value), 'yellow') ) return # send notifications if it's a live session if jh.is_live(): jesse_logger.error( '{}: {}'.format(exc_type.__name__, exc_value) ) if jh.is_live() or jh.is_collecting_data(): logging.error("Uncaught Exception:", exc_info=(exc_type, exc_value, exc_traceback)) else: print('=' * 30 + ' EXCEPTION TRACEBACK:') traceback.print_tb(exc_traceback, file=sys.stdout) print("=" * 73) print( '\n', jh.color('Uncaught Exception:', 'red'), jh.color('{}: {}'.format(exc_type.__name__, exc_value), 'yellow') ) if jh.is_paper_trading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}'.format( 'storage/logs/paper-trade.txt' ), 'red' ) ) elif jh.is_livetrading(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}'.format( 'storage/logs/live-trade.txt' ), 'red' ) ) elif jh.is_collecting_data(): print( jh.color( 'An uncaught exception was raised. Check the log file at:\n{}'.format( 'storage/logs/collect.txt' ), 'red' ) )
def main(): """ Main command-line execution loop. """ try: # Parse command line options parser, options, arguments = parse_options() # Handle regular args vs -- args arguments = parser.largs remainder_arguments = parser.rargs # Allow setting of arbitrary env keys. # This comes *before* the "specific" env_options so that those may # override these ones. Specific should override generic, if somebody # was silly enough to specify the same key in both places. # E.g. "fab --set shell=foo --shell=bar" should have env.shell set to # 'bar', not 'foo'. for pair in _escape_split(',', options.env_settings): pair = _escape_split('=', pair) # "--set x" => set env.x to True # "--set x=" => set env.x to "" key = pair[0] value = True if len(pair) == 2: value = pair[1] state.env[key] = value # Update env with any overridden option values # NOTE: This needs to remain the first thing that occurs # post-parsing, since so many things hinge on the values in env. for option in env_options: state.env[option.dest] = getattr(options, option.dest) # Handle --hosts, --roles, --exclude-hosts (comma separated string => # list) for key in ['hosts', 'roles', 'exclude_hosts']: if key in state.env and isinstance(state.env[key], basestring): state.env[key] = state.env[key].split(',') # Handle output control level show/hide update_output_levels(show=options.show, hide=options.hide) # Handle version number option if options.show_version: print("Fabric %s" % state.env.version) print("ssh (library) %s" % ssh.__version__) sys.exit(0) # Load settings from user settings file, into shared env dict. state.env.update(load_settings(state.env.rcfile)) # Find local fabfile path or abort fabfile = find_fabfile() if not fabfile and not remainder_arguments: abort("""Couldn't find any fabfiles! Remember that -f can be used to specify fabfile path, and use -h for help.""") # Store absolute path to fabfile in case anyone needs it state.env.real_fabfile = fabfile # Load fabfile (which calls its module-level code, including # tweaks to env values) and put its commands in the shared commands # dict default = None if fabfile: docstring, callables, default = load_fabfile(fabfile) state.commands.update(callables) # Handle case where we were called bare, i.e. just "fab", and print # a help message. actions = (options.list_commands, options.shortlist, options.display, arguments, remainder_arguments, default) if not any(actions): parser.print_help() sys.exit(1) # Abort if no commands found if not state.commands and not remainder_arguments: abort("Fabfile didn't contain any commands!") # Now that we're settled on a fabfile, inform user. if state.output.debug: if fabfile: print("Using fabfile '%s'" % fabfile) else: print("No fabfile loaded -- remainder command only") # Shortlist is now just an alias for the "short" list format; # it overrides use of --list-format if somebody were to specify both if options.shortlist: options.list_format = 'short' options.list_commands = True # List available commands if options.list_commands: show_commands(docstring, options.list_format) # Handle show (command-specific help) option if options.display: display_command(options.display) # If user didn't specify any commands to run, show help if not (arguments or remainder_arguments or default): parser.print_help() sys.exit(0) # Or should it exit with error (1)? # Parse arguments into commands to run (plus args/kwargs/hosts) commands_to_run = parse_arguments(arguments) # Parse remainders into a faux "command" to execute remainder_command = parse_remainder(remainder_arguments) # Figure out if any specified task names are invalid unknown_commands = [] for tup in commands_to_run: if crawl(tup[0], state.commands) is None: unknown_commands.append(tup[0]) # Abort if any unknown commands were specified if unknown_commands: warn("Command(s) not found:\n%s" \ % indent(unknown_commands)) show_commands(None, options.list_format, 1) # Generate remainder command and insert into commands, commands_to_run if remainder_command: r = '<remainder>' state.commands[r] = lambda: api.run(remainder_command) commands_to_run.append((r, [], {}, [], [], [])) # Ditto for a default, if found if not commands_to_run and default: commands_to_run.append((default.name, [], {}, [], [], [])) if state.output.debug: names = ", ".join(x[0] for x in commands_to_run) print("Commands to run: %s" % names) # At this point all commands must exist, so execute them in order. for name, args, kwargs, arg_hosts, arg_roles, arg_exclude_hosts in commands_to_run: execute(name, hosts=arg_hosts, roles=arg_roles, exclude_hosts=arg_exclude_hosts, *args, **kwargs) # If we got here, no errors occurred, so print a final note. if state.output.status: print("\nDone.") except SystemExit: # a number of internal functions might raise this one. raise except KeyboardInterrupt: if state.output.status: sys.stderr.write("\nStopped.\n") sys.exit(1) except: sys.excepthook(*sys.exc_info()) # we might leave stale threads if we don't explicitly exit() sys.exit(1) finally: disconnect_all() sys.exit(0)
# Do not print the first 3 lines of the stack trace to hide the # machinery above. This unfortunately means we don't print chained # exceptions. sys.stderr.write(''.join( traceback.format_list(traceback.extract_tb(etb)[3:]))) sys.stderr.write(''.join(traceback.format_exception_only(et, ev))) if __name__ == '__main__': conn = XferClientConnection() xfer = XferIO(conn) mod_name = '/box/program.py' spec = importlib.util.spec_from_file_location('__main__', mod_name) m = importlib.util.module_from_spec(spec) sys.argv[0] = mod_name m.__dict__['input'] = xfer.input for k, v in conn.start().items(): m.__dict__[k] = v old_stdout = io.open(1, 'w') with contextlib.redirect_stdout(xfer): with contextlib.redirect_stderr(xfer): sys.excepthook = excepthook try: spec.loader.exec_module(m) except Exception as e: sys.excepthook(*sys.exc_info())
def run_with_except_hook(*args2, **kwargs2): try: run_original(*args2, **kwargs2) except Exception: sys.excepthook(*sys.exc_info())
def __on_error(self, err): sys.excepthook(type(err), err, getattr(err, "__traceback__"))
def run(self): while not self._quiting: try: data = self.sock.recv(8192) except socket.timeout: self._handle_timeouts() continue self.log("NetIO < %s" % repr(data)) try: resp = None for i in range(2): try: tmp = data if tmp[:2] == '\x00\x00': tmp = zlib.decompressobj().decompress(tmp[2:]) self.log("UnZip | %s" % repr(tmp)) resp = ResponseResolver(tmp) except: sys.excepthook(*sys.exc_info()) self.crypt = None self.session = None else: break if not resp: raise AniDBPacketCorruptedError, "Either decrypting, decompressing or parsing the packet failed" cmd = self._cmd_dequeue(resp) resp = resp.resolve(cmd) resp.parse() if resp.rescode in ('200', '201'): self.session = resp.attrs['sesskey'] if resp.rescode in ('209', ): print "sorry encryption is not supported" raise # self.crypt=aes(md5(resp.req.apipassword+resp.attrs['salt']).digest()) if resp.rescode in ('203', '403', '500', '501', '503', '506'): self.session = None self.crypt = None if resp.rescode in ('504', '555'): self.banned = True print "AniDB API informs that user or client is banned:", resp.resstr resp.handle() if not cmd or not cmd.mode: self._resp_queue(resp) else: self.tags.remove(resp.restag) except: sys.excepthook(*sys.exc_info()) print "Avoiding flood by paranoidly panicing: Aborting link thread, killing connection, releasing waiters and quiting" self.sock.close() try: cmd.waiter.release() except: pass for tag, cmd in self.cmd_queue.items(): try: cmd.waiter.release() except: pass sys.exit()
def assertImageApproved(image, standardFile, message=None, **kwargs): """Check that an image test result matches a pre-approved standard. If the result does not match, then the user can optionally invoke a GUI to compare the images and decide whether to fail the test or save the new image as the standard. This function will automatically clone the test-data repository into ~/.pyqtgraph/test-data. However, it is up to the user to ensure this repository is kept up to date and to commit/push new images after they are saved. Run the test with the environment variable PYQTGRAPH_AUDIT=1 to bring up the auditing GUI. Parameters ---------- image : (h, w, 4) ndarray standardFile : str The name of the approved test image to check against. This file name is relative to the root of the pyqtgraph test-data repository and will be automatically fetched. message : str A string description of the image. It is recommended to describe specific features that an auditor should look for when deciding whether to fail a test. Extra keyword arguments are used to set the thresholds for automatic image comparison (see ``assertImageMatch()``). """ if isinstance(image, QtGui.QWidget): w = image # just to be sure the widget size is correct (new window may be resized): QtGui.QApplication.processEvents() graphstate = scenegraphState(w, standardFile) qimg = QtGui.QImage(w.size(), QtGui.QImage.Format.Format_ARGB32) qimg.fill(QtCore.Qt.GlobalColor.transparent) painter = QtGui.QPainter(qimg) w.render(painter) painter.end() image = fn.imageToArray(qimg, copy=False, transpose=False) # the standard images seem to have their Red and Blue swapped if sys.byteorder == 'little': # transpose B,G,R,A to R,G,B,A image = image[..., [2, 1, 0, 3]] else: # transpose A,R,G,B to A,B,G,R image = image[..., [0, 3, 2, 1]] if message is None: code = inspect.currentframe().f_back.f_code message = "%s::%s" % (code.co_filename, code.co_name) # Make sure we have a test data repo available, possibly invoking git dataPath = getTestDataRepo() # Read the standard image if it exists stdFileName = os.path.join(dataPath, standardFile + '.png') if not os.path.isfile(stdFileName): stdImage = None else: pxm = QtGui.QPixmap() pxm.load(stdFileName) stdImage = fn.imageToArray(pxm.toImage(), copy=True, transpose=False) # If the test image does not match, then we go to audit if requested. try: if stdImage is None: raise Exception("No reference image saved for this test.") if image.shape[2] != stdImage.shape[2]: raise Exception( "Test result has different channel count than standard image" "(%d vs %d)" % (image.shape[2], stdImage.shape[2])) if image.shape != stdImage.shape: # Allow im1 to be an integer multiple larger than im2 to account # for high-resolution displays ims1 = np.array(image.shape).astype(float) ims2 = np.array(stdImage.shape).astype(float) sr = ims1 / ims2 if ims1[0] > ims2[0] else ims2 / ims1 if (sr[0] != sr[1] or not np.allclose(sr, np.round(sr)) or sr[0] < 1): raise TypeError("Test result shape %s is not an integer factor" " different than standard image shape %s." % (ims1, ims2)) sr = np.round(sr).astype(int) image = fn.downsample(image, sr[0], axis=(0, 1)).astype(image.dtype) assertImageMatch(image, stdImage, **kwargs) if bool(os.getenv('PYQTGRAPH_PRINT_TEST_STATE', False)): print(graphstate) if os.getenv('PYQTGRAPH_AUDIT_ALL') == '1': raise Exception( "Image test passed, but auditing due to PYQTGRAPH_AUDIT_ALL evnironment variable." ) except Exception: if stdFileName in gitStatus(dataPath): print("\n\nWARNING: unit test failed against modified standard " "image %s.\nTo revert this file, run `cd %s; git checkout " "%s`\n" % (stdFileName, dataPath, standardFile)) if os.getenv('PYQTGRAPH_AUDIT') == '1' or os.getenv( 'PYQTGRAPH_AUDIT_ALL') == '1': sys.excepthook(*sys.exc_info()) getTester().test(image, stdImage, message) stdPath = os.path.dirname(stdFileName) print('Saving new standard image to "%s"' % stdFileName) if not os.path.isdir(stdPath): os.makedirs(stdPath) img = fn.makeQImage(image, alpha=True, transpose=False) img.save(stdFileName) else: if stdImage is None: raise Exception("Test standard %s does not exist. Set " "PYQTGRAPH_AUDIT=1 to add this image." % stdFileName) else: if os.getenv('TRAVIS') is not None: saveFailedTest(image, stdImage, standardFile, upload=True) elif os.getenv('CI') is not None: standardFile = os.path.join( os.getenv("SCREENSHOT_DIR", "screenshots"), standardFile) saveFailedTest(image, stdImage, standardFile) print(graphstate) raise
def amplitude_scatter_plot(self, suffix): """ Make a scatter plot of upstream versus downstream data """ amp_dict_ds = self.amp.amplitudes[suffix]["amplitude_dict_downstream"] amp_dict_us = self.amp.amplitudes[suffix]["amplitude_dict_upstream"] amp_list_us = [] amp_list_ds = [] amp_list_delta = [] suffix_label = self.get_suffix_label(suffix) for key, amp_ds in amp_dict_ds.iteritems(): try: amp_us = amp_dict_us[key] except KeyError: sys.excepthook(*sys.exc_info()) continue amp_delta = amp_us - amp_ds amp_list_us.append(amp_us) amp_list_ds.append(amp_ds) amp_list_delta.append(amp_delta) canvas = common.make_root_canvas("amplitude_residuals") canvas.Draw() n_points = min(len(amp_list_us), 10000) # no more than 10k points in the scatter hist, graph = common.make_root_graph( "delta amplitude scatter", amp_list_us, "US " + suffix_label + " Amplitude [mm]", amp_list_delta, suffix_label + " US Amplitude - DS Amplitude [mm]", xmin=0., xmax=100., ymin=-50., ymax=50.) hist.SetTitle(self.config_anal['name']) hist.Draw() graph.Draw("P") canvas.Update() for format in ["eps", "png", "root"]: canvas.Print(self.plot_dir + "amplitude_delta_" + suffix + "_scatter." + format) canvas = common.make_root_canvas("delta_amplitude_hist") canvas.Draw() canvas.SetFrameFillColor(utilities.utilities.get_frame_fill()) hist = common.make_root_histogram("delta amplitude hist", amp_list_us, suffix_label + " US Amplitude [mm]", 100, amp_list_delta, suffix_label + " US Amplitude - DS Amplitude [mm]", 100, xmin=0., xmax=100., ymin=-50., ymax=50.) hist.SetTitle(self.config_anal['name']) hist.Draw("COLZ") canvas.Update() for format in ["eps", "png", "root"]: canvas.Print(self.plot_dir + "amplitude_delta_" + suffix + "_hist." + format)
def printExcept(): """print exception type, value and traceback on stderr""" sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
def test_notification_manager_no_gui_with_threading(): """ Direct test of the notification manager. This does not test the integration with the gui, but test that exceptions and warnings from threads are correctly captured. """ def _warn(): time.sleep(0.01) warnings.showwarning('this is a warning', UserWarning, '', 0) def _raise(): time.sleep(0.01) with pytest.raises(PurposefulException): raise PurposefulException("this is an exception") if PY38_OR_HIGHER: previous_threading_exhook = threading.excepthook with notification_manager: notification_manager.records.clear() # save all of the events that get emitted store: List[Notification] = [] _append = lambda e: store.append(e) # lambda needed on py3.7 # noqa notification_manager.notification_ready.connect(_append) # Test exception inside threads if PY38_OR_HIGHER: # `threading.excepthook` available only for Python >= 3.8 assert (threading.excepthook == notification_manager.receive_thread_error) exception_thread = threading.Thread(target=_raise) exception_thread.start() time.sleep(0.02) if PY38_OR_HIGHER: threading.excepthook(sys.exc_info()) else: sys.excepthook(*sys.exc_info()) assert len(notification_manager.records) == 1 assert store[-1].type == 'error' # Test warning inside threads assert warnings.showwarning == notification_manager.receive_warning warning_thread = threading.Thread(target=_warn) warning_thread.start() for _ in range(100): time.sleep(0.01) if (len(notification_manager.records) == 2 and store[-1].type == 'warning'): break else: raise AssertionError("Thread notification not received in time") # make sure we've restored the threading except hook if PY38_OR_HIGHER: assert threading.excepthook == previous_threading_exhook assert all(isinstance(x, Notification) for x in store)
def _error_handler(task): try: task.result() except Exception: sys.excepthook(*sys.exc_info())
def complete(self, text=None, line_buffer=None, cursor_pos=None): """Find completions for the given text and line context. This is called successively with state == 0, 1, 2, ... until it returns None. The completion should begin with 'text'. Note that both the text and the line_buffer are optional, but at least one of them must be given. Parameters ---------- text : string, optional Text to perform the completion on. If not given, the line buffer is split using the instance's CompletionSplitter object. line_buffer : string, optional If not given, the completer attempts to obtain the current line buffer via readline. This keyword allows clients which are requesting for text completions in non-readline contexts to inform the completer of the entire text. cursor_pos : int, optional Index of the cursor in the full line buffer. Should be provided by remote frontends where kernel has no access to frontend state. Returns ------- text : str Text that was actually used in the completion. matches : list A list of completion matches. """ #io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg # if the cursor position isn't given, the only sane assumption we can # make is that it's at the end of the line (the common case) if cursor_pos is None: cursor_pos = len(line_buffer) if text is None else len(text) # if text is either None or an empty string, rely on the line buffer if not text: text = self.splitter.split_line(line_buffer, cursor_pos) # If no line buffer is given, assume the input text is all there was if line_buffer is None: line_buffer = text self.line_buffer = line_buffer self.text_until_cursor = self.line_buffer[:cursor_pos] #io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg # Start with a clean slate of completions self.matches[:] = [] custom_res = self.dispatch_custom_completer(text) if custom_res is not None: # did custom completers produce something? self.matches = custom_res else: # Extend the list of completions with the results of each # matcher, so we return results to the user from all # namespaces. if self.merge_completions: self.matches = [] for matcher in self.matchers: try: self.matches.extend(matcher(text)) except: # Show the ugly traceback if the matcher causes an # exception, but do NOT crash the kernel! sys.excepthook(*sys.exc_info()) else: for matcher in self.matchers: self.matches = matcher(text) if self.matches: break # FIXME: we should extend our api to return a dict with completions for # different types of objects. The rlcomplete() method could then # simply collapse the dict into a list for readline, but we'd have # richer completion semantics in other evironments. self.matches = sorted(set(self.matches)) #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg return text, self.matches
def send_exception_to_crash_reporter(e: BaseException): sys.excepthook(type(e), e, e.__traceback__)
def output(out=None): """ Prints diagnostic information (to stdout by default, but a filename or stream can be specified). """### import sys if out == None: out = sys.stdout openfile = isinstance(out, str) if openfile: out = open(out, 'w') v = {} for i in 'sys ctypes Numeric numarray numpy PIL OpenGL pygame setuptools VisionEgg scipy matplotlib win32api pyaudio pyreadline IPython BCPy2000'.split( ): if i == 'PIL': i = 'Image' try: exec 'import ' + i except ImportError: v[i] = '** NOT INSTALLED **' except: out.write('failed to import %s\n' % str(i)) a = list(sys.exc_info()) a[2] = a[2].tb_next.tb_next sys.excepthook(*a) else: m = sys.modules[i] ver = getattr(m, '__version__', getattr(m, 'version', getattr(m, 'VERSION', '?'))) ver = getattr(ver, 'ver', ver) if callable(ver): ver = ver() v[i] = ver if i in v: out.write('%30s: %s\n' % (i, str(v[i]))) out.write('\n') p = {} try: import platform except: pass else: for i in 'architecture machine platform processor system'.split(): p[i] = getattr(platform, i, '') k = "platform." + i if callable(p[i]): p[i] = p[i]() k += "()" out.write('%30s = %s\n' % (k, repr(p[i]))) try: sp = sys.platform except: sp = '?' out.write("%30s = %s\n" % ("sys.platform", repr(sp))) out.write('\n') out.write("sys.path:\n") for i in sys.path: out.write(" %s\n" % i) import os out.write('\n') out.write("os.getcwd():\n %s\n" % os.getcwd()) out.write('\n') out.write(" os.environ:\n") for k, v in sorted(os.environ.items()): out.write("%30s: %s\n" % (k, v)) if openfile: out.close()
def run(self): while not self._quiting: try: data = self.sock.recv(8192) except socket.timeout: self._handle_timeouts() continue except OSError as error: logger.exception('Exception: %s', error) break logger.debug("NetIO < %r", data) try: for i in range(2): try: tmp = data resp = None if tmp[:2] == b'\x00\x00': tmp = zlib.decompressobj().decompress(tmp[2:]) logger.debug("UnZip | %r", tmp) resp = ResponseResolver(tmp) except Exception as e: logger.exception('Exception: %s', e) sys.excepthook(*sys.exc_info()) self.crypt = None self.session = None else: break if not resp: raise AniDBPacketCorruptedError( "Either decrypting, decompressing or parsing the packet failed" ) cmd = self._cmd_dequeue(resp) resp = resp.resolve(cmd) resp.parse() if resp.rescode in (b'200', b'201'): self.session = resp.attrs[b'sesskey'] if resp.rescode in (b'209', ): logger.error("sorry encryption is not supported") raise AniDBError() # self.crypt=aes(md5(resp.req.apipassword+resp.attrs['salt']).digest()) if resp.rescode in (b'203', b'403', b'500', b'501', b'503', b'506'): self.session = None self.crypt = None if resp.rescode in (b'504', b'555'): self.banned = True logger.critical( ("AniDB API informs that user or client is banned:", resp.resstr)) resp.handle() if not cmd or not cmd.mode: self._resp_queue(resp) else: self.tags.remove(resp.restag) except: sys.excepthook(*sys.exc_info()) logger.error( "Avoiding flood by paranoidly panicing: Aborting link thread, killing connection, releasing waiters and quiting" ) self.sock.close() try: cmd.waiter.release() except: pass for tag, cmd in self.cmd_queue.items(): try: cmd.waiter.release() except: pass sys.exit() if self._quiting: self.QuitProcessed = True
def run(self, task_id, resume_job=False, logging_thread=None): """ This function is executed to run this job. :param int task_id: :param bool resume_job: :param sisyphus.worker.LoggingThread logging_thread: """ logging.debug("Task name: %s id: %s" % (self.name(), task_id)) job = self._job logging.info("Start Job: %s Task: %s" % (job, self.name())) logging.info("Inputs:") for i in self._job._sis_inputs: logging.info(str(i)) # each input must be at least X seconds old # if an input file is too young it's may not synced in a network filesystem yet try: input_age = time.time() - os.stat(i.get_path()).st_mtime time.sleep(max(0, gs.WAIT_PERIOD_MTIME_OF_INPUTS - input_age)) except FileNotFoundError: logging.warning('Input path does not exist: %s' % i.get_path()) if i.creator and gs.ENABLE_LAST_USAGE: # mark that input was used try: os.unlink( os.path.join(i.creator, gs.JOB_LAST_USER, os.getlogin())) except OSError as e: if e.errno not in (2, 13): # 2: file not found # 13: permission denied raise e try: user_path = os.path.join(i.creator, gs.JOB_LAST_USER, os.getlogin()) os.symlink(os.path.abspath(job._sis_path()), user_path) os.chmod(user_path, 0o775) except OSError as e: if e.errno not in (2, 13, 17): # 2: file not found # 13: permission denied # 17: file exists raise e tools.get_system_informations(sys.stdout) sys.stdout.flush() try: if resume_job: if self._resume is not None: task = self._resume else: task = self._start logging.warning( 'No resume function set (changed tasks after job was initialized?) ' 'Fallback to normal start function: %s' % task) else: task = self._start assert task is not None, "Error loading task" # save current directory and change into work directory with tools.execute_in_dir(self.path(gs.JOB_WORK_DIR)): f = getattr(self._job, task) # get job arguments for arg_id in self._get_arg_idx_for_task_id(task_id): args = self._args[arg_id] if not isinstance(args, (list, tuple)): args = [args] logging.info("-" * 60) logging.info("Starting subtask for arg id: %d args: %s" % (arg_id, str(args))) logging.info("-" * 60) f(*args) except sp.CalledProcessError as e: if e.returncode == 137: # TODO move this into engine class logging.error( "Command got killed by SGE (probably out of memory):") logging.error("Cmd: %s" % e.cmd) logging.error("Args: %s" % str(e.args)) logging.error("Return-Code: %s" % e.returncode) logging_thread.out_of_memory = True logging_thread.stop() else: logging.error("Executed command failed:") logging.error("Cmd: %s" % e.cmd) logging.error("Args: %s" % str(e.args)) logging.error("Return-Code: %s" % e.returncode) logging_thread.stop() self.error(task_id, True) except Exception as e: # Job failed logging.error("Job failed, traceback:") sys.excepthook(*sys.exc_info()) logging_thread.stop() self.error(task_id, True) # TODO handle failed job else: # Job finished normally logging_thread.stop() if not self.continuable: self.finished(task_id, True) sys.stdout.flush() sys.stderr.flush() logging.info("Job finished successful")
def write_events(event_fifo_name, events_file, _debug_out): """Read from a file containing mission events and send them to a named fifo as events""" event_count = 0 try: event_fifo = open(event_fifo_name, "w", 0) except: print "Unable to open event.fifo for write\n" else: try: while True: event_line = events_file.readline() #print event_line #print event_line #_x=input('continue') if len(event_line) > 0: if (event_line[:9] == "<mission>" or event_line[:9] == "<mission " or event_line[:15] == "<scoring_event>"): try: event_line.index('xmlns') except ValueError: event_line = event_line.replace( '>', ' xmlns="mynamespace">', 1) event_count += 1 event_str = event_line else: event_str += event_line if ("</mission>" in event_line or "</scoring_event>" in event_line): current_event = "<EVENT>" + event_str + "</EVENT>" while True: #if random.randint (0, 100) > 98: if random.randint(0, 100) > 101: # hang up the fifo and re-open periodically to test reconnect # behavior #print 'CLOSED!',event_count event_fifo.close() event_fifo = open(event_fifo_name, "w", 0) #print 'RE-OPENED' try: xml_fifo_io2.send_text(event_fifo, current_event) if (event_count > 0 and ((event_count % 3000) == 0)): print str(event_count) + " events sent\n" break except IOError: event_fifo = open(event_fifo_name, "w", 0) except IOError: pass events_file.close() if (debug_out is not None): debug_out.close() try: event_fifo.close() except Exception, e: sys.excepthook(sys.exc_info()[0], sys.exc_info[1], sys.exc_info[2]) print "write_events exiting after %d events\n" % event_count
def main(): kcci = get_network("KCCI") kimt = get_network("KIMT") get_precip_totals(kcci, 'KCCI') get_precip_totals(kimt, 'KIMT') computeOthers(kcci) computeOthers(kimt) tmpfp = tempfile.mktemp() of = open(tmpfp, 'w') of.write("sid,ts,tmpf,dwpf,relh,feel,alti,altiTend,drctTxt,sped,sknt,") of.write("drct,20gu,gmph,gtim,pday,pmonth,tmpf_min,tmpf_max,max_sknt,") of.write("drct_max,max_sped,max_drctTxt,max_srad\n") for sid in kcci.keys(): v = kcci[sid] try: s = ("%s,%s,%s,%s,%s," "") % (v.get('id'), v.get('ticks'), formatter(v.get('tmpf'), 0, -99), formatter(v.get('dwpf'), 0, -99), formatter(v.get('relh'), 0, -99)) s += "%s,%s,%s,%s," % (formatter(v.get('feel'), 0, -99), formatter(v.get('pres'), 2, -99), v.get('altiTend'), v.get('drctTxt')) s += "%s,%s,%s,%s," % (formatter(v.get('sped'), 0, -99), formatter(v.get('sknt'), 0, -99), formatter(v.get('drct'), 0, -99), formatter(v.get('20gu'), 0, -99)) s += "%s,%s,%s,%s," % (formatter(v.get('gmph'), 0, -99), v.get('gtim'), formatter(v.get('pday'), 2, -99), formatter(v.get('pmonth'), 2, -99)) s += "%s,%s,%s," % (formatter(v.get('sknt'), 0, -99), formatter(v.get('drct'), 0, -99), formatter(v.get('20gu'), 0, -99)) s += "%s,%s,%s,%s\n" % (formatter(v.get('max_drct'), 0, -99), formatter(v.get('max_sped'), 0, -99), v.get('max_drctTxt'), formatter(v.get('max_srad'), 0, -99)) of.write(s.replace("'", "")) except Exception as _exp: print(kcci[sid]) print(sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) sys.exc_traceback = None of.close() subprocess.call(("/home/ldm/bin/pqinsert -p 'data c %s csv/kcci.dat " "bogus dat' %s") % (tstr, tmpfp), shell=True) os.remove(tmpfp) of = open(tmpfp, 'w') of.write("sid,ts,tmpf,dwpf,relh,feel,alti,altiTend,drctTxt,sped,sknt,") of.write("drct,20gu,gmph,gtim,pday,pmonth,tmpf_min,tmpf_max,max_sknt,") of.write("drct_max,max_sped,max_drctTxt,srad,max_srad,online\n") for sid in kcci.keys(): v = kcci[sid] v['online'] = 1 if (now - v['valid']).seconds > 3600: v['online'] = 0 try: s = "%s,%s,%s,%s,%s," % (v.get('id'), v.get('ticks'), formatter(v.get('tmpf'), 0), formatter(v.get('dwpf'), 0), formatter(v.get('relh'), 0)) s += "%s,%s,%s,%s," % (formatter(v.get('feel'), 0), formatter(v.get('pres'), 2), v.get('altiTend'), v.get('drctTxt')) s += "%s,%s,%s,%s," % (formatter(v.get('sped'), 0), formatter(v.get('sknt'), 0), formatter(v.get('drct'), 0), formatter(v.get('20gu'), 0)) s += "%s,%s,%s,%s," % (formatter(v.get('gmph'), 0), v.get('gtim'), formatter(v.get('pday'), 2), formatter(v.get('pmonth'), 2)) s += "%s,%s,%s," % (formatter(v.get('sknt'), 0), formatter(v.get('drct'), 0), formatter(v.get('20gu'), 0)) s += "%s,%s,%s,%s,%s,%s\n" % (formatter(v.get('max_drct'), 0), formatter(v.get('max_sped'), 0), v.get('max_drctTxt'), formatter(v.get('srad'), 0), formatter(v.get('max_srad'), 0), v.get('online')) of.write(s.replace("'", "")) except Exception as _exp: print(kcci[sid]) print(sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) sys.exc_traceback = None of.close() subprocess.call(("/home/ldm/bin/pqinsert -p 'data c %s csv/kcci2.dat " "bogus dat' %s") % (tstr, tmpfp), shell=True) os.remove(tmpfp) of = open(tmpfp, 'w') of.write("sid,ts,tmpf,dwpf,relh,feel,alti,altiTend,drctTxt,sped,sknt,") of.write("drct,20gu,gmph,gtim,pday,pmonth,tmpf_min,tmpf_max,max_sknt,") of.write("drct_max,max_sped,max_drctTxt,srad,max_srad,online\n") for sid in kimt.keys(): v = kimt[sid] v['online'] = 1 if (now - v['valid']).seconds > 3600: v['online'] = 0 try: s = "%s,%s,%s,%s,%s," % (v.get('id'), v.get('ticks'), formatter(v.get('tmpf'), 0), formatter(v.get('dwpf'), 0), formatter(v.get('relh'), 0)) s += "%s,%s,%s,%s," % (formatter(v.get('feel'), 0), formatter(v.get('pres'), 2), v.get('altiTend'), v.get('drctTxt')) s += "%s,%s,%s,%s," % (formatter(v.get('sped'), 0), formatter(v.get('sknt'), 0), formatter(v.get('drct'), 0), formatter(v.get('20gu'), 0)) s += "%s,%s,%s,%s," % (formatter(v.get('gmph'), 0), v.get('gtim'), formatter(v.get('pday'), 2), formatter(v.get('pmonth'), 2)) s += "%s,%s,%s," % (formatter(v.get('sknt'), 0), formatter(v.get('drct'), 0), formatter(v.get('20gu'), 0)) s += "%s,%s,%s,%s,%s,%s\n" % (formatter(v.get('max_drct'), 0), formatter(v.get('max_sped'), 0), v.get('max_drctTxt'), formatter(v.get('srad'), 0), formatter(v.get('max_srad'), 0), v.get('online')) of.write(s.replace("'", "")) except Exception as _exp: print(kimt[sid]) print(sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) sys.exc_traceback = None of.close() cmd = ("/home/ldm/bin/pqinsert -p 'data c %s csv/kimt.dat " "bogus dat' %s") % (tstr, tmpfp) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.stdout.read() os.remove(tmpfp) # Do KCCI stuff in WXC format of = open('/tmp/wxc_snet8.txt', 'w') of.write("""Weather Central 001d0300 Surface Data 25 5 Station 52 CityName 20 CityShort 2 State 7 Lat 8 Lon 2 Day 4 Hour 4 AirTemp 4 AirDewp 4 Feels Like 4 Wind Direction Degrees 4 Wind Direction Text 4 Wind Speed 4 Today Max Temp 4 Today Min Temp 4 Today Max Wind Gust MPH 3 Today Max Wind Direction Text 6 Today Precip 6 Month Precip 8 Time of Today Gust 6 Last 2 Day Precip 6 Last 3 Day Precip 6 Last 7 Day Precip 6 Last 14 Day Precip """) for sid in kcci.keys(): try: of.write("%5s %-52s %-20.20s %2s %7.2f %8.2f " % ( sid, st.sts[sid]['plot_name'], st.sts[sid]['plot_name'], 'IA', st.sts[sid]['lat'], st.sts[sid]['lon'])) of.write("%2s %4s %4.0f %4.0f %4.0f %4.0f %4s " % ( kcci[sid]['valid'].day, kcci[sid]['valid'].hour, kcci[sid].get('tmpf') or -99, kcci[sid].get('dwpf') or -99, kcci[sid].get('feel') or -99, kcci[sid].get('drct') or -99, kcci[sid].get('drctTxt'))) of.write("%4.0f %4.0f %4.0f %4.0f %3s " % ( kcci[sid].get('sknt') or -99, kcci[sid].get('max_tmpf') or -99, kcci[sid].get('min_tmpf') or -99, kcci[sid].get('max_sped') or -99, kcci[sid].get('max_drctTxt'))) of.write("%6.2f %6.2f %8s %6.2f %6.2f %6.2f %6.2f\n" % ( kcci[sid]['pday'], kcci[sid]['pmonth'], kcci[sid].get('gtim2'), kcci[sid].get('p2day', 0), kcci[sid].get('p3day', 0), kcci[sid].get('p7day', 0), kcci[sid].get('p14day', 0))) except Exception as _exp: print(kcci[sid]) print(sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) sys.exc_traceback = None of.close() pqstr = 'data c 000000000000 wxc/wxc_snet8.txt bogus txt' subprocess.call(("/home/ldm/bin/pqinsert -p '%s' " "/tmp/wxc_snet8.txt") % (pqstr, ), shell=True) os.remove("/tmp/wxc_snet8.txt")
""" self.asyncBlock = state self.emit(SIGNAL("blockingStateChanged(bool)"), self.asyncBlock) if not self.isBlocking(): self.scheduleSignalProcessing() def isBlocking(self): """ Is this widget blocking signal processing. Widget is blocking if asyncBlock value is True or any AsyncCall objects in asyncCalls list has blocking flag set """ return self.asyncBlock or any(a.blocking for a in self.asyncCalls) def asyncExceptionHandler(self, (etype, value, tb)): import traceback sys.excepthook(etype, value, tb) def asyncFinished(self, async, string): """ Remove async from asyncCalls, update blocking state """ index = self.asyncCalls.index(async) async = self.asyncCalls.pop(index) if async .blocking and not self.isBlocking(): # if we are responsible for unblocking self.emit(SIGNAL("blockingStateChanged(bool)"), False) self.scheduleSignalProcessing() async .disconnect(async, SIGNAL("finished(PyQt_PyObject, QString)"), self.asyncFinished)
def test_excepthook(self): with test.support.captured_output("stderr") as stderr: sys.excepthook(1, '1', 1) self.assertTrue("TypeError: print_exception(): Exception expected for " \ "value, str found" in stderr.getvalue())
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): '''Run forkserver.''' if preload: if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: spawn.import_main_path(main_path) finally: del process.current_process()._inheriting for modname in preload: try: __import__(modname) except ImportError: pass # close sys.stdin if sys.stdin is not None: try: sys.stdin.close() sys.stdin = open(os.devnull) except (OSError, ValueError): pass # ignoring SIGCHLD means no need to reap zombie processes handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() selector.register(listener, selectors.EVENT_READ) selector.register(alive_r, selectors.EVENT_READ) while True: try: while True: rfds = [key.fileobj for (key, events) in selector.select()] if rfds: break if alive_r in rfds: # EOF because no more client processes left assert os.read(alive_r, 1) == b'' raise SystemExit assert listener in rfds with listener.accept()[0] as s: code = 1 if os.fork() == 0: try: _serve_one(s, listener, alive_r, handler) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: os._exit(code) except InterruptedError: pass except OSError as e: if e.errno != errno.ECONNABORTED: raise
def assertImageApproved(image, standardFile, message=None, **kwargs): """Check that an image test result matches a pre-approved standard. If the result does not match, then the user can optionally invoke a GUI to compare the images and decide whether to fail the test or save the new image as the standard. Run the test with the environment variable PYQTGRAPH_AUDIT=1 to bring up the auditing GUI. Parameters ---------- image : (h, w, 4) ndarray standardFile : str The name of the approved test image to check against. This file name is relative to the root of the pyqtgraph test-data repository and will be automatically fetched. message : str A string description of the image. It is recommended to describe specific features that an auditor should look for when deciding whether to fail a test. Extra keyword arguments are used to set the thresholds for automatic image comparison (see ``assertImageMatch()``). """ if isinstance(image, QtGui.QWidget): # just to be sure the widget size is correct (new window may be resized): QtGui.QApplication.processEvents() graphstate = scenegraphState(image, standardFile) image = getImageFromWidget(image) if message is None: code = inspect.currentframe().f_back.f_code message = "%s::%s" % (code.co_filename, code.co_name) # Make sure we have a test data repo available dataPath = getTestDataDirectory() # Read the standard image if it exists stdFileName = os.path.join(dataPath, standardFile + '.png') if not os.path.isfile(stdFileName): stdImage = None else: qimg = QtGui.QImage(stdFileName) qimg = qimg.convertToFormat(QtGui.QImage.Format.Format_RGBA8888) stdImage = fn.ndarray_from_qimage(qimg).copy() del qimg # If the test image does not match, then we go to audit if requested. try: if stdImage is None: raise Exception("No reference image saved for this test.") if image.shape[2] != stdImage.shape[2]: raise Exception("Test result has different channel count than standard image" "(%d vs %d)" % (image.shape[2], stdImage.shape[2])) if image.shape != stdImage.shape: # Allow im1 to be an integer multiple larger than im2 to account # for high-resolution displays ims1 = np.array(image.shape).astype(float) ims2 = np.array(stdImage.shape).astype(float) sr = ims1 / ims2 if ims1[0] > ims2[0] else ims2 / ims1 if (sr[0] != sr[1] or not np.allclose(sr, np.round(sr)) or sr[0] < 1): raise TypeError("Test result shape %s is not an integer factor" " different than standard image shape %s." % (ims1, ims2)) sr = np.round(sr).astype(int) image = fn.downsample(image, sr[0], axis=(0, 1)).astype(image.dtype) assertImageMatch(image, stdImage, **kwargs) if bool(os.getenv('PYQTGRAPH_PRINT_TEST_STATE', False)): print(graphstate) if os.getenv('PYQTGRAPH_AUDIT_ALL') == '1': raise Exception("Image test passed, but auditing due to PYQTGRAPH_AUDIT_ALL evnironment variable.") except Exception: if os.getenv('PYQTGRAPH_AUDIT') == '1' or os.getenv('PYQTGRAPH_AUDIT_ALL') == '1': sys.excepthook(*sys.exc_info()) getTester().test(image, stdImage, message) stdPath = os.path.dirname(stdFileName) print('Saving new standard image to "%s"' % stdFileName) if not os.path.isdir(stdPath): os.makedirs(stdPath) qimg = fn.ndarray_to_qimage(image, QtGui.QImage.Format.Format_RGBA8888) qimg.save(stdFileName) del qimg else: if stdImage is None: raise Exception("Test standard %s does not exist. Set " "PYQTGRAPH_AUDIT=1 to add this image." % stdFileName) if os.getenv('CI') is not None: standardFile = os.path.join(os.getenv("SCREENSHOT_DIR", "screenshots"), standardFile) saveFailedTest(image, stdImage, standardFile) print(graphstate) raise
def _report_error(self, exc_info): sys.excepthook(exc_info[0], exc_info[1], exc_info[2])
def wing_debug_hook(*args, **kwargs): if __debug__ and 'WINGDB_ACTIVE' in os.environ: exc_type, exc_value, traceback = sys.exc_info() sys.excepthook(exc_type, exc_value, traceback) return old_technical_500_response(*args, **kwargs)
def preprocess(self, file): self.assertPyparsing() self.buildParser() ## we need this so that evalExpr works properly self.currentFile = file packStack = [(None,None)] ## stack for #pragma pack push/pop self.packList[file] = [(0,None)] packing = None ## current packing value text = self.files[file] ## First join together lines split by \\n text = Literal('\\\n').suppress().transformString(text) #self.ppDirective = Combine("#" + Word(alphas).leaveWhitespace()) + restOfLine # define the structure of a macro definition name = Word(alphas+'_', alphanums+'_')('name') self.ppDefine = name.setWhitespaceChars(' \t')("macro") + Optional(lparen + delimitedList(name) + rparen).setWhitespaceChars(' \t')('args') + SkipTo(LineEnd())('value') self.ppDefine.setParseAction(self.processMacroDefn) #self.updateMacroDefns() #self.updateFnMacroDefns() # define pattern for scanning through the input string #self.macroExpander = (self.macroExpr | self.fnMacroExpr) ## Comb through lines, process all directives lines = text.split('\n') result = [] #macroExpander = (quotedString | self.macroExpander) directive = re.compile(r'\s*#([a-zA-Z]+)(.*)$') ifTrue = [True] ifHit = [] for i in range(len(lines)): line = lines[i] newLine = '' m = directive.match(line) if m is None: # regular code line if ifTrue[-1]: # only include if we are inside the correct section of an IF block #line = macroExpander.transformString(line) # expand all known macros newLine = self.expandMacros(line) else: # macro line d = m.groups()[0] rest = m.groups()[1] #print "PREPROCESS:", d, rest if d == 'ifdef': d = 'if' rest = 'defined '+rest elif d == 'ifndef': d = 'if' rest = '!defined '+rest ## Evaluate 'defined' operator before expanding macros if d in ['if', 'elif']: def pa(t): return ['0', '1'][t['name'] in self.defs['macros'] or t['name'] in self.defs['fnmacros']] rest = ( Keyword('defined') + (name | lparen + name + rparen) ).setParseAction(pa).transformString(rest) elif d in ['define', 'undef']: macroName, rest = re.match(r'\s*([a-zA-Z_][a-zA-Z0-9_]*)(.*)$', rest).groups() ## Expand macros if needed if rest is not None and (all(ifTrue) or d in ['if', 'elif']): rest = self.expandMacros(rest) if d == 'elif': if ifHit[-1] or not all(ifTrue[:-1]): ev = False else: ev = self.evalPreprocessorExpr(rest) if self.verbose: print " "*(len(ifTrue)-2) + line, rest, ev ifTrue[-1] = ev ifHit[-1] = ifHit[-1] or ev elif d == 'else': if self.verbose: print " "*(len(ifTrue)-2) + line, not ifHit[-1] ifTrue[-1] = (not ifHit[-1]) and all(ifTrue[:-1]) ifHit[-1] = True elif d == 'endif': ifTrue.pop() ifHit.pop() if self.verbose: print " "*(len(ifTrue)-1) + line elif d == 'if': if all(ifTrue): ev = self.evalPreprocessorExpr(rest) else: ev = False if self.verbose: print " "*(len(ifTrue)-1) + line, rest, ev ifTrue.append(ev) ifHit.append(ev) elif d == 'define': if not ifTrue[-1]: continue if self.verbose: print " "*(len(ifTrue)) + "define:", macroName, rest try: self.ppDefine.parseString(macroName+ ' ' + rest) ## macro is registered here except: print "Error processing macro definition:", macroName, rest print " ", sys.exc_info()[1] elif d == 'undef': if not ifTrue[-1]: continue try: self.remDef('macros', macroName.strip()) #self.macroListString = '|'.join(self.defs['macros'].keys() + self.defs['fnmacros'].keys()) #self.updateMacroDefns() except: if sys.exc_info()[0] is not KeyError: sys.excepthook(*sys.exc_info()) print "Error removing macro definition '%s'" % macroName.strip() elif d == 'pragma': ## Check for changes in structure packing if not ifTrue[-1]: continue m = re.match(r'\s+pack\s*\(([^\)]+)\)', rest) if m is None: continue opts = [s.strip() for s in m.groups()[0].split(',')] pushpop = id = val = None for o in opts: if o in ['push', 'pop']: pushpop = o elif o.isdigit(): val = int(o) else: id = o if val is not None: packing = val if pushpop == 'push': packStack.append((packing, id)) elif opts[0] == 'pop': if id is None: packStack.pop() else: ind = None for i in range(len(packStack)): if packStack[i][1] == id: ind = i break if ind is not None: packStack = packStack[:ind] if val is None: packing = packStack[-1][0] else: packing = int(opts[0]) if self.verbose: print ">> Packing changed to %s at line %d" % (str(packing), i) self.packList[file].append((i, packing)) else: pass ## Ignore any other directives result.append(newLine) self.files[file] = '\n'.join(result)
def __set__(self, inst, value): try: self.prop.__set__(inst, value) except AttributeError: # We should never reraise this particular exception. Thus catch it here. sys.excepthook(*sys.exc_info())
class Grid(object): """ An interface to job lifecycle management. """ def __init__(self, config_file=gc3utils.Default.CONFIG_FILE_LOCATION, default_output_dir=None): cfg = gc3libs.config.Config(config_file) self.mw = gc3utils.gcli.Gcli(cfg) self.default_output_dir = default_output_dir def save(self, job): """ Save a job using gc3utils' persistence mechanism. """ # update state so that it is correctly saved, but do not use set_state() # so the job history is not altered job.state = _get_state_from_gc3utils_job_status(job.status) job.jobid = job.unique_token # XXX gc3utils.Job.persist_job(job) def submit(self, application, job=None): """ Submit an instance of the given `application`, and store it into `job` (or a new instance of the `Job` class if `job` is `None`). After successful submission, persist job to permanent storage. Return (string) id of submitted job. """ job = self.mw.gsub(application, job) return job.unique_token def update_state(self, job): """ Update running status of `job`. In case update fails, state is set to 'UNKNOWN'. Return job state. """ st = self.mw.gstat(job) # `gstat` returns list(!?) if len(st) == 0: job.set_state('UNKNOWN') job.set_info("Could not update job status.") else: job.set_state(_get_state_from_gc3utils_job_status(job.status)) return job.state def get_output(self, job, output_dir=None): """ Retrieve job's output files into `output_dir`. If `output_dir` is `None` (default), then use `self.output_dir`. """ # `job_local_dir` is where gc3utils will retrieve the output if output_dir is not None: job.job_local_dir = output_dir elif self.default_output_dir is not None: job.job_local_dir = self.default_output_dir else: job.job_local_dir = os.getcwd() self.mw.gget(job) job.output_retrieved_to = os.path.join(job.job_local_dir, job.jobid) def progress(self, job, can_submit=True, can_retrieve=True): """ Update the job's state and take appropriate action; return the (possibly changed) job state. If optional argument `can_submit` is `True` (default), will try to submit jobs in state ``NEW``. If optional argument `can_retrieve` is `False` (default), will try to fetch job results back. """ # update status of SUBMITTED/RUNNING jobs before launching new ones, otherwise # we would be checking the status of some jobs twice... if job.state == 'SUBMITTED' or job.state == 'RUNNING' or job.state == 'UNKNOWN': # update state try: self.update_state(job) except Exception, x: logger.error( "Ignoring error in updating state of job '%s': %s: %s" % (job.id, x.__class__.__name__, str(x)), exc_info=True) if can_submit and job.state == 'NEW': # try to submit; go to 'SUBMITTED' if successful, 'FAILED' if not try: self.submit(job.application, job) job.set_state('SUBMITTED') except Exception, x: logger.error("Error in submitting job '%s': %s: %s" % (job.id, x.__class__.__name__, str(x))) sys.excepthook(*sys.exc_info()) job.set_state('NEW') job.set_info("Submission failed: %s" % str(x))