def check_grads(model): assert np.isfinite(model.l_alpha()) assert np.isfinite(model.l_valpha()) x = model.grad_l_vmu() assert np.isfinite(x).all() import pdb try: # Main update rules log.info('xi update:', check_grad(model, 'xi', model.l_xi, model.grad_l_xi)) log.info('valpha update:', check_grad(model, 'valpha', model.l_valpha, model.grad_l_valpha)) log.info('alpha update:', check_grad(model, 'alpha', model.l_alpha, model.grad_l_alpha)) log.info('vmu update:', check_grad(model, 'vmu', model.l_vmu, model.tangent_grad_l_vmu)) f = lambda: avk(model.V, model.xi) g = lambda: deriv_avk(model.V, model.xi) log.info('avk_xi', check_grad(model, 'xi', f, g)) f = lambda: np.sum(model.e_squared_norm_batch()) g = lambda: np.sum(model.grad_e_squared_norm_xi()) log.info('grad_esn_xi', check_grad(model, 'xi', f, g)) f = lambda: np.sum(model.rho_batch()) g = lambda: np.sum(model.deriv_rho_xi()) log.info('deriv_rho_xi', check_grad(model, 'xi', f, g)) except Exception, e: log.error(e) pdb.post_mortem()
def runDebug(self, exc_info): if flags.can_touch_runtime_system("switch console") and self._intf_tty_num != 1: iutil.vtActivate(1) iutil.eintr_retry_call(os.open, "/dev/console", os.O_RDWR) # reclaim stdin iutil.eintr_ignore(os.dup2, 0, 1) # reclaim stdout iutil.eintr_ignore(os.dup2, 0, 2) # reclaim stderr # ^ # | # +------ dup2 is magic, I tells ya! # bring back the echo import termios si = sys.stdin.fileno() attr = termios.tcgetattr(si) attr[3] = attr[3] & termios.ECHO termios.tcsetattr(si, termios.TCSADRAIN, attr) print("\nEntering debugger...") print("Use 'continue' command to quit the debugger and get back to " "the main window") import pdb pdb.post_mortem(exc_info.stack) if flags.can_touch_runtime_system("switch console") and self._intf_tty_num != 1: iutil.vtActivate(self._intf_tty_num)
def dispatch(self, service_name, method, params): try: auth = getattr(self, 'auth_provider', None) logger = logging.getLogger('result') start_time = end_time = 0 if logger.isEnabledFor(logging.DEBUG_RPC_ANSWER): self.log('service', tuple(replace_request_password(params)), depth=None, fn='%s.%s'%(service_name,method)) if logger.isEnabledFor(logging.DEBUG_RPC): start_time = time.time() result = ExportService.getService(service_name).dispatch(method, auth, params) if logger.isEnabledFor(logging.DEBUG_RPC): end_time = time.time() if not logger.isEnabledFor(logging.DEBUG_RPC_ANSWER): self.log('service (%.3fs)' % (end_time - start_time), tuple(replace_request_password(params)), depth=1, fn='%s.%s'%(service_name,method)) self.log('execution time', '%.3fs' % (end_time - start_time), channel=logging.DEBUG_RPC_ANSWER) self.log('result', result, channel=logging.DEBUG_RPC_ANSWER) return result except Exception, e: self.log('exception', tools.exception_to_unicode(e)) tb = getattr(e, 'traceback', sys.exc_info()) tb_s = "".join(traceback.format_exception(*tb)) if tools.config['debug_mode'] and isinstance(tb[2], types.TracebackType): import pdb pdb.post_mortem(tb[2]) raise OpenERPDispatcherException(e, tb_s)
def execution_loop(tasks, options, tasks_executed, tasks_loaded): from time import sleep logging.info('Execute start (%s tasks)' % len(tasks)) while tasks: upnext = [] # tasks that can be run for i in xrange(options.execute_nr_wait_cycles): max_cannot_run = min(len(tasks), 128) for i in xrange(max_cannot_run): # The argument for this is the following: # if T' is dependent on the result of T, it is better if the # processor that ran T, also runs T'. By having everyone else # push T' to the end of tasks, this is more likely to happen. # # Furthermore, this avoids always querying the same tasks. if tasks[0].can_run(): break tasks.append(tasks.pop(0)) while tasks and tasks[0].can_run(): upnext.append(tasks.pop(0)) if upnext: break logging.info('waiting 12 secs for an open task...') sleep(options.execute_wait_cycle_time_secs) if not upnext: logging.info('No tasks can be run!') break for t in upnext: if t.can_load(): logging.info('Loadable %s...' % t.name) tasks_loaded[t.name] += 1 continue locked = False try: locked = t.lock() if t.can_load(): # This can be true if the task ran between the check above and this one logging.info('Loadable %s...' % t.name) tasks_loaded[t.name] += 1 elif locked: logging.info('Executing %s...' % t.name) t.run() tasks_executed[t.name] += 1 if options.aggressive_unload: t.unload_recursive() else: logging.info('Already in execution %s...' % t.name) except Exception, e: if options.pdb: import pdb, sys _,_, tb = sys.exc_info() pdb.post_mortem(tb) else: import itertools logging.critical('Exception while running %s: %s' % (t.name,e)) for other in itertools.chain(upnext, tasks): for dep in other.dependencies(): if dep is t: logging.critical('Other tasks are dependent on this one! Parallel processors will be held waiting!') raise finally:
def Run(self): with test_lib.FakeTime(42, increment=1): hunt_urn = self.StartHunt( description="the hunt", output_plugins=[ output_plugin.OutputPluginDescriptor( plugin_name=standard_test.FailingDummyHuntOutputPlugin. __name__) ]) self.client_ids = self.SetupClients(2) for index, client_id in enumerate(self.client_ids): self.AssignTasksToClients(client_ids=[client_id]) self.RunHunt(failrate=-1) with test_lib.FakeTime(100042 + index * 100): try: self.ProcessHuntOutputPlugins() except process_results.ResultsProcessingError: if flags.FLAGS.debug: pdb.post_mortem() self.Check( "GET", "/api/hunts/%s/output-plugins/" "FailingDummyHuntOutputPlugin_0/errors" % hunt_urn.Basename(), replace={hunt_urn.Basename(): "H:123456"})
def main(argv=sys.argv, out=sys.stdout): setup_log_ = functools.partial(setup_log, str_formatter='%(asctime)s [%(levelname)s] %(message)s') try: return pbparser_runner( argv=argv[1:], parser=get_parser(), args_runner_func=args_runner, contract_runner_func=resolved_tool_contract_runner, alog=logging.getLogger(__name__), setup_log_func=setup_log_) # FIXME is there a more central place to deal with this? except Exception as e: type, value, tb = sys.exc_info() traceback.print_exc(file=sys.stderr) # Note: if kt.args.usePdb # This won't work. If an exception is raised in parseArgs, # then kt.args is not defined yet. if '--pdb' in argv: try: # this has better integration with ipython and is nicer # pip install ipdb import ipdb ipdb.post_mortem(tb) except ImportError: import pdb pdb.post_mortem(tb) else: # exit non-zero raise
def __call__(self, environ, start_response): try: return self.app(environ, start_response) except: traceback.print_exception(*sys.exc_info()) pdb.post_mortem(sys.exc_info()[2]) raise
def handle_pdb(event): import pdb pdb.post_mortem(tb) # This level of interest seems to indicate the user might # want to debug this error if it occurs again. if (filename, line_number) in previously_seen_error_locations: previously_seen_error_locations.remove((filename, line_number))
def pdb_postmortem(): try: yield except Exception: type, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb)
def main_wrapper(module, args): """This wraps the main method of an action so that: - backtrace is not printed by default - backtrace is printed is --backtrace was given - a pdb session is run if --pdb was given """ try: module.do(args) except Exception as e: if args.pdb: traceback = sys.exc_info()[2] print "" print "### Exception:", e print "### Starting a debugger" try: #pylint: disable-msg=F0401 import ipdb ipdb.post_mortem(traceback) sys.exit(0) except ImportError: import pdb pdb.post_mortem(traceback) sys.exit(0) if args.backtrace: raise message = str(e) if message.endswith("\n"): message = message[:-1] ui.error(e.__class__.__name__, message) sys.exit(2)
def GuessAddressSpace(self, base_as=None, **kwargs): """Loads an address space by stacking valid ASes on top of each other (priority order first). """ base_as = base_as error = addrspace.AddrSpaceError() address_spaces = sorted(addrspace.BaseAddressSpace.classes.values(), key=lambda x: x.order) while 1: logging.debug("Voting round") found = False for cls in address_spaces: # Only try address spaces which claim to support images. if not cls.metadata("image"): continue logging.debug("Trying %s ", cls) try: base_as = cls(base=base_as, session=self.session, **kwargs) logging.debug("Succeeded instantiating %s", base_as) found = True break except (AssertionError, addrspace.ASAssertionError, IOError), e: logging.debug("Failed instantiating %s: %s", cls.__name__, e) error.append_reason(cls.__name__, e) continue except Exception, e: logging.error("Fatal Error: %s", e) if self.session.debug: pdb.post_mortem() raise
def run(self): """Main thread for processing messages.""" self.OnStartup() # As long as our output queue has some room we can process some # input messages: while True: message = self._in_queue.get() # A message of None is our terminal message. if message is None: break try: self.HandleMessage(message) # Catch any errors and keep going here except Exception as e: # pylint: disable=broad-except logging.warn("%s", e) self.SendReply( rdf_flows.GrrStats( status=rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR, error_message=utils.SmartUnicode(e)), request_id=message.request_id, response_id=message.response_id, session_id=message.session_id, task_id=message.task_id, message_type=rdf_flows.GrrMessage.Type.STATUS) if flags.FLAGS.debug: pdb.post_mortem()
def main(): parser = setup_parser() argcomplete.autocomplete(parser) options = parser.parse_args() _setup_logger(options) # Support the deprecated -c option if getattr(options, 'config', None) is not None: options.configs.append(options.config) if options.subparser in ('report', 'logs', 'metrics', 'run'): _default_region(options) _default_account_id(options) try: command = options.command if not callable(command): command = getattr( importlib.import_module(command.rsplit('.', 1)[0]), command.rsplit('.', 1)[-1]) # Set the process name to something cleaner process_name = [os.path.basename(sys.argv[0])] process_name.extend(sys.argv[1:]) setproctitle(' '.join(process_name)) command(options) except Exception: if not options.debug: raise traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1])
def shell(self): while self.active: try: raw = raw_input('pmr2cli> ') if not raw: continue rawargs = raw.split(None, 1) command = rawargs.pop(0) obj = getattr(self, 'do_' + command, None) if callable(obj): obj(*rawargs) else: print("Invalid command, try 'help'.") except EOFError: self.active = False print('') except KeyboardInterrupt: print('\nGot interrupt signal.') self.active = False except ValueError: print("Couldn't decode json.") # print("Status was %d") % self.last_response.status_code print("Use console to check `self.last_response` for details.") except: print(traceback.format_exc()) if self.debug: pdb.post_mortem()
def addFailure(self, test, err): super(ExceptionTestResultMixin, self).addFailure(test, err) exctype, value, tb = err self.stream.writeln() self.stream.writeln(self.separator1) self.stream.writeln(">>> %s" % (self.getDescription(test))) self.stream.writeln(self.separator2) self.stream.writeln(self._exc_info_to_string(err, test).rstrip()) self.stream.writeln(self.separator1) self.stream.writeln() ## Skip test runner traceback levels #while tb and self._is_relevant_tb_level(tb): # tb = tb.tb_next # Really hacky way to jump up a couple of frames. # I'm sure it's not that difficult to do properly, # but I havn't figured out how. #p = pdb.Pdb() #p.reset() #p.setup(None, tb) #p.do_up(None) #p.do_up(None) #p.cmdloop() # It would be good if we could make sure we're in the correct frame here pdb.post_mortem(tb)
def reset_index(): try: main() except: import pdb, traceback, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1])
def run(main, argv=None): """Begins executing the program. Args: main: The main function to execute. It takes an single argument "argv", which is a list of command line arguments with parsed flags removed. argv: A non-empty list of the command line arguments including program name, sys.argv is used if None. - Parses command line flags with the flag module. - If there are any errors, prints usage(). - Calls main() with the remaining arguments. - If main() raises a UsageError, prints usage and the error message. """ try: argv = _run_init(sys.argv if argv is None else argv) try: _run_main(main, argv) except UsageError as error: usage(shorthelp=True, detailed_error=error, exitcode=error.exitcode) except: if FLAGS.pdb_post_mortem: traceback.print_exc() pdb.post_mortem() raise except Exception as e: _call_exception_handlers(e) raise
def debugger_hook(exc, value, tb): if (not hasattr(sys.stderr, "isatty") or not sys.stderr.isatty() or exc in (SyntaxError, IndentationError, KeyboardInterrupt)): sys.__excepthook__(exc, value, tb) else: import pdb pdb.post_mortem(tb)
def main(argv=None): # New user interactive session (with extra bells and whistles). user_session = session.InteractiveSession() user_session.session_list.append(user_session) # Alow all special plugins to run. user_session.privileged = True def global_arg_cb(global_flags, _): if global_flags.version: print("This is Rekall Version %s (%s)" % ( constants.VERSION, constants.CODENAME)) print(rekall.get_versions()) sys.exit(0) with user_session.GetRenderer().start(): plugin_cls, flags = args.parse_args( argv=argv, global_arg_cb=global_arg_cb, user_session=user_session) # Install any quotas the user requested. user_session = quotas.wrap_session(user_session) try: # Run the plugin with plugin specific args. user_session.RunPlugin(plugin_cls, **config.RemoveGlobalOptions(flags)) except Exception as e: logging.fatal("%s. Try --debug for more information." % e) if getattr(flags, "debug", None): pdb.post_mortem(sys.exc_info()[2]) raise finally: user_session.Flush()
def __init__(self, base=None, filename=None, **kwargs): self.as_assert(base == None, 'Must be first Address Space') super(Win32FileAddressSpace, self).__init__(**kwargs) path = filename or (self.session and self.session.GetParameter( "filename")) self.as_assert(path, "Filename must be specified in session (e.g. " "session.GetParameter('filename', 'MyFile.raw').") self.fname = path self.fhandle = win32file.CreateFile( path, win32file.GENERIC_READ | win32file.GENERIC_WRITE, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE, None, win32file.OPEN_EXISTING, win32file.FILE_ATTRIBUTE_NORMAL, None) # Try to get the memory runs from the winpmem driver. try: self.ParseMemoryRuns() except Exception: if self.session.debug: import pdb pdb.post_mortem() self.runs.insert((0, 0, win32file.GetFileSize(self.fhandle))) # IO on windows is extremely slow so we are better off using a # cache. self.cache = utils.FastStore(1000)
def __call__(self): if '-D' in sys.argv: sys.argv.remove('-D') self.debug = True try: if len(sys.argv)==1: self.parser.print_help() elif len(sys.argv)==2 and sys.argv[1].startswith('-'): self.options, self.args = self.parser.parse_args() else: command_name = sys.argv[1] command = self.get_command(command_name) if command: command(self)() else: self.parser.print_help() except SystemExit: pass except KeyboardInterrupt: print 'Keyboard interrupt' except: if self.debug: exc_info = sys.exc_info() traceback.print_exception(*exc_info) sys.stderr.write('\nStarting pdb:\n') pdb.post_mortem(exc_info[2]) else: raise
def test_A_constraints(): import pdb, traceback, sys try: dims = [4, 8] N_rand = 10 eps = 1e-5 tol = 1e-3 np.set_printoptions(precision=3) for dim in dims: block_dim = int(dim/4) # Generate initial data D = np.eye(block_dim) Dinv = np.linalg.inv(D) Q = 0.5*np.eye(block_dim) mu = np.ones((block_dim, 1)) As, bs, Cs, ds, Fs, gradFs, Gs, gradGs = \ A_constraints(block_dim, D, Dinv, Q, mu) tol = 1e-3 eps = 1e-4 N_rand = 10 for (g, gradg) in zip(Gs, gradGs): for i in range(N_rand): X = np.random.rand(dim, dim) val = g(X) grad = gradg(X) print "grad:\n", grad num_grad = numerical_derivative(g, X, eps) print "num_grad:\n", num_grad assert np.sum(np.abs(grad - num_grad)) < tol except: type, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb)
def main(): """ Process NMEC with precompiled denoiser and Fielnet """ def denoising_func(x): orig_shape = x.shape x = 1. - x/255. x = denoiser.predict(x, verbose=0) x = x.reshape(orig_shape) return x try: featext = load_verbatimnet('fc7', paramsfile='/fileserver/iam/iam-processed/models/fiel_657.hdf5') featext.compile(loss='mse', optimizer='sgd') featext_func = lambda x: featext.predict(x, verbose=0.0) print "Making the denoiser" denoiser = conv4p_model() denoiser.load_weights('/fileserver/iam/models/conv4p_linet56-iambin-tifs.hdf5') hdf5file='/fileserver/nmec-handwriting/flat_nmec_bin_uint8.hdf5' with h5py.File(hdf5file, "r") as data_file: features = extract_features_for_corpus(data_file,featext_func,shingle_dim=(56,56),transform=denoising_func) with h5py.File("output_features.hdf5", "w") as feature_file: for document_id, document_features in features.iteritems(): feature_file.create_dataset(document_id, data=document_features) except Exception as e: print e pdb.post_mortem()
def pdb(self): # pragma: no cover ''' Run a pdb post-mortem on any pending exception information ''' import pdb if self.next_exc: pdb.post_mortem(self.next_exc.__traceback__)
def __exit__(self, type_, value, traceback_): host = socket.gethostname() if isinstance(value, Exception): error = type_.__name__ temp = '{host} encountered {error}: {value} in {task}' event = temp.format(host=host, error=error, value=value, task=self.name) info = [] # traceback tb_items = traceback.format_tb(traceback_) tb_items.append("%s: %s\n" % (error, value)) tb_str = '\n'.join(tb_items) info.append(tb_str) # object info if self.crash_info_func: info.extend(self.crash_info_func()) self.send(event, info) # drop into pdb if self.debug: traceback.print_exc() pdb.post_mortem(traceback_) else: event = '{host} finished {task}'.format(host=host, task=self.name) self.send(event)
def destroy_tile(x, y): # If the old tile was a multitile head or pointer if (g.map[x][y] and (type(g.map[x][y]) == MultiTileHead or type(g.map[x][y]) == MultiTilePointer)): # Get the destroy value if type(g.map[x][y]) == MultiTileHead: destroy_value = c.IMAGES[g.map[x][y].type].destroy multi_tile = {"x": x, "y": y, "width": g.map[x][y].width, "height": g.map[x][y].height} else: target_x, target_y = g.map[x][y].target destroy_value = c.IMAGES[g.map[target_x][target_y].type].destroy try: multi_tile = {"x": target_x, "y": target_y, "width": g.map[target_x][target_y].width, "height": g.map[target_x][target_y].height} except: import pdb, sys e, m, tb = sys.exc_info() pdb.post_mortem(tb) for i in range(multi_tile["x"], multi_tile["x"] + multi_tile["width"]): for j in range(multi_tile["y"], multi_tile["y"] + multi_tile["height"]): if i == x and j == y: continue if len(destroy_value) > 2: make_tile(destroy_value[2], i, j) else: make_tile(destroy_value[1], i, j) make_tile(destroy_value[1], x, y) else: make_tile(c.IMAGES[g.map[x][y].type].destroy[1], x, y)
def post_mortem(exc_info): err = exc_info[1] if isinstance(err, (doctest.UnexpectedException, doctest.DocTestFailure)): if isinstance(err, doctest.UnexpectedException): exc_info = err.exc_info # Print out location info if the error was in a doctest if exc_info[2].tb_frame.f_code.co_filename == '<string>': print_doctest_location(err) else: print_doctest_location(err) # Hm, we have a DocTestFailure exception. We need to # generate our own traceback try: exec ('raise ValueError' '("Expected and actual output are different")' ) in err.test.globs except: exc_info = sys.exc_info() print "%s:" % (exc_info[0], ) print exc_info[1] pdb.post_mortem(exc_info[2]) raise zope.testing.testrunner.interfaces.EndRun()
def test_Q_constraints(): import pdb, traceback, sys try: dims = [4, 8] N_rand = 10 eps = 1e-4 tol = 1e-3 for dim in dims: block_dim = int(dim/4) # Generate initial data D = np.eye(block_dim) Dinv = np.linalg.inv(D) B = np.eye(block_dim) A = 0.5 * np.eye(block_dim) c = 0.5 As, bs, Cs, ds, Fs, gradFs, Gs, gradGs = \ Q_constraints(block_dim, A, B, D, c) N_rand = 10 for (g, gradg) in zip(Gs, gradGs): for i in range(N_rand): X = np.random.rand(dim, dim) val = g(X) grad = gradg(X) print "grad:\n", grad num_grad = numerical_derivative(g, X, eps) print "num_grad:\n", num_grad assert np.sum(np.abs(grad - num_grad)) < tol except: type, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb)
def add(self, pipe, descriptor): """ store a rendered content object on the filesystem. """ # sometimes we want to have content in the uri db # but we don't actually want to store it... either because # of an error during rendering or based on configuration. if descriptor.isGhost(): return descriptors = descriptor.getContainedDescriptors() structure = self.getStructure( pipe ) for descriptor in descriptors: if descriptor.isGhost(): # if a child view errors then it has no content continue content_path = structure.getContentPathFromDescriptor( descriptor ) #if content_path.endswith(sep): #log.warning('invalid content path detected %s ... fixing'%content_path) # content_path = content_path[:-1] try: self.storeDescriptor( pipe, content_path, descriptor ) except: if not DEPLOYMENT_DEBUG: raise import pdb, sys, traceback traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) return True
def runner(files, test_filter, debug): runner = ImmediateTestRunner(verbosity=VERBOSE, debug=debug, progress=progress) suite = unittest.TestSuite() for file in files: s = get_suite(file) # See if the levels match dolevel = (level == 0) or level >= getattr(s, "level", 0) if s is not None and dolevel: s = filter_testcases(s, test_filter) suite.addTest(s) try: r = runner.run(suite) if timesfn: r.print_times(open(timesfn, "w")) if VERBOSE: print ("Wrote timing data to", timesfn) if timetests: r.print_times(sys.stdout, timetests) except: if debugger: print ("%s:" % (sys.exc_info()[0], )) print (sys.exc_info()[1]) pdb.post_mortem(sys.exc_info()[2]) else: raise
def unexpectedErrorAlertPdb(): import pdb traceback.print_exc() pdb.post_mortem(sys.exc_info()[2]) return True
def RunOnce(self): """Processes one set of messages from Task Scheduler. The worker processes new jobs from the task master. For each job we retrieve the session from the Task Scheduler. Returns: Total number of messages processed by this call. """ start_time = time.time() processed = 0 queue_manager = queue_manager_lib.QueueManager(token=self.token) for queue in self.queues: # Freezeing the timestamp used by queue manager to query/delete # notifications to avoid possible race conditions. queue_manager.FreezeTimestamp() fetch_messages_start = time.time() notifications = queue_manager.GetNotifications(queue) stats_collector_instance.Get().RecordEvent( "worker_time_to_retrieve_notifications", time.time() - fetch_messages_start) stuck_flows = [] for n in notifications: if n.in_progress: stuck_flows.append(n) # Process stuck flows first if stuck_flows: self.ProcessStuckFlows(stuck_flows, queue_manager) notifications_available = [] for notification in notifications: # Filter out session ids we already tried to lock but failed. if notification.session_id not in self.queued_flows: notifications_available.append(notification) try: # If we spent too much time processing what we have so far, the # active_sessions list might not be current. We therefore break here # so we can re-fetch a more up to date version of the list, and try # again later. The risk with running with an old active_sessions list # is that another worker could have already processed this message, # and when we try to process it, there is nothing to do - costing us a # lot of processing time. This is a tradeoff between checking the data # store for current information and processing out of date # information. processed += self.ProcessMessages( notifications_available, queue_manager, self.RUN_ONCE_MAX_SECONDS - (time.time() - start_time)) # We need to keep going no matter what. except Exception as e: # pylint: disable=broad-except logging.error("Error processing message %s. %s.", e, traceback.format_exc()) stats_collector_instance.Get().IncrementCounter("grr_worker_exceptions") if flags.FLAGS.pdb_post_mortem: pdb.post_mortem() queue_manager.UnfreezeTimestamp() # If we have spent too much time, stop. if (time.time() - start_time) > self.RUN_ONCE_MAX_SECONDS: return processed return processed
total_time = time.time() - start_time stats.STATS.RecordEvent("ui_renderer_latency", total_time, fields=[renderer_name]) except access_control.UnauthorizedAccess, e: result = http.HttpResponse(content_type="text/html") result = renderers.Renderer.GetPlugin("UnauthorizedRenderer")().Layout( request, result, exception=e) except Exception: stats.STATS.IncrementCounter("ui_renderer_failure", fields=[renderer_name]) if flags.FLAGS.debug: pdb.post_mortem() raise if not isinstance(result, http.HttpResponse): raise RuntimeError("Renderer returned invalid response %r" % result) return result def RedirectToRemoteHelp(path): """Redirect to GitHub-hosted documentation.""" target_path = os.path.join( config_lib.CONFIG["AdminUI.github_docs_location"], path.replace(".html", ".adoc"))
log.info("Cloning repository: %s", repo_uri) repo = pygit2.clone_repository(repo_uri, temp_dir.path) else: repo = pygit2.Repository(repo_uri) load_available() policy_repo = PolicyRepo(repo_uri, repo, matcher) change_count = 0 with contextlib.closing(transport(stream_uri, assume)) as t: if after is None and isinstance(t, IndexedTransport): after = t.last() for change in policy_repo.delta_stream(after=after, before=before): change_count += 1 t.send(change) log.info("Streamed %d policy repo changes", change_count) return change_count if __name__ == '__main__': try: cli() except SystemExit: raise except KeyboardInterrupt: raise except: # NOQA import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1])
def debug(self): if self.debugger: import pdb tb = self.exc_info[2] pdb.post_mortem(tb)
args = parser.parse_args() receiver = Receiver(args) while True: # receive data packet_type = receiver.recv_data() # if fin break if packet_type == "fin": break # wait for a while time.sleep(5) receiver.send_fin() receiver.recv_finack() # flush packet receiver.file += b''.join(receiver.buf_list) print("flush") # write to file receiver.write_file(args.file_path, os.path.splitext(receiver.filename)[-1]) return 0 if __name__ == '__main__': import sys, traceback, pdb try: main() except: type, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb)
def waf_entry_point(current_directory, version, wafdir): Logs.init_log() if Context.WAFVERSION != version: Logs.error('Waf script %r and library %r do not match (directory %r)', version, Context.WAFVERSION, wafdir) sys.exit(1) Context.waf_dir = wafdir Context.run_dir = Context.launch_dir = current_directory start_dir = current_directory no_climb = os.environ.get('NOCLIMB') if len(sys.argv) > 1: potential_wscript = os.path.join(current_directory, sys.argv[1]) if os.path.basename( potential_wscript) == Context.WSCRIPT_FILE and os.path.isfile( potential_wscript): path = os.path.normpath(os.path.dirname(potential_wscript)) start_dir = os.path.abspath(path) no_climb = True sys.argv.pop(1) ctx = Context.create_context('options') (options, commands, env) = ctx.parse_cmd_args(allow_unknown=True) if options.top: start_dir = Context.run_dir = Context.top_dir = options.top no_climb = True if options.out: Context.out_dir = options.out if not no_climb: for k in no_climb_commands: for y in commands: if y.startswith(k): no_climb = True break cur = start_dir while cur: try: lst = os.listdir(cur) except OSError: lst = [] Logs.error('Directory %r is unreadable!', cur) if Options.lockfile in lst: env = ConfigSet.ConfigSet() try: env.load(os.path.join(cur, Options.lockfile)) ino = os.stat(cur)[stat.ST_INO] except EnvironmentError: pass else: for x in (env.run_dir, env.top_dir, env.out_dir): if not x: continue if Utils.is_win32: if cur == x: load = True break else: try: ino2 = os.stat(x)[stat.ST_INO] except OSError: pass else: if ino == ino2: load = True break else: Logs.warn('invalid lock file in %s', cur) load = False if load: Context.run_dir = env.run_dir Context.top_dir = env.top_dir Context.out_dir = env.out_dir break if not Context.run_dir: if Context.WSCRIPT_FILE in lst: Context.run_dir = cur next = os.path.dirname(cur) if next == cur: break cur = next if no_climb: break if not Context.run_dir: if options.whelp: Logs.warn( 'These are the generic options (no wscript/project found)') ctx.parser.print_help() sys.exit(0) Logs.error( 'Waf: Run from a folder containing a %r file (or try -h for the generic options)', Context.WSCRIPT_FILE) sys.exit(1) try: os.chdir(Context.run_dir) except OSError: Logs.error('Waf: The folder %r is unreadable', Context.run_dir) sys.exit(1) try: set_main_module( os.path.normpath( os.path.join(Context.run_dir, Context.WSCRIPT_FILE))) except Errors.WafError as e: Logs.pprint('RED', e.verbose_msg) Logs.error(str(e)) sys.exit(1) except Exception as e: Logs.error('Waf: The wscript in %r is unreadable', Context.run_dir) traceback.print_exc(file=sys.stdout) sys.exit(2) if options.profile: import cProfile, pstats cProfile.runctx( 'from waflib import Scripting; Scripting.run_commands()', {}, {}, 'profi.txt') p = pstats.Stats('profi.txt') p.sort_stats('time').print_stats(75) else: try: try: run_commands() except: if options.pdb: import pdb type, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb) else: raise except Errors.WafError as e: if Logs.verbose > 1: Logs.pprint('RED', e.verbose_msg) Logs.error(e.msg) sys.exit(1) except SystemExit: raise except Exception as e: traceback.print_exc(file=sys.stdout) sys.exit(2) except KeyboardInterrupt: Logs.pprint('RED', 'Interrupted') sys.exit(68)
with open(cmdfname, 'rb+') as f: mm = mmap.mmap(f.fileno(), cmdfsize, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE) cmdent = cmdfsize // sizeof(zhpe.xdm_cmd) cmds = zhpe.xdm_cmd * cmdent cmd = cmds.from_buffer(mm, 0) for i in range(0, cmdent): print('{:#x} {}'.format(i, cmd[i])) # end with with open(cmpfname, 'rb+') as f: mm = mmap.mmap(f.fileno(), cmdfsize, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE) cmpent = cmpfsize // sizeof(zhpe.xdm_cmpl) cmps = zhpe.xdm_cmpl * cmpent cmp = cmps.from_buffer(mm, 0) for i in range(0, cmpent): print('{:#x} {}'.format(i, cmp[i])) # end with if __name__ == '__main__': try: main() except: if args.post_mortem: post_mortem() else: raise
def update_test(t): # the update-eval tests refer to graphs on http://example.org rdflib_sparql_module.SPARQL_LOAD_GRAPHS = False uri, name, comment, data, graphdata, query, res, syntax = t if uri in skiptests: raise SkipTest() try: g = Dataset() if not res: if syntax: with bopen(query[7:]) as f: translateUpdate(parseUpdate(f)) else: try: with bopen(query[7:]) as f: translateUpdate(parseUpdate(f)) raise AssertionError("Query shouldn't have parsed!") except: pass # negative syntax test return resdata, resgraphdata = res # read input graphs if data: g.default_context.load(data, format=_fmt(data)) if graphdata: for x, l in graphdata: g.load(x, publicID=URIRef(l), format=_fmt(x)) with bopen(query[7:]) as f: req = translateUpdate(parseUpdate(f)) evalUpdate(g, req) # read expected results resg = Dataset() if resdata: resg.default_context.load(resdata, format=_fmt(resdata)) if resgraphdata: for x, l in resgraphdata: resg.load(x, publicID=URIRef(l), format=_fmt(x)) eq( set(x.identifier for x in g.contexts() if x != g.default_context), set(x.identifier for x in resg.contexts() if x != resg.default_context), "named graphs in datasets do not match", ) assert isomorphic( g.default_context, resg.default_context), "Default graphs are not isomorphic" for x in g.contexts(): if x == g.default_context: continue assert isomorphic(x, resg.get_context( x.identifier)), ("Graphs with ID %s are not isomorphic" % x.identifier) except Exception as e: if isinstance(e, AssertionError): failed_tests.append(uri) fails[str(e)] += 1 else: error_tests.append(uri) errors[str(e)] += 1 if DEBUG_ERROR and not isinstance(e, AssertionError) or DEBUG_FAIL: print("======================================") print(uri) print(name) print(comment) if not res: if syntax: print("Positive syntax test") else: print("Negative syntax test") if data: print("----------------- DATA --------------------") print(">>>", data) print(bopen_read_close(data[7:])) if graphdata: print("----------------- GRAPHDATA --------------------") for x, l in graphdata: print(">>>", x, l) print(bopen_read_close(x[7:])) print("----------------- Request -------------------") print(">>>", query) print(bopen_read_close(query[7:])) if res: if resdata: print("----------------- RES DATA --------------------") print(">>>", resdata) print(bopen_read_close(resdata[7:])) if resgraphdata: print( "----------------- RES GRAPHDATA -------------------") for x, l in resgraphdata: print(">>>", x, l) print(bopen_read_close(x[7:])) print("------------- MY RESULT ----------") print(g.serialize(format="trig")) try: pq = translateUpdate(parseUpdate(bopen_read_close(query[7:]))) print("----------------- Parsed ------------------") pprintAlgebra(pq) # print pq except: print("(parser error)") print(decodeStringEscape(str(e))) import pdb pdb.post_mortem(sys.exc_info()[2]) raise
def report_internal_error( err: Exception, file: Optional[str], line: int, errors: Errors, options: Options, stdout: Optional[TextIO] = None, stderr: Optional[TextIO] = None, ) -> None: """Report internal error and exit. This optionally starts pdb or shows a traceback. """ stdout = (stdout or sys.stdout) stderr = (stderr or sys.stderr) # Dump out errors so far, they often provide a clue. # But catch unexpected errors rendering them. try: for msg in errors.new_messages(): print(msg) except Exception as e: print("Failed to dump errors:", repr(e), file=stderr) # Compute file:line prefix for official-looking error messages. if file: if line: prefix = '{}:{}: '.format(file, line) else: prefix = '{}: '.format(file) else: prefix = '' # Print "INTERNAL ERROR" message. print( '{}error: INTERNAL ERROR --'.format(prefix), 'Please try using mypy master on Github:\n' 'https://mypy.rtfd.io/en/latest/common_issues.html#using-a-development-mypy-build', file=stderr) if options.show_traceback: print('Please report a bug at https://github.com/python/mypy/issues', file=stderr) else: print( 'If this issue continues with mypy master, ' 'please report a bug at https://github.com/python/mypy/issues', file=stderr) print('version: {}'.format(mypy_version), file=stderr) # If requested, drop into pdb. This overrides show_tb. if options.pdb: print('Dropping into pdb', file=stderr) import pdb pdb.post_mortem(sys.exc_info()[2]) # If requested, print traceback, else print note explaining how to get one. if options.raise_exceptions: raise err if not options.show_traceback: if not options.pdb: print('{}: note: please use --show-traceback to print a traceback ' 'when reporting a bug'.format(prefix), file=stderr) else: tb = traceback.extract_stack()[:-2] tb2 = traceback.extract_tb(sys.exc_info()[2]) print('Traceback (most recent call last):') for s in traceback.format_list(tb + tb2): print(s.rstrip('\n')) print('{}: {}'.format(type(err).__name__, err), file=stdout) print('{}: note: use --pdb to drop into pdb'.format(prefix), file=stderr) # Exit. The caller has nothing more to say. # We use exit code 2 to signal that this is no ordinary error. raise SystemExit(2)
def exec_py_file(self, filename, args=None, ins=None, outs=None, errs=None): _, current_state = self.get_current_worker_and_state() if ins: current_state.sys_stdin = ins if outs: current_state.sys_stdout = outs if errs: current_state.sys_stderr = errs file_path = os.path.relpath(filename) namespace = dict(locals(), **globals()) namespace['__name__'] = '__main__' namespace['__file__'] = os.path.abspath(file_path) namespace['_stash'] = self.stash saved_sys_argv = sys.argv[:] # First argument is the script name sys.argv = [os.path.basename(filename)] + (args or []) # Set current os environ to the threading environ saved_os_environ = os.environ os.environ = dict(current_state.environ) # Honor any leading vars, e.g. A=42 echo $A os.environ.update(current_state.temporary_environ) # This needs to be done after environ due to possible leading PYTHONPATH var saved_sys_path = sys.path sys.path = current_state.sys_path[:] self.handle_PYTHONPATH() # Make sure PYTHONPATH is honored try: execfile(file_path, namespace, namespace) current_state.return_value = 0 except SystemExit as e: current_state.return_value = e.code except Exception as e: current_state.return_value = 1 etype, evalue, tb = sys.exc_info() err_msg = '%s: %s\n' % (repr(etype), evalue) if self.debug: self.logger.debug(err_msg) self.stash.write_message(err_msg) if self.py_traceback or self.py_pdb: import traceback traceback.print_exception(etype, evalue, tb) if self.py_pdb: import pdb pdb.post_mortem(tb) finally: # Thread specific vars are not modified, e.g. current_state.environ is unchanged. # This means the vars cannot be changed inside a python script. It can only be # done through shell command, e.g. NEW_VAR=42 sys.argv = saved_sys_argv sys.path = saved_sys_path os.environ = saved_os_environ
def query_test(t): uri, name, comment, data, graphdata, query, resfile, syntax = t # the query-eval tests refer to graphs to load by resolvable filenames rdflib_sparql_module.SPARQL_LOAD_GRAPHS = True if uri in skiptests: raise SkipTest() def skip(reason="(none)"): print("Skipping %s from now on." % uri) with bopen("skiptests.list", "a") as f: f.write("%s\t%s\n" % (uri, reason)) try: g = Dataset() if data: g.default_context.load(data, format=_fmt(data)) if graphdata: for x in graphdata: g.load(x, format=_fmt(x)) if not resfile: # no result - syntax test if syntax: translateQuery(parseQuery(bopen_read_close(query[7:])), base=urljoin(query, ".")) else: # negative syntax test try: translateQuery( parseQuery(bopen_read_close(query[7:])), base=urljoin(query, "."), ) assert False, "Query should not have parsed!" except: pass # it's fine - the query should not parse return # eval test - carry out query res2 = g.query(bopen_read_close(query[7:]), base=urljoin(query, ".")) if resfile.endswith("ttl"): resg = Graph() resg.load(resfile, format="turtle", publicID=resfile) res = RDFResultParser().parse(resg) elif resfile.endswith("rdf"): resg = Graph() resg.load(resfile, publicID=resfile) res = RDFResultParser().parse(resg) else: with bopen(resfile[7:]) as f: if resfile.endswith("srj"): res = Result.parse(f, format="json") elif resfile.endswith("tsv"): res = Result.parse(f, format="tsv") elif resfile.endswith("csv"): res = Result.parse(f, format="csv") # CSV is lossy, round-trip our own resultset to # lose the same info :) # write bytes, read strings... s = BytesIO() res2.serialize(s, format="csv") s.seek(0) res2 = Result.parse(s, format="csv") s.close() else: res = Result.parse(f, format="xml") if not DETAILEDASSERT: eq(res.type, res2.type, "Types do not match") if res.type == "SELECT": eq(set(res.vars), set(res2.vars), "Vars do not match") comp = bindingsCompatible(set(res), set(res2)) assert comp, "Bindings do not match" elif res.type == "ASK": eq(res.askAnswer, res2.askAnswer, "Ask answer does not match") elif res.type in ("DESCRIBE", "CONSTRUCT"): assert isomorphic(res.graph, res2.graph), "graphs are not isomorphic!" else: raise Exception("Unknown result type: %s" % res.type) else: eq( res.type, res2.type, "Types do not match: %r != %r" % (res.type, res2.type), ) if res.type == "SELECT": eq( set(res.vars), set(res2.vars), "Vars do not match: %r != %r" % (set(res.vars), set(res2.vars)), ) assert bindingsCompatible(set(res), set(res2)), ( "Bindings do not match: \nexpected:\n%s\n!=\ngot:\n%s" % ( res.serialize(format="txt", namespace_manager=g.namespace_manager), res2.serialize(format="txt", namespace_manager=g.namespace_manager), )) elif res.type == "ASK": eq( res.askAnswer, res2.askAnswer, "Ask answer does not match: %r != %r" % (res.askAnswer, res2.askAnswer), ) elif res.type in ("DESCRIBE", "CONSTRUCT"): assert isomorphic(res.graph, res2.graph), "graphs are not isomorphic!" else: raise Exception("Unknown result type: %s" % res.type) except Exception as e: if isinstance(e, AssertionError): failed_tests.append(uri) fails[str(e)] += 1 else: error_tests.append(uri) errors[str(e)] += 1 if DEBUG_ERROR and not isinstance(e, AssertionError) or DEBUG_FAIL: print("======================================") print(uri) print(name) print(comment) if not resfile: if syntax: print("Positive syntax test") else: print("Negative syntax test") if data: print("----------------- DATA --------------------") print(">>>", data) print(bopen_read_close(data[7:])) if graphdata: print("----------------- GRAPHDATA --------------------") for x in graphdata: print(">>>", x) print(bopen_read_close(x[7:])) print("----------------- Query -------------------") print(">>>", query) print(bopen_read_close(query[7:])) if resfile: print("----------------- Res -------------------") print(">>>", resfile) print(bopen_read_close(resfile[7:])) try: pq = parseQuery(bopen_read_close(query[7:])) print("----------------- Parsed ------------------") pprintAlgebra(translateQuery(pq, base=urljoin(query, "."))) except: print("(parser error)") print(decodeStringEscape(str(e))) import pdb pdb.post_mortem(sys.exc_info()[2]) # pdb.set_trace() # nose.tools.set_trace() raise
def drop_into_pdb(app, exception): """ https://gist.github.com/alonho/4389137 """ traceback.print_exc() pdb.post_mortem(sys.exc_info()[2])
def Execute(self, message): """This function parses the RDFValue from the server. The Run method will be called with the specified RDFValue. Args: message: The GrrMessage that we are called to process. Returns: Upon return a callback will be called on the server to register the end of the function and pass back exceptions. Raises: RuntimeError: The arguments from the server do not match the expected rdf type. """ self.message = message if message: self.require_fastpoll = message.require_fastpoll args = None try: if self.message.args_rdf_name: if not self.in_rdfvalue: raise RuntimeError("Did not expect arguments, got %s." % self.message.args_rdf_name) if self.in_rdfvalue.__name__ != self.message.args_rdf_name: raise RuntimeError("Unexpected arg type %s != %s." % (self.message.args_rdf_name, self.in_rdfvalue.__name__)) args = self.message.payload # Only allow authenticated messages in the client if self._authentication_required and ( self.message.auth_state != rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED): raise RuntimeError("Message for %s was not Authenticated." % self.message.name) self.cpu_start = self.proc.cpu_times() self.cpu_limit = self.message.cpu_limit if getattr(flags.FLAGS, "debug_client_actions", False): pdb.set_trace() try: self.Run(args) # Ensure we always add CPU usage even if an exception occurred. finally: used = self.proc.cpu_times() self.cpu_used = (used.user - self.cpu_start.user, used.system - self.cpu_start.system) except NetworkBytesExceededError as e: self.SetStatus( rdf_flows.GrrStatus.ReturnedStatus.NETWORK_LIMIT_EXCEEDED, "%r: %s" % (e, e), traceback.format_exc()) # We want to report back all errors and map Python exceptions to # Grr Errors. except Exception as e: # pylint: disable=broad-except self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR, "%r: %s" % (e, e), traceback.format_exc()) if flags.FLAGS.pdb_post_mortem: self.DisableNanny() pdb.post_mortem() if self.status.status != rdf_flows.GrrStatus.ReturnedStatus.OK: logging.info("Job Error (%s): %s", self.__class__.__name__, self.status.error_message) if self.status.backtrace: logging.debug(self.status.backtrace) if self.cpu_used: self.status.cpu_time_used.user_cpu_time = self.cpu_used[0] self.status.cpu_time_used.system_cpu_time = self.cpu_used[1] # This returns the error status of the Actions to the flow. self.SendReply(self.status, message_type=rdf_flows.GrrMessage.Type.STATUS) self._RunGC()
def run_account(account, region, policies_config, output_path, cache_period, cache_path, metrics, dryrun, debug): """Execute a set of policies on an account. """ logging.getLogger('custodian.output').setLevel(logging.ERROR + 1) CONN_CACHE.session = None CONN_CACHE.time = None load_available() # allow users to specify interpolated output paths if '{' not in output_path: output_path = os.path.join(output_path, account['name'], region) cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region)) config = Config.empty( region=region, cache=cache_path, cache_period=cache_period, dryrun=dryrun, output_dir=output_path, account_id=account['account_id'], metrics_enabled=metrics, log_group=None, profile=None, external_id=None) env_vars = account_tags(account) if account.get('role'): if isinstance(account['role'], six.string_types): config['assume_role'] = account['role'] config['external_id'] = account.get('external_id') else: env_vars.update( _get_env_creds(get_session(account, 'custodian', region), region)) elif account.get('profile'): config['profile'] = account['profile'] policies = PolicyCollection.from_data(policies_config, config) policy_counts = {} success = True st = time.time() with environ(**env_vars): for p in policies: # Extend policy execution conditions with account information p.conditions.env_vars['account'] = account # Variable expansion and non schema validation (not optional) p.expand_variables(p.get_variables(account.get('vars', {}))) p.validate() if p.region and p.region != region: continue log.debug( "Running policy:%s account:%s region:%s", p.name, account['name'], region) try: resources = p.run() policy_counts[p.name] = resources and len(resources) or 0 if not resources: continue if not config.dryrun and p.execution_mode != 'pull': log.info("Ran account:%s region:%s policy:%s provisioned time:%0.2f", account['name'], region, p.name, time.time() - st) continue log.info( "Ran account:%s region:%s policy:%s matched:%d time:%0.2f", account['name'], region, p.name, len(resources), time.time() - st) except ClientError as e: success = False if e.response['Error']['Code'] == 'AccessDenied': log.warning('Access denied api:%s policy:%s account:%s region:%s', e.operation_name, p.name, account['name'], region) return policy_counts, success log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) continue except Exception as e: success = False log.error( "Exception running policy:%s account:%s region:%s error:%s", p.name, account['name'], region, e) if not debug: continue import traceback, pdb, sys traceback.print_exc() pdb.post_mortem(sys.exc_info()[-1]) raise return policy_counts, success
self.document.attachevent(self.callback) def write(self, text): if len(text) >= self.block.width: text = text[:self.block.width] self.l += 1 if self.l >= self.block.height: self.block.detach(index=0, _notify=False) self.block.detach(index=0, _notify=False) self.l -= 1 self.block.attach([Text(text), Newline()]) def callback(self, e): if e.type == 'resize': self.block.height = self.document.height self.block.width = self.document.width self.block._notify() self.write('%s: %s, %s' % (e.type, e.args, e.flags)) s = System() try: a = s.getdocument() b = TestView(a) s.start() except KeyboardInterrupt: s.cleanup() except: s.cleanup() pdb.post_mortem(sys.exc_traceback)
n_jobs=-1, max_iter=1000, ).fit(results.train[field]) train_acc = cluster_classify(results.train[field], results.train.label, 10, kmeans) valid_acc = cluster_classify(results.valid[field], results.valid.label, 10, kmeans) print('\t{}: train_acc={:.04f}, valid_acc={:.04f}'.format( field, train_acc, valid_acc)) checkpoint_folder = osp.dirname(FLAGS.snapshot) figure_filename = osp.join(checkpoint_folder, FLAGS.tsne_figure_name) print('Savign TSNE plot at "{}"'.format(figure_filename)) make_tsne_plot(valid_results.posterior_pres, valid_results.label, figure_filename) if __name__ == '__main__': try: logging.set_verbosity(logging.INFO) tf.app.run() except Exception as err: # pylint: disable=broad-except FLAGS = flags.FLAGS last_traceback = sys.exc_info()[2] traceback.print_tb(last_traceback) print(err) pdb.post_mortem(last_traceback)
def run_commandline(self, argv=None) -> Optional[Run]: """ Run the command-line interface of this experiment. If ``argv`` is omitted it defaults to ``sys.argv``. Parameters ---------- argv Command-line as string or list of strings like ``sys.argv``. Returns ------- The Run object corresponding to the finished run. """ argv = ensure_wellformed_argv(argv) short_usage, usage, internal_usage = self.get_usage() args = docopt(internal_usage, [str(a) for a in argv[1:]], help=False) cmd_name = args.get("COMMAND") or self.default_command config_updates, named_configs = get_config_updates(args["UPDATE"]) err = self._check_command(cmd_name) if not args["help"] and err: print(short_usage) print(err) sys.exit(1) if self._handle_help(args, usage): sys.exit() try: return self.run( cmd_name, config_updates, named_configs, info={}, meta_info={}, options=args, ) except Exception as e: if self.current_run: debug = self.current_run.debug else: # The usual command line options are applied after the run # object is built completely. Some exceptions (e.g. # ConfigAddedError) are raised before this. In these cases, # the debug flag must be checked manually. debug = args.get("--debug", False) if debug: # Debug: Don't change behavior, just re-raise exception raise elif self.current_run and self.current_run.pdb: # Print exception and attach pdb debugger import traceback import pdb traceback.print_exception(*sys.exc_info()) pdb.post_mortem() else: # Handle pretty printing of exceptions. This includes # filtering the stacktrace and printing the usage, as # specified by the exceptions attributes if isinstance(e, SacredError): print(format_sacred_error(e, short_usage), file=sys.stderr) else: print_filtered_stacktrace() sys.exit(1)
def Control(self): """Handle POSTS.""" if not master.MASTER_WATCHER.IsMaster(): # We shouldn't be getting requests from the client unless we # are the active instance. stats.STATS.IncrementCounter("frontend_inactive_request_count", fields=["http"]) logging.info("Request sent to inactive frontend from %s", self.client_address[0]) # Get the api version try: api_version = int(cgi.parse_qs(self.path.split("?")[1])["api"][0]) except (ValueError, KeyError, IndexError): # The oldest api version we support if not specified. api_version = 3 with GRRHTTPServerHandler.active_counter_lock: GRRHTTPServerHandler.active_counter += 1 stats.STATS.SetGaugeValue("frontend_active_count", self.active_counter, fields=["http"]) try: length = int(self.headers.getheader("content-length")) request_comms = rdf_flows.ClientCommunication( self._GetPOSTData(length)) # If the client did not supply the version in the protobuf we use the get # parameter. if not request_comms.api_version: request_comms.api_version = api_version # Reply using the same version we were requested with. responses_comms = rdf_flows.ClientCommunication( api_version=request_comms.api_version) source_ip = ipaddr.IPAddress(self.client_address[0]) if source_ip.version == 6: source_ip = source_ip.ipv4_mapped or source_ip request_comms.orig_request = rdf_flows.HttpRequest( raw_headers=utils.SmartStr(self.headers), source_ip=utils.SmartStr(source_ip)) source, nr_messages = self.server.frontend.HandleMessageBundles( request_comms, responses_comms) logging.info( "HTTP request from %s (%s), %d bytes - %d messages received," " %d messages sent.", source, utils.SmartStr(source_ip), length, nr_messages, responses_comms.num_messages) self.Send(responses_comms.SerializeToString()) except communicator.UnknownClientCert: # "406 Not Acceptable: The server can only generate a response that is not # accepted by the client". This is because we can not encrypt for the # client appropriately. self.Send("Enrollment required", status=406) except Exception as e: if flags.FLAGS.debug: pdb.post_mortem() logging.error("Had to respond with status 500: %s.", e) self.Send("Error", status=500) finally: with GRRHTTPServerHandler.active_counter_lock: GRRHTTPServerHandler.active_counter -= 1 stats.STATS.SetGaugeValue("frontend_active_count", self.active_counter, fields=["http"])
def addFailure(self, test, err): traceback.print_exception(*err) pdb.post_mortem(err[2]) super(DebugTestResult, self).addFailure(test, err)
def run_smoketests(raiden_service: RaidenService, test_config: Dict, debug: bool = False): """ Test that the assembled raiden_service correctly reflects the configuration from the smoketest_genesis. """ try: chain = raiden_service.chain assert ( raiden_service.default_registry.address == to_canonical_address( test_config['contracts']['registry_address'])) assert (raiden_service.default_secret_registry.address == to_canonical_address( test_config['contracts']['secret_registry_address'])) token_network_added_events = raiden_service.default_registry.filter_token_added_events( ) token_addresses = [ event['args']['token_address'] for event in token_network_added_events ] assert token_addresses == [test_config['contracts']['token_address']] if test_config.get('transport') == 'udp': assert len(chain.address_to_discovery.keys()) == 1, repr( chain.address_to_discovery) assert (list( chain.address_to_discovery.keys())[0] == to_canonical_address( test_config['contracts']['discovery_address'])) discovery = list(chain.address_to_discovery.values())[0] assert discovery.endpoint_by_address( raiden_service.address) != TEST_ENDPOINT token_networks = views.get_token_network_addresses_for( views.state_from_raiden(raiden_service), raiden_service.default_registry.address, ) assert len(token_networks) == 1 channel_state = views.get_channelstate_for( views.state_from_raiden(raiden_service), raiden_service.default_registry.address, token_networks[0], unhexlify(TEST_PARTNER_ADDRESS), ) distributable = channel.get_distributable( channel_state.our_state, channel_state.partner_state, ) assert distributable == TEST_DEPOSIT_AMOUNT assert distributable == channel_state.our_state.contract_balance assert channel.get_status(channel_state) == CHANNEL_STATE_OPENED # Run API test run_restapi_smoketests() except Exception: error = traceback.format_exc() if debug: import pdb pdb.post_mortem() return error
def main(argv=None): if argv is None: argv = sys.argv[1:] class ArgumentParserWithDefaults(argparse.ArgumentParser): ''' From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help ''' def add_argument(self, *args, help=None, default=None, **kwargs): if help is not None: kwargs['help'] = help if default is not None and args[0] != '-h': kwargs['default'] = default if help is not None: kwargs['help'] += ' (default: {})'.format(default) super().add_argument(*args, **kwargs) parser = ArgumentParserWithDefaults( formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("--debug", dest="debug", help="Enable interactive debugger on error", action='store_true') parser.add_argument("-c", "--chart_output", dest="chart_output", help="Chart output directory", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory", required=True) parser.add_argument("-s", "--sim-output", dest="sim_output", help="Sim output directory", required=True) parser.add_argument("-w", "--window-size", dest="window_size", help="Minutes over which to collect data", default=3, type=int) parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) if 'multiprocessing' in sys.modules: import multiprocessing_logging multiprocessing_logging.install_mp_handler() if args.debug: import pdb, traceback try: return main_method(args) except: extype, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb) else: return main_method(args)
def main(): parameters = OrderedDict() try: optparser = optparse.OptionParser() optparser.add_option("-w", "--win", default="50", type='int', help="Window size") optparser.add_option("-b", "--batch", default="10", type='int', help="Batch size") optparser.add_option("-s", "--hidden_size", default="100", type='int', help="lstm hidden size") optparser.add_option("-n", "--num_layers", default="1", type='int', help="Number of encoder-decoder layers") optparser.add_option("-l", "--lr", default="0.001", type='float', help="Learning rate") optparser.add_option("-e", "--epoch", default="1", type='int', help="Number of epochs") optparser.add_option("-t", "--dir_training", default="../train", type='string', help="Training data directory") optparser.add_option("-v", "--dir_test", default="../test", type='string', help="Test data directory") optparser.add_option("-p", "--helper", default="th", help="Training helper") optparser.add_option("-d", "--dropout", default="0.5", type='float', help="Keep prob dropout, if 1 no dropout") optparser.add_option( "-o", "--overlap", default="1", type='int', help= "the overlap that we need to slid the window; it should be less than window size" ) opts = optparser.parse_args()[0] parameters['win'] = opts.win parameters['batch'] = opts.batch parameters['num_layers'] = opts.num_layers parameters['hidden_size'] = opts.hidden_size parameters['lr'] = opts.lr parameters['epoch'] = opts.epoch parameters['dir_training'] = opts.dir_training parameters['dir_test'] = opts.dir_test parameters['helper'] = opts.helper parameters['dropout'] = opts.dropout parameters['overlap'] = opts.overlap model = GGNNModel(parameters) model.train() # except: typ, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb)
def debug_excepthook(exc_type, exc, exc_tb): import pdb loop = asyncio.get_event_loop() loop.stop() pdb.post_mortem(exc_tb) sys.__excepthook__(exc_type, exc, exc_tb)
def tk_exception_handler(self, exc, val, tb): original_exception_handler(self, exc, val, tb) import pdb pdb.post_mortem()
def Main(arguments=None): """Start the tool.""" multiprocessing.freeze_support() front_end = PsortFrontend() arg_parser = argparse.ArgumentParser( description=(u'PSORT - Application to read, filter and process ' u'output from a plaso storage file.'), add_help=False) tool_group = arg_parser.add_argument_group('Optional Arguments For Psort') output_group = arg_parser.add_argument_group( 'Optional Arguments For Output Modules') analysis_group = arg_parser.add_argument_group( 'Optional Arguments For Analysis Modules') tool_group.add_argument('-d', '--debug', action='store_true', dest='debug', default=False, help='Fall back to debug shell if psort fails.') tool_group.add_argument( '-q', '--quiet', action='store_true', dest='quiet', default=False, help='Don\'t print out counter information after processing.') tool_group.add_argument('-h', '--help', action='help', help='Show this help message and exit.') tool_group.add_argument( '-a', '--include_all', action='store_false', dest='dedup', default=True, help=( 'By default the tool removes duplicate entries from the output. ' 'This parameter changes that behavior so all events are included.' )) tool_group.add_argument( '-o', '--output_format', '--output-format', metavar='FORMAT', dest='output_format', default='dynamic', help=('The output format or "-o list" to see a list of available ' 'output formats.')) tool_group.add_argument( '--analysis', metavar='PLUGIN_LIST', dest='analysis_plugins', default='', action='store', type=unicode, help=('A comma separated list of analysis plugin names to be loaded ' 'or "--analysis list" to see a list of available plugins.')) tool_group.add_argument( '-z', '--zone', metavar='TIMEZONE', default='UTC', dest='timezone', help= ('The timezone of the output or "-z list" to see a list of available ' 'timezones.')) tool_group.add_argument('-w', '--write', metavar='OUTPUTFILE', dest='write', help='Output filename. Defaults to stdout.') tool_group.add_argument( '--slice', metavar='DATE', dest='slice', type=str, default='', action='store', help= ('Create a time slice around a certain date. This parameter, if ' 'defined will display all events that happened X minutes before and ' 'after the defined date. X is controlled by the parameter ' '--slice_size but defaults to 5 minutes.')) tool_group.add_argument( '--slicer', dest='slicer', action='store_true', default=False, help= ('Create a time slice around every filter match. This parameter, if ' 'defined will save all X events before and after a filter match has ' 'been made. X is defined by the --slice_size parameter.')) tool_group.add_argument( '--slice_size', dest='slice_size', type=int, default=5, action='store', help=( 'Defines the slice size. In the case of a regular time slice it ' 'defines the number of minutes the slice size should be. In the ' 'case of the --slicer it determines the number of events before ' 'and after a filter match has been made that will be included in ' 'the result set. The default value is 5]. See --slice or --slicer ' 'for more details about this option.')) tool_group.add_argument( '-v', '--version', dest='version', action='version', version='log2timeline - psort version {0:s}'.format( plaso.GetVersion()), help='Show the current version of psort.') front_end.AddStorageFileOptions(tool_group) tool_group.add_argument( 'filter', nargs='?', action='store', metavar='FILTER', default=None, type=unicode, help=('A filter that can be used to filter the dataset before it ' 'is written into storage. More information about the filters' ' and it\'s usage can be found here: http://plaso.kiddaland.' 'net/usage/filters')) if arguments is None: arguments = sys.argv[1:] # Add the output module options. if '-o' in arguments: argument_index = arguments.index('-o') + 1 elif '--output_format' in arguments: argument_index = arguments.index('--output_format') + 1 elif '--output-format' in arguments: argument_index = arguments.index('--output-format') + 1 else: argument_index = 0 if argument_index > 0: module_names = arguments[argument_index] front_end.AddOutputModuleOptions(output_group, [module_names]) # Add the analysis plugin options. if '--analysis' in arguments: argument_index = arguments.index('--analysis') + 1 # Get the names of the analysis plugins that should be loaded. plugin_names = arguments[argument_index] try: front_end.AddAnalysisPluginOptions(analysis_group, plugin_names) except errors.BadConfigOption as exception: arg_parser.print_help() print u'' logging.error('{0:s}'.format(exception)) return False options = arg_parser.parse_args(args=arguments) format_str = '[%(levelname)s] %(message)s' if getattr(options, 'debug', False): logging.basicConfig(level=logging.DEBUG, format=format_str) else: logging.basicConfig(level=logging.INFO, format=format_str) if options.timezone == 'list': front_end.ListTimeZones() return True if options.analysis_plugins == 'list': front_end.ListAnalysisPlugins() return True if options.output_format == 'list': front_end.ListOutputModules() return True try: front_end.ParseOptions(options) except errors.BadConfigOption as exception: arg_parser.print_help() print u'' logging.error(u'{0:s}'.format(exception)) return False if front_end.preferred_encoding == 'ascii': logging.warning( u'The preferred encoding of your system is ASCII, which is not optimal ' u'for the typically non-ASCII characters that need to be parsed and ' u'processed. The tool will most likely crash and die, perhaps in a way ' u'that may not be recoverable. A five second delay is introduced to ' u'give you time to cancel the runtime and reconfigure your preferred ' u'encoding, otherwise continue at own risk.') time.sleep(5) try: counter = front_end.ParseStorage(options) if not options.quiet: logging.info(frontend_utils.FormatHeader('Counter')) for element, count in counter.most_common(): logging.info(frontend_utils.FormatOutputString(element, count)) except IOError as exception: # Piping results to "|head" for instance causes an IOError. if u'Broken pipe' not in exception: logging.error( u'Processing stopped early: {0:s}.'.format(exception)) except KeyboardInterrupt: pass # Catching every remaining exception in case we are debugging. except Exception as exception: if not options.debug: raise logging.error(u'{0:s}'.format(exception)) pdb.post_mortem() return True
def main(args: Sequence[str] = sys.argv[1:]) -> int: """ This is the console_scripts entry point for pydoctor CLI. @param args: Command line arguments to run the CLI. """ options, args = parse_args(args) exitcode = 0 if options.configfile: readConfigFile(options) cache = prepareCache(clearCache=options.clear_intersphinx_cache, enableCache=options.enable_intersphinx_cache, cachePath=options.intersphinx_cache_path, maxAge=options.intersphinx_cache_max_age) try: # step 1: make/find the system if options.systemclass: systemclass = findClassFromDottedName(options.systemclass, '--system-class', model.System) else: systemclass = zopeinterface.ZopeInterfaceSystem system = systemclass(options) system.fetchIntersphinxInventories(cache) if options.htmlsourcebase: if options.projectbasedirectory is None: error("you must specify --project-base-dir " "when using --html-viewsource-base") system.sourcebase = options.htmlsourcebase # step 1.5: check that we're actually going to accomplish something here args = list(args) + options.modules + options.packages if options.makehtml == MAKE_HTML_DEFAULT: if not options.testing and not options.makeintersphinx: options.makehtml = True else: options.makehtml = False # Support source date epoch: # https://reproducible-builds.org/specs/source-date-epoch/ try: system.buildtime = datetime.datetime.utcfromtimestamp( int(os.environ['SOURCE_DATE_EPOCH'])) except ValueError as e: error(str(e)) except KeyError: pass if options.buildtime: try: system.buildtime = datetime.datetime.strptime( options.buildtime, BUILDTIME_FORMAT) except ValueError as e: error(str(e)) # step 2: add any packages and modules if args: prependedpackage = None if options.prependedpackage: for m in options.prependedpackage.split('.'): prependedpackage = system.Package(system, m, prependedpackage) system.addObject(prependedpackage) initmodule = system.Module(system, '__init__', prependedpackage) system.addObject(initmodule) added_paths = set() for arg in args: path = resolve_path(arg) if path in added_paths: continue if options.projectbasedirectory is not None: # Note: Path.is_relative_to() was only added in Python 3.9, # so we have to use this workaround for now. try: path.relative_to(options.projectbasedirectory) except ValueError as ex: error(f"Source path lies outside base directory: {ex}") if path.is_dir(): system.msg('addPackage', f"adding directory {path}") system.addPackage(str(path), prependedpackage) elif path.is_file(): system.msg('addModuleFromPath', f"adding module {path}") system.addModuleFromPath(prependedpackage, str(path)) elif path.exists(): error(f"Source path is neither file nor directory: {path}") else: error(f"Source path does not exist: {path}") added_paths.add(path) else: error("No source paths given.") # step 3: move the system to the desired state if system.options.projectname is None: name = '/'.join(ro.name for ro in system.rootobjects) system.msg('warning', f"Guessing '{name}' for project name.", thresh=0) system.projectname = name else: system.projectname = system.options.projectname system.process() # step 4: make html, if desired if options.makehtml: options.makeintersphinx = True from pydoctor import templatewriter if options.htmlwriter: writerclass = findClassFromDottedName( options.htmlwriter, '--html-writer', templatewriter.TemplateWriter) else: writerclass = templatewriter.TemplateWriter system.msg( 'html', 'writing html to %s using %s.%s' % (options.htmloutput, writerclass.__module__, writerclass.__name__)) writer = writerclass(options.htmloutput) writer.prepOutputDirectory() if options.htmlsubjects: subjects = [] for fn in options.htmlsubjects: subjects.append(system.allobjects[fn]) elif options.htmlsummarypages: writer.writeModuleIndex(system) subjects = [] else: writer.writeModuleIndex(system) subjects = system.rootobjects writer.writeIndividualFiles(subjects) if system.docstring_syntax_errors: def p(msg: str) -> None: system.msg('docstring-summary', msg, thresh=-1, topthresh=1) p("these %s objects' docstrings contain syntax errors:" % (len(system.docstring_syntax_errors), )) exitcode = 2 for fn in sorted(system.docstring_syntax_errors): p(' ' + fn) if system.violations and options.warnings_as_errors: # Update exit code if the run has produced warnings. exitcode = 3 if options.makeintersphinx: if not options.makehtml: subjects = system.rootobjects # Generate Sphinx inventory. sphinx_inventory = SphinxInventoryWriter( logger=system.msg, project_name=system.projectname, project_version=system.options.projectversion, ) if not os.path.exists(options.htmloutput): os.makedirs(options.htmloutput) sphinx_inventory.generate( subjects=subjects, basepath=options.htmloutput, ) except: if options.pdb: import pdb pdb.post_mortem(sys.exc_info()[2]) raise return exitcode
def _score_test(self, test: Test, outcome: Outcome) -> Tuple[bool, bool]: """ Given a test and the test's outcome, determine if the test met expectations and log pertinent information """ # Helper for logging result def _result_was(): result_was = ("{} (result was {})".format( test.__qualname__, type(result).__qualname__)) return result_was # scoring outcomes result_pass = True sim_failed = False try: outcome.get() except Exception as e: result = remove_traceback_frames(e, ['_score_test', 'get']) else: result = TestSuccess() if (isinstance(result, TestSuccess) and not test.expect_fail and not test.expect_error): self.log.info("Test Passed: %s" % test.__qualname__) elif (isinstance(result, AssertionError) and test.expect_fail): self.log.info("Test failed as expected: " + _result_was()) elif (isinstance(result, TestSuccess) and test.expect_error): self.log.error("Test passed but we expected an error: " + _result_was()) result_pass = False elif isinstance(result, TestSuccess): self.log.error("Test passed but we expected a failure: " + _result_was()) result_pass = False elif isinstance(result, SimFailure): if isinstance(result, test.expect_error): self.log.info("Test errored as expected: " + _result_was()) else: self.log.error( "Test error has lead to simulator shutting us " "down", exc_info=result) result_pass = False # whether we expected it or not, the simulation has failed unrecoverably sim_failed = True elif test.expect_error: if isinstance(result, test.expect_error): self.log.info("Test errored as expected: " + _result_was()) else: self.log.error("Test errored with unexpected type: " + _result_was(), exc_info=result) result_pass = False else: self.log.error("Test Failed: " + _result_was(), exc_info=result) result_pass = False if _pdb_on_exception: pdb.post_mortem(result.__traceback__) return result_pass, sim_failed
def run(self, cmd_args): """entry point for all commands :param cmd_args: list of string arguments from command line :param extra_config: dict of extra argument values (by argument name) This is parameter is only used by explicit API call. return codes: 0: tasks executed successfully 1: one or more tasks failed 2: error while executing a task 3: error before task execution starts, in this case the Reporter is not used. So be aware if you expect a different formatting (like JSON) from the Reporter. """ # get list of available commands sub_cmds = self.get_cmds() # special parameters that dont run anything if cmd_args: if cmd_args[0] == "--version": self.print_version() return 0 if cmd_args[0] == "--help": Help.print_usage(sub_cmds.to_dict()) return 0 # get "global vars" from cmd-line args = self.process_args(cmd_args) # get specified sub-command or use default='run' if len(args) == 0 or args[0] not in sub_cmds: specified_run = False cmd_name = 'run' else: specified_run = True cmd_name = args.pop(0) # execute command command = sub_cmds.get_plugin(cmd_name)( task_loader=self.task_loader, cmds=sub_cmds, config=self.config, ) try: return command.parse_execute(args) # dont show traceback for user errors. except (CmdParseError, InvalidDodoFile, InvalidCommand, InvalidTask) as err: if isinstance(err, InvalidCommand): err.cmd_used = cmd_name if specified_run else None err.bin_name = self.BIN_NAME sys.stderr.write("ERROR: %s\n" % str(err)) return 3 except Exception: if command.pdb: # pragma: no cover import pdb pdb.post_mortem(sys.exc_info()[2]) sys.stderr.write(traceback.format_exc()) return 3
def exception_hook(self, exctype, excvalue, tb): """Handle uncaught python exceptions. It'll try very hard to write all open tabs to a file, and then exit gracefully. """ exc = (exctype, excvalue, tb) qapp = QApplication.instance() if not self._quitter.quit_status['crash']: log.misc.error( "ARGH, there was an exception while the crash " "dialog is already shown:", exc_info=exc) return log.misc.error("Uncaught exception", exc_info=exc) is_ignored_exception = (exctype is bdb.BdbQuit or not issubclass(exctype, Exception)) if self._args.pdb_postmortem: pdb.post_mortem(tb) if is_ignored_exception or self._args.pdb_postmortem: # pdb exit, KeyboardInterrupt, ... status = 0 if is_ignored_exception else 2 try: self._quitter.shutdown(status) return except Exception: log.init.exception("Error while shutting down") qapp.quit() return self._quitter.quit_status['crash'] = False info = self._get_exception_info() try: objreg.get('ipc-server').ignored = True except Exception: log.destroy.exception("Error while ignoring ipc") try: self._app.lastWindowClosed.disconnect( self._quitter.on_last_window_closed) except TypeError: log.destroy.exception("Error while preventing shutdown") global is_crashing is_crashing = True self._app.closeAllWindows() if self._args.no_err_windows: crashdialog.dump_exception_info(exc, info.pages, info.cmd_history, info.objects) else: self._crash_dialog = crashdialog.ExceptionCrashDialog( self._args.debug, info.pages, info.cmd_history, exc, info.objects) ret = self._crash_dialog.exec_() if ret == QDialog.Accepted: # restore self._quitter.restart(info.pages) # We might risk a segfault here, but that's better than continuing to # run in some undefined state, so we only do the most needed shutdown # here. qInstallMessageHandler(None) self.destroy_crashlogfile() sys.exit(usertypes.Exit.exception)