def check_grads(model):
assert np.isfinite(model.l_alpha())
assert np.isfinite(model.l_valpha())
x = model.grad_l_vmu()
assert np.isfinite(x).all()
import pdb
try:
# Main update rules
log.info('xi update:', check_grad(model, 'xi', model.l_xi, model.grad_l_xi))
log.info('valpha update:', check_grad(model, 'valpha', model.l_valpha, model.grad_l_valpha))
log.info('alpha update:', check_grad(model, 'alpha', model.l_alpha, model.grad_l_alpha))
log.info('vmu update:', check_grad(model, 'vmu', model.l_vmu, model.tangent_grad_l_vmu))
f = lambda: avk(model.V, model.xi)
g = lambda: deriv_avk(model.V, model.xi)
log.info('avk_xi', check_grad(model, 'xi', f, g))
f = lambda: np.sum(model.e_squared_norm_batch())
g = lambda: np.sum(model.grad_e_squared_norm_xi())
log.info('grad_esn_xi', check_grad(model, 'xi', f, g))
f = lambda: np.sum(model.rho_batch())
g = lambda: np.sum(model.deriv_rho_xi())
log.info('deriv_rho_xi', check_grad(model, 'xi', f, g))
except Exception, e:
log.error(e)
pdb.post_mortem()
def runDebug(self, exc_info):
if flags.can_touch_runtime_system("switch console") and self._intf_tty_num != 1:
iutil.vtActivate(1)
iutil.eintr_retry_call(os.open, "/dev/console", os.O_RDWR) # reclaim stdin
iutil.eintr_ignore(os.dup2, 0, 1) # reclaim stdout
iutil.eintr_ignore(os.dup2, 0, 2) # reclaim stderr
# ^
# |
# +------ dup2 is magic, I tells ya!
# bring back the echo
import termios
si = sys.stdin.fileno()
attr = termios.tcgetattr(si)
attr[3] = attr[3] & termios.ECHO
termios.tcsetattr(si, termios.TCSADRAIN, attr)
print("\nEntering debugger...")
print("Use 'continue' command to quit the debugger and get back to " "the main window")
import pdb
pdb.post_mortem(exc_info.stack)
if flags.can_touch_runtime_system("switch console") and self._intf_tty_num != 1:
iutil.vtActivate(self._intf_tty_num)
def dispatch(self, service_name, method, params):
try:
auth = getattr(self, 'auth_provider', None)
logger = logging.getLogger('result')
start_time = end_time = 0
if logger.isEnabledFor(logging.DEBUG_RPC_ANSWER):
self.log('service', tuple(replace_request_password(params)), depth=None, fn='%s.%s'%(service_name,method))
if logger.isEnabledFor(logging.DEBUG_RPC):
start_time = time.time()
result = ExportService.getService(service_name).dispatch(method, auth, params)
if logger.isEnabledFor(logging.DEBUG_RPC):
end_time = time.time()
if not logger.isEnabledFor(logging.DEBUG_RPC_ANSWER):
self.log('service (%.3fs)' % (end_time - start_time), tuple(replace_request_password(params)), depth=1, fn='%s.%s'%(service_name,method))
self.log('execution time', '%.3fs' % (end_time - start_time), channel=logging.DEBUG_RPC_ANSWER)
self.log('result', result, channel=logging.DEBUG_RPC_ANSWER)
return result
except Exception, e:
self.log('exception', tools.exception_to_unicode(e))
tb = getattr(e, 'traceback', sys.exc_info())
tb_s = "".join(traceback.format_exception(*tb))
if tools.config['debug_mode'] and isinstance(tb[2], types.TracebackType):
import pdb
pdb.post_mortem(tb[2])
raise OpenERPDispatcherException(e, tb_s)
def execution_loop(tasks, options, tasks_executed, tasks_loaded):
from time import sleep
logging.info('Execute start (%s tasks)' % len(tasks))
while tasks:
upnext = [] # tasks that can be run
for i in xrange(options.execute_nr_wait_cycles):
max_cannot_run = min(len(tasks), 128)
for i in xrange(max_cannot_run):
# The argument for this is the following:
# if T' is dependent on the result of T, it is better if the
# processor that ran T, also runs T'. By having everyone else
# push T' to the end of tasks, this is more likely to happen.
#
# Furthermore, this avoids always querying the same tasks.
if tasks[0].can_run():
break
tasks.append(tasks.pop(0))
while tasks and tasks[0].can_run():
upnext.append(tasks.pop(0))
if upnext:
break
logging.info('waiting 12 secs for an open task...')
sleep(options.execute_wait_cycle_time_secs)
if not upnext:
logging.info('No tasks can be run!')
break
for t in upnext:
if t.can_load():
logging.info('Loadable %s...' % t.name)
tasks_loaded[t.name] += 1
continue
locked = False
try:
locked = t.lock()
if t.can_load(): # This can be true if the task ran between the check above and this one
logging.info('Loadable %s...' % t.name)
tasks_loaded[t.name] += 1
elif locked:
logging.info('Executing %s...' % t.name)
t.run()
tasks_executed[t.name] += 1
if options.aggressive_unload:
t.unload_recursive()
else:
logging.info('Already in execution %s...' % t.name)
except Exception, e:
if options.pdb:
import pdb, sys
_,_, tb = sys.exc_info()
pdb.post_mortem(tb)
else:
import itertools
logging.critical('Exception while running %s: %s' % (t.name,e))
for other in itertools.chain(upnext, tasks):
for dep in other.dependencies():
if dep is t:
logging.critical('Other tasks are dependent on this one! Parallel processors will be held waiting!')
raise
finally:
def Run(self):
with test_lib.FakeTime(42, increment=1):
hunt_urn = self.StartHunt(
description="the hunt",
output_plugins=[
output_plugin.OutputPluginDescriptor(
plugin_name=standard_test.FailingDummyHuntOutputPlugin.
__name__)
])
self.client_ids = self.SetupClients(2)
for index, client_id in enumerate(self.client_ids):
self.AssignTasksToClients(client_ids=[client_id])
self.RunHunt(failrate=-1)
with test_lib.FakeTime(100042 + index * 100):
try:
self.ProcessHuntOutputPlugins()
except process_results.ResultsProcessingError:
if flags.FLAGS.debug:
pdb.post_mortem()
self.Check(
"GET",
"/api/hunts/%s/output-plugins/"
"FailingDummyHuntOutputPlugin_0/errors" % hunt_urn.Basename(),
replace={hunt_urn.Basename(): "H:123456"})
def main(argv=sys.argv, out=sys.stdout):
setup_log_ = functools.partial(setup_log,
str_formatter='%(asctime)s [%(levelname)s] %(message)s')
try:
return pbparser_runner(
argv=argv[1:],
parser=get_parser(),
args_runner_func=args_runner,
contract_runner_func=resolved_tool_contract_runner,
alog=logging.getLogger(__name__),
setup_log_func=setup_log_)
# FIXME is there a more central place to deal with this?
except Exception as e:
type, value, tb = sys.exc_info()
traceback.print_exc(file=sys.stderr)
# Note: if kt.args.usePdb
# This won't work. If an exception is raised in parseArgs,
# then kt.args is not defined yet.
if '--pdb' in argv:
try:
# this has better integration with ipython and is nicer
# pip install ipdb
import ipdb
ipdb.post_mortem(tb)
except ImportError:
import pdb
pdb.post_mortem(tb)
else:
# exit non-zero
raise
def __call__(self, environ, start_response):
try:
return self.app(environ, start_response)
except:
traceback.print_exception(*sys.exc_info())
pdb.post_mortem(sys.exc_info()[2])
raise
def handle_pdb(event):
import pdb
pdb.post_mortem(tb)
# This level of interest seems to indicate the user might
# want to debug this error if it occurs again.
if (filename, line_number) in previously_seen_error_locations:
previously_seen_error_locations.remove((filename, line_number))
def pdb_postmortem():
try:
yield
except Exception:
type, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
def main_wrapper(module, args):
"""This wraps the main method of an action so that:
- backtrace is not printed by default
- backtrace is printed is --backtrace was given
- a pdb session is run if --pdb was given
"""
try:
module.do(args)
except Exception as e:
if args.pdb:
traceback = sys.exc_info()[2]
print ""
print "### Exception:", e
print "### Starting a debugger"
try:
#pylint: disable-msg=F0401
import ipdb
ipdb.post_mortem(traceback)
sys.exit(0)
except ImportError:
import pdb
pdb.post_mortem(traceback)
sys.exit(0)
if args.backtrace:
raise
message = str(e)
if message.endswith("\n"):
message = message[:-1]
ui.error(e.__class__.__name__, message)
sys.exit(2)
def GuessAddressSpace(self, base_as=None, **kwargs):
"""Loads an address space by stacking valid ASes on top of each other
(priority order first).
"""
base_as = base_as
error = addrspace.AddrSpaceError()
address_spaces = sorted(addrspace.BaseAddressSpace.classes.values(),
key=lambda x: x.order)
while 1:
logging.debug("Voting round")
found = False
for cls in address_spaces:
# Only try address spaces which claim to support images.
if not cls.metadata("image"):
continue
logging.debug("Trying %s ", cls)
try:
base_as = cls(base=base_as, session=self.session,
**kwargs)
logging.debug("Succeeded instantiating %s", base_as)
found = True
break
except (AssertionError, addrspace.ASAssertionError, IOError), e:
logging.debug("Failed instantiating %s: %s",
cls.__name__, e)
error.append_reason(cls.__name__, e)
continue
except Exception, e:
logging.error("Fatal Error: %s", e)
if self.session.debug:
pdb.post_mortem()
raise
def run(self):
"""Main thread for processing messages."""
self.OnStartup()
# As long as our output queue has some room we can process some
# input messages:
while True:
message = self._in_queue.get()
# A message of None is our terminal message.
if message is None:
break
try:
self.HandleMessage(message)
# Catch any errors and keep going here
except Exception as e: # pylint: disable=broad-except
logging.warn("%s", e)
self.SendReply(
rdf_flows.GrrStats(
status=rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR,
error_message=utils.SmartUnicode(e)),
request_id=message.request_id,
response_id=message.response_id,
session_id=message.session_id,
task_id=message.task_id,
message_type=rdf_flows.GrrMessage.Type.STATUS)
if flags.FLAGS.debug:
pdb.post_mortem()
def main():
parser = setup_parser()
argcomplete.autocomplete(parser)
options = parser.parse_args()
_setup_logger(options)
# Support the deprecated -c option
if getattr(options, 'config', None) is not None:
options.configs.append(options.config)
if options.subparser in ('report', 'logs', 'metrics', 'run'):
_default_region(options)
_default_account_id(options)
try:
command = options.command
if not callable(command):
command = getattr(
importlib.import_module(command.rsplit('.', 1)[0]),
command.rsplit('.', 1)[-1])
# Set the process name to something cleaner
process_name = [os.path.basename(sys.argv[0])]
process_name.extend(sys.argv[1:])
setproctitle(' '.join(process_name))
command(options)
except Exception:
if not options.debug:
raise
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def shell(self):
while self.active:
try:
raw = raw_input('pmr2cli> ')
if not raw:
continue
rawargs = raw.split(None, 1)
command = rawargs.pop(0)
obj = getattr(self, 'do_' + command, None)
if callable(obj):
obj(*rawargs)
else:
print("Invalid command, try 'help'.")
except EOFError:
self.active = False
print('')
except KeyboardInterrupt:
print('\nGot interrupt signal.')
self.active = False
except ValueError:
print("Couldn't decode json.")
# print("Status was %d") % self.last_response.status_code
print("Use console to check `self.last_response` for details.")
except:
print(traceback.format_exc())
if self.debug:
pdb.post_mortem()
def addFailure(self, test, err):
super(ExceptionTestResultMixin, self).addFailure(test, err)
exctype, value, tb = err
self.stream.writeln()
self.stream.writeln(self.separator1)
self.stream.writeln(">>> %s" % (self.getDescription(test)))
self.stream.writeln(self.separator2)
self.stream.writeln(self._exc_info_to_string(err, test).rstrip())
self.stream.writeln(self.separator1)
self.stream.writeln()
## Skip test runner traceback levels
#while tb and self._is_relevant_tb_level(tb):
# tb = tb.tb_next
# Really hacky way to jump up a couple of frames.
# I'm sure it's not that difficult to do properly,
# but I havn't figured out how.
#p = pdb.Pdb()
#p.reset()
#p.setup(None, tb)
#p.do_up(None)
#p.do_up(None)
#p.cmdloop()
# It would be good if we could make sure we're in the correct frame here
pdb.post_mortem(tb)
def reset_index():
try:
main()
except:
import pdb, traceback, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def run(main, argv=None):
"""Begins executing the program.
Args:
main: The main function to execute. It takes an single argument "argv",
which is a list of command line arguments with parsed flags removed.
argv: A non-empty list of the command line arguments including program name,
sys.argv is used if None.
- Parses command line flags with the flag module.
- If there are any errors, prints usage().
- Calls main() with the remaining arguments.
- If main() raises a UsageError, prints usage and the error message.
"""
try:
argv = _run_init(sys.argv if argv is None else argv)
try:
_run_main(main, argv)
except UsageError as error:
usage(shorthelp=True, detailed_error=error, exitcode=error.exitcode)
except:
if FLAGS.pdb_post_mortem:
traceback.print_exc()
pdb.post_mortem()
raise
except Exception as e:
_call_exception_handlers(e)
raise
def debugger_hook(exc, value, tb):
if (not hasattr(sys.stderr, "isatty") or
not sys.stderr.isatty() or exc in (SyntaxError, IndentationError, KeyboardInterrupt)):
sys.__excepthook__(exc, value, tb)
else:
import pdb
pdb.post_mortem(tb)
def main(argv=None):
# New user interactive session (with extra bells and whistles).
user_session = session.InteractiveSession()
user_session.session_list.append(user_session)
# Alow all special plugins to run.
user_session.privileged = True
def global_arg_cb(global_flags, _):
if global_flags.version:
print("This is Rekall Version %s (%s)" % (
constants.VERSION, constants.CODENAME))
print(rekall.get_versions())
sys.exit(0)
with user_session.GetRenderer().start():
plugin_cls, flags = args.parse_args(
argv=argv, global_arg_cb=global_arg_cb,
user_session=user_session)
# Install any quotas the user requested.
user_session = quotas.wrap_session(user_session)
try:
# Run the plugin with plugin specific args.
user_session.RunPlugin(plugin_cls, **config.RemoveGlobalOptions(flags))
except Exception as e:
logging.fatal("%s. Try --debug for more information." % e)
if getattr(flags, "debug", None):
pdb.post_mortem(sys.exc_info()[2])
raise
finally:
user_session.Flush()
def __init__(self, base=None, filename=None, **kwargs):
self.as_assert(base == None, 'Must be first Address Space')
super(Win32FileAddressSpace, self).__init__(**kwargs)
path = filename or (self.session and self.session.GetParameter(
"filename"))
self.as_assert(path, "Filename must be specified in session (e.g. "
"session.GetParameter('filename', 'MyFile.raw').")
self.fname = path
self.fhandle = win32file.CreateFile(
path,
win32file.GENERIC_READ | win32file.GENERIC_WRITE,
win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE,
None,
win32file.OPEN_EXISTING,
win32file.FILE_ATTRIBUTE_NORMAL,
None)
# Try to get the memory runs from the winpmem driver.
try:
self.ParseMemoryRuns()
except Exception:
if self.session.debug:
import pdb
pdb.post_mortem()
self.runs.insert((0, 0, win32file.GetFileSize(self.fhandle)))
# IO on windows is extremely slow so we are better off using a
# cache.
self.cache = utils.FastStore(1000)
def __call__(self):
if '-D' in sys.argv:
sys.argv.remove('-D')
self.debug = True
try:
if len(sys.argv)==1:
self.parser.print_help()
elif len(sys.argv)==2 and sys.argv[1].startswith('-'):
self.options, self.args = self.parser.parse_args()
else:
command_name = sys.argv[1]
command = self.get_command(command_name)
if command:
command(self)()
else:
self.parser.print_help()
except SystemExit:
pass
except KeyboardInterrupt:
print 'Keyboard interrupt'
except:
if self.debug:
exc_info = sys.exc_info()
traceback.print_exception(*exc_info)
sys.stderr.write('\nStarting pdb:\n')
pdb.post_mortem(exc_info[2])
else:
raise
def test_A_constraints():
import pdb, traceback, sys
try:
dims = [4, 8]
N_rand = 10
eps = 1e-5
tol = 1e-3
np.set_printoptions(precision=3)
for dim in dims:
block_dim = int(dim/4)
# Generate initial data
D = np.eye(block_dim)
Dinv = np.linalg.inv(D)
Q = 0.5*np.eye(block_dim)
mu = np.ones((block_dim, 1))
As, bs, Cs, ds, Fs, gradFs, Gs, gradGs = \
A_constraints(block_dim, D, Dinv, Q, mu)
tol = 1e-3
eps = 1e-4
N_rand = 10
for (g, gradg) in zip(Gs, gradGs):
for i in range(N_rand):
X = np.random.rand(dim, dim)
val = g(X)
grad = gradg(X)
print "grad:\n", grad
num_grad = numerical_derivative(g, X, eps)
print "num_grad:\n", num_grad
assert np.sum(np.abs(grad - num_grad)) < tol
except:
type, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
def main():
""" Process NMEC with precompiled denoiser and Fielnet
"""
def denoising_func(x):
orig_shape = x.shape
x = 1. - x/255.
x = denoiser.predict(x, verbose=0)
x = x.reshape(orig_shape)
return x
try:
featext = load_verbatimnet('fc7', paramsfile='/fileserver/iam/iam-processed/models/fiel_657.hdf5')
featext.compile(loss='mse', optimizer='sgd')
featext_func = lambda x: featext.predict(x, verbose=0.0)
print "Making the denoiser"
denoiser = conv4p_model()
denoiser.load_weights('/fileserver/iam/models/conv4p_linet56-iambin-tifs.hdf5')
hdf5file='/fileserver/nmec-handwriting/flat_nmec_bin_uint8.hdf5'
with h5py.File(hdf5file, "r") as data_file:
features = extract_features_for_corpus(data_file,featext_func,shingle_dim=(56,56),transform=denoising_func)
with h5py.File("output_features.hdf5", "w") as feature_file:
for document_id, document_features in features.iteritems():
feature_file.create_dataset(document_id, data=document_features)
except Exception as e:
print e
pdb.post_mortem()
def pdb(self): # pragma: no cover
'''
Run a pdb post-mortem on any pending exception information
'''
import pdb
if self.next_exc:
pdb.post_mortem(self.next_exc.__traceback__)
def __exit__(self, type_, value, traceback_):
host = socket.gethostname()
if isinstance(value, Exception):
error = type_.__name__
temp = '{host} encountered {error}: {value} in {task}'
event = temp.format(host=host, error=error, value=value,
task=self.name)
info = []
# traceback
tb_items = traceback.format_tb(traceback_)
tb_items.append("%s: %s\n" % (error, value))
tb_str = '\n'.join(tb_items)
info.append(tb_str)
# object info
if self.crash_info_func:
info.extend(self.crash_info_func())
self.send(event, info)
# drop into pdb
if self.debug:
traceback.print_exc()
pdb.post_mortem(traceback_)
else:
event = '{host} finished {task}'.format(host=host, task=self.name)
self.send(event)
def destroy_tile(x, y):
# If the old tile was a multitile head or pointer
if (g.map[x][y] and (type(g.map[x][y]) == MultiTileHead or
type(g.map[x][y]) == MultiTilePointer)):
# Get the destroy value
if type(g.map[x][y]) == MultiTileHead:
destroy_value = c.IMAGES[g.map[x][y].type].destroy
multi_tile = {"x": x, "y": y,
"width": g.map[x][y].width,
"height": g.map[x][y].height}
else:
target_x, target_y = g.map[x][y].target
destroy_value = c.IMAGES[g.map[target_x][target_y].type].destroy
try:
multi_tile = {"x": target_x, "y": target_y,
"width": g.map[target_x][target_y].width,
"height": g.map[target_x][target_y].height}
except:
import pdb, sys
e, m, tb = sys.exc_info()
pdb.post_mortem(tb)
for i in range(multi_tile["x"], multi_tile["x"] + multi_tile["width"]):
for j in range(multi_tile["y"], multi_tile["y"] + multi_tile["height"]):
if i == x and j == y:
continue
if len(destroy_value) > 2:
make_tile(destroy_value[2], i, j)
else:
make_tile(destroy_value[1], i, j)
make_tile(destroy_value[1], x, y)
else:
make_tile(c.IMAGES[g.map[x][y].type].destroy[1], x, y)
def post_mortem(exc_info):
err = exc_info[1]
if isinstance(err, (doctest.UnexpectedException, doctest.DocTestFailure)):
if isinstance(err, doctest.UnexpectedException):
exc_info = err.exc_info
# Print out location info if the error was in a doctest
if exc_info[2].tb_frame.f_code.co_filename == '<string>':
print_doctest_location(err)
else:
print_doctest_location(err)
# Hm, we have a DocTestFailure exception. We need to
# generate our own traceback
try:
exec ('raise ValueError'
'("Expected and actual output are different")'
) in err.test.globs
except:
exc_info = sys.exc_info()
print "%s:" % (exc_info[0], )
print exc_info[1]
pdb.post_mortem(exc_info[2])
raise zope.testing.testrunner.interfaces.EndRun()
def test_Q_constraints():
import pdb, traceback, sys
try:
dims = [4, 8]
N_rand = 10
eps = 1e-4
tol = 1e-3
for dim in dims:
block_dim = int(dim/4)
# Generate initial data
D = np.eye(block_dim)
Dinv = np.linalg.inv(D)
B = np.eye(block_dim)
A = 0.5 * np.eye(block_dim)
c = 0.5
As, bs, Cs, ds, Fs, gradFs, Gs, gradGs = \
Q_constraints(block_dim, A, B, D, c)
N_rand = 10
for (g, gradg) in zip(Gs, gradGs):
for i in range(N_rand):
X = np.random.rand(dim, dim)
val = g(X)
grad = gradg(X)
print "grad:\n", grad
num_grad = numerical_derivative(g, X, eps)
print "num_grad:\n", num_grad
assert np.sum(np.abs(grad - num_grad)) < tol
except:
type, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
def add(self, pipe, descriptor):
"""
store a rendered content object on the filesystem.
"""
# sometimes we want to have content in the uri db
# but we don't actually want to store it... either because
# of an error during rendering or based on configuration.
if descriptor.isGhost():
return
descriptors = descriptor.getContainedDescriptors()
structure = self.getStructure( pipe )
for descriptor in descriptors:
if descriptor.isGhost(): # if a child view errors then it has no content
continue
content_path = structure.getContentPathFromDescriptor( descriptor )
#if content_path.endswith(sep):
#log.warning('invalid content path detected %s ... fixing'%content_path)
# content_path = content_path[:-1]
try:
self.storeDescriptor( pipe, content_path, descriptor )
except:
if not DEPLOYMENT_DEBUG:
raise
import pdb, sys, traceback
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
return True
def runner(files, test_filter, debug):
runner = ImmediateTestRunner(verbosity=VERBOSE, debug=debug,
progress=progress)
suite = unittest.TestSuite()
for file in files:
s = get_suite(file)
# See if the levels match
dolevel = (level == 0) or level >= getattr(s, "level", 0)
if s is not None and dolevel:
s = filter_testcases(s, test_filter)
suite.addTest(s)
try:
r = runner.run(suite)
if timesfn:
r.print_times(open(timesfn, "w"))
if VERBOSE:
print ("Wrote timing data to", timesfn)
if timetests:
r.print_times(sys.stdout, timetests)
except:
if debugger:
print ("%s:" % (sys.exc_info()[0], ))
print (sys.exc_info()[1])
pdb.post_mortem(sys.exc_info()[2])
else:
raise
def unexpectedErrorAlertPdb():
import pdb
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[2])
return True
def RunOnce(self):
"""Processes one set of messages from Task Scheduler.
The worker processes new jobs from the task master. For each job
we retrieve the session from the Task Scheduler.
Returns:
Total number of messages processed by this call.
"""
start_time = time.time()
processed = 0
queue_manager = queue_manager_lib.QueueManager(token=self.token)
for queue in self.queues:
# Freezeing the timestamp used by queue manager to query/delete
# notifications to avoid possible race conditions.
queue_manager.FreezeTimestamp()
fetch_messages_start = time.time()
notifications = queue_manager.GetNotifications(queue)
stats_collector_instance.Get().RecordEvent(
"worker_time_to_retrieve_notifications",
time.time() - fetch_messages_start)
stuck_flows = []
for n in notifications:
if n.in_progress:
stuck_flows.append(n)
# Process stuck flows first
if stuck_flows:
self.ProcessStuckFlows(stuck_flows, queue_manager)
notifications_available = []
for notification in notifications:
# Filter out session ids we already tried to lock but failed.
if notification.session_id not in self.queued_flows:
notifications_available.append(notification)
try:
# If we spent too much time processing what we have so far, the
# active_sessions list might not be current. We therefore break here
# so we can re-fetch a more up to date version of the list, and try
# again later. The risk with running with an old active_sessions list
# is that another worker could have already processed this message,
# and when we try to process it, there is nothing to do - costing us a
# lot of processing time. This is a tradeoff between checking the data
# store for current information and processing out of date
# information.
processed += self.ProcessMessages(
notifications_available, queue_manager,
self.RUN_ONCE_MAX_SECONDS - (time.time() - start_time))
# We need to keep going no matter what.
except Exception as e: # pylint: disable=broad-except
logging.error("Error processing message %s. %s.", e,
traceback.format_exc())
stats_collector_instance.Get().IncrementCounter("grr_worker_exceptions")
if flags.FLAGS.pdb_post_mortem:
pdb.post_mortem()
queue_manager.UnfreezeTimestamp()
# If we have spent too much time, stop.
if (time.time() - start_time) > self.RUN_ONCE_MAX_SECONDS:
return processed
return processed
total_time = time.time() - start_time
stats.STATS.RecordEvent("ui_renderer_latency",
total_time,
fields=[renderer_name])
except access_control.UnauthorizedAccess, e:
result = http.HttpResponse(content_type="text/html")
result = renderers.Renderer.GetPlugin("UnauthorizedRenderer")().Layout(
request, result, exception=e)
except Exception:
stats.STATS.IncrementCounter("ui_renderer_failure",
fields=[renderer_name])
if flags.FLAGS.debug:
pdb.post_mortem()
raise
if not isinstance(result, http.HttpResponse):
raise RuntimeError("Renderer returned invalid response %r" % result)
return result
def RedirectToRemoteHelp(path):
"""Redirect to GitHub-hosted documentation."""
target_path = os.path.join(
config_lib.CONFIG["AdminUI.github_docs_location"],
path.replace(".html", ".adoc"))
log.info("Cloning repository: %s", repo_uri)
repo = pygit2.clone_repository(repo_uri, temp_dir.path)
else:
repo = pygit2.Repository(repo_uri)
load_available()
policy_repo = PolicyRepo(repo_uri, repo, matcher)
change_count = 0
with contextlib.closing(transport(stream_uri, assume)) as t:
if after is None and isinstance(t, IndexedTransport):
after = t.last()
for change in policy_repo.delta_stream(after=after, before=before):
change_count += 1
t.send(change)
log.info("Streamed %d policy repo changes", change_count)
return change_count
if __name__ == '__main__':
try:
cli()
except SystemExit:
raise
except KeyboardInterrupt:
raise
except: # NOQA
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def debug(self):
if self.debugger:
import pdb
tb = self.exc_info[2]
pdb.post_mortem(tb)
args = parser.parse_args()
receiver = Receiver(args)
while True:
# receive data
packet_type = receiver.recv_data()
# if fin break
if packet_type == "fin":
break
# wait for a while
time.sleep(5)
receiver.send_fin()
receiver.recv_finack()
# flush packet
receiver.file += b''.join(receiver.buf_list)
print("flush")
# write to file
receiver.write_file(args.file_path,
os.path.splitext(receiver.filename)[-1])
return 0
if __name__ == '__main__':
import sys, traceback, pdb
try:
main()
except:
type, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
def waf_entry_point(current_directory, version, wafdir):
Logs.init_log()
if Context.WAFVERSION != version:
Logs.error('Waf script %r and library %r do not match (directory %r)',
version, Context.WAFVERSION, wafdir)
sys.exit(1)
Context.waf_dir = wafdir
Context.run_dir = Context.launch_dir = current_directory
start_dir = current_directory
no_climb = os.environ.get('NOCLIMB')
if len(sys.argv) > 1:
potential_wscript = os.path.join(current_directory, sys.argv[1])
if os.path.basename(
potential_wscript) == Context.WSCRIPT_FILE and os.path.isfile(
potential_wscript):
path = os.path.normpath(os.path.dirname(potential_wscript))
start_dir = os.path.abspath(path)
no_climb = True
sys.argv.pop(1)
ctx = Context.create_context('options')
(options, commands, env) = ctx.parse_cmd_args(allow_unknown=True)
if options.top:
start_dir = Context.run_dir = Context.top_dir = options.top
no_climb = True
if options.out:
Context.out_dir = options.out
if not no_climb:
for k in no_climb_commands:
for y in commands:
if y.startswith(k):
no_climb = True
break
cur = start_dir
while cur:
try:
lst = os.listdir(cur)
except OSError:
lst = []
Logs.error('Directory %r is unreadable!', cur)
if Options.lockfile in lst:
env = ConfigSet.ConfigSet()
try:
env.load(os.path.join(cur, Options.lockfile))
ino = os.stat(cur)[stat.ST_INO]
except EnvironmentError:
pass
else:
for x in (env.run_dir, env.top_dir, env.out_dir):
if not x:
continue
if Utils.is_win32:
if cur == x:
load = True
break
else:
try:
ino2 = os.stat(x)[stat.ST_INO]
except OSError:
pass
else:
if ino == ino2:
load = True
break
else:
Logs.warn('invalid lock file in %s', cur)
load = False
if load:
Context.run_dir = env.run_dir
Context.top_dir = env.top_dir
Context.out_dir = env.out_dir
break
if not Context.run_dir:
if Context.WSCRIPT_FILE in lst:
Context.run_dir = cur
next = os.path.dirname(cur)
if next == cur:
break
cur = next
if no_climb:
break
if not Context.run_dir:
if options.whelp:
Logs.warn(
'These are the generic options (no wscript/project found)')
ctx.parser.print_help()
sys.exit(0)
Logs.error(
'Waf: Run from a folder containing a %r file (or try -h for the generic options)',
Context.WSCRIPT_FILE)
sys.exit(1)
try:
os.chdir(Context.run_dir)
except OSError:
Logs.error('Waf: The folder %r is unreadable', Context.run_dir)
sys.exit(1)
try:
set_main_module(
os.path.normpath(
os.path.join(Context.run_dir, Context.WSCRIPT_FILE)))
except Errors.WafError as e:
Logs.pprint('RED', e.verbose_msg)
Logs.error(str(e))
sys.exit(1)
except Exception as e:
Logs.error('Waf: The wscript in %r is unreadable', Context.run_dir)
traceback.print_exc(file=sys.stdout)
sys.exit(2)
if options.profile:
import cProfile, pstats
cProfile.runctx(
'from waflib import Scripting; Scripting.run_commands()', {}, {},
'profi.txt')
p = pstats.Stats('profi.txt')
p.sort_stats('time').print_stats(75)
else:
try:
try:
run_commands()
except:
if options.pdb:
import pdb
type, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
else:
raise
except Errors.WafError as e:
if Logs.verbose > 1:
Logs.pprint('RED', e.verbose_msg)
Logs.error(e.msg)
sys.exit(1)
except SystemExit:
raise
except Exception as e:
traceback.print_exc(file=sys.stdout)
sys.exit(2)
except KeyboardInterrupt:
Logs.pprint('RED', 'Interrupted')
sys.exit(68)
with open(cmdfname, 'rb+') as f:
mm = mmap.mmap(f.fileno(), cmdfsize, mmap.MAP_SHARED,
mmap.PROT_READ | mmap.PROT_WRITE)
cmdent = cmdfsize // sizeof(zhpe.xdm_cmd)
cmds = zhpe.xdm_cmd * cmdent
cmd = cmds.from_buffer(mm, 0)
for i in range(0, cmdent):
print('{:#x} {}'.format(i, cmd[i]))
# end with
with open(cmpfname, 'rb+') as f:
mm = mmap.mmap(f.fileno(), cmdfsize, mmap.MAP_SHARED,
mmap.PROT_READ | mmap.PROT_WRITE)
cmpent = cmpfsize // sizeof(zhpe.xdm_cmpl)
cmps = zhpe.xdm_cmpl * cmpent
cmp = cmps.from_buffer(mm, 0)
for i in range(0, cmpent):
print('{:#x} {}'.format(i, cmp[i]))
# end with
if __name__ == '__main__':
try:
main()
except:
if args.post_mortem:
post_mortem()
else:
raise
def update_test(t):
# the update-eval tests refer to graphs on http://example.org
rdflib_sparql_module.SPARQL_LOAD_GRAPHS = False
uri, name, comment, data, graphdata, query, res, syntax = t
if uri in skiptests:
raise SkipTest()
try:
g = Dataset()
if not res:
if syntax:
with bopen(query[7:]) as f:
translateUpdate(parseUpdate(f))
else:
try:
with bopen(query[7:]) as f:
translateUpdate(parseUpdate(f))
raise AssertionError("Query shouldn't have parsed!")
except:
pass # negative syntax test
return
resdata, resgraphdata = res
# read input graphs
if data:
g.default_context.load(data, format=_fmt(data))
if graphdata:
for x, l in graphdata:
g.load(x, publicID=URIRef(l), format=_fmt(x))
with bopen(query[7:]) as f:
req = translateUpdate(parseUpdate(f))
evalUpdate(g, req)
# read expected results
resg = Dataset()
if resdata:
resg.default_context.load(resdata, format=_fmt(resdata))
if resgraphdata:
for x, l in resgraphdata:
resg.load(x, publicID=URIRef(l), format=_fmt(x))
eq(
set(x.identifier for x in g.contexts() if x != g.default_context),
set(x.identifier for x in resg.contexts()
if x != resg.default_context),
"named graphs in datasets do not match",
)
assert isomorphic(
g.default_context,
resg.default_context), "Default graphs are not isomorphic"
for x in g.contexts():
if x == g.default_context:
continue
assert isomorphic(x, resg.get_context(
x.identifier)), ("Graphs with ID %s are not isomorphic" %
x.identifier)
except Exception as e:
if isinstance(e, AssertionError):
failed_tests.append(uri)
fails[str(e)] += 1
else:
error_tests.append(uri)
errors[str(e)] += 1
if DEBUG_ERROR and not isinstance(e, AssertionError) or DEBUG_FAIL:
print("======================================")
print(uri)
print(name)
print(comment)
if not res:
if syntax:
print("Positive syntax test")
else:
print("Negative syntax test")
if data:
print("----------------- DATA --------------------")
print(">>>", data)
print(bopen_read_close(data[7:]))
if graphdata:
print("----------------- GRAPHDATA --------------------")
for x, l in graphdata:
print(">>>", x, l)
print(bopen_read_close(x[7:]))
print("----------------- Request -------------------")
print(">>>", query)
print(bopen_read_close(query[7:]))
if res:
if resdata:
print("----------------- RES DATA --------------------")
print(">>>", resdata)
print(bopen_read_close(resdata[7:]))
if resgraphdata:
print(
"----------------- RES GRAPHDATA -------------------")
for x, l in resgraphdata:
print(">>>", x, l)
print(bopen_read_close(x[7:]))
print("------------- MY RESULT ----------")
print(g.serialize(format="trig"))
try:
pq = translateUpdate(parseUpdate(bopen_read_close(query[7:])))
print("----------------- Parsed ------------------")
pprintAlgebra(pq)
# print pq
except:
print("(parser error)")
print(decodeStringEscape(str(e)))
import pdb
pdb.post_mortem(sys.exc_info()[2])
raise
def report_internal_error(
err: Exception,
file: Optional[str],
line: int,
errors: Errors,
options: Options,
stdout: Optional[TextIO] = None,
stderr: Optional[TextIO] = None,
) -> None:
"""Report internal error and exit.
This optionally starts pdb or shows a traceback.
"""
stdout = (stdout or sys.stdout)
stderr = (stderr or sys.stderr)
# Dump out errors so far, they often provide a clue.
# But catch unexpected errors rendering them.
try:
for msg in errors.new_messages():
print(msg)
except Exception as e:
print("Failed to dump errors:", repr(e), file=stderr)
# Compute file:line prefix for official-looking error messages.
if file:
if line:
prefix = '{}:{}: '.format(file, line)
else:
prefix = '{}: '.format(file)
else:
prefix = ''
# Print "INTERNAL ERROR" message.
print(
'{}error: INTERNAL ERROR --'.format(prefix),
'Please try using mypy master on Github:\n'
'https://mypy.rtfd.io/en/latest/common_issues.html#using-a-development-mypy-build',
file=stderr)
if options.show_traceback:
print('Please report a bug at https://github.com/python/mypy/issues',
file=stderr)
else:
print(
'If this issue continues with mypy master, '
'please report a bug at https://github.com/python/mypy/issues',
file=stderr)
print('version: {}'.format(mypy_version), file=stderr)
# If requested, drop into pdb. This overrides show_tb.
if options.pdb:
print('Dropping into pdb', file=stderr)
import pdb
pdb.post_mortem(sys.exc_info()[2])
# If requested, print traceback, else print note explaining how to get one.
if options.raise_exceptions:
raise err
if not options.show_traceback:
if not options.pdb:
print('{}: note: please use --show-traceback to print a traceback '
'when reporting a bug'.format(prefix),
file=stderr)
else:
tb = traceback.extract_stack()[:-2]
tb2 = traceback.extract_tb(sys.exc_info()[2])
print('Traceback (most recent call last):')
for s in traceback.format_list(tb + tb2):
print(s.rstrip('\n'))
print('{}: {}'.format(type(err).__name__, err), file=stdout)
print('{}: note: use --pdb to drop into pdb'.format(prefix),
file=stderr)
# Exit. The caller has nothing more to say.
# We use exit code 2 to signal that this is no ordinary error.
raise SystemExit(2)
def exec_py_file(self,
filename,
args=None,
ins=None,
outs=None,
errs=None):
_, current_state = self.get_current_worker_and_state()
if ins:
current_state.sys_stdin = ins
if outs:
current_state.sys_stdout = outs
if errs:
current_state.sys_stderr = errs
file_path = os.path.relpath(filename)
namespace = dict(locals(), **globals())
namespace['__name__'] = '__main__'
namespace['__file__'] = os.path.abspath(file_path)
namespace['_stash'] = self.stash
saved_sys_argv = sys.argv[:]
# First argument is the script name
sys.argv = [os.path.basename(filename)] + (args or [])
# Set current os environ to the threading environ
saved_os_environ = os.environ
os.environ = dict(current_state.environ)
# Honor any leading vars, e.g. A=42 echo $A
os.environ.update(current_state.temporary_environ)
# This needs to be done after environ due to possible leading PYTHONPATH var
saved_sys_path = sys.path
sys.path = current_state.sys_path[:]
self.handle_PYTHONPATH() # Make sure PYTHONPATH is honored
try:
execfile(file_path, namespace, namespace)
current_state.return_value = 0
except SystemExit as e:
current_state.return_value = e.code
except Exception as e:
current_state.return_value = 1
etype, evalue, tb = sys.exc_info()
err_msg = '%s: %s\n' % (repr(etype), evalue)
if self.debug:
self.logger.debug(err_msg)
self.stash.write_message(err_msg)
if self.py_traceback or self.py_pdb:
import traceback
traceback.print_exception(etype, evalue, tb)
if self.py_pdb:
import pdb
pdb.post_mortem(tb)
finally:
# Thread specific vars are not modified, e.g. current_state.environ is unchanged.
# This means the vars cannot be changed inside a python script. It can only be
# done through shell command, e.g. NEW_VAR=42
sys.argv = saved_sys_argv
sys.path = saved_sys_path
os.environ = saved_os_environ
def query_test(t):
uri, name, comment, data, graphdata, query, resfile, syntax = t
# the query-eval tests refer to graphs to load by resolvable filenames
rdflib_sparql_module.SPARQL_LOAD_GRAPHS = True
if uri in skiptests:
raise SkipTest()
def skip(reason="(none)"):
print("Skipping %s from now on." % uri)
with bopen("skiptests.list", "a") as f:
f.write("%s\t%s\n" % (uri, reason))
try:
g = Dataset()
if data:
g.default_context.load(data, format=_fmt(data))
if graphdata:
for x in graphdata:
g.load(x, format=_fmt(x))
if not resfile:
# no result - syntax test
if syntax:
translateQuery(parseQuery(bopen_read_close(query[7:])),
base=urljoin(query, "."))
else:
# negative syntax test
try:
translateQuery(
parseQuery(bopen_read_close(query[7:])),
base=urljoin(query, "."),
)
assert False, "Query should not have parsed!"
except:
pass # it's fine - the query should not parse
return
# eval test - carry out query
res2 = g.query(bopen_read_close(query[7:]), base=urljoin(query, "."))
if resfile.endswith("ttl"):
resg = Graph()
resg.load(resfile, format="turtle", publicID=resfile)
res = RDFResultParser().parse(resg)
elif resfile.endswith("rdf"):
resg = Graph()
resg.load(resfile, publicID=resfile)
res = RDFResultParser().parse(resg)
else:
with bopen(resfile[7:]) as f:
if resfile.endswith("srj"):
res = Result.parse(f, format="json")
elif resfile.endswith("tsv"):
res = Result.parse(f, format="tsv")
elif resfile.endswith("csv"):
res = Result.parse(f, format="csv")
# CSV is lossy, round-trip our own resultset to
# lose the same info :)
# write bytes, read strings...
s = BytesIO()
res2.serialize(s, format="csv")
s.seek(0)
res2 = Result.parse(s, format="csv")
s.close()
else:
res = Result.parse(f, format="xml")
if not DETAILEDASSERT:
eq(res.type, res2.type, "Types do not match")
if res.type == "SELECT":
eq(set(res.vars), set(res2.vars), "Vars do not match")
comp = bindingsCompatible(set(res), set(res2))
assert comp, "Bindings do not match"
elif res.type == "ASK":
eq(res.askAnswer, res2.askAnswer, "Ask answer does not match")
elif res.type in ("DESCRIBE", "CONSTRUCT"):
assert isomorphic(res.graph,
res2.graph), "graphs are not isomorphic!"
else:
raise Exception("Unknown result type: %s" % res.type)
else:
eq(
res.type,
res2.type,
"Types do not match: %r != %r" % (res.type, res2.type),
)
if res.type == "SELECT":
eq(
set(res.vars),
set(res2.vars),
"Vars do not match: %r != %r" %
(set(res.vars), set(res2.vars)),
)
assert bindingsCompatible(set(res), set(res2)), (
"Bindings do not match: \nexpected:\n%s\n!=\ngot:\n%s" % (
res.serialize(format="txt",
namespace_manager=g.namespace_manager),
res2.serialize(format="txt",
namespace_manager=g.namespace_manager),
))
elif res.type == "ASK":
eq(
res.askAnswer,
res2.askAnswer,
"Ask answer does not match: %r != %r" %
(res.askAnswer, res2.askAnswer),
)
elif res.type in ("DESCRIBE", "CONSTRUCT"):
assert isomorphic(res.graph,
res2.graph), "graphs are not isomorphic!"
else:
raise Exception("Unknown result type: %s" % res.type)
except Exception as e:
if isinstance(e, AssertionError):
failed_tests.append(uri)
fails[str(e)] += 1
else:
error_tests.append(uri)
errors[str(e)] += 1
if DEBUG_ERROR and not isinstance(e, AssertionError) or DEBUG_FAIL:
print("======================================")
print(uri)
print(name)
print(comment)
if not resfile:
if syntax:
print("Positive syntax test")
else:
print("Negative syntax test")
if data:
print("----------------- DATA --------------------")
print(">>>", data)
print(bopen_read_close(data[7:]))
if graphdata:
print("----------------- GRAPHDATA --------------------")
for x in graphdata:
print(">>>", x)
print(bopen_read_close(x[7:]))
print("----------------- Query -------------------")
print(">>>", query)
print(bopen_read_close(query[7:]))
if resfile:
print("----------------- Res -------------------")
print(">>>", resfile)
print(bopen_read_close(resfile[7:]))
try:
pq = parseQuery(bopen_read_close(query[7:]))
print("----------------- Parsed ------------------")
pprintAlgebra(translateQuery(pq, base=urljoin(query, ".")))
except:
print("(parser error)")
print(decodeStringEscape(str(e)))
import pdb
pdb.post_mortem(sys.exc_info()[2])
# pdb.set_trace()
# nose.tools.set_trace()
raise
def drop_into_pdb(app, exception):
"""
https://gist.github.com/alonho/4389137
"""
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[2])
def Execute(self, message):
"""This function parses the RDFValue from the server.
The Run method will be called with the specified RDFValue.
Args:
message: The GrrMessage that we are called to process.
Returns:
Upon return a callback will be called on the server to register
the end of the function and pass back exceptions.
Raises:
RuntimeError: The arguments from the server do not match the expected
rdf type.
"""
self.message = message
if message:
self.require_fastpoll = message.require_fastpoll
args = None
try:
if self.message.args_rdf_name:
if not self.in_rdfvalue:
raise RuntimeError("Did not expect arguments, got %s." %
self.message.args_rdf_name)
if self.in_rdfvalue.__name__ != self.message.args_rdf_name:
raise RuntimeError("Unexpected arg type %s != %s." %
(self.message.args_rdf_name,
self.in_rdfvalue.__name__))
args = self.message.payload
# Only allow authenticated messages in the client
if self._authentication_required and (
self.message.auth_state !=
rdf_flows.GrrMessage.AuthorizationState.AUTHENTICATED):
raise RuntimeError("Message for %s was not Authenticated." %
self.message.name)
self.cpu_start = self.proc.cpu_times()
self.cpu_limit = self.message.cpu_limit
if getattr(flags.FLAGS, "debug_client_actions", False):
pdb.set_trace()
try:
self.Run(args)
# Ensure we always add CPU usage even if an exception occurred.
finally:
used = self.proc.cpu_times()
self.cpu_used = (used.user - self.cpu_start.user,
used.system - self.cpu_start.system)
except NetworkBytesExceededError as e:
self.SetStatus(
rdf_flows.GrrStatus.ReturnedStatus.NETWORK_LIMIT_EXCEEDED,
"%r: %s" % (e, e), traceback.format_exc())
# We want to report back all errors and map Python exceptions to
# Grr Errors.
except Exception as e: # pylint: disable=broad-except
self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.GENERIC_ERROR,
"%r: %s" % (e, e), traceback.format_exc())
if flags.FLAGS.pdb_post_mortem:
self.DisableNanny()
pdb.post_mortem()
if self.status.status != rdf_flows.GrrStatus.ReturnedStatus.OK:
logging.info("Job Error (%s): %s", self.__class__.__name__,
self.status.error_message)
if self.status.backtrace:
logging.debug(self.status.backtrace)
if self.cpu_used:
self.status.cpu_time_used.user_cpu_time = self.cpu_used[0]
self.status.cpu_time_used.system_cpu_time = self.cpu_used[1]
# This returns the error status of the Actions to the flow.
self.SendReply(self.status,
message_type=rdf_flows.GrrMessage.Type.STATUS)
self._RunGC()
def run_account(account, region, policies_config, output_path,
cache_period, cache_path, metrics, dryrun, debug):
"""Execute a set of policies on an account.
"""
logging.getLogger('custodian.output').setLevel(logging.ERROR + 1)
CONN_CACHE.session = None
CONN_CACHE.time = None
load_available()
# allow users to specify interpolated output paths
if '{' not in output_path:
output_path = os.path.join(output_path, account['name'], region)
cache_path = os.path.join(cache_path, "%s-%s.cache" % (account['account_id'], region))
config = Config.empty(
region=region, cache=cache_path,
cache_period=cache_period, dryrun=dryrun, output_dir=output_path,
account_id=account['account_id'], metrics_enabled=metrics,
log_group=None, profile=None, external_id=None)
env_vars = account_tags(account)
if account.get('role'):
if isinstance(account['role'], six.string_types):
config['assume_role'] = account['role']
config['external_id'] = account.get('external_id')
else:
env_vars.update(
_get_env_creds(get_session(account, 'custodian', region), region))
elif account.get('profile'):
config['profile'] = account['profile']
policies = PolicyCollection.from_data(policies_config, config)
policy_counts = {}
success = True
st = time.time()
with environ(**env_vars):
for p in policies:
# Extend policy execution conditions with account information
p.conditions.env_vars['account'] = account
# Variable expansion and non schema validation (not optional)
p.expand_variables(p.get_variables(account.get('vars', {})))
p.validate()
if p.region and p.region != region:
continue
log.debug(
"Running policy:%s account:%s region:%s",
p.name, account['name'], region)
try:
resources = p.run()
policy_counts[p.name] = resources and len(resources) or 0
if not resources:
continue
if not config.dryrun and p.execution_mode != 'pull':
log.info("Ran account:%s region:%s policy:%s provisioned time:%0.2f",
account['name'], region, p.name, time.time() - st)
continue
log.info(
"Ran account:%s region:%s policy:%s matched:%d time:%0.2f",
account['name'], region, p.name, len(resources),
time.time() - st)
except ClientError as e:
success = False
if e.response['Error']['Code'] == 'AccessDenied':
log.warning('Access denied api:%s policy:%s account:%s region:%s',
e.operation_name, p.name, account['name'], region)
return policy_counts, success
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
continue
except Exception as e:
success = False
log.error(
"Exception running policy:%s account:%s region:%s error:%s",
p.name, account['name'], region, e)
if not debug:
continue
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
raise
return policy_counts, success
self.document.attachevent(self.callback)
def write(self, text):
if len(text) >= self.block.width:
text = text[:self.block.width]
self.l += 1
if self.l >= self.block.height:
self.block.detach(index=0, _notify=False)
self.block.detach(index=0, _notify=False)
self.l -= 1
self.block.attach([Text(text), Newline()])
def callback(self, e):
if e.type == 'resize':
self.block.height = self.document.height
self.block.width = self.document.width
self.block._notify()
self.write('%s: %s, %s' % (e.type, e.args, e.flags))
s = System()
try:
a = s.getdocument()
b = TestView(a)
s.start()
except KeyboardInterrupt:
s.cleanup()
except:
s.cleanup()
pdb.post_mortem(sys.exc_traceback)
n_jobs=-1,
max_iter=1000,
).fit(results.train[field])
train_acc = cluster_classify(results.train[field], results.train.label,
10, kmeans)
valid_acc = cluster_classify(results.valid[field], results.valid.label,
10, kmeans)
print('\t{}: train_acc={:.04f}, valid_acc={:.04f}'.format(
field, train_acc, valid_acc))
checkpoint_folder = osp.dirname(FLAGS.snapshot)
figure_filename = osp.join(checkpoint_folder, FLAGS.tsne_figure_name)
print('Savign TSNE plot at "{}"'.format(figure_filename))
make_tsne_plot(valid_results.posterior_pres, valid_results.label,
figure_filename)
if __name__ == '__main__':
try:
logging.set_verbosity(logging.INFO)
tf.app.run()
except Exception as err: # pylint: disable=broad-except
FLAGS = flags.FLAGS
last_traceback = sys.exc_info()[2]
traceback.print_tb(last_traceback)
print(err)
pdb.post_mortem(last_traceback)
def run_commandline(self, argv=None) -> Optional[Run]:
"""
Run the command-line interface of this experiment.
If ``argv`` is omitted it defaults to ``sys.argv``.
Parameters
----------
argv
Command-line as string or list of strings like ``sys.argv``.
Returns
-------
The Run object corresponding to the finished run.
"""
argv = ensure_wellformed_argv(argv)
short_usage, usage, internal_usage = self.get_usage()
args = docopt(internal_usage, [str(a) for a in argv[1:]], help=False)
cmd_name = args.get("COMMAND") or self.default_command
config_updates, named_configs = get_config_updates(args["UPDATE"])
err = self._check_command(cmd_name)
if not args["help"] and err:
print(short_usage)
print(err)
sys.exit(1)
if self._handle_help(args, usage):
sys.exit()
try:
return self.run(
cmd_name,
config_updates,
named_configs,
info={},
meta_info={},
options=args,
)
except Exception as e:
if self.current_run:
debug = self.current_run.debug
else:
# The usual command line options are applied after the run
# object is built completely. Some exceptions (e.g.
# ConfigAddedError) are raised before this. In these cases,
# the debug flag must be checked manually.
debug = args.get("--debug", False)
if debug:
# Debug: Don't change behavior, just re-raise exception
raise
elif self.current_run and self.current_run.pdb:
# Print exception and attach pdb debugger
import traceback
import pdb
traceback.print_exception(*sys.exc_info())
pdb.post_mortem()
else:
# Handle pretty printing of exceptions. This includes
# filtering the stacktrace and printing the usage, as
# specified by the exceptions attributes
if isinstance(e, SacredError):
print(format_sacred_error(e, short_usage), file=sys.stderr)
else:
print_filtered_stacktrace()
sys.exit(1)
def Control(self):
"""Handle POSTS."""
if not master.MASTER_WATCHER.IsMaster():
# We shouldn't be getting requests from the client unless we
# are the active instance.
stats.STATS.IncrementCounter("frontend_inactive_request_count",
fields=["http"])
logging.info("Request sent to inactive frontend from %s",
self.client_address[0])
# Get the api version
try:
api_version = int(cgi.parse_qs(self.path.split("?")[1])["api"][0])
except (ValueError, KeyError, IndexError):
# The oldest api version we support if not specified.
api_version = 3
with GRRHTTPServerHandler.active_counter_lock:
GRRHTTPServerHandler.active_counter += 1
stats.STATS.SetGaugeValue("frontend_active_count",
self.active_counter,
fields=["http"])
try:
length = int(self.headers.getheader("content-length"))
request_comms = rdf_flows.ClientCommunication(
self._GetPOSTData(length))
# If the client did not supply the version in the protobuf we use the get
# parameter.
if not request_comms.api_version:
request_comms.api_version = api_version
# Reply using the same version we were requested with.
responses_comms = rdf_flows.ClientCommunication(
api_version=request_comms.api_version)
source_ip = ipaddr.IPAddress(self.client_address[0])
if source_ip.version == 6:
source_ip = source_ip.ipv4_mapped or source_ip
request_comms.orig_request = rdf_flows.HttpRequest(
raw_headers=utils.SmartStr(self.headers),
source_ip=utils.SmartStr(source_ip))
source, nr_messages = self.server.frontend.HandleMessageBundles(
request_comms, responses_comms)
logging.info(
"HTTP request from %s (%s), %d bytes - %d messages received,"
" %d messages sent.", source, utils.SmartStr(source_ip),
length, nr_messages, responses_comms.num_messages)
self.Send(responses_comms.SerializeToString())
except communicator.UnknownClientCert:
# "406 Not Acceptable: The server can only generate a response that is not
# accepted by the client". This is because we can not encrypt for the
# client appropriately.
self.Send("Enrollment required", status=406)
except Exception as e:
if flags.FLAGS.debug:
pdb.post_mortem()
logging.error("Had to respond with status 500: %s.", e)
self.Send("Error", status=500)
finally:
with GRRHTTPServerHandler.active_counter_lock:
GRRHTTPServerHandler.active_counter -= 1
stats.STATS.SetGaugeValue("frontend_active_count",
self.active_counter,
fields=["http"])
def addFailure(self, test, err):
traceback.print_exception(*err)
pdb.post_mortem(err[2])
super(DebugTestResult, self).addFailure(test, err)
def run_smoketests(raiden_service: RaidenService,
test_config: Dict,
debug: bool = False):
""" Test that the assembled raiden_service correctly reflects the configuration from the
smoketest_genesis. """
try:
chain = raiden_service.chain
assert (
raiden_service.default_registry.address == to_canonical_address(
test_config['contracts']['registry_address']))
assert (raiden_service.default_secret_registry.address ==
to_canonical_address(
test_config['contracts']['secret_registry_address']))
token_network_added_events = raiden_service.default_registry.filter_token_added_events(
)
token_addresses = [
event['args']['token_address']
for event in token_network_added_events
]
assert token_addresses == [test_config['contracts']['token_address']]
if test_config.get('transport') == 'udp':
assert len(chain.address_to_discovery.keys()) == 1, repr(
chain.address_to_discovery)
assert (list(
chain.address_to_discovery.keys())[0] == to_canonical_address(
test_config['contracts']['discovery_address']))
discovery = list(chain.address_to_discovery.values())[0]
assert discovery.endpoint_by_address(
raiden_service.address) != TEST_ENDPOINT
token_networks = views.get_token_network_addresses_for(
views.state_from_raiden(raiden_service),
raiden_service.default_registry.address,
)
assert len(token_networks) == 1
channel_state = views.get_channelstate_for(
views.state_from_raiden(raiden_service),
raiden_service.default_registry.address,
token_networks[0],
unhexlify(TEST_PARTNER_ADDRESS),
)
distributable = channel.get_distributable(
channel_state.our_state,
channel_state.partner_state,
)
assert distributable == TEST_DEPOSIT_AMOUNT
assert distributable == channel_state.our_state.contract_balance
assert channel.get_status(channel_state) == CHANNEL_STATE_OPENED
# Run API test
run_restapi_smoketests()
except Exception:
error = traceback.format_exc()
if debug:
import pdb
pdb.post_mortem()
return error
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
class ArgumentParserWithDefaults(argparse.ArgumentParser):
'''
From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help
'''
def add_argument(self, *args, help=None, default=None, **kwargs):
if help is not None:
kwargs['help'] = help
if default is not None and args[0] != '-h':
kwargs['default'] = default
if help is not None:
kwargs['help'] += ' (default: {})'.format(default)
super().add_argument(*args, **kwargs)
parser = ArgumentParserWithDefaults(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("--debug",
dest="debug",
help="Enable interactive debugger on error",
action='store_true')
parser.add_argument("-c",
"--chart_output",
dest="chart_output",
help="Chart output directory",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory",
required=True)
parser.add_argument("-s",
"--sim-output",
dest="sim_output",
help="Sim output directory",
required=True)
parser.add_argument("-w",
"--window-size",
dest="window_size",
help="Minutes over which to collect data",
default=3,
type=int)
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
if 'multiprocessing' in sys.modules:
import multiprocessing_logging
multiprocessing_logging.install_mp_handler()
if args.debug:
import pdb, traceback
try:
return main_method(args)
except:
extype, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
else:
return main_method(args)
def main():
parameters = OrderedDict()
try:
optparser = optparse.OptionParser()
optparser.add_option("-w",
"--win",
default="50",
type='int',
help="Window size")
optparser.add_option("-b",
"--batch",
default="10",
type='int',
help="Batch size")
optparser.add_option("-s",
"--hidden_size",
default="100",
type='int',
help="lstm hidden size")
optparser.add_option("-n",
"--num_layers",
default="1",
type='int',
help="Number of encoder-decoder layers")
optparser.add_option("-l",
"--lr",
default="0.001",
type='float',
help="Learning rate")
optparser.add_option("-e",
"--epoch",
default="1",
type='int',
help="Number of epochs")
optparser.add_option("-t",
"--dir_training",
default="../train",
type='string',
help="Training data directory")
optparser.add_option("-v",
"--dir_test",
default="../test",
type='string',
help="Test data directory")
optparser.add_option("-p",
"--helper",
default="th",
help="Training helper")
optparser.add_option("-d",
"--dropout",
default="0.5",
type='float',
help="Keep prob dropout, if 1 no dropout")
optparser.add_option(
"-o",
"--overlap",
default="1",
type='int',
help=
"the overlap that we need to slid the window; it should be less than window size"
)
opts = optparser.parse_args()[0]
parameters['win'] = opts.win
parameters['batch'] = opts.batch
parameters['num_layers'] = opts.num_layers
parameters['hidden_size'] = opts.hidden_size
parameters['lr'] = opts.lr
parameters['epoch'] = opts.epoch
parameters['dir_training'] = opts.dir_training
parameters['dir_test'] = opts.dir_test
parameters['helper'] = opts.helper
parameters['dropout'] = opts.dropout
parameters['overlap'] = opts.overlap
model = GGNNModel(parameters)
model.train() #
except:
typ, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
def debug_excepthook(exc_type, exc, exc_tb):
import pdb
loop = asyncio.get_event_loop()
loop.stop()
pdb.post_mortem(exc_tb)
sys.__excepthook__(exc_type, exc, exc_tb)
def tk_exception_handler(self, exc, val, tb):
original_exception_handler(self, exc, val, tb)
import pdb
pdb.post_mortem()
def Main(arguments=None):
"""Start the tool."""
multiprocessing.freeze_support()
front_end = PsortFrontend()
arg_parser = argparse.ArgumentParser(
description=(u'PSORT - Application to read, filter and process '
u'output from a plaso storage file.'),
add_help=False)
tool_group = arg_parser.add_argument_group('Optional Arguments For Psort')
output_group = arg_parser.add_argument_group(
'Optional Arguments For Output Modules')
analysis_group = arg_parser.add_argument_group(
'Optional Arguments For Analysis Modules')
tool_group.add_argument('-d',
'--debug',
action='store_true',
dest='debug',
default=False,
help='Fall back to debug shell if psort fails.')
tool_group.add_argument(
'-q',
'--quiet',
action='store_true',
dest='quiet',
default=False,
help='Don\'t print out counter information after processing.')
tool_group.add_argument('-h',
'--help',
action='help',
help='Show this help message and exit.')
tool_group.add_argument(
'-a',
'--include_all',
action='store_false',
dest='dedup',
default=True,
help=(
'By default the tool removes duplicate entries from the output. '
'This parameter changes that behavior so all events are included.'
))
tool_group.add_argument(
'-o',
'--output_format',
'--output-format',
metavar='FORMAT',
dest='output_format',
default='dynamic',
help=('The output format or "-o list" to see a list of available '
'output formats.'))
tool_group.add_argument(
'--analysis',
metavar='PLUGIN_LIST',
dest='analysis_plugins',
default='',
action='store',
type=unicode,
help=('A comma separated list of analysis plugin names to be loaded '
'or "--analysis list" to see a list of available plugins.'))
tool_group.add_argument(
'-z',
'--zone',
metavar='TIMEZONE',
default='UTC',
dest='timezone',
help=
('The timezone of the output or "-z list" to see a list of available '
'timezones.'))
tool_group.add_argument('-w',
'--write',
metavar='OUTPUTFILE',
dest='write',
help='Output filename. Defaults to stdout.')
tool_group.add_argument(
'--slice',
metavar='DATE',
dest='slice',
type=str,
default='',
action='store',
help=
('Create a time slice around a certain date. This parameter, if '
'defined will display all events that happened X minutes before and '
'after the defined date. X is controlled by the parameter '
'--slice_size but defaults to 5 minutes.'))
tool_group.add_argument(
'--slicer',
dest='slicer',
action='store_true',
default=False,
help=
('Create a time slice around every filter match. This parameter, if '
'defined will save all X events before and after a filter match has '
'been made. X is defined by the --slice_size parameter.'))
tool_group.add_argument(
'--slice_size',
dest='slice_size',
type=int,
default=5,
action='store',
help=(
'Defines the slice size. In the case of a regular time slice it '
'defines the number of minutes the slice size should be. In the '
'case of the --slicer it determines the number of events before '
'and after a filter match has been made that will be included in '
'the result set. The default value is 5]. See --slice or --slicer '
'for more details about this option.'))
tool_group.add_argument(
'-v',
'--version',
dest='version',
action='version',
version='log2timeline - psort version {0:s}'.format(
plaso.GetVersion()),
help='Show the current version of psort.')
front_end.AddStorageFileOptions(tool_group)
tool_group.add_argument(
'filter',
nargs='?',
action='store',
metavar='FILTER',
default=None,
type=unicode,
help=('A filter that can be used to filter the dataset before it '
'is written into storage. More information about the filters'
' and it\'s usage can be found here: http://plaso.kiddaland.'
'net/usage/filters'))
if arguments is None:
arguments = sys.argv[1:]
# Add the output module options.
if '-o' in arguments:
argument_index = arguments.index('-o') + 1
elif '--output_format' in arguments:
argument_index = arguments.index('--output_format') + 1
elif '--output-format' in arguments:
argument_index = arguments.index('--output-format') + 1
else:
argument_index = 0
if argument_index > 0:
module_names = arguments[argument_index]
front_end.AddOutputModuleOptions(output_group, [module_names])
# Add the analysis plugin options.
if '--analysis' in arguments:
argument_index = arguments.index('--analysis') + 1
# Get the names of the analysis plugins that should be loaded.
plugin_names = arguments[argument_index]
try:
front_end.AddAnalysisPluginOptions(analysis_group, plugin_names)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error('{0:s}'.format(exception))
return False
options = arg_parser.parse_args(args=arguments)
format_str = '[%(levelname)s] %(message)s'
if getattr(options, 'debug', False):
logging.basicConfig(level=logging.DEBUG, format=format_str)
else:
logging.basicConfig(level=logging.INFO, format=format_str)
if options.timezone == 'list':
front_end.ListTimeZones()
return True
if options.analysis_plugins == 'list':
front_end.ListAnalysisPlugins()
return True
if options.output_format == 'list':
front_end.ListOutputModules()
return True
try:
front_end.ParseOptions(options)
except errors.BadConfigOption as exception:
arg_parser.print_help()
print u''
logging.error(u'{0:s}'.format(exception))
return False
if front_end.preferred_encoding == 'ascii':
logging.warning(
u'The preferred encoding of your system is ASCII, which is not optimal '
u'for the typically non-ASCII characters that need to be parsed and '
u'processed. The tool will most likely crash and die, perhaps in a way '
u'that may not be recoverable. A five second delay is introduced to '
u'give you time to cancel the runtime and reconfigure your preferred '
u'encoding, otherwise continue at own risk.')
time.sleep(5)
try:
counter = front_end.ParseStorage(options)
if not options.quiet:
logging.info(frontend_utils.FormatHeader('Counter'))
for element, count in counter.most_common():
logging.info(frontend_utils.FormatOutputString(element, count))
except IOError as exception:
# Piping results to "|head" for instance causes an IOError.
if u'Broken pipe' not in exception:
logging.error(
u'Processing stopped early: {0:s}.'.format(exception))
except KeyboardInterrupt:
pass
# Catching every remaining exception in case we are debugging.
except Exception as exception:
if not options.debug:
raise
logging.error(u'{0:s}'.format(exception))
pdb.post_mortem()
return True
def main(args: Sequence[str] = sys.argv[1:]) -> int:
"""
This is the console_scripts entry point for pydoctor CLI.
@param args: Command line arguments to run the CLI.
"""
options, args = parse_args(args)
exitcode = 0
if options.configfile:
readConfigFile(options)
cache = prepareCache(clearCache=options.clear_intersphinx_cache,
enableCache=options.enable_intersphinx_cache,
cachePath=options.intersphinx_cache_path,
maxAge=options.intersphinx_cache_max_age)
try:
# step 1: make/find the system
if options.systemclass:
systemclass = findClassFromDottedName(options.systemclass,
'--system-class',
model.System)
else:
systemclass = zopeinterface.ZopeInterfaceSystem
system = systemclass(options)
system.fetchIntersphinxInventories(cache)
if options.htmlsourcebase:
if options.projectbasedirectory is None:
error("you must specify --project-base-dir "
"when using --html-viewsource-base")
system.sourcebase = options.htmlsourcebase
# step 1.5: check that we're actually going to accomplish something here
args = list(args) + options.modules + options.packages
if options.makehtml == MAKE_HTML_DEFAULT:
if not options.testing and not options.makeintersphinx:
options.makehtml = True
else:
options.makehtml = False
# Support source date epoch:
# https://reproducible-builds.org/specs/source-date-epoch/
try:
system.buildtime = datetime.datetime.utcfromtimestamp(
int(os.environ['SOURCE_DATE_EPOCH']))
except ValueError as e:
error(str(e))
except KeyError:
pass
if options.buildtime:
try:
system.buildtime = datetime.datetime.strptime(
options.buildtime, BUILDTIME_FORMAT)
except ValueError as e:
error(str(e))
# step 2: add any packages and modules
if args:
prependedpackage = None
if options.prependedpackage:
for m in options.prependedpackage.split('.'):
prependedpackage = system.Package(system, m,
prependedpackage)
system.addObject(prependedpackage)
initmodule = system.Module(system, '__init__',
prependedpackage)
system.addObject(initmodule)
added_paths = set()
for arg in args:
path = resolve_path(arg)
if path in added_paths:
continue
if options.projectbasedirectory is not None:
# Note: Path.is_relative_to() was only added in Python 3.9,
# so we have to use this workaround for now.
try:
path.relative_to(options.projectbasedirectory)
except ValueError as ex:
error(f"Source path lies outside base directory: {ex}")
if path.is_dir():
system.msg('addPackage', f"adding directory {path}")
system.addPackage(str(path), prependedpackage)
elif path.is_file():
system.msg('addModuleFromPath', f"adding module {path}")
system.addModuleFromPath(prependedpackage, str(path))
elif path.exists():
error(f"Source path is neither file nor directory: {path}")
else:
error(f"Source path does not exist: {path}")
added_paths.add(path)
else:
error("No source paths given.")
# step 3: move the system to the desired state
if system.options.projectname is None:
name = '/'.join(ro.name for ro in system.rootobjects)
system.msg('warning',
f"Guessing '{name}' for project name.",
thresh=0)
system.projectname = name
else:
system.projectname = system.options.projectname
system.process()
# step 4: make html, if desired
if options.makehtml:
options.makeintersphinx = True
from pydoctor import templatewriter
if options.htmlwriter:
writerclass = findClassFromDottedName(
options.htmlwriter, '--html-writer',
templatewriter.TemplateWriter)
else:
writerclass = templatewriter.TemplateWriter
system.msg(
'html', 'writing html to %s using %s.%s' %
(options.htmloutput, writerclass.__module__,
writerclass.__name__))
writer = writerclass(options.htmloutput)
writer.prepOutputDirectory()
if options.htmlsubjects:
subjects = []
for fn in options.htmlsubjects:
subjects.append(system.allobjects[fn])
elif options.htmlsummarypages:
writer.writeModuleIndex(system)
subjects = []
else:
writer.writeModuleIndex(system)
subjects = system.rootobjects
writer.writeIndividualFiles(subjects)
if system.docstring_syntax_errors:
def p(msg: str) -> None:
system.msg('docstring-summary',
msg,
thresh=-1,
topthresh=1)
p("these %s objects' docstrings contain syntax errors:" %
(len(system.docstring_syntax_errors), ))
exitcode = 2
for fn in sorted(system.docstring_syntax_errors):
p(' ' + fn)
if system.violations and options.warnings_as_errors:
# Update exit code if the run has produced warnings.
exitcode = 3
if options.makeintersphinx:
if not options.makehtml:
subjects = system.rootobjects
# Generate Sphinx inventory.
sphinx_inventory = SphinxInventoryWriter(
logger=system.msg,
project_name=system.projectname,
project_version=system.options.projectversion,
)
if not os.path.exists(options.htmloutput):
os.makedirs(options.htmloutput)
sphinx_inventory.generate(
subjects=subjects,
basepath=options.htmloutput,
)
except:
if options.pdb:
import pdb
pdb.post_mortem(sys.exc_info()[2])
raise
return exitcode
def _score_test(self, test: Test, outcome: Outcome) -> Tuple[bool, bool]:
"""
Given a test and the test's outcome, determine if the test met expectations and log pertinent information
"""
# Helper for logging result
def _result_was():
result_was = ("{} (result was {})".format(
test.__qualname__,
type(result).__qualname__))
return result_was
# scoring outcomes
result_pass = True
sim_failed = False
try:
outcome.get()
except Exception as e:
result = remove_traceback_frames(e, ['_score_test', 'get'])
else:
result = TestSuccess()
if (isinstance(result, TestSuccess) and not test.expect_fail
and not test.expect_error):
self.log.info("Test Passed: %s" % test.__qualname__)
elif (isinstance(result, AssertionError) and test.expect_fail):
self.log.info("Test failed as expected: " + _result_was())
elif (isinstance(result, TestSuccess) and test.expect_error):
self.log.error("Test passed but we expected an error: " +
_result_was())
result_pass = False
elif isinstance(result, TestSuccess):
self.log.error("Test passed but we expected a failure: " +
_result_was())
result_pass = False
elif isinstance(result, SimFailure):
if isinstance(result, test.expect_error):
self.log.info("Test errored as expected: " + _result_was())
else:
self.log.error(
"Test error has lead to simulator shutting us "
"down",
exc_info=result)
result_pass = False
# whether we expected it or not, the simulation has failed unrecoverably
sim_failed = True
elif test.expect_error:
if isinstance(result, test.expect_error):
self.log.info("Test errored as expected: " + _result_was())
else:
self.log.error("Test errored with unexpected type: " +
_result_was(),
exc_info=result)
result_pass = False
else:
self.log.error("Test Failed: " + _result_was(), exc_info=result)
result_pass = False
if _pdb_on_exception:
pdb.post_mortem(result.__traceback__)
return result_pass, sim_failed
def run(self, cmd_args):
"""entry point for all commands
:param cmd_args: list of string arguments from command line
:param extra_config: dict of extra argument values (by argument name)
This is parameter is only used by explicit API call.
return codes:
0: tasks executed successfully
1: one or more tasks failed
2: error while executing a task
3: error before task execution starts,
in this case the Reporter is not used.
So be aware if you expect a different formatting (like JSON)
from the Reporter.
"""
# get list of available commands
sub_cmds = self.get_cmds()
# special parameters that dont run anything
if cmd_args:
if cmd_args[0] == "--version":
self.print_version()
return 0
if cmd_args[0] == "--help":
Help.print_usage(sub_cmds.to_dict())
return 0
# get "global vars" from cmd-line
args = self.process_args(cmd_args)
# get specified sub-command or use default='run'
if len(args) == 0 or args[0] not in sub_cmds:
specified_run = False
cmd_name = 'run'
else:
specified_run = True
cmd_name = args.pop(0)
# execute command
command = sub_cmds.get_plugin(cmd_name)(
task_loader=self.task_loader,
cmds=sub_cmds,
config=self.config,
)
try:
return command.parse_execute(args)
# dont show traceback for user errors.
except (CmdParseError, InvalidDodoFile, InvalidCommand,
InvalidTask) as err:
if isinstance(err, InvalidCommand):
err.cmd_used = cmd_name if specified_run else None
err.bin_name = self.BIN_NAME
sys.stderr.write("ERROR: %s\n" % str(err))
return 3
except Exception:
if command.pdb: # pragma: no cover
import pdb
pdb.post_mortem(sys.exc_info()[2])
sys.stderr.write(traceback.format_exc())
return 3
def exception_hook(self, exctype, excvalue, tb):
"""Handle uncaught python exceptions.
It'll try very hard to write all open tabs to a file, and then exit
gracefully.
"""
exc = (exctype, excvalue, tb)
qapp = QApplication.instance()
if not self._quitter.quit_status['crash']:
log.misc.error(
"ARGH, there was an exception while the crash "
"dialog is already shown:",
exc_info=exc)
return
log.misc.error("Uncaught exception", exc_info=exc)
is_ignored_exception = (exctype is bdb.BdbQuit
or not issubclass(exctype, Exception))
if self._args.pdb_postmortem:
pdb.post_mortem(tb)
if is_ignored_exception or self._args.pdb_postmortem:
# pdb exit, KeyboardInterrupt, ...
status = 0 if is_ignored_exception else 2
try:
self._quitter.shutdown(status)
return
except Exception:
log.init.exception("Error while shutting down")
qapp.quit()
return
self._quitter.quit_status['crash'] = False
info = self._get_exception_info()
try:
objreg.get('ipc-server').ignored = True
except Exception:
log.destroy.exception("Error while ignoring ipc")
try:
self._app.lastWindowClosed.disconnect(
self._quitter.on_last_window_closed)
except TypeError:
log.destroy.exception("Error while preventing shutdown")
global is_crashing
is_crashing = True
self._app.closeAllWindows()
if self._args.no_err_windows:
crashdialog.dump_exception_info(exc, info.pages, info.cmd_history,
info.objects)
else:
self._crash_dialog = crashdialog.ExceptionCrashDialog(
self._args.debug, info.pages, info.cmd_history, exc,
info.objects)
ret = self._crash_dialog.exec_()
if ret == QDialog.Accepted: # restore
self._quitter.restart(info.pages)
# We might risk a segfault here, but that's better than continuing to
# run in some undefined state, so we only do the most needed shutdown
# here.
qInstallMessageHandler(None)
self.destroy_crashlogfile()
sys.exit(usertypes.Exit.exception)
