def main():
"""
Launch Zulip Terminal.
"""
args = parse_args()
if args.debug:
save_stdout()
if args.profile:
import cProfile
prof = cProfile.Profile()
prof.enable()
try:
Controller(args.config_file, args.theme).main()
except Exception:
# A unexpected exception occurred, open the debugger in debug mode
if args.debug:
import pudb
pudb.post_mortem()
finally:
if args.debug:
restore_stdout()
if args.profile:
prof.disable()
prof.dump_stats("/tmp/profile.data")
print("Profile data saved to /tmp/profile.data")
print("You can visualize it using e.g."
"`snakeviz /tmp/profile.data`")
print("\nThanks for using the Zulip-Terminal interface.\n")
sys.exit(1)
def handler(environ, start_response):
try:
return app(environ, start_response)
except:
import pudb
pudb.post_mortem()
raise
def handle_noargs(self, **options): # We don't use **options pylint: disable-msg=W0613
'Called by NoArgsCommand'
one_day = datetime.timedelta(1)
average_date = datetime.date.today() - one_day
end_date = Campaign.objects.aggregate( Min('start_date') )['start_date__min']
average_user = Indicator.objects.average_user()
try:
while average_date > end_date:
print 'Calculating mean for all indicators on %s' % average_date
for campaign in Campaign.objects.all():
for indicator in Indicator.objects.all_regular_indicators(campaign):
indicator.average(average_date)
for group in Group.objects.all():
indicator.average(average_date, group=group)
Indicator.objects.calculate_day_average(campaign, group.avg_user, average_date)
Indicator.objects.calculate_day_average(campaign, average_user, average_date)
average_date -= one_day
except Exception:
import pudb; pudb.post_mortem()
def after_step(context, step):
if BEHAVE_DEBUG_ON_ERROR and step.status == 'failed':
import pudb
pudb.post_mortem(tb=step.exc_traceback,
e_type=None,
e_value=None)
return
def process_exception(exception):
qlog.critical('EXCEPTION while processing HTTP request: "{}"\n{}'.format(
type(exception).__name__,
str(traceback.format_exc())
))
if not global_arguments['-g']:
return render_template('exception.html')
# Try to launch a postmortem debugger.
# The order of preference is: pudb, ipdb, pdb
# pudb will not work on Windows machines, so it can be suppressed with the -p option.
# ipdb may not be installed on all machines, so it falls back to pdb if not installed
e_traceback = sys.exc_info()[2]
debugger_launched = False
if global_arguments['-p']:
try:
# Use pudb debugger if available
import pudb
pudb.post_mortem(e_traceback)
debugger_launched = True
except:
print('Could not launch pudb. Falling back to ipdb/pdb.')
pass
if not debugger_launched:
try:
# Use iPython debugger if available
import ipdb
ipdb.post_mortem(e_traceback)
except:
# Fall back to pdb
import pdb
pdb.post_mortem(e_traceback)
def test_fast_computation(self):
from edge.utils import device, cuda
if device != cuda:
self.assertTrue(
False, "This test should be run with device=cuda. "
"See edge.utils.device.py to set this.")
x = np.linspace(0, 1, 9000, dtype=np.float32).reshape((-1, 1))
y = np.exp(-x**2).reshape(-1)
gp = MaternGP(x,
y,
noise_constraint=(0, 1e-3),
value_structure_discount_factor=0.5)
x_ = np.linspace(0, 1, 20, dtype=np.float32).reshape((-1, 1))
y_ = np.exp(-x_**2).reshape(-1)
#with gpytorch.settings.fast_computations(solves=False):
try:
pred = gp.predict(x_).mean.cpu().numpy()
except Exception as e:
if DEBUG:
try:
import pudb
pudb.post_mortem()
except ImportError:
pass
self.assertTrue(
False, f'Prediction failed with the following error: {str(e)}')
self.assertTrue(True)
def debug(self, err):
ec, ev, tb = err
stdout = sys.stdout
sys.stdout = sys.__stdout__
try:
pudb.post_mortem(tb)
finally:
sys.stdout = stdout
def handle(self, *args, **options):
try:
self._handle(*args, **options)
except Exception:
print "-" * TERMINAL_WIDTH
traceback.print_exc()
print "-" * TERMINAL_WIDTH
debugger.post_mortem()
def debug(self, err):
ec, ev, tb = err
stdout = sys.stdout
sys.stdout = sys.__stdout__
try:
pudb.post_mortem(tb)
finally:
sys.stdout = stdout
def handle(self, *args, **options):
try:
self._handle(*args, **options)
except Exception:
print "-"*TERMINAL_WIDTH
traceback.print_exc()
print "-"*TERMINAL_WIDTH
debugger.post_mortem()
def debug(self, err):
import sys # FIXME why is this import here?
ec, ev, tb = err
stdout = sys.stdout
sys.stdout = sys.__stdout__
try:
pudb.post_mortem((ec, ev,tb))
finally:
sys.stdout = stdout
def post_mortem(config, info):
if config['debug_mode'] and isinstance(info[2], types.TracebackType):
try:
import pudb as pdb
except ImportError:
try:
import ipdb as pdb
except ImportError:
import pdb
pdb.post_mortem(info[2])
def post_mortem(config, info):
if config['debug_mode'] and isinstance(info[2], types.TracebackType):
try:
import pudb as pdb
except ImportError:
try:
import ipdb as pdb
except ImportError:
import pdb
pdb.post_mortem(info[2])
def pdb_post_postmortem(tb=None):
try:
import pudb
pudb.post_mortem(tb)
except ImportError:
try:
import ipdb
ipdb.post_mortem(tb)
except ImportError:
import pdb
pdb.post_mortem(tb)
def pdb_post_postmortem(tb=None):
try:
import pudb
pudb.post_mortem(tb)
except ImportError:
try:
import ipdb
ipdb.post_mortem(tb)
except ImportError:
import pdb
pdb.post_mortem(tb)
def __call__(self, type, value, tb):
if self.instance is None:
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
sys.__excepthook__(type, value, tb)
else:
import traceback
# from IPython.core import ultratb
# self.instance = ultratb.FormattedTB(mode='Plain',
# color_scheme='Linux', call_pdb=1)
import pudb
traceback.print_exception(type, value, tb)
pudb.post_mortem(tb)
def handle(self, jobname, **options):
try:
getattr(jobs, jobname)()
except Exception, e:
try:
if options.get('pdb'):
post_mortem()
else:
tb = "\n".join(traceback.format_exception(*(sys.exc_info())))
mail_admins("Exception in job %s" % jobname, "\n".join(traceback.format_exception(*(sys.exc_info()))))
except:
print tb
finally:
raise e
def handle(self, jobname, **options):
try:
getattr(jobs, jobname)()
except Exception as e:
try:
if options.get('pdb'):
post_mortem()
else:
tb = "\n".join(traceback.format_exception(*(sys.exc_info())))
mail_admins("Exception in job %s" % jobname, "\n".join(traceback.format_exception(*(sys.exc_info()))))
except:
print(tb)
finally:
raise e
def excepthook(self,
exception_type,
exception_obj,
tb,
die=True,
stdout=True,
level=50):
"""
:param exception_type:
:param exception_obj:
:param tb:
:param die:
:param stdout:
:param level:
:return: logdict see github/threefoldtech/jumpscaleX_core/docs/Internals/logging_errorhandling/logdict.md
"""
if isinstance(exception_obj, self.tools.exceptions.RemoteException):
print(
self.tools.text_replace(
"{RED}*****Remote Exception*****{RESET}"))
logdict = exception_obj.data
self.tools.log2stdout(logdict)
exception_obj.data = None
exception_obj.exception = None
# logdict = self.tools.log(tb=tb, level=level, exception=exception_obj, stdout=stdout)
try:
logdict = self.tools.log(tb=tb,
level=level,
exception=exception_obj,
stdout=stdout)
except Exception as e:
self.tools.pprint("{RED}ERROR IN LOG HANDLER")
print(e)
ttype, msg, tb = sys.exc_info()
traceback.print_exception(etype=ttype, tb=tb, value=msg)
self.tools.pprint("{RESET}")
raise e
sys.exit(1)
exception_obj._logdict = logdict
if self.debug and tb:
# exception_type, exception_obj, tb = sys.exc_info()
pudb.post_mortem(tb)
if die is False:
return logdict
else:
sys.exit(1)
def mainloop(): # cli=True):
if IMPORT_ERRORS:
for each in IMPORT_ERRORS:
LOG.warning("Import issue: {}".format(each))
try:
Game().run()
except (KeyboardInterrupt, SystemExit):
sys.stdout.flush()
sys.stderr.flush()
except Exception as err:
# Unhandeld exception
if 0: # TODO
LOG.exception(err)
pdb.post_mortem() # 'e' to view
else:
raise
def error_caught(force=False):
if Config.is_debug() or force:
import traceback
import sys
mod = Config.get_debugger()
type_, value, tb = sys.exc_info()
traceback.print_exc()
if tb:
print("Post mortem")
mod.post_mortem(tb)
else:
print("Lets debug. Hit n to get to the previous scope.")
mod.set_trace()
return True
return False
def mainloop(): # cli=True):
if IMPORT_ERRORS:
for each in IMPORT_ERRORS:
LOG.warning("Import issue: {}".format(each))
try:
Game().run()
except (KeyboardInterrupt, SystemExit):
sys.stdout.flush()
sys.stderr.flush()
except Exception as err:
# Unhandeld exception
if 0: # TODO
LOG.exception(err)
pdb.post_mortem() # 'e' to view
else:
raise
def test_large_ds_matern(self):
x = np.linspace(0, 1, 9000, dtype=np.float32).reshape((-1, 1))
y = np.exp(-x ** 2).reshape(-1)
gp = MaternGP(
x, y, noise_constraint=(0, 1e-3)
)
x_ = np.linspace(0, 1, 20, dtype=np.float32).reshape((-1, 1))
y_ = np.exp(-x_ ** 2).reshape(-1)
try:
pred = gp.predict(x_).mean.cpu().numpy()
except Exception as e:
if DEBUG:
import pudb
pudb.post_mortem()
self.assertTrue(False, f'Prediction failed with the following error: {str(e)}')
self.assertTrue(True)
def main() -> None:
"""
Launch Zulip Terminal.
"""
args = parse_args()
if args.config_file:
zuliprc_path = args.config_file
else:
zuliprc_path = '~/zuliprc'
zterm = parse_zuliprc(zuliprc_path)
if args.profile:
import cProfile
prof = cProfile.Profile()
prof.enable()
try:
Controller(zuliprc_path, zterm['theme']).main()
except Exception as e:
if args.debug:
# A unexpected exception occurred, open the debugger in debug mode
import pudb
pudb.post_mortem()
sys.stdout.flush()
traceback.print_exc(file=sys.stderr)
print("Zulip Terminal has crashed!", file=sys.stderr)
print("You can ask for help at:", file=sys.stderr)
print("https://chat.zulip.org/#narrow/stream/206-zulip-terminal",
file=sys.stderr)
print("\nThanks for using the Zulip-Terminal interface.\n")
sys.stderr.flush()
finally:
if args.profile:
prof.disable()
prof.dump_stats("/tmp/profile.data")
print("Profile data saved to /tmp/profile.data")
print("You can visualize it using e.g."
"`snakeviz /tmp/profile.data`")
sys.exit(1)
def exc_info_hook(exc_type, value, tb):
"""An exception hook that starts IPdb automatically on error if in interactive mode."""
if hasattr(sys, 'ps1') or not sys.stderr.isatty() or exc_type == KeyboardInterrupt:
# we are in interactive mode, we don't have a tty-like
# device,, or the user triggered a KeyboardInterrupt,
# so we call the default hook
sys.__excepthook__(exc_type, value, tb)
else:
import traceback
# import ipdb
import pudb
# we are NOT in interactive mode, print the exception
traceback.print_exception(exc_type, value, tb)
print
# then start the debugger in post-mortem mode.
# pdb.pm() # deprecated
# ipdb.post_mortem(tb) # more modern
pudb.post_mortem(tb)
def main():
print(__doc__)
try:
Controller()
except KeyboardInterrupt:
print("Interrupted")
except SystemExit as e:
pass
except:
import traceback
try:
import pudb as mod
except ImportError:
try:
import ipdb as mod
except ImportError:
import pdb as mod
type, value, tb = sys.exc_info()
traceback.print_exc()
mod.post_mortem(tb)
def exc_info_hook(exc_type, value, tb):
"""An exception hook that starts IPdb automatically on error if in interactive mode."""
if hasattr(
sys,
'ps1') or not sys.stderr.isatty() or exc_type == KeyboardInterrupt:
# we are in interactive mode, we don't have a tty-like
# device,, or the user triggered a KeyboardInterrupt,
# so we call the default hook
sys.__excepthook__(exc_type, value, tb)
else:
import traceback
# import ipdb
import pudb
# we are NOT in interactive mode, print the exception
traceback.print_exception(exc_type, value, tb)
print
# then start the debugger in post-mortem mode.
# pdb.pm() # deprecated
# ipdb.post_mortem(tb) # more modern
pudb.post_mortem(tb)
def main():
print(__doc__)
try:
Controller()
except KeyboardInterrupt:
print("Interrupted")
except SystemExit as e:
pass
except:
import traceback
try:
import pudb as mod
except ImportError:
try:
import ipdb as mod
except ImportError:
import pdb as mod
type, value, tb = sys.exc_info()
traceback.print_exc()
mod.post_mortem(tb)
def excepthook(self, ttype, err, tb, die=True):
""" every fatal error in jumpscale or by python itself will result in an exception
in this function the exception is caught.
@ttype : is the description of the error
@tb : can be a python data object or a Event
"""
# print ("jumpscale EXCEPTIONHOOK")
if self.inException:
print(
"**ERROR IN EXCEPTION HANDLING ROUTINES, which causes recursive errorhandling behavior.**"
)
print(err)
sys.exit(1)
return
self._j.core.tools.log(msg=err, tb=tb, level=40)
if die:
if self._j.core.myenv.debug:
pudb.post_mortem(tb)
self._j.core.tools.pprint("{RED}CANNOT CONTINUE{RESET}")
sys.exit(1)
else:
print("WARNING IGNORE EXCEPTIONHOOK, NEED TO IMPLEMENT: #TODO:")
print('Launching postmortem debugger...')
import traceback
traceback.print_exc()
e_traceback = sys.exc_info()[2]
# Try to launch a postmortem debugger.
# The order of preference is: pudb, ipdb, pdb
# pudb will not work on Windows machines, so it can be suppressed with the -p option.
# ipdb may not be installed on all machines, so it falls back to pdb if not installed
debugger_launched = False
if arguments['-p']:
try:
# Use pudb debugger if available
import pudb
pudb.post_mortem(e_traceback)
debugger_launched = True
except:
print('Could not launch pudb. Falling back to ipdb/pdb.')
pass
if not debugger_launched:
try:
# Use iPython debugger if available
import ipdb
ipdb.post_mortem(e_traceback)
except:
# Fall back to pdb
import pdb
pdb.post_mortem(e_traceback)
def process_exception(self, request, exception):
if isinstance(exception, Http404):
return
print('PudbMiddleware: activating on "{0}"'.format(repr(exception)))
pudb.post_mortem()
def main(options: Optional[List[str]] = None) -> None:
"""
Launch Zulip Terminal.
"""
argv = options if options is not None else sys.argv[1:]
args = parse_args(argv)
set_encoding('utf-8')
if args.debug:
print("NOTE: Debug mode enabled; API calls being logged to {}.".format(
in_color("blue", API_CALL_LOG_FILENAME)))
requests_logfile_handler = logging.FileHandler(API_CALL_LOG_FILENAME)
requests_logger.addHandler(requests_logfile_handler)
else:
requests_logger.addHandler(logging.NullHandler())
if args.profile:
import cProfile
prof = cProfile.Profile()
prof.enable()
if args.version:
print('Zulip Terminal ' + ZT_VERSION)
sys.exit(0)
if args.list_themes:
available_themes = all_themes()
print('Themes available:')
for theme in available_themes:
print(' {}'.format(theme))
sys.exit(0)
if args.config_file:
zuliprc_path = args.config_file
else:
zuliprc_path = '~/zuliprc'
try:
zterm = parse_zuliprc(zuliprc_path)
if args.autohide:
zterm['autohide'] = (args.autohide, 'on command line')
if args.theme:
theme_to_use = (args.theme, 'on command line')
else:
theme_to_use = zterm['theme']
available_themes = all_themes()
theme_aliases = aliased_themes()
is_valid_theme = (theme_to_use[0] in available_themes
or theme_to_use[0] in theme_aliases)
if not is_valid_theme:
print("Invalid theme '{}' was specified {}.".format(*theme_to_use))
print("The following themes are available:")
for theme in available_themes:
print(" ", theme)
print("Specify theme in zuliprc file or override "
"using -t/--theme options on command line.")
sys.exit(1)
if theme_to_use[0] not in available_themes:
# theme must be an alias, as it is valid
real_theme_name = theme_aliases[theme_to_use[0]]
theme_to_use = (real_theme_name, "{} (by alias '{}')".format(
theme_to_use[1], theme_to_use[0]))
if args.color_depth:
zterm['color-depth'] = (args.color_depth, 'on command line')
color_depth = int(zterm['color-depth'][0])
print("Loading with:")
print(" theme '{}' specified {}.".format(*theme_to_use))
complete, incomplete = complete_and_incomplete_themes()
if theme_to_use[0] in incomplete:
print(
in_color(
'yellow', " WARNING: Incomplete theme; "
"results may vary!\n"
" (you could try: {})".format(", ".join(complete))))
print(" autohide setting '{}' specified {}.".format(
*zterm['autohide']))
print(" footlinks setting '{}' specified {}.".format(
*zterm['footlinks']))
print(" color depth setting '{}' specified {}.".format(
*zterm['color-depth']))
# For binary settings
# Specify setting in order True, False
valid_settings = {
'autohide': ['autohide', 'no_autohide'],
'notify': ['enabled', 'disabled'],
'footlinks': ['enabled', 'disabled'],
'color-depth': ['1', '16', '256']
}
boolean_settings = dict() # type: Dict[str, bool]
for setting, valid_values in valid_settings.items():
if zterm[setting][0] not in valid_values:
print("Invalid {} setting '{}' was specified {}.".format(
setting, *zterm[setting]))
print("The following options are available:")
for option in valid_values:
print(" ", option)
print("Specify the {} option in zuliprc file.".format(setting))
sys.exit(1)
if setting == 'color-depth':
break
boolean_settings[setting] = (zterm[setting][0] == valid_values[0])
if color_depth == 1:
theme_data = theme_with_monochrome_added(THEMES[theme_to_use[0]])
else:
theme_data = THEMES[theme_to_use[0]]
Controller(zuliprc_path, theme_data, color_depth, args.explore,
**boolean_settings).main()
except ServerConnectionFailure as e:
print(
in_color('red',
"\nError connecting to Zulip server: {}.".format(e)))
# Acts as separator between logs
zt_logger.info("\n\n" + str(e) + "\n\n")
zt_logger.exception(e)
sys.exit(1)
except (display_common.AttrSpecError, display_common.ScreenError) as e:
# NOTE: Strictly this is not necessarily just a theme error
# FIXME: Add test for this - once loading takes place after UI setup
print(in_color('red', "\nPossible theme error: {}.".format(e)))
# Acts as separator between logs
zt_logger.info("\n\n" + str(e) + "\n\n")
zt_logger.exception(e)
sys.exit(1)
except Exception as e:
zt_logger.info("\n\n" + str(e) + "\n\n")
zt_logger.exception(e)
if args.debug:
sys.stdout.flush()
traceback.print_exc(file=sys.stderr)
run_debugger = input("Run Debugger? (y/n): ")
if run_debugger in ["y", "Y", "yes"]:
# Open PUDB Debuuger
import pudb
pudb.post_mortem()
if hasattr(e, 'extra_info'):
print(
"\n" + in_color("red", e.extra_info), # type: ignore
file=sys.stderr)
print(in_color(
"red", "\nZulip Terminal has crashed!"
"\nPlease refer to " + TRACEBACK_LOG_FILENAME +
" for full log of the error."),
file=sys.stderr)
print("You can ask for help at:", file=sys.stderr)
print("https://chat.zulip.org/#narrow/stream/206-zulip-terminal",
file=sys.stderr)
print("\nThanks for using the Zulip-Terminal interface.\n")
sys.stderr.flush()
finally:
if args.profile:
prof.disable()
import tempfile
with tempfile.NamedTemporaryFile(prefix="zulip_term_profile.",
suffix=".dat",
delete=False) as profile_file:
profile_path = profile_file.name
# Dump stats only after temporary file is closed (for Win NT+ case)
prof.dump_stats(profile_path)
print("Profile data saved to {0}.\n"
"You can visualize it using e.g. `snakeviz {0}`".format(
profile_path))
sys.exit(1)
def process_exception(self, request, exception):
if isinstance(exception, Http404):
return
print('PudbMiddleware: activating on "{0}"'.format(repr(exception)))
pudb.post_mortem()
def debug_pudb(exc_info):
import pudb
pudb.post_mortem(exc_info[2], exc_info[1], exc_info[0])
def main(options: Optional[List[str]] = None) -> None:
"""
Launch Zulip Terminal.
"""
argv = options if options is not None else sys.argv[1:]
args = parse_args(argv)
set_encoding('utf-8')
if args.profile:
import cProfile
prof = cProfile.Profile()
prof.enable()
if args.version:
print('Zulip Terminal ' + ZT_VERSION)
sys.exit(0)
if args.config_file:
zuliprc_path = args.config_file
else:
zuliprc_path = '~/zuliprc'
try:
zterm = parse_zuliprc(zuliprc_path)
if args.theme:
theme_to_use = (args.theme, 'on command line')
else:
theme_to_use = zterm['theme']
available_themes = all_themes()
if theme_to_use[0] not in available_themes:
print("Invalid theme '{}' was specified {}.".format(*theme_to_use))
print("The following themes are available:")
for theme in available_themes:
print(" ", theme)
print("Specify theme in zuliprc file or override "
"using -t/--theme options on command line.")
sys.exit(1)
valid_autohide_settings = {'autohide', 'no_autohide'}
if zterm['autohide'][0] not in valid_autohide_settings:
print("Invalid autohide setting '{}' was specified {}.".format(
*zterm['autohide']))
print("The following options are available:")
for option in valid_autohide_settings:
print(" ", option)
print("Specify the autohide option in zuliprc file.")
sys.exit(1)
autohide_setting = (zterm['autohide'][0] == 'autohide')
print("Loading with:")
print(" theme '{}' specified {}.".format(*theme_to_use))
complete, incomplete = complete_and_incomplete_themes()
if theme_to_use[0] in incomplete:
print(
in_color(
'yellow', " WARNING: Incomplete theme; "
"results may vary!\n"
" (you could try: {})".format(", ".join(complete))))
print(" autohide setting '{}' specified {}.".format(
*zterm['autohide']))
Controller(zuliprc_path, THEMES[theme_to_use[0]],
autohide_setting).main()
except ServerConnectionFailure as e:
print(
in_color('red',
"\nError connecting to Zulip server: {}.".format(e)))
# Acts as separator between logs
logging.info("\n\n" + str(e) + "\n\n")
logging.exception(e)
sys.exit(1)
except Exception as e:
logging.info("\n\n" + str(e) + "\n\n")
logging.exception(e)
if args.debug:
sys.stdout.flush()
traceback.print_exc(file=sys.stderr)
run_debugger = input("Run Debugger? (y/n): ")
if run_debugger in ["y", "Y", "yes"]:
# Open PUDB Debuuger
import pudb
pudb.post_mortem()
if hasattr(e, 'extra_info'):
print(
"\n" + in_color("red", e.extra_info), # type: ignore
file=sys.stderr)
print(in_color(
"red", "\nZulip Terminal has crashed!"
"\nPlease refer to " + LOG_FILENAME + " for full log of"
" the error."),
file=sys.stderr)
print("You can ask for help at:", file=sys.stderr)
print("https://chat.zulip.org/#narrow/stream/206-zulip-terminal",
file=sys.stderr)
print("\nThanks for using the Zulip-Terminal interface.\n")
sys.stderr.flush()
finally:
if args.profile:
prof.disable()
prof.dump_stats("/tmp/profile.data")
print("Profile data saved to /tmp/profile.data")
print("You can visualize it using e.g."
"`snakeviz /tmp/profile.data`")
sys.exit(1)
def start(self):
self._log_info("start", data=self.worker_obj)
# initial waiting state
self.worker_obj.last_update = j.data.time.epoch
self.worker_obj.current_job = 2147483647
self.worker_obj.state = "WAITING"
self.worker_obj.pid = os.getpid()
self.worker_obj.save()
last_info_push = j.data.time.epoch
def queue_results(job):
"""
add job results to queues, if there
:param job: job obj
"""
for queue_name in job.return_queues:
queue = j.clients.redis.queue_get(redisclient=j.core.db,
key="myjobs:%s" % queue_name)
data = {
"id": job.id,
"result": job.result,
"state": job.state._string
}
queue.put(j.data.serializers.json.dumps(data))
while True:
res = None
if self.onetime:
while not res:
res = self.queue_jobs_start.get(timeout=0)
gevent.sleep(0.1)
self._log_debug("jobget from queue")
else:
res = self.queue_jobs_start.get(timeout=20)
self.worker_obj = self.model_worker.get(self.id)
if self.worker_obj.halt or res == b"halt":
return self.stop()
if res is None:
if j.data.time.epoch > last_info_push + 20:
# print(self.worker_obj)
# self._log_info("queue request timeout, no data, continue", data=self.worker_obj)
self._state_set()
last_info_push = j.data.time.epoch
else:
# self._log_debug("queue has data")
jobid = int(res)
try:
job = self.model_job.get(jobid)
except Exception as e:
if not self.model_job.exists(jobid):
self._log_warning("job with:%s did not exist" % jobid)
continue
raise
self.job = job
job.time_start = j.data.time.epoch
skip = False
relaunch = False
for dep_id in job.dependencies:
job_deb = self.model_job.get(dep_id)
if job_deb.state in ["ERROR"]:
job.state = job_deb.state
msg = f"cannot run because dependency failed: {job_deb.id}"
job.result = msg
log = j.core.tools.log(msg, stdout=False)
log["dependency_failure"] = job_deb.id
job.error = log
job.time_stop = j.data.time.epoch
job.save()
skip = True
elif job_deb.state not in ["OK", "DONE"]:
skip = True
# put the job back at end of queue, it needs to be done could not do yet
relaunch = True
if skip and relaunch:
if self.queue_jobs_start.qsize() == 0:
# means we are waiting for some jobs to finish, lets wait 1 sec before we throw this job back on queue
# self._log_info("job queue empty, will wait 1 sec to relaunch the job", data=job)
time.sleep(1)
job.state = "NEW"
job.save()
self.queue_jobs_start.put(job.id)
if not skip:
self.worker_obj.last_update = j.data.time.epoch
self.worker_obj.current_job = jobid
if job is None:
self._log_error("ERROR: job:%s not found" % jobid)
j.shell()
else:
# now have job
action = self.model_action.get(job.action_id)
kwargs = job.kwargs
self.worker_obj.last_update = j.data.time.epoch
self.worker_obj.current_job = jobid # set current jobid
self.worker_obj.state = "busy"
self.worker_obj.save()
if self.showout:
self._log_info("execute", data=job)
try:
exec(action.code)
# better not to use eval but the JSX coderunner?
method = eval(action.methodname)
except Exception as e:
tb = sys.exc_info()[-1]
logdict = j.core.tools.log(
tb=tb,
exception=e,
msg="cannot compile action",
data=action.code,
stdout=self.showout)
job.error = logdict
job.state = "ERROR"
job.time_stop = j.data.time.epoch
job.save()
queue_results(job)
if self.debug:
pudb.post_mortem(tb)
if self.onetime:
return
continue
try:
if job.dependencies != []:
res = deadline(job.timeout)(method)(
process=self, **kwargs)
else:
res = deadline(job.timeout)(method)(**kwargs)
except BaseException as e:
tb = sys.exc_info()[-1]
if isinstance(e, gevent.Timeout):
msg = "time out"
e = j.exceptions.Base(msg)
job.error_cat = "TIMEOUT"
else:
msg = "cannot execute action"
logdict = j.core.tools.log(tb=tb,
exception=e,
msg=msg,
data=action.code,
stdout=self.showout)
job.state = "ERROR"
job.time_stop = j.data.time.epoch
job.error = logdict
job.save()
queue_results(job)
if self.debug:
pudb.post_mortem(tb)
if self.onetime:
return
continue
try:
job.result = j.data.serializers.json.dumps(res)
except Exception as e:
e.message_pub = "could not json serialize result of job"
try:
job.error = e.logdict
except Exception as e:
job.error = str(e)
job.state = "ERROR"
job.time_stop = j.data.time.epoch
job.save()
queue_results(job)
if self.showout:
self._log_error("ERROR:%s" % e,
exception=e,
data=job)
if self.onetime:
return
continue
job.time_stop = j.data.time.epoch
job.state = "OK"
if self.showout:
self._log("OK", data=job)
job.save()
queue_results(job)
if self.queue_jobs_start.qsize() == 0:
# make sure we already set here, otherwise no need because a new job is waiting anyhow
self._state_set()
gevent.sleep(0)
if self.onetime:
self._state_set(state="HALTED")
self.worker_obj.save()
# need to make sure all gets processed
return
def start(self):
while True:
res = None
if self.onetime:
while not res:
res = self.queue_jobs_start.get(timeout=0)
gevent.sleep(0.1)
print("jobget")
else:
res = self.queue_jobs_start.get(timeout=10)
if res == None:
if self.showout:
self._log_info("queue request timeout, no data, continue")
# have to fetch this again because was waiting on queue
if self.data.halt:
# model_worker.
print("WORKER REMOVE SELF:%s" % self.data.id)
return
else:
jobid = int(res)
# update worker has been active
self.data = self.model_worker.get(self.data.id)
if res == b"halt":
return
self.data.last_update = j.data.time.epoch
self.data.current_job = jobid
self.data.save()
job = self.model_job.get(obj_id=jobid, die=False)
if job == None:
self._log_error("ERROR: job:%s not found" % jobid)
else:
# now have job
action = self.model_action.get(job.action_id, die=False)
if action == None:
raise j.exceptions.Base("ERROR: action:%s not found" %
job.action_id)
kwargs = job.kwargs # j.data.serializers.json.loads(job.kwargs)
args = job.args
self.data.last_update = j.data.time.epoch
self.data.current_job = jobid # set current jobid
self.data.save()
if self.showout:
self._log_info("execute", data=job)
try:
exec(action.code)
# better not to use eval but the JSX coderunner?
method = eval(action.methodname)
except Exception as e:
tb = sys.exc_info()[-1]
logdict = j.core.tools.log(tb=tb,
exception=e,
msg="cannot compile action",
data=action.code,
stdout=self.showout)
job.error = logdict
job.state = "ERROR"
job.time_stop = j.data.time.epoch
job.save()
if self.debug:
pudb.post_mortem(tb)
if self.onetime:
return
continue
try:
res = deadline(job.timeout)(method)(*args, **kwargs)
except BaseException as e:
tb = sys.exc_info()[-1]
if isinstance(e, gevent.Timeout):
msg = "time out"
e = j.exceptions.Base(msg)
else:
msg = "cannot execute action"
job.time_stop = j.data.time.epoch
logdict = j.core.tools.log(tb=tb,
exception=e,
msg=msg,
data=action.code,
stdout=self.showout)
job.error = logdict
job.state = "ERROR"
job.save()
if self.debug:
pudb.post_mortem(tb)
if self.onetime:
return
continue
try:
job.result = j.data.serializers.json.dumps(res)
except Exception as e:
job.error = (
str(e) +
"\nCOULD NOT SERIALIZE RESULT OF THE METHOD, make sure json can be used on result"
)
job.state = "ERROR"
job.time_stop = j.data.time.epoch
job.save()
if self.showout:
self._log_error("ERROR:%s" % e,
exception=e,
data=job)
if self.onetime:
return
continue
job.time_stop = j.data.time.epoch
job.state = "OK"
if self.showout:
self._log("OK", data=job)
job.save()
self.data.current_job = 2147483647
self.data.save()
gevent.sleep(0)
if self.onetime:
return
def my_excepthook(exception_type, exception_obj, tb):
Tools.log(msg=exception_obj, tb=tb, level=40)
if MyEnv.debug:
pudb.post_mortem(tb)
Tools.pprint("{RED}CANNOT CONTINUE{RESET}")
sys.exit(1)
def hook(typ, value, tb):
import pudb
pudb.post_mortem(tb)
log_col = inquisitor.collectors.LoggingCollector()
log_col.setFormatter(log_formatter)
exc_catcher.collectors.append(log_col)
logger.addHandler(log_col)
rh = inquisitor.utils.ReportManager("bug_info", "bug_info_{no}.xml",
max_files_size=5 * inquisitor.utils.MiB)
xml_h = inquisitor.handlers.XMLFileDumpHandler(reportmanager=rh)
log_h = inquisitor.handlers.LogTracebackHandler()
exc_catcher.handlers = [xml_h, log_h]
exc_catcher.watch_sys_excepthook()
exc_catcher.watch_tkinter_report_callback_exception()
if args_from_parser.cli:
tk_h = inquisitor.handlers.StreamMessageHandler()
exc_catcher.handlers.append(tk_h)
else:
try:
import pudb
except:
logger.info("Could not import pudb, disabling debugging")
args = {}
else:
args = {"ask_start_db": True, "db_cmd": lambda ty, e, tb: pudb.post_mortem(tb, ty, e)}
tk_h = inquisitor.handlers.TkinterMessageHandler(**args)
exc_catcher.handlers.append(tk_h)
exc_catcher.enabled = args_from_parser.nobuginfo
if args_from_parser.cli:
exit(main_cli(args_from_parser, exc_catcher))
else:
exit(main_gui(args_from_parser, exc_catcher))
def f():
fail
try:
f()
except:
from pudb import post_mortem
post_mortem()
def main(options: Optional[List[str]]=None) -> None:
"""
Launch Zulip Terminal.
"""
argv = options if options is not None else sys.argv[1:]
args = parse_args(argv)
set_encoding('utf-8')
if args.profile:
import cProfile
prof = cProfile.Profile()
prof.enable()
if args.version:
print('Zulip Terminal ' + ZT_VERSION)
sys.exit(0)
if args.config_file:
zuliprc_path = args.config_file
else:
zuliprc_path = '~/zuliprc'
try:
zterm = parse_zuliprc(zuliprc_path)
if args.autohide:
zterm['autohide'] = (args.autohide, 'on command line')
if args.theme:
theme_to_use = (args.theme, 'on command line')
else:
theme_to_use = zterm['theme']
available_themes = all_themes()
if theme_to_use[0] not in available_themes:
print("Invalid theme '{}' was specified {}."
.format(*theme_to_use))
print("The following themes are available:")
for theme in available_themes:
print(" ", theme)
print("Specify theme in zuliprc file or override "
"using -t/--theme options on command line.")
sys.exit(1)
print("Loading with:")
print(" theme '{}' specified {}.".format(*theme_to_use))
complete, incomplete = complete_and_incomplete_themes()
if theme_to_use[0] in incomplete:
print(in_color('yellow',
" WARNING: Incomplete theme; "
"results may vary!\n"
" (you could try: {})".
format(", ".join(complete))))
print(" autohide setting '{}' specified {}."
.format(*zterm['autohide']))
# For binary settings
# Specify setting in order True, False
valid_settings = {
'autohide': ['autohide', 'no_autohide'],
'notify': ['enabled', 'disabled'],
}
boolean_settings = dict() # type: Dict[str, bool]
for setting, valid_values in valid_settings.items():
if zterm[setting][0] not in valid_values:
print("Invalid {} setting '{}' was specified {}."
.format(setting, *zterm[setting]))
print("The following options are available:")
for option in valid_values:
print(" ", option)
print("Specify the {} option in zuliprc file.".format(setting))
sys.exit(1)
boolean_settings[setting] = (zterm[setting][0] == valid_values[0])
color_depth = int(args.color_depth)
if color_depth == 1:
theme_data = theme_with_monochrome_added(THEMES[theme_to_use[0]])
else:
theme_data = THEMES[theme_to_use[0]]
Controller(zuliprc_path,
theme_data,
int(args.color_depth),
**boolean_settings).main()
except ServerConnectionFailure as e:
print(in_color('red',
"\nError connecting to Zulip server: {}.".format(e)))
# Acts as separator between logs
logging.info("\n\n" + str(e) + "\n\n")
logging.exception(e)
sys.exit(1)
except (display_common.AttrSpecError, display_common.ScreenError) as e:
# NOTE: Strictly this is not necessarily just a theme error
# FIXME: Add test for this - once loading takes place after UI setup
print(in_color('red', "\nPossible theme error: {}.".format(e)))
# Acts as separator between logs
logging.info("\n\n" + str(e) + "\n\n")
logging.exception(e)
sys.exit(1)
except Exception as e:
logging.info("\n\n" + str(e) + "\n\n")
logging.exception(e)
if args.debug:
sys.stdout.flush()
traceback.print_exc(file=sys.stderr)
run_debugger = input("Run Debugger? (y/n): ")
if run_debugger in ["y", "Y", "yes"]:
# Open PUDB Debuuger
import pudb
pudb.post_mortem()
if hasattr(e, 'extra_info'):
print("\n" + in_color("red", e.extra_info), # type: ignore
file=sys.stderr)
print(in_color("red", "\nZulip Terminal has crashed!"
"\nPlease refer to " + LOG_FILENAME + " for full log of"
" the error."), file=sys.stderr)
print("You can ask for help at:", file=sys.stderr)
print("https://chat.zulip.org/#narrow/stream/206-zulip-terminal",
file=sys.stderr)
print("\nThanks for using the Zulip-Terminal interface.\n")
sys.stderr.flush()
finally:
if args.profile:
prof.disable()
prof.dump_stats("/tmp/profile.data")
print("Profile data saved to /tmp/profile.data")
print("You can visualize it using e.g."
"`snakeviz /tmp/profile.data`")
sys.exit(1)
def main(options: Optional[List[str]] = None) -> None:
"""
Launch Zulip Terminal.
"""
argv = options if options is not None else sys.argv[1:]
args = parse_args(argv)
if args.profile:
import cProfile
prof = cProfile.Profile()
prof.enable()
if args.config_file:
zuliprc_path = args.config_file
else:
zuliprc_path = '~/zuliprc'
try:
zterm = parse_zuliprc(zuliprc_path)
if args.theme:
theme_to_use = (args.theme, 'on command line')
else:
theme_to_use = zterm['theme']
valid_themes = THEMES.keys()
if theme_to_use[0] not in valid_themes:
print("Invalid theme '{}' was specified {}.".format(*theme_to_use))
print("The following themes are available:")
for theme in valid_themes:
print(" ", theme)
print("Specify theme in zuliprc file or override "
"using -t/--theme options on command line.")
sys.exit(1)
valid_autohide_settings = {'autohide', 'no_autohide'}
if zterm['autohide'][0] not in valid_autohide_settings:
print("Invalid autohide setting '{}' was specified {}.".format(
*zterm['autohide']))
print("The following options are available:")
for option in valid_autohide_settings:
print(" ", option)
print("Specify the autohide option in zuliprc file.")
sys.exit(1)
autohide_setting = (zterm['autohide'][0] == 'autohide')
print("Loading with:")
print(" theme '{}' specified {}.".format(*theme_to_use))
print(" autohide setting '{}' specified {}.".format(
*zterm['autohide']))
Controller(zuliprc_path, THEMES[theme_to_use[0]],
autohide_setting).main()
except ServerConnectionFailure as e:
print("\n\033[91mError connecting to Zulip server: {}.".format(e))
sys.exit(1)
except Exception as e:
if args.debug:
# A unexpected exception occurred, open the debugger in debug mode
import pudb
pudb.post_mortem()
sys.stdout.flush()
traceback.print_exc(file=sys.stderr)
print("Zulip Terminal has crashed!", file=sys.stderr)
print("You can ask for help at:", file=sys.stderr)
print("https://chat.zulip.org/#narrow/stream/206-zulip-terminal",
file=sys.stderr)
print("\nThanks for using the Zulip-Terminal interface.\n")
sys.stderr.flush()
finally:
if args.profile:
prof.disable()
prof.dump_stats("/tmp/profile.data")
print("Profile data saved to /tmp/profile.data")
print("You can visualize it using e.g."
"`snakeviz /tmp/profile.data`")
sys.exit(1)
def debug_pudb(exc_info):
import pudb
pudb.post_mortem(exc_info[2], exc_info[1], exc_info[0])
