def __init__(self, **kwargs):
super(WebServer, self).__init__()
if not debug_mode:
Logger.redirect_all_logging_to_file(root.common.web.log_file, backups=root.common.web.log_backups)
self.application = web.Application(
[
("/service", ServiceHandler, {"server": self}),
("/update", UpdateHandler, {"server": self}),
("/logs.html?.*", LogsHandler, {"server": self}),
(r"/((js|css|fonts|img|maps)/.*)", web.StaticFileHandler, {"path": root.common.web.root}),
(r"/(.+\.html)", web.StaticFileHandler, {"path": root.common.web.root}),
("/", web.RedirectHandler, {"url": "/status.html", "permanent": True}),
("", web.RedirectHandler, {"url": "/status.html", "permanent": True}),
],
template_path=root.common.web.templates,
gzip=not debug_mode,
)
self._port = kwargs.get("port", root.common.web.port)
self.application.listen(self._port)
self.masters = {}
self.motor = motor.MotorClient("mongodb://" + kwargs.get("mongodb", root.common.mongodb_logging_address))
self.db = self.motor.veles
self.ensure_mongo_indexes()
self.mongo_drop_time_threshold = kwargs.get("mongo_drop_time_threshold", root.common.web.drop_time)
self.mongo_dropper = PeriodicCallback(self.drop_old_mongo_records, self.mongo_drop_time_threshold * 1000)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-b",
"--backend",
nargs='?',
default=root.common.graphics.matplotlib.backend,
help="Matplotlib drawing backend. \"no\" value disable"
"s any real plotting (useful with --dump).")
parser.add_argument("-e",
"--endpoint",
required=True,
help="ZeroMQ endpoint to receive updates from.")
parser.add_argument("--webagg-discovery-fifo",
nargs='?',
default=None,
help="Matplotlib drawing backend.")
LOG_LEVEL_MAP = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warning": logging.WARNING,
"error": logging.ERROR
}
parser.add_argument("-v",
"--verbose",
type=str,
default="info",
choices=LOG_LEVEL_MAP.keys(),
help="set verbosity level [default: %(default)s]")
parser.add_argument("-d",
"--dump",
type=str,
default="",
help="Dump incoming messages to this directory.")
cmdargs = parser.parse_args()
log_level = LOG_LEVEL_MAP[cmdargs.verbose]
Logger.setup_logging(level=log_level)
if log_level == logging.DEBUG:
setup_pickle_debug()
client = GraphicsClient(cmdargs.backend,
cmdargs.endpoint,
webagg_fifo=cmdargs.webagg_discovery_fifo,
dump_dir=cmdargs.dump)
if log_level == logging.DEBUG:
client.debug("Activated pickle debugging")
if cmdargs.backend == "WebAgg":
client_thread = threading.Thread(target=client.run)
client_thread.start()
reactor.run()
client_thread.join()
else:
client.run()
def main():
args = parse_args()
Logger.setup_logging(logging.INFO if not args.quiet else logging.WARNING)
logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
logger.info("Loading snapshots...")
first, second = tuple(load_snapshot(p) for p in (args.first, args.second))
if type(first) != type(second) or first.checksum != second.checksum: # pylint: disable=W1504
raise ValueError("Cannot compare different workflows")
logger.info("Comparing snapshots...")
diffs = list(get_diffs(first.units_in_dependency_order, second.units_in_dependency_order))
logger.info("Sorting the results...")
diffs = sort_diffs(diffs, args.sort)
logger.info("Printing the results...")
print_table(diffs)
def main():
args = parse_args()
Logger.setup_logging(logging.INFO if not args.quiet else logging.WARNING)
logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
logger.info("Loading snapshots...")
first, second = tuple(load_snapshot(p) for p in (args.first, args.second))
if (type(first) != type(second) or # pylint: disable=W1504
first.checksum != second.checksum):
raise ValueError("Cannot compare different workflows")
logger.info("Comparing snapshots...")
diffs = list(get_diffs(first.units_in_dependency_order,
second.units_in_dependency_order))
logger.info("Sorting the results...")
diffs = sort_diffs(diffs, args.sort)
logger.info("Printing the results...")
print_table(diffs)
def __init__(self, args=None, own_reactor=True):
super(ForgeClient, self).__init__()
self.own_reactor = own_reactor
if args is None:
parser = ForgeClient.init_parser()
args = parser.parse_args()
for k, v in getattr(args, "__dict__", args).items():
setattr(self, k, v)
try:
Logger.setup_logging(
logging.DEBUG if self.verbose else logging.INFO)
except Logger.LoggerHasBeenAlreadySetUp:
pass
if self.action not in ("assist",):
self._validate_base()
if not self.base.endswith('/'):
self.base += '/'
if self.action in ("details", "fetch") and not self.name:
raise ValueError("Package name may not be empty.")
self.action = getattr(self, self.action)
def __init__(self, **kwargs):
super(WebServer, self).__init__()
if not debug_mode:
Logger.redirect_all_logging_to_file(
root.common.web.log_file, backups=root.common.web.log_backups)
self.application = web.Application(
[("/service", ServiceHandler, {
"server": self
}), ("/update", UpdateHandler, {
"server": self
}), ("/logs.html?.*", LogsHandler, {
"server": self
}),
(r"/((js|css|fonts|img|maps)/.*)", web.StaticFileHandler, {
'path': root.common.web.root
}),
(r"/(.+\.html)", web.StaticFileHandler, {
'path': root.common.web.root
}),
("/", web.RedirectHandler, {
"url": "/status.html",
"permanent": True
}),
("", web.RedirectHandler, {
"url": "/status.html",
"permanent": True
})],
template_path=root.common.web.templates,
gzip=not debug_mode)
self._port = kwargs.get("port", root.common.web.port)
self.application.listen(self._port)
self.masters = {}
self.motor = motor.MotorClient(
"mongodb://" +
kwargs.get("mongodb", root.common.mongodb_logging_address))
self.db = self.motor.veles
self.ensure_mongo_indexes()
self.mongo_drop_time_threshold = kwargs.get(
"mongo_drop_time_threshold", root.common.web.drop_time)
self.mongo_dropper = PeriodicCallback(
self.drop_old_mongo_records, self.mongo_drop_time_threshold * 1000)
def _process_special_args(self):
if "--frontend" in sys.argv:
try:
self._open_frontend()
except KeyboardInterrupt:
return Main.EXIT_FAILURE
return self._process_special_args()
if self.interactive:
for opt in "forge", "--version", "--help", "--dump-config":
if opt in self.argv:
raise ValueError(
"\"%s\" is not supported in interactive mode" % opt)
return None
if len(sys.argv) > 1 and sys.argv[1] == "forge":
from veles.forge.forge_client import __run__ as forge_run
del sys.argv[1]
action = sys.argv[1]
try:
forge_run()
return Main.EXIT_SUCCESS
except Exception as e:
if isinstance(e, SystemExit):
raise from_none(e)
self.exception("Failed to run forge %s", action)
return Main.EXIT_FAILURE
if "--version" in sys.argv:
self._print_version()
return Main.EXIT_SUCCESS
if "--html-help" in sys.argv:
veles.__html__()
return Main.EXIT_SUCCESS
if "--help" in sys.argv:
# help text requires UTF-8, but the default codec is ascii over ssh
Logger.ensure_utf8_streams()
if "--dump-config" in sys.argv:
self.info("Scanning for the plugins...")
self.debug("Loaded plugins: %s", veles.__plugins__)
root.print_()
return Main.EXIT_SUCCESS
return None
def _process_special_args(self):
if "--frontend" in sys.argv:
try:
self._open_frontend()
except KeyboardInterrupt:
return Main.EXIT_FAILURE
return self._process_special_args()
if self.interactive:
for opt in "forge", "--version", "--help", "--dump-config":
if opt in self.argv:
raise ValueError(
"\"%s\" is not supported in interactive mode" % opt)
return None
if len(sys.argv) > 1 and sys.argv[1] == "forge":
from veles.forge.forge_client import __run__ as forge_run
del sys.argv[1]
action = sys.argv[1]
try:
forge_run()
return Main.EXIT_SUCCESS
except Exception as e:
if isinstance(e, SystemExit):
raise from_none(e)
self.exception("Failed to run forge %s", action)
return Main.EXIT_FAILURE
if "--version" in sys.argv:
self._print_version()
return Main.EXIT_SUCCESS
if "--html-help" in sys.argv:
veles.__html__()
return Main.EXIT_SUCCESS
if "--help" in sys.argv:
# help text requires UTF-8, but the default codec is ascii over ssh
Logger.ensure_utf8_streams()
if "--dump-config" in sys.argv:
self.info("Scanning for the plugins...")
self.debug("Loaded plugins: %s", veles.__plugins__)
root.print_()
return Main.EXIT_SUCCESS
return None
def __init__(self, args=None):
super(ForgeServer, self).__init__()
if args is None:
parser = ForgeServer.init_parser()
args = parser.parse_args()
Logger.setup_logging(
logging.INFO if not args.verbose else logging.DEBUG)
for key, val in args.__dict__.items():
setattr(self, key, val)
self.suburi = getattr(args, "suburi", "/")
if not self.suburi.endswith("/"):
raise ValueError("--suburi must end with a slash (\"/\")")
self.thread_pool = ThreadPoolExecutor(4)
self._stop = IOLoop.instance().stop
IOLoop.instance().stop = self.stop
self._tokens = None
self._tokens_timestamp = None
self._emails = None
self.tokens_file = getattr(self, "tokens_file",
root.common.forge.tokens_file)
self.pending_file = getattr(self, "pending_file",
root.common.forge.pending_file)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-b", "--backend", nargs='?',
default=root.common.graphics.matplotlib.backend,
help="Matplotlib drawing backend. \"no\" value disable"
"s any real plotting (useful with --dump).")
parser.add_argument("-e", "--endpoint", required=True,
help="ZeroMQ endpoint to receive updates from.")
parser.add_argument("--webagg-discovery-fifo", nargs='?',
default=None, help="Matplotlib drawing backend.")
LOG_LEVEL_MAP = {"debug": logging.DEBUG, "info": logging.INFO,
"warning": logging.WARNING, "error": logging.ERROR}
parser.add_argument("-v", "--verbose", type=str, default="info",
choices=LOG_LEVEL_MAP.keys(),
help="set verbosity level [default: %(default)s]")
parser.add_argument("-d", "--dump", type=str, default="",
help="Dump incoming messages to this directory.")
cmdargs = parser.parse_args()
log_level = LOG_LEVEL_MAP[cmdargs.verbose]
Logger.setup_logging(level=log_level)
if log_level == logging.DEBUG:
setup_pickle_debug()
client = GraphicsClient(cmdargs.backend, cmdargs.endpoint,
webagg_fifo=cmdargs.webagg_discovery_fifo,
dump_dir=cmdargs.dump)
if log_level == logging.DEBUG:
client.debug("Activated pickle debugging")
if cmdargs.backend == "WebAgg":
client_thread = threading.Thread(target=client.run)
client_thread.start()
reactor.run()
client_thread.join()
else:
client.run()
raise PermissionError(pidfile)
if os.path.exists(full_pidfile):
real_pidfile = os.readlink(full_pidfile)
pid = int(real_pidfile.split('.')[-1])
print("Replacing PID %d..." % pid)
try:
os.kill(pid, 0)
except OSError:
os.remove(real_pidfile)
os.remove(full_pidfile)
print_("Detected a stale lock file %s" % real_pidfile,
file=sys.stderr)
else:
raise FileExistsError(full_pidfile)
print("Daemonizing, PID will be referenced by ", full_pidfile)
try:
sys.stdout.flush()
except BrokenPipeError:
pass
with daemon.DaemonContext(pidfile=pidfile,
stderr=sys.stderr,
uid=args.user,
gid=args.group):
log_file = root.common.web.log_file
Logger.setup_logging(level=logging.INFO)
Logger.redirect_all_logging_to_file(log_file, backups=9)
main(port=args.port)
else:
Logger.setup_logging(level=logging.DEBUG)
main(port=args.port)
if not debug_mode:
pidfile = root.common.web.pidfile
full_pidfile = pidfile + ".lock"
if args.user is None and not os.access(os.path.dirname(full_pidfile), os.W_OK):
raise PermissionError(pidfile)
if os.path.exists(full_pidfile):
real_pidfile = os.readlink(full_pidfile)
pid = int(real_pidfile.split(".")[-1])
print("Replacing PID %d..." % pid)
try:
os.kill(pid, 0)
except OSError:
os.remove(real_pidfile)
os.remove(full_pidfile)
print_("Detected a stale lock file %s" % real_pidfile, file=sys.stderr)
else:
raise FileExistsError(full_pidfile)
print("Daemonizing, PID will be referenced by ", full_pidfile)
try:
sys.stdout.flush()
except BrokenPipeError:
pass
with daemon.DaemonContext(pidfile=pidfile, stderr=sys.stderr, uid=args.user, gid=args.group):
log_file = root.common.web.log_file
Logger.setup_logging(level=logging.INFO)
Logger.redirect_all_logging_to_file(log_file, backups=9)
main(port=args.port)
else:
Logger.setup_logging(level=logging.DEBUG)
main(port=args.port)
