def main(argv, cfg):
usage = "%(prog)s [-a | [workdir [workdir [...]]]"
parser = ArgumentParser(usage=usage, prog=argv.pop(0))
parser.add_argument('-a', '--all', action='store_true', help="list all workdirs")
parser.add_argument('workdirs', nargs='*', default=[])
args = parser.parse_args(argv)
if args.all:
args.workdirs.extend(sorted(cfg.workdirs))
if not args.workdirs:
for wd in sorted(cfg.workdirs):
print(wd)
return
for name in args.workdirs:
if name not in cfg.workdirs:
print("No such workdir:", name, file=sys.stderr)
continue
known = call(cfg.url + '/workdir/' + url_quote(name))
for jid in workdir_jids(cfg, name):
show_job(known, jid)
try:
latest = os.readlink(os.path.join(cfg.workdirs[name], name + '-LATEST'))
except OSError:
latest = None
if latest:
show_job(known, jid, name + '-LATEST')
def cmd_abort(argv):
parser = ArgumentParser(prog=argv.pop(0))
parser.add_argument('-q', '--quiet', action='store_true', help="no output")
args = parser.parse_args(argv)
from accelerator.build import Automata
a = Automata(cfg.url)
res = a.abort()
if not args.quiet:
print("Killed %d running job%s." % (res.killed, '' if res.killed == 1 else 's'))
def main(argv, cfg):
parser = ArgumentParser(
prog=argv.pop(0),
usage="%(prog)s [options] [script]",
formatter_class=RawTextHelpFormatter,
)
parser.add_argument('-f', '--flags', default='', help="comma separated list of flags", )
parser.add_argument('-q', '--quick', action='store_true', help="skip method updates and checking workdirs for new jobs", )
parser.add_argument('-c', '--concurrency', action='append', metavar='SPEC', help="set max concurrency for methods, either method=N\nor just N to set for all other methods", )
parser.add_argument('-w', '--workdir', default=None, help="build in this workdir\nset_workdir() and workdir= override this.", )
parser.add_argument('-W', '--just_wait',action='store_true', help="just wait for running job, don't run any build script", )
parser.add_argument('-F', '--fullpath', action='store_true', help="print full path to jobdirs")
parser.add_argument('--verbose', default='status', help="verbosity style {no, status, dots, log}")
parser.add_argument('--quiet', action='store_true', help="same as --verbose=no")
parser.add_argument('--horizon', default=None, help="time horizon - dates after this are not visible in\nurd.latest")
parser.add_argument('script', default='build' , help="build script to run. default \"build\".\nsearches under all method directories in alphabetical\norder if it does not contain a dot.\nprefixes build_ to last element unless specified.\npackage name suffixes are ok.\nso for example \"test_methods.tests\" expands to\n\"accelerator.test_methods.build_tests\".", nargs='?')
options = parser.parse_args(argv)
if '.' in options.script:
options.package, options.script = options.script.rsplit('.', 1)
else:
options.package = None
options.verbose = {'no': False, 'status': True, 'dots': 'dots', 'log': 'log'}[options.verbose]
if options.quiet: options.verbose = False
concurrency_map = {}
for v in options.concurrency or ():
if v.isnumeric():
concurrency_map['-default-'] = int(v)
else:
try:
method, v = v.split('=', 1)
concurrency_map[method] = int(v)
except ValueError:
raise Exception('Bad concurrency spec %r' % (v,))
options.concurrency_map = concurrency_map
try:
run_automata(options, cfg)
return 0
except (JobError, ServerError):
# If it's a JobError we don't care about the local traceback,
# we want to see the job traceback, and maybe know what line
# we built the job on.
# If it's a ServerError we just want the line and message.
print_minimal_traceback()
except Exception:
# For the rest we still don't want to see stuff from this
# file and earlier.
print_user_part_traceback()
return 1
def main(argv, cfg):
global authdict, allow_passwordless, db
parser = ArgumentParser(prog=argv.pop(0))
parser.add_argument(
'--path',
type=str,
default='urd.db',
help=
'database directory (can be relative to project directory) (default: urd.db)',
)
parser.add_argument('--allow-passwordless',
action='store_true',
help='accept any pass for users not in passwd.')
parser.add_argument('--quiet', action='store_true', help='less chatty.')
args = parser.parse_args(argv)
if not args.quiet:
print('-' * 79)
print(args)
print()
auth_fn = os.path.join(args.path, 'passwd')
authdict = readauth(auth_fn)
allow_passwordless = args.allow_passwordless
if not authdict and not args.allow_passwordless:
raise Exception(
'No users in %r and --allow-passwordless not specified.' %
(auth_fn, ))
db = DB(args.path, not args.quiet)
bottle.install(jsonify)
kw = dict(debug=False,
reloader=False,
quiet=args.quiet,
server=WaitressServer)
listen = cfg.urd_listen
if not listen:
raise Exception('urd not configured in this project')
if isinstance(listen, tuple):
kw['host'], kw['port'] = listen
else:
from accelerator.server import check_socket
check_socket(listen)
kw['host'] = listen
kw['port'] = 0
bottle.run(**kw)
def main(argv):
from os import makedirs, listdir, chdir
from os.path import exists, join, realpath
from sys import version_info
from argparse import RawTextHelpFormatter
from accelerator.compat import ArgumentParser
from accelerator.error import UserError
from accelerator.extras import DotDict
parser = ArgumentParser(
prog=argv.pop(0),
description=r'''
creates an accelerator project directory.
defaults to the current directory.
creates accelerator.conf, a method dir, a workdir and result dir.
both the method directory and workdir will be named <NAME>,
"dev" by default.
'''.replace('\t', ''),
formatter_class=RawTextHelpFormatter,
)
parser.add_argument('--slices', default=None, type=int, help='override slice count detection')
parser.add_argument('--name', default='dev', help='name of method dir and workdir, default "dev"')
parser.add_argument('--input', default='# /some/path where you want import methods to look.', help='input directory')
parser.add_argument('--force', action='store_true', help='go ahead even though directory is not empty, or workdir\nexists with incompatible slice count')
parser.add_argument('--tcp', default=False, metavar='HOST/PORT', nargs='?', help='listen on TCP instead of unix sockets.\nspecify HOST (can be IP) to listen on that host\nspecify PORT to use range(PORT, PORT + 3)\nspecify both as HOST:PORT')
parser.add_argument('--no-git', action='store_true', help='don\'t create git repository')
parser.add_argument('directory', default='.', help='project directory to create. default "."', metavar='DIR', nargs='?')
options = parser.parse_args(argv)
assert options.name
assert '/' not in options.name
if options.tcp is False:
listen = DotDict(
board='.socket.dir/board',
server='.socket.dir/server',
urd='.socket.dir/urd',
)
else:
hostport = options.tcp or ''
if hostport.endswith(']'): # ipv6
host, port = hostport, None
elif ':' in hostport:
host, port = hostport.rsplit(':', 1)
elif hostport.isdigit():
host, port = '', hostport
else:
host, port = hostport, None
if port:
port = int(port)
else:
port = find_free_ports(0x3000, 0x8000)
listen = DotDict(
server='%s:%d' % (host, port,),
board='%s:%d' % (host, port + 1,),
urd='%s:%d' % (host, port + 2,),
)
if not options.input.startswith('#'):
options.input = quote(realpath(options.input))
prefix = realpath(options.directory)
workdir = join(prefix, 'workdirs', options.name)
slices_conf = join(workdir, '.slices')
try:
with open(slices_conf, 'r') as fh:
workdir_slices = int(fh.read())
except IOError:
workdir_slices = None
if workdir_slices and options.slices is None:
options.slices = workdir_slices
if options.slices is None:
from multiprocessing import cpu_count
options.slices = cpu_count()
if workdir_slices and workdir_slices != options.slices and not options.force:
raise UserError('Workdir %r has %d slices, refusing to continue with %d slices' % (workdir, workdir_slices, options.slices,))
if not options.force and exists(options.directory) and listdir(options.directory):
raise UserError('Directory %r is not empty.' % (options.directory,))
if not exists(options.directory):
makedirs(options.directory)
chdir(options.directory)
for dir_to_make in ('.socket.dir', 'urd.db',):
if not exists(dir_to_make):
makedirs(dir_to_make, 0o750)
for dir_to_make in (workdir, 'results',):
if not exists(dir_to_make):
makedirs(dir_to_make)
with open(slices_conf, 'w') as fh:
fh.write('%d\n' % (options.slices,))
method_dir = options.name
if not exists(method_dir):
makedirs(method_dir)
with open(join(method_dir, '__init__.py'), 'w') as fh:
pass
with open(join(method_dir, 'methods.conf'), 'w') as fh:
fh.write('example\n')
with open(join(method_dir, 'a_example.py'), 'w') as fh:
fh.write(a_example)
with open(join(method_dir, 'build.py'), 'w') as fh:
fh.write(build_script)
with open('accelerator.conf', 'w') as fh:
fh.write(config_template.format(
name=quote(options.name),
slices=options.slices,
input=options.input,
major=version_info.major,
minor=version_info.minor,
micro=version_info.micro,
listen=DotDict({k: quote(v) for k, v in listen.items()}),
))
if not options.no_git:
git(method_dir)
def main():
# As of python 3.8 the default start_method is 'spawn' on macOS.
# This doesn't work for us. 'fork' is fairly unsafe on macOS,
# but it's better than not working at all. See
# https://bugs.python.org/issue33725
# for more information.
import multiprocessing
if hasattr(multiprocessing, 'set_start_method'):
# If possible, make the forkserver (used by database updates) pre-import everthing
if hasattr(multiprocessing, 'set_forkserver_preload'):
multiprocessing.set_forkserver_preload(
['accelerator', 'accelerator.server'])
multiprocessing.set_start_method('fork')
from accelerator import g
g.running = 'shell'
from accelerator.autoflush import AutoFlush
main_argv, argv = split_args(sys.argv[1:])
sys.stdout = AutoFlush(sys.stdout)
sys.stderr = AutoFlush(sys.stderr)
aliases = {
'cat': 'grep ""',
}
aliases.update(parse_user_config() or ())
while argv and argv[0] in aliases:
try:
expanded = shlex.split(aliases[argv[0]])
except ValueError as e:
raise ValueError('Failed to expand alias %s (%r): %s' % (
argv[0],
aliases[argv[0]],
e,
))
more_main_argv, argv = split_args(expanded + argv[1:])
main_argv.extend(more_main_argv)
epilog = ['commands:', '']
cmdlen = max(len(cmd) for cmd in COMMANDS)
template = ' %%%ds %%s' % (cmdlen, )
for cmd, func in sorted(COMMANDS.items()):
epilog.append(template % (
cmd,
func.help,
))
epilog.append('')
epilog.append('aliases:')
epilog.extend('%s = %s' % item for item in sorted(aliases.items()))
epilog.append('')
epilog.append('use %(prog)s <command> --help for <command> usage')
parser = ArgumentParser(
usage='%(prog)s [--config CONFIG_FILE] command [args]',
epilog='\n'.join(epilog),
formatter_class=RawDescriptionHelpFormatter,
)
parser.add_argument('--config',
metavar='CONFIG_FILE',
help='configuration file')
parser.add_argument('--version',
action='store_true',
help='alias for the version command')
args = parser.parse_args(main_argv)
if args.version:
sys.exit(cmd_version(()))
args.command = argv.pop(0) if argv else None
if args.command not in COMMANDS:
parser.print_help(file=sys.stderr)
print(file=sys.stderr)
if args.command is not None:
print('Unknown command "%s"' % (args.command, ), file=sys.stderr)
sys.exit(2)
config_fn = args.config
if args.command == 'init':
config_fn = False
cmd = COMMANDS[args.command]
debug_cmd = getattr(cmd, 'is_debug', False)
try:
setup(config_fn, debug_cmd)
argv.insert(0, '%s %s' % (
basename(sys.argv[0]),
args.command,
))
return cmd(argv)
except UserError as e:
print(e, file=sys.stderr)
return 1
except IOError as e:
if e.errno == errno.EPIPE and debug_cmd:
return
else:
raise
def main(argv, config):
g.running = 'server'
parser = ArgumentParser(prog=argv.pop(0))
parser.add_argument('--debug', action='store_true')
parser.add_argument(
'--debuggable',
action='store_true',
help=
'make breakpoint() work in methods. note that this makes a failing method kill the whole server.'
)
options = parser.parse_args(argv)
config.debuggable = options.debuggable
# all forks belong to the same happy family
try:
os.setpgrp()
except OSError:
print(
"Failed to create process group - there is probably already one (daemontools).",
file=sys.stderr)
# Set a low (but not too low) open file limit to make
# dispatch.update_valid_fds faster.
# The runners will set the highest limit they can
# before actually running any methods.
r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE)
r1 = min(r1, r2, 1024)
resource.setrlimit(resource.RLIMIT_NOFILE, (r1, r2))
# Start the board-server in a separate process so it can't interfere.
# Even if it dies we don't care.
try:
if not isinstance(config.board_listen, tuple):
# Don't bother if something is already listening.
check_socket(config.board_listen)
Process(target=board.run, args=(config, ), name='board-server').start()
except Exception:
pass
iowrapper.main()
# setup statmsg sink and tell address using ENV
statmsg_rd, statmsg_wr = socket.socketpair(socket.AF_UNIX,
socket.SOCK_DGRAM)
os.environ['BD_STATUS_FD'] = str(statmsg_wr.fileno())
def buf_up(fh, opt):
sock = socket.fromfd(fh.fileno(), socket.AF_UNIX, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, opt, 256 * 1024)
# does not close fh, because fromfd dups the fd (but not the underlying socket)
sock.close()
buf_up(statmsg_wr, socket.SO_SNDBUF)
buf_up(statmsg_rd, socket.SO_RCVBUF)
t = DeadlyThread(target=statmsg_sink,
args=(statmsg_rd, ),
name="statmsg sink")
t.daemon = True
t.start()
# do all main-stuff, i.e. run server
sys.stdout = autoflush.AutoFlush(sys.stdout)
sys.stderr = autoflush.AutoFlush(sys.stderr)
atexit.register(exitfunction)
signal.signal(signal.SIGTERM, exitfunction)
signal.signal(signal.SIGINT, exitfunction)
signal.signal(signal.SIGUSR1, siginfo)
signal.siginterrupt(signal.SIGUSR1, False)
if hasattr(signal, 'pthread_sigmask'):
signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGUSR1})
if hasattr(signal, 'SIGINFO'):
signal.signal(signal.SIGINFO, siginfo)
signal.siginterrupt(signal.SIGINFO, False)
if isinstance(config.listen, tuple):
server = ThreadedHTTPServer(config.listen, XtdHandler)
else:
check_socket(config.listen)
# We want the socket to be world writeable, protect it with dir permissions.
u = os.umask(0)
server = ThreadedUnixHTTPServer(config.listen, XtdHandler)
os.umask(u)
if config.get('urd_local'):
from accelerator import urd
t = DeadlyThread(target=urd.main,
args=(['urd', '--quiet',
'--allow-passwordless'], config),
name='urd')
t.daemon = True
t.start()
ctrl = control.Main(config, options, config.url)
print()
ctrl.print_workdirs()
print()
XtdHandler.ctrl = ctrl
job_tracking[None].workdir = ctrl.target_workdir
for n in (
"project_directory",
"result_directory",
"input_directory",
):
v = config.get(n)
n = n.replace("_", " ")
print("%17s: %s" % (
n,
v,
))
for n in (
"board",
"urd",
):
v = config.get(n + '_listen')
if v and not config.get(n + '_local', True):
extra = ' (remote)'
else:
extra = ''
print("%17s: %s%s" % (
n,
v,
extra,
))
print()
print("Serving on %s\n" % (config.listen, ), file=sys.stderr)
server.serve_forever()
def main():
# Several commands use SIGUSR1 which (naturally...) defaults to killing the
# process, so start by blocking that to minimise the race time.
if hasattr(signal, 'pthread_sigmask'):
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGUSR1})
else:
# Or if we can't block it, just ignore it.
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
# As of python 3.8 the default start_method is 'spawn' on macOS.
# This doesn't work for us. 'fork' is fairly unsafe on macOS,
# but it's better than not working at all. See
# https://bugs.python.org/issue33725
# for more information.
import multiprocessing
if hasattr(multiprocessing, 'set_start_method'):
# If possible, make the forkserver (used by database updates) pre-import everthing
if hasattr(multiprocessing, 'set_forkserver_preload'):
multiprocessing.set_forkserver_preload(
['accelerator', 'accelerator.server'])
multiprocessing.set_start_method('fork')
from accelerator import g
g.running = 'shell'
from accelerator.autoflush import AutoFlush
main_argv, argv = split_args(sys.argv[1:])
sys.stdout = AutoFlush(sys.stdout)
sys.stderr = AutoFlush(sys.stderr)
# configuration defaults
aliases = {
'cat': 'grep -e ""',
}
colour_d = {
'warning': ('RED', ),
'highlight': ('BOLD', ),
'grep/highlight': ('RED', ),
'info': ('BRIGHTBLUE', ),
'infohighlight': (
'BOLD',
'BRIGHTBLUE',
),
'separator': (
'CYAN',
'UNDERLINE',
),
'header': (
'BRIGHTBLUE',
'BOLD',
),
}
parse_user_config(aliases, colour_d)
colour._names.update(colour_d)
used_aliases = []
while argv and argv[0] in aliases:
alias = argv[0]
if alias == 'noalias': # save the user from itself
break
try:
expanded = shlex.split(aliases[alias])
except ValueError as e:
raise ValueError('Failed to expand alias %s (%r): %s' % (
argv[0],
aliases[argv[0]],
e,
))
more_main_argv, argv = split_args(expanded + argv[1:])
main_argv.extend(more_main_argv)
if expanded and alias == expanded[0]:
break
used_aliases.append(alias)
if alias in used_aliases[:-1]:
raise ValueError('Alias loop: %r' % (used_aliases, ))
while argv and argv[0] == 'noalias':
argv.pop(0)
epilog = ['commands:', '']
cmdlen = max(len(cmd) for cmd in COMMANDS)
template = ' %%%ds %%s' % (cmdlen, )
for cmd, func in sorted(COMMANDS.items()):
epilog.append(template % (
cmd,
func.help,
))
epilog.append('')
epilog.append('aliases:')
epilog.extend(' %s = %s' % item for item in sorted(aliases.items()))
epilog.append('')
epilog.append('use "' +
colour('%(prog)s <command> --help', 'help/highlight') +
'" for <command> usage')
epilog.append('try "' + colour('%(prog)s intro', 'help/highlight') +
'" for an introduction')
parser = ArgumentParser(
usage='%(prog)s [--config CONFIG_FILE] command [args]',
epilog='\n'.join(epilog),
formatter_class=RawDescriptionHelpFormatter,
)
parser.add_argument('--config',
metavar='CONFIG_FILE',
help='configuration file')
parser.add_argument('--version',
action='store_true',
help='alias for the version command')
args = parser.parse_args(main_argv)
if args.version:
sys.exit(cmd_version(()))
args.command = argv.pop(0) if argv else None
if args.command not in COMMANDS:
parser.print_help(file=sys.stderr)
if args.command is not None:
print(file=sys.stderr)
print('Unknown command "%s"' % (args.command, ), file=sys.stderr)
sys.exit(2)
config_fn = args.config
if args.command in (
'init',
'intro',
'version',
):
config_fn = False
cmd = COMMANDS[args.command]
debug_cmd = getattr(cmd, 'is_debug', False)
try:
setup(config_fn, debug_cmd)
argv.insert(0, '%s %s' % (
basename(sys.argv[0]),
args.command,
))
return cmd(argv)
except UserError as e:
print(e, file=sys.stderr)
return 1
except OSError as e:
if e.errno == errno.EPIPE:
return 1
else:
raise
except KeyboardInterrupt:
# Exiting with KeyboardInterrupt causes python to print a traceback.
# We don't want that, but we do want to exit from SIGINT (so the
# calling process can know that happened).
signal.signal(signal.SIGINT, signal.SIG_DFL)
os.kill(os.getpid(), signal.SIGINT)
# If that didn't work let's re-raise the KeyboardInterrupt.
raise
def main(argv):
from os import makedirs, listdir, chdir
from os.path import exists, join, realpath
from sys import version_info
from argparse import RawDescriptionHelpFormatter
from accelerator.compat import ArgumentParser
from accelerator.error import UserError
parser = ArgumentParser(
prog=argv.pop(0),
description=r'''
creates an accelerator project directory.
defaults to the current directory.
creates accelerator.conf, a method dir, a workdir and result dir.
both the method directory and workdir will be named <NAME>,
"dev" by default.
'''.replace('\t', ''),
formatter_class=RawDescriptionHelpFormatter,
)
parser.add_argument('--slices', default=None, type=int, help='override slice count detection')
parser.add_argument('--name', default='dev', help='name of method dir and workdir, default "dev"')
parser.add_argument('--input', default='# /some/path where you want import methods to look.', help='input directory')
parser.add_argument('--force', action='store_true', help='go ahead even though directory is not empty, or workdir exists with incompatible slice count')
parser.add_argument('directory', default='.', help='project directory to create. default "."', metavar='DIR', nargs='?')
options = parser.parse_args(argv)
assert options.name
assert '/' not in options.name
if not options.input.startswith('#'):
options.input = quote(realpath(options.input))
prefix = realpath(options.directory)
workdir = join(prefix, 'workdirs', options.name)
slices_conf = join(workdir, '.slices')
try:
with open(slices_conf, 'r') as fh:
workdir_slices = int(fh.read())
except IOError:
workdir_slices = None
if workdir_slices and options.slices is None:
options.slices = workdir_slices
if options.slices is None:
from multiprocessing import cpu_count
options.slices = cpu_count()
if workdir_slices and workdir_slices != options.slices and not options.force:
raise UserError('Workdir %r has %d slices, refusing to continue with %d slices' % (workdir, workdir_slices, options.slices,))
if not options.force and exists(options.directory) and listdir(options.directory):
raise UserError('Directory %r is not empty.' % (options.directory,))
if not exists(options.directory):
makedirs(options.directory)
chdir(options.directory)
for dir_to_make in ('.socket.dir', 'urd.db',):
if not exists(dir_to_make):
makedirs(dir_to_make, 0o750)
for dir_to_make in (workdir, 'results',):
if not exists(dir_to_make):
makedirs(dir_to_make)
with open(slices_conf, 'w') as fh:
fh.write('%d\n' % (options.slices,))
method_dir = options.name
if not exists(method_dir):
makedirs(method_dir)
with open(join(method_dir, '__init__.py'), 'w') as fh:
pass
with open(join(method_dir, 'methods.conf'), 'w') as fh:
fh.write('example\n')
with open(join(method_dir, 'a_example.py'), 'w') as fh:
fh.write(a_example)
with open(join(method_dir, 'build.py'), 'w') as fh:
fh.write(build_script)
with open('accelerator.conf', 'w') as fh:
fh.write(config_template.format(
name=quote(options.name),
slices=options.slices,
input=options.input,
major=version_info.major,
minor=version_info.minor,
micro=version_info.micro,
))