def main(argv, cfg):
global authdict, allow_passwordless, db
parser = ArgumentParser(prog=argv.pop(0))
parser.add_argument(
'--path',
type=str,
default='urd.db',
help=
'database directory (can be relative to project directory) (default: urd.db)',
)
parser.add_argument('--allow-passwordless',
action='store_true',
help='accept any pass for users not in passwd.')
parser.add_argument('--quiet', action='store_true', help='less chatty.')
args = parser.parse_args(argv)
if not args.quiet:
print('-' * 79)
print(args)
print()
auth_fn = os.path.join(args.path, 'passwd')
authdict = readauth(auth_fn)
allow_passwordless = args.allow_passwordless
if not authdict and not args.allow_passwordless:
raise Exception(
'No users in %r and --allow-passwordless not specified.' %
(auth_fn, ))
db = DB(args.path, not args.quiet)
bottle.install(jsonify)
kw = dict(debug=False,
reloader=False,
quiet=args.quiet,
server=WaitressServer)
listen = cfg.urd_listen
if not listen:
raise Exception('urd not configured in this project')
if isinstance(listen, tuple):
kw['host'], kw['port'] = listen
else:
from accelerator.server import check_socket
check_socket(listen)
kw['host'] = listen
kw['port'] = 0
bottle.run(**kw)
def run(cfg, from_shell=False):
project = os.path.split(cfg.project_directory)[1]
setproctitle('ax board for %s on %s' % (
project,
cfg.board_listen,
))
def call_s(*path):
return call(os.path.join(cfg.url, *map(url_quote, path)))
def call_u(*path):
return call(os.path.join(cfg.urd, *map(url_quote, path)),
server_name='urd')
@bottle.get('/')
@view('main')
def main_page():
return dict(
project=project,
workdirs=cfg.workdirs,
)
@bottle.get('/results')
def results():
res = {}
for fn in os.listdir(cfg.result_directory):
if fn.endswith('_'):
continue
ffn = os.path.join(cfg.result_directory, fn)
try:
jobid, name = os.readlink(ffn).split('/')[-2:]
res[fn] = dict(
jobid=jobid,
name=name,
ts=os.lstat(ffn).st_mtime,
size=os.stat(ffn).st_size,
)
except OSError:
continue
bottle.response.content_type = 'application/json; charset=UTF-8'
bottle.response.set_header('Cache-Control', 'no-cache')
return json.dumps(res)
@bottle.get('/results/<name>')
def file(name):
return bottle.static_file(name, root=cfg.result_directory)
@bottle.get('/status')
@view('status')
def status():
status = call_s('status/full')
if 'short' in bottle.request.query:
if status.idle:
return 'idle'
else:
t, msg, _ = status.current
return '%s (%s)' % (
msg,
fmttime(status.report_t - t, short=True),
)
else:
status.tree = list(
fix_stacks(status.pop('status_stacks', ()), status.report_t))
return status
@bottle.get('/last_error')
@view('last_error')
def last_error():
return call_s('last_error')
@bottle.get('/job/<jobid>/method.tar.gz/')
@bottle.get('/job/<jobid>/method.tar.gz/<name:path>')
def job_method(jobid, name=None):
job = get_job(jobid)
with tarfile.open(job.filename('method.tar.gz'), 'r:gz') as tar:
if name:
info = tar.getmember(name)
else:
members = [info for info in tar.getmembers() if info.isfile()]
if len(members) == 1 and not name:
info = members[0]
else:
return template('job_method_list',
members=members,
job=job)
bottle.response.content_type = 'text/plain; charset=UTF-8'
return tar.extractfile(info).read()
@bottle.get('/job/<jobid>/<name:path>')
def job_file(jobid, name):
job = get_job(jobid)
res = bottle.static_file(name, root=job.path)
if not res.content_type and res.status_code < 400:
# bottle default is text/html, which is probably wrong.
res.content_type = 'text/plain'
return res
@bottle.get('/job/<jobid>')
@bottle.get('/job/<jobid>/')
@view('job')
def job(jobid):
job = get_job(jobid)
try:
post = job.post
except IOError:
post = None
if post:
aborted = False
files = [fn for fn in job.files() if fn[0] != '/']
subjobs = [Job(jobid) for jobid in post.subjobs]
current = call_s('job_is_current', job)
else:
aborted = True
current = False
files = None
subjobs = None
return dict(
job=job,
aborted=aborted,
current=current,
output=os.path.exists(job.filename('OUTPUT')),
datasets=job.datasets,
params=job.params,
subjobs=subjobs,
files=files,
)
@bottle.get('/dataset/<dsid:path>')
@view('dataset', ds_json)
def dataset(dsid):
ds = Dataset(dsid.rstrip('/'))
q = bottle.request.query
if q.column:
lines = int(q.lines or 10)
it = ds.iterate(None, q.column)
it = itertools.islice(it, lines)
t = ds.columns[q.column].type
if t in (
'datetime',
'date',
'time',
):
it = map(str, it)
elif t in (
'bytes',
'pickle',
):
it = map(repr, it)
res = list(it)
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(res)
else:
return dict(ds=ds)
def load_workdir(jobs, name):
known = call_s('workdir', name)
jobs[name + '-LATEST'] = None # Sorts first
try:
latest = os.readlink(
os.path.join(cfg.workdirs[name], name + '-LATEST'))
except OSError:
latest = None
for jid in workdir_jids(cfg, name):
jobs[jid] = job_data(known, jid)
if latest in jobs:
jobs[name + '-LATEST'] = jobs[latest]
else:
del jobs[name + '-LATEST']
return jobs
@bottle.get('/workdir/<name>')
@view('workdir', 'jobs')
def workdir(name):
return dict(name=name,
jobs=load_workdir(collections.OrderedDict(), name))
@bottle.get('/workdir')
@bottle.get('/workdir/')
@view('workdir', 'jobs')
def all_workdirs():
jobs = collections.OrderedDict()
for name in sorted(cfg.workdirs):
load_workdir(jobs, name)
return dict(name='ALL', jobs=jobs)
@bottle.get('/methods')
@view('methods')
def methods():
methods = call_s('methods')
by_package = collections.defaultdict(list)
for name, data in sorted(methods.items()):
by_package[data.package].append(name)
by_package.pop('accelerator.test_methods', None)
return dict(methods=methods, by_package=by_package)
@bottle.get('/method/<name>')
@view('method', 'data')
def method(name):
methods = call_s('methods')
if name not in methods:
return bottle.HTTPError(404, 'Method %s not found' % (name, ))
return dict(name=name, data=methods[name], cfg=cfg)
@bottle.get('/urd')
@bottle.get('/urd/')
@view('urd', 'lists')
def urd():
return dict(
lists=call_u('list'),
project=project,
)
@bottle.get('/urd/<user>/<build>')
@bottle.get('/urd/<user>/<build>/')
@view('urdlist', 'timestamps')
def urdlist(user, build):
key = user + '/' + build
return dict(
key=key,
timestamps=call_u(key, 'since/0'),
)
@bottle.get('/urd/<user>/<build>/<ts>')
@view('urditem', 'entry')
def urditem(user, build, ts):
key = user + '/' + build + '/' + ts
d = call_u(key)
return dict(key=key, entry=d)
bottle.TEMPLATE_PATH = [os.path.join(os.path.dirname(__file__), 'board')]
if from_shell:
kw = {'reloader': True}
else:
kw = {'quiet': True}
kw['server'] = WaitressServer
listen = cfg.board_listen
if isinstance(listen, tuple):
kw['host'], kw['port'] = listen
else:
from accelerator.server import check_socket
check_socket(listen)
kw['host'] = listen
kw['port'] = 0
bottle.run(**kw)