def maybe_create_sirepo_user(module, email, display_name):
u = module.unchecked_user_by_user_name(email)
if u:
# Fully registered email user
assert sirepo.auth_db.UserRegistration.search_by(uid=u).display_name, \
f'uid={u} authorized AuthEmailUser record but no UserRegistration.display_name'
return u
m = module.AuthEmailUser.search_by(unverified_email=email)
if m:
# Email user that needs to complete registration (no display_name but have unverified_email)
assert sirepo.auth.need_complete_registration(m), \
'email={email} has no display_name but does not need to complete registration'
pkcli.command_error(
'email={} needs complete registration but we do not have their uid (in cookie)',
email,
)
# Completely new Sirepo user
u = sirepo.auth.create_new_user(
lambda u: sirepo.auth.user_registration(u,
display_name=display_name),
module,
)
module.AuthEmailUser(
unverified_email=email,
uid=u,
user_name=email,
).save()
return u
def _normalize_path_arg(path, **kw):
"""Normalizes the path, to bytes or str depending on the Python version
Args:
path (object): path to normalize
**kwargs: Supports `py.path.local.check_`
.. _py.path.local.check_: http://py.readthedocs.io/en/latest/path.html#py._path.svnwc.SvnWCCommandPath.check
"""
idx = lambda x: tuple(sorted(x.items()))
a = py.path.local(path)
if not a.check(**kw):
msg = {
idx({
'file': True,
'exists': True
}): 'Error, file "{}" does not exist',
idx({'exists': False}): 'Error, file "{}" exists',
}.get(idx(kw), 'Error with file "{}"')
pkcli.command_error(msg, path)
a = str(path)
if isinstance(a, six.text_type):
a = a.encode()
return a
def primes(max_prime, timeit=0):
"""Compute primes less than `max_prime`
Args:
max_prime (int): maximum number of primes
timeit (int): run timeit function with this many loops
Returns:
object: list of primes (list) or timeit result (str)
"""
max_prime = int(max_prime)
if max_prime <= 0:
pkcli.command_error('{}: max_prime must be positive', max_prime)
timeit = int(timeit)
if timeit < 0:
pkcli.command_error('{}: timeit must be non-negative', timeit)
primes = _sundaram3(max_prime)
if timeit == 0:
return primes
import timeit as t
secs = t.timeit(
stmt='_sundaram3({})'.format(max_prime),
number=timeit,
setup='from pykern.pkcli.pkexample import _sundaram3',
)
return 'computed {:,} primes in {:.3f} seconds'.format(len(primes), secs)
def cfg_job_queue(value):
"""Return job queue class based on name
Args:
value (object): May be class or str.
Returns:
object: `Background` or `Celery` class.
"""
if isinstance(value, type) and issubclass(value, (Celery, Background)):
# Already initialized but may call initializer with original object
return value
if value == 'Celery':
from sirepo import celery_tasks
err = None
try:
if not celery_tasks.celery.control.ping():
err = 'You need to start Celery:\nsirepo service celery'
except Exception:
err = 'You need to start Rabbit:\nsirepo service rabbitmq'
if err:
#TODO(robnagler) really should be pkconfig.Error() or something else
# but this prints a nice message. Don't call sys.exit, not nice
import pykern.pkcli
pkcli.command_error(err)
return Celery
elif value == 'Background':
signal.signal(signal.SIGCHLD, Background.sigchld_handler)
return Background
else:
pkcli.command_error('{}: unknown job_queue', value)
def cfg_job_queue(value):
"""Return job queue class based on name
Args:
value (object): May be class or str.
Returns:
object: `Background` or `Celery` class.
"""
if isinstance(value, type) and issubclass(value, (Celery, Background)):
# Already initialized but may call initializer with original object
return value
if value == 'Celery':
from sirepo import celery_tasks
err = None
try:
if not celery_tasks.celery.control.ping():
err = 'You need to start Celery:\nsirepo service celery'
except Exception:
err = 'You need to start Rabbit:\nsirepo service rabbitmq'
if err:
#TODO(robnagler) really should be pkconfig.Error() or something else
# but this prints a nice message. Don't call sys.exit, not nice
import pykern.pkcli
pkcli.command_error(err)
return Celery
elif value == 'Background':
signal.signal(signal.SIGCHLD, Background.sigchld_handler)
return Background
else:
pkcli.command_error('{}: unknown job_queue', value)
def _license(name, which):
"""Returns matching license or command_error"""
try:
return LICENSES[name][which]
except KeyError:
pkcli.command_error(
'{}: unknown license name. Valid licenses: {}',
name,
' '.join(sorted(LICENSES.values())),
)
def _init_do(host, comp):
from rsconf import db
host = host.lower()
dbt = db.T()
for c, hosts in dbt.channel_hosts().items():
for h in hosts:
if h == host.lower():
return comp.host_init(dbt.host_db(c, h), h)
pkcli.command_error('{}: host not found in rsconf_db', host)
def test_command_error(capsys):
from pykern import pkcli
from pykern import pkconfig
pkconfig.reset_state_for_testing()
with pytest.raises(pkcli.CommandError) as e:
pkcli.command_error('{abc}', abc='abcdef')
assert 'abcdef' in str(e.value), \
'When passed a format, command_error should output formatted result'
_dev('p2', ['some-mod', 'command-error'], None, r'raising CommandError',
capsys)
def _cmd_init(*args):
"""Create git repo locally and on remote
"""
from pykern import pkcli
import os.path
#TODO(robnagler) add -public
if os.path.exists(_GIT_DIR):
pkcli.command_error('already initialized (.git directory exists)')
#TODO(robnagler) configure bitbucket locally for each repo
_init_python_user_base()
_init_git()
def _cmd_init(*args):
"""Create git repo locally and on remote
"""
from pykern import pkcli
import os.path
#TODO(robnagler) add -public
if os.path.exists(_GIT_DIR):
pkcli.command_error('already initialized (.git directory exists)')
#TODO(robnagler) configure bitbucket locally for each repo
_init_python_user_base()
_init_git()
def _args(tests):
paths = []
flags = []
for t in tests:
if '=' in t:
a, b = t.split('=')
if a == 'case':
flags.extend(('-k', b))
else:
pkcli.command_error('unsupported option={}'.format(t))
else:
paths.append(t)
return _find(paths), flags
def _cmd_run(*args):
"""Execute run.py or run.sh
"""
from pykern import pkcli
import os.path
missing = []
# Prefer _BASH, which may call run.py
for x in (_BASH, _PYTHON):
if os.path.exists(x[1]):
_rsmanifest()
msg = ': ' + ' '.join(args) if args else ''
_git_commit('run' + msg, check_init=True)
return _call(x)
missing.append(x[1])
pkcli.command_error('{}: neither run file exists', missing)
def _git_auth():
"""Get git user.name
Returns:
str: configured user name
"""
from pykern import pkcli
import netrc
try:
b = netrc.netrc().authenticators(_GIT_REMOTE)
if b:
return (b[0], b[2])
except netrc.NetrcParseError:
pass
pkcli.command_error('missing login info {}; please "git login"', _GIT_REMOTE)
def _cmd_run(*args):
"""Execute run.py or run.sh
"""
from pykern import pkcli
import os.path
missing = []
# Prefer _BASH, which may call run.py
for x in (_BASH, _PYTHON):
if os.path.exists(x[1]):
_rsmanifest()
msg = ': ' + ' '.join(args) if args else ''
_git_commit('run' + msg, check_init=True)
return _call(x)
missing.append(x[1])
pkcli.command_error('{}: neither run file exists', missing)
def _git_auth():
"""Get git user.name
Returns:
str: configured user name
"""
from pykern import pkcli
import netrc
try:
b = netrc.netrc().authenticators(_GIT_REMOTE)
if b:
return (b[0], b[2])
except netrc.NetrcParseError:
pass
pkcli.command_error('missing login info {}; please "git login"',
_GIT_REMOTE)
def _cfg_emails(value):
"""Parse a list of emails separated by comma, colons, semicolons or spaces.
Args:
value (object): if list or tuple, use verbatim; else split
Returns:
list: validated emails
"""
import pyisemail
try:
if not isinstance(value, (list, tuple)):
value = re.split(r'[,;:\s]+', value)
except Exception:
pkcli.command_error('{}: invalid email list', value)
for v in value:
if not pyisemail.is_email(value):
pkcli.command_error('{}: invalid email', v)
def default_command(force=False):
"""Generate index.html files in mm-dd subdirectories
Args:
force (bool): force thumbs and indexes even if they exist
"""
if _DIR_RE.search(os.getcwd()):
_one_dir(force)
else:
dirs = list(glob.iglob(_MM_DD))
if not dirs:
dirs = list(glob.iglob(_YYYY_MM_DD))
if not dirs:
pkcli.command_error('no directories matching YYYY or MM-DD')
for d in sorted(dirs):
with pkio.save_chdir(d):
_one_dir(force)
def _cfg_emails(value):
"""Parse a list of emails separated by comma, colons, semicolons or spaces.
Args:
value (object): if list or tuple, use verbatim; else split
Returns:
list: validated emails
"""
import pyisemail
try:
if not isinstance(value, (list, tuple)):
value = re.split(r'[,;:\s]+', value)
except Exception:
pkcli.command_error('{}: invalid email list', value)
for v in value:
if not pyisemail.is_email(value):
pkcli.command_error('{}: invalid email', v)
def rabbitmq():
assert pkconfig.channel_in('dev')
run_dir = _run_dir().join('rabbitmq').ensure(dir=True)
with pkio.save_chdir(run_dir):
cmd = [
'docker',
'run',
'--env=RABBITMQ_NODE_IP_ADDRESS=' + cfg.ip,
'--net=host',
'--rm',
'--volume={}:/var/lib/rabbitmq'.format(run_dir),
'rabbitmq:management',
]
try:
pksubprocess.check_call_with_signals(cmd)
except OSError as e:
if e.errno == errno.ENOENT:
pkcli.command_error('docker is not installed')
def init_class(app, *args, **kwargs):
"""Verify celery & rabbit are running"""
if pkconfig.channel_in('dev'):
return CeleryJob
for x in range(10):
err = None
try:
if not celery_tasks.celery.control.ping():
err = 'You need to start Celery:\nsirepo service celery'
except Exception:
err = 'You need to start Rabbit:\nsirepo service rabbitmq'
# Rabbit doesn't have a long timeout, but celery ping does
time.sleep(.5)
if not err:
return CeleryJob
#TODO(robnagler) really should be pkconfig.Error() or something else
# but this prints a nice message. Don't call sys.exit, not nice
pkcli.command_error(err)
def init_class(app, uwsgi):
"""Verify celery & rabbit are running"""
if pkconfig.channel_in('dev'):
return CeleryJob
for x in range(10):
err = None
try:
if not celery_tasks.celery.control.ping():
err = 'You need to start Celery:\nsirepo service celery'
except Exception:
err = 'You need to start Rabbit:\nsirepo service rabbitmq'
# Rabbit doesn't have a long timeout, but celery ping does
time.sleep(.5)
if not err:
return CeleryJob
#TODO(robnagler) really should be pkconfig.Error() or something else
# but this prints a nice message. Don't call sys.exit, not nice
pkcli.command_error(err)
def rabbitmq():
assert pkconfig.channel_in('dev')
run_dir = _run_dir().join('rabbitmq').ensure(dir=True)
with pkio.save_chdir(run_dir):
cmd = [
'docker',
'run',
'--env=RABBITMQ_NODE_IP_ADDRESS=' + cfg.ip,
'--net=host',
'--rm',
'--volume={}:/var/lib/rabbitmq'.format(run_dir),
'rabbitmq:management',
]
try:
pksubprocess.check_call_with_signals(cmd)
except OSError as e:
if e.errno == errno.ENOENT:
pkcli.command_error('docker is not installed')
def uwsgi():
"""Starts UWSGI server"""
in_dev = pkconfig.channel_in('dev')
if in_dev:
from sirepo import server, runner
# uwsgi doesn't pass signals right so can't use _Background
if not issubclass(server.cfg.job_queue, runner.Celery):
pkcli.command_error('uwsgi only works if sirepo.server.cfg.job_queue=_Celery')
db_dir =_db_dir()
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
values['logto'] = None if in_dev else str(run_dir.join('uwsgi.log'))
# uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
pksubprocess.check_call_with_signals(cmd)
def echo(suffix, prefix='howdy: '):
"""Concatenate prefix and suffix
Args:
to_echo (str): what to print and must be at least five chars
prefix (str): what to put in front of `to_echo` ["howdy: "]
Returns:
str: prefix + suffix
"""
if len(suffix) < 5:
# We raise argument and other errors with command_error, which
# raises `argh.CommandError` in a nice way. We use Unix-style
# error messages, that is, problematic object followed by colon
# followed by an error message and any other details.
pkcli.command_error('{}: suffix is too short (< 5 chars)', suffix)
# Instead of printing messages in the simple case, we just
# return the value. This way the function can be used in
# other contexts. argh prints the return of the function to stdout.
return prefix + suffix
def _assert_celery():
"""Verify celery & rabbit are running"""
from sirepo import celery_tasks
import time
for x in range(10):
err = None
try:
if not celery_tasks.celery.control.ping():
err = 'You need to start Celery:\nsirepo service celery'
except Exception:
err = 'You need to start Rabbit:\nsirepo service rabbitmq'
# Rabbit doesn't have a long timeout, but celery ping does
time.sleep(.5)
if not err:
return
#TODO(robnagler) really should be pkconfig.Error() or something else
# but this prints a nice message. Don't call sys.exit, not nice
pkcli.command_error(err)
def uwsgi():
"""Starts UWSGI server"""
in_dev = pkconfig.channel_in('dev')
if in_dev:
from sirepo import server, runner
# uwsgi doesn't pass signals right so can't use _Background
if not issubclass(server.cfg.job_queue, runner.Celery):
pkcli.command_error(
'uwsgi only works if sirepo.server.cfg.job_queue=_Celery')
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
values['logto'] = None if in_dev else str(run_dir.join('uwsgi.log'))
# uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
pksubprocess.check_call_with_signals(cmd)
def _git_commit(msg, check_init=False):
"""Write rsmanifest and commit all files
Args:
check_init (bool): make sure git is initialized
"""
#TODO(robnagler) do every run(?)
from pykern import pkcli
import os.path
import subprocess
if check_init:
if not os.path.exists(_GIT_DIR):
pkcli.command_error('not initialized, please call "init"')
_git_auth()
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(['git', 'commit', '-m', msg])
c = ['git', 'push']
if not check_init:
c.extend(['-u', 'origin', 'master'])
subprocess.check_call(c)
def _git_commit(msg, check_init=False):
"""Write rsmanifest and commit all files
Args:
check_init (bool): make sure git is initialized
"""
#TODO(robnagler) do every run(?)
from pykern import pkcli
import os.path
import subprocess
if check_init:
if not os.path.exists(_GIT_DIR):
pkcli.command_error('not initialized, please call "init"')
_git_auth()
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(['git', 'commit', '-m', msg])
c = ['git', 'push']
if not check_init:
c.extend(['-u', 'origin', 'master'])
subprocess.check_call(c)
def cfg_job_queue(value):
"""Return job queue class based on name
Args:
value (object): May be class or str.
Returns:
object: `Background` or `Celery` class.
"""
if isinstance(value, type) and issubclass(value, (Celery, Background)):
# Already initialized but may call initializer with original object
return value
if value == 'Celery':
if pkconfig.channel_in('dev'):
_assert_celery()
return Celery
elif value == 'Background':
signal.signal(signal.SIGCHLD, Background.sigchld_handler)
return Background
else:
pkcli.command_error('{}: unknown job_queue', value)
def _git_api_request(method, url, ctx):
from pykern import pkcli
import requests
user, pw = _git_auth()
ctx['method'] = method
ctx['user'] = user
ctx['pass'] = pw
ctx['host'] = _GIT_REMOTE
ctx['url'] = ('https://api.{host}/2.0/' + url).format(**ctx)
x = dict(
url=ctx['url'],
method=ctx['method'],
auth=(user, pw),
)
if 'json' in ctx:
x['json'] = ctx['json']
r = requests.request(**x)
# Will return 2xx so best test for now
if not r.ok:
pkcli.command_error('{}: post failed: {} {}', ctx['url'], r, r.text)
return r, ctx
def _git_api_request(method, url, ctx):
from pykern import pkcli
import requests
user, pw = _git_auth()
ctx['method'] = method
ctx['user'] = user
ctx['pass'] = pw
ctx['host'] = _GIT_REMOTE
ctx['url'] = ('https://api.{host}/2.0/' + url).format(**ctx)
x = dict(
url=ctx['url'],
method=ctx['method'],
auth=(user, pw),
)
if 'json' in ctx:
x['json'] = ctx['json']
r = requests.request(**x)
# Will return 2xx so best test for now
if not r.ok:
pkcli.command_error('{}: post failed: {} {}', ctx['url'], r, r.text)
return r, ctx
def _cfg_ip(value):
try:
socket.inet_aton(value)
return value
except socket.error:
pkcli.command_error('{}: ip is not a valid IPv4 address', value)
def _cfg_ip(value):
try:
socket.inet_aton(value)
return value
except socket.error:
pkcli.command_error('{}: ip is not a valid IPv4 address', value)
def create_user(email, display_name):
"""Create a JupyterHubUser
This is idempotent. It will create a Sirepo email user if none exists for
the email before creating a jupyterhub user
It will update the user's display_name if the one supplied is different than
the one in the db.
Args:
email (str): Email of user to create.
display_name (str): UserRegistration display_name
Returns:
user_name (str): The jupyterhub user_name of the user
"""
import pyisemail
import sirepo.auth
import sirepo.auth_db
import sirepo.server
import sirepo.sim_api.jupyterhublogin
import sirepo.template
def maybe_create_sirepo_user(module, email, display_name):
u = module.unchecked_user_by_user_name(email)
if u:
# Fully registered email user
assert sirepo.auth_db.UserRegistration.search_by(uid=u).display_name, \
f'uid={u} authorized AuthEmailUser record but no UserRegistration.display_name'
return u
m = module.AuthEmailUser.search_by(unverified_email=email)
if m:
# Email user that needs to complete registration (no display_name but have unverified_email)
assert sirepo.auth.need_complete_registration(m), \
'email={email} has no display_name but does not need to complete registration'
pkcli.command_error(
'email={} needs complete registration but we do not have their uid (in cookie)',
email,
)
# Completely new Sirepo user
u = sirepo.auth.create_new_user(
lambda u: sirepo.auth.user_registration(u,
display_name=display_name),
module,
)
module.AuthEmailUser(
unverified_email=email,
uid=u,
user_name=email,
).save()
return u
if not pyisemail.is_email(email):
pkcli.command_error('invalid email={}', email)
sirepo.server.init()
sirepo.template.assert_sim_type('jupyterhublogin')
with sirepo.auth_db.session_and_lock():
u = maybe_create_sirepo_user(
sirepo.auth.get_module('email'),
email,
display_name,
)
with sirepo.auth.set_user_outside_of_http_request(u):
n = sirepo.sim_api.jupyterhublogin.create_user(check_dir=True)
return PKDict(email=email, jupyterhub_user_name=n)
def test_command_error(capsys):
with pytest.raises(argh.CommandError) as e:
pkcli.command_error('{abc}', abc='abcdef')
assert 'abcdef' in e.value, \
'When passed a format, command_error should output formatted result'
def test_command_error(capsys):
with pytest.raises(argh.CommandError) as e:
pkcli.command_error('{abc}', abc='abcdef')
assert 'abcdef' in e.value, \
'When passed a format, command_error should output formatted result'
def default_command(*args):
"""Run tests one at a time with py.test.
Searches in ``tests`` sub-directory if not provided a
list of tests.
Arguments are directories or files, which are searched for _test.py
files.
An argument which is ``case=<pattern>``, is passed to pytest
as ``-k <pattern>``.
Writes failures to ``<base>_test.log``
Args:
args (str): test dirs, files, options
Returns:
str: passed=N if all passed, else raises `pkcli.Error`
"""
from pykern import pkcli
from pykern import pksubprocess
from pykern import pkio
from pykern import pkunit
import os
import sys
e = PKDict(os.environ)
n = 0
f = []
c = []
paths, flags = _args(args)
for t in paths:
n += 1
o = t.replace('.py', '.log')
m = 'pass'
try:
sys.stdout.write(t)
sys.stdout.flush()
pksubprocess.check_call_with_signals(
['py.test', '--tb=native', '-v', '-s', '-rs', t] + flags,
output=o,
env=PKDict(os.environ, ).pkupdate({pkunit.TEST_FILE_ENV: t}),
#TODO(robnagler) not necessary
# recursive_kill=True,
)
except Exception as e:
if isinstance(e, RuntimeError) and 'exit(5)' in e.args[0]:
# 5 means test was skipped
# see http://doc.pytest.org/en/latest/usage.html#possible-exit-codes
m = 'skipped'
else:
m = 'FAIL {}'.format(o)
f.append(o)
sys.stdout.write(' ' + m + '\n')
if len(f) > 5:
sys.stdout.write('too many failures aborting\n')
break
if n == 0:
pkcli.command_error('no tests found')
if len(f) > 0:
# Avoid dumping too many test logs
for o in f[:5]:
sys.stdout.write(pkio.read_text(o))
sys.stdout.flush()
pkcli.command_error('FAILED={} passed={}'.format(len(f), n - len(f)))
return 'passed={}'.format(n)
