def maybe_create_sirepo_user(module, email, display_name): u = module.unchecked_user_by_user_name(email) if u: # Fully registered email user assert sirepo.auth_db.UserRegistration.search_by(uid=u).display_name, \ f'uid={u} authorized AuthEmailUser record but no UserRegistration.display_name' return u m = module.AuthEmailUser.search_by(unverified_email=email) if m: # Email user that needs to complete registration (no display_name but have unverified_email) assert sirepo.auth.need_complete_registration(m), \ 'email={email} has no display_name but does not need to complete registration' pkcli.command_error( 'email={} needs complete registration but we do not have their uid (in cookie)', email, ) # Completely new Sirepo user u = sirepo.auth.create_new_user( lambda u: sirepo.auth.user_registration(u, display_name=display_name), module, ) module.AuthEmailUser( unverified_email=email, uid=u, user_name=email, ).save() return u
def _normalize_path_arg(path, **kw): """Normalizes the path, to bytes or str depending on the Python version Args: path (object): path to normalize **kwargs: Supports `py.path.local.check_` .. _py.path.local.check_: http://py.readthedocs.io/en/latest/path.html#py._path.svnwc.SvnWCCommandPath.check """ idx = lambda x: tuple(sorted(x.items())) a = py.path.local(path) if not a.check(**kw): msg = { idx({ 'file': True, 'exists': True }): 'Error, file "{}" does not exist', idx({'exists': False}): 'Error, file "{}" exists', }.get(idx(kw), 'Error with file "{}"') pkcli.command_error(msg, path) a = str(path) if isinstance(a, six.text_type): a = a.encode() return a
def primes(max_prime, timeit=0): """Compute primes less than `max_prime` Args: max_prime (int): maximum number of primes timeit (int): run timeit function with this many loops Returns: object: list of primes (list) or timeit result (str) """ max_prime = int(max_prime) if max_prime <= 0: pkcli.command_error('{}: max_prime must be positive', max_prime) timeit = int(timeit) if timeit < 0: pkcli.command_error('{}: timeit must be non-negative', timeit) primes = _sundaram3(max_prime) if timeit == 0: return primes import timeit as t secs = t.timeit( stmt='_sundaram3({})'.format(max_prime), number=timeit, setup='from pykern.pkcli.pkexample import _sundaram3', ) return 'computed {:,} primes in {:.3f} seconds'.format(len(primes), secs)
def cfg_job_queue(value): """Return job queue class based on name Args: value (object): May be class or str. Returns: object: `Background` or `Celery` class. """ if isinstance(value, type) and issubclass(value, (Celery, Background)): # Already initialized but may call initializer with original object return value if value == 'Celery': from sirepo import celery_tasks err = None try: if not celery_tasks.celery.control.ping(): err = 'You need to start Celery:\nsirepo service celery' except Exception: err = 'You need to start Rabbit:\nsirepo service rabbitmq' if err: #TODO(robnagler) really should be pkconfig.Error() or something else # but this prints a nice message. Don't call sys.exit, not nice import pykern.pkcli pkcli.command_error(err) return Celery elif value == 'Background': signal.signal(signal.SIGCHLD, Background.sigchld_handler) return Background else: pkcli.command_error('{}: unknown job_queue', value)
def _license(name, which): """Returns matching license or command_error""" try: return LICENSES[name][which] except KeyError: pkcli.command_error( '{}: unknown license name. Valid licenses: {}', name, ' '.join(sorted(LICENSES.values())), )
def _init_do(host, comp): from rsconf import db host = host.lower() dbt = db.T() for c, hosts in dbt.channel_hosts().items(): for h in hosts: if h == host.lower(): return comp.host_init(dbt.host_db(c, h), h) pkcli.command_error('{}: host not found in rsconf_db', host)
def test_command_error(capsys): from pykern import pkcli from pykern import pkconfig pkconfig.reset_state_for_testing() with pytest.raises(pkcli.CommandError) as e: pkcli.command_error('{abc}', abc='abcdef') assert 'abcdef' in str(e.value), \ 'When passed a format, command_error should output formatted result' _dev('p2', ['some-mod', 'command-error'], None, r'raising CommandError', capsys)
def _cmd_init(*args): """Create git repo locally and on remote """ from pykern import pkcli import os.path #TODO(robnagler) add -public if os.path.exists(_GIT_DIR): pkcli.command_error('already initialized (.git directory exists)') #TODO(robnagler) configure bitbucket locally for each repo _init_python_user_base() _init_git()
def _args(tests): paths = [] flags = [] for t in tests: if '=' in t: a, b = t.split('=') if a == 'case': flags.extend(('-k', b)) else: pkcli.command_error('unsupported option={}'.format(t)) else: paths.append(t) return _find(paths), flags
def _cmd_run(*args): """Execute run.py or run.sh """ from pykern import pkcli import os.path missing = [] # Prefer _BASH, which may call run.py for x in (_BASH, _PYTHON): if os.path.exists(x[1]): _rsmanifest() msg = ': ' + ' '.join(args) if args else '' _git_commit('run' + msg, check_init=True) return _call(x) missing.append(x[1]) pkcli.command_error('{}: neither run file exists', missing)
def _git_auth(): """Get git user.name Returns: str: configured user name """ from pykern import pkcli import netrc try: b = netrc.netrc().authenticators(_GIT_REMOTE) if b: return (b[0], b[2]) except netrc.NetrcParseError: pass pkcli.command_error('missing login info {}; please "git login"', _GIT_REMOTE)
def _cfg_emails(value): """Parse a list of emails separated by comma, colons, semicolons or spaces. Args: value (object): if list or tuple, use verbatim; else split Returns: list: validated emails """ import pyisemail try: if not isinstance(value, (list, tuple)): value = re.split(r'[,;:\s]+', value) except Exception: pkcli.command_error('{}: invalid email list', value) for v in value: if not pyisemail.is_email(value): pkcli.command_error('{}: invalid email', v)
def default_command(force=False): """Generate index.html files in mm-dd subdirectories Args: force (bool): force thumbs and indexes even if they exist """ if _DIR_RE.search(os.getcwd()): _one_dir(force) else: dirs = list(glob.iglob(_MM_DD)) if not dirs: dirs = list(glob.iglob(_YYYY_MM_DD)) if not dirs: pkcli.command_error('no directories matching YYYY or MM-DD') for d in sorted(dirs): with pkio.save_chdir(d): _one_dir(force)
def rabbitmq(): assert pkconfig.channel_in('dev') run_dir = _run_dir().join('rabbitmq').ensure(dir=True) with pkio.save_chdir(run_dir): cmd = [ 'docker', 'run', '--env=RABBITMQ_NODE_IP_ADDRESS=' + cfg.ip, '--net=host', '--rm', '--volume={}:/var/lib/rabbitmq'.format(run_dir), 'rabbitmq:management', ] try: pksubprocess.check_call_with_signals(cmd) except OSError as e: if e.errno == errno.ENOENT: pkcli.command_error('docker is not installed')
def init_class(app, *args, **kwargs): """Verify celery & rabbit are running""" if pkconfig.channel_in('dev'): return CeleryJob for x in range(10): err = None try: if not celery_tasks.celery.control.ping(): err = 'You need to start Celery:\nsirepo service celery' except Exception: err = 'You need to start Rabbit:\nsirepo service rabbitmq' # Rabbit doesn't have a long timeout, but celery ping does time.sleep(.5) if not err: return CeleryJob #TODO(robnagler) really should be pkconfig.Error() or something else # but this prints a nice message. Don't call sys.exit, not nice pkcli.command_error(err)
def init_class(app, uwsgi): """Verify celery & rabbit are running""" if pkconfig.channel_in('dev'): return CeleryJob for x in range(10): err = None try: if not celery_tasks.celery.control.ping(): err = 'You need to start Celery:\nsirepo service celery' except Exception: err = 'You need to start Rabbit:\nsirepo service rabbitmq' # Rabbit doesn't have a long timeout, but celery ping does time.sleep(.5) if not err: return CeleryJob #TODO(robnagler) really should be pkconfig.Error() or something else # but this prints a nice message. Don't call sys.exit, not nice pkcli.command_error(err)
def uwsgi(): """Starts UWSGI server""" in_dev = pkconfig.channel_in('dev') if in_dev: from sirepo import server, runner # uwsgi doesn't pass signals right so can't use _Background if not issubclass(server.cfg.job_queue, runner.Celery): pkcli.command_error('uwsgi only works if sirepo.server.cfg.job_queue=_Celery') db_dir =_db_dir() run_dir = _run_dir() with pkio.save_chdir(run_dir): values = dict(pkcollections.map_items(cfg)) values['logto'] = None if in_dev else str(run_dir.join('uwsgi.log')) # uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml for f in ('uwsgi.py', 'uwsgi.yml'): output = run_dir.join(f) values[f.replace('.', '_')] = str(output) pkjinja.render_resource(f, values, output=output) cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']] pksubprocess.check_call_with_signals(cmd)
def echo(suffix, prefix='howdy: '): """Concatenate prefix and suffix Args: to_echo (str): what to print and must be at least five chars prefix (str): what to put in front of `to_echo` ["howdy: "] Returns: str: prefix + suffix """ if len(suffix) < 5: # We raise argument and other errors with command_error, which # raises `argh.CommandError` in a nice way. We use Unix-style # error messages, that is, problematic object followed by colon # followed by an error message and any other details. pkcli.command_error('{}: suffix is too short (< 5 chars)', suffix) # Instead of printing messages in the simple case, we just # return the value. This way the function can be used in # other contexts. argh prints the return of the function to stdout. return prefix + suffix
def _assert_celery(): """Verify celery & rabbit are running""" from sirepo import celery_tasks import time for x in range(10): err = None try: if not celery_tasks.celery.control.ping(): err = 'You need to start Celery:\nsirepo service celery' except Exception: err = 'You need to start Rabbit:\nsirepo service rabbitmq' # Rabbit doesn't have a long timeout, but celery ping does time.sleep(.5) if not err: return #TODO(robnagler) really should be pkconfig.Error() or something else # but this prints a nice message. Don't call sys.exit, not nice pkcli.command_error(err)
def uwsgi(): """Starts UWSGI server""" in_dev = pkconfig.channel_in('dev') if in_dev: from sirepo import server, runner # uwsgi doesn't pass signals right so can't use _Background if not issubclass(server.cfg.job_queue, runner.Celery): pkcli.command_error( 'uwsgi only works if sirepo.server.cfg.job_queue=_Celery') run_dir = _run_dir() with pkio.save_chdir(run_dir): values = dict(pkcollections.map_items(cfg)) values['logto'] = None if in_dev else str(run_dir.join('uwsgi.log')) # uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml for f in ('uwsgi.py', 'uwsgi.yml'): output = run_dir.join(f) values[f.replace('.', '_')] = str(output) pkjinja.render_resource(f, values, output=output) cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']] pksubprocess.check_call_with_signals(cmd)
def _git_commit(msg, check_init=False): """Write rsmanifest and commit all files Args: check_init (bool): make sure git is initialized """ #TODO(robnagler) do every run(?) from pykern import pkcli import os.path import subprocess if check_init: if not os.path.exists(_GIT_DIR): pkcli.command_error('not initialized, please call "init"') _git_auth() subprocess.check_call(['git', 'add', '.']) subprocess.check_call(['git', 'commit', '-m', msg]) c = ['git', 'push'] if not check_init: c.extend(['-u', 'origin', 'master']) subprocess.check_call(c)
def cfg_job_queue(value): """Return job queue class based on name Args: value (object): May be class or str. Returns: object: `Background` or `Celery` class. """ if isinstance(value, type) and issubclass(value, (Celery, Background)): # Already initialized but may call initializer with original object return value if value == 'Celery': if pkconfig.channel_in('dev'): _assert_celery() return Celery elif value == 'Background': signal.signal(signal.SIGCHLD, Background.sigchld_handler) return Background else: pkcli.command_error('{}: unknown job_queue', value)
def _git_api_request(method, url, ctx): from pykern import pkcli import requests user, pw = _git_auth() ctx['method'] = method ctx['user'] = user ctx['pass'] = pw ctx['host'] = _GIT_REMOTE ctx['url'] = ('https://api.{host}/2.0/' + url).format(**ctx) x = dict( url=ctx['url'], method=ctx['method'], auth=(user, pw), ) if 'json' in ctx: x['json'] = ctx['json'] r = requests.request(**x) # Will return 2xx so best test for now if not r.ok: pkcli.command_error('{}: post failed: {} {}', ctx['url'], r, r.text) return r, ctx
def _cfg_ip(value): try: socket.inet_aton(value) return value except socket.error: pkcli.command_error('{}: ip is not a valid IPv4 address', value)
def create_user(email, display_name): """Create a JupyterHubUser This is idempotent. It will create a Sirepo email user if none exists for the email before creating a jupyterhub user It will update the user's display_name if the one supplied is different than the one in the db. Args: email (str): Email of user to create. display_name (str): UserRegistration display_name Returns: user_name (str): The jupyterhub user_name of the user """ import pyisemail import sirepo.auth import sirepo.auth_db import sirepo.server import sirepo.sim_api.jupyterhublogin import sirepo.template def maybe_create_sirepo_user(module, email, display_name): u = module.unchecked_user_by_user_name(email) if u: # Fully registered email user assert sirepo.auth_db.UserRegistration.search_by(uid=u).display_name, \ f'uid={u} authorized AuthEmailUser record but no UserRegistration.display_name' return u m = module.AuthEmailUser.search_by(unverified_email=email) if m: # Email user that needs to complete registration (no display_name but have unverified_email) assert sirepo.auth.need_complete_registration(m), \ 'email={email} has no display_name but does not need to complete registration' pkcli.command_error( 'email={} needs complete registration but we do not have their uid (in cookie)', email, ) # Completely new Sirepo user u = sirepo.auth.create_new_user( lambda u: sirepo.auth.user_registration(u, display_name=display_name), module, ) module.AuthEmailUser( unverified_email=email, uid=u, user_name=email, ).save() return u if not pyisemail.is_email(email): pkcli.command_error('invalid email={}', email) sirepo.server.init() sirepo.template.assert_sim_type('jupyterhublogin') with sirepo.auth_db.session_and_lock(): u = maybe_create_sirepo_user( sirepo.auth.get_module('email'), email, display_name, ) with sirepo.auth.set_user_outside_of_http_request(u): n = sirepo.sim_api.jupyterhublogin.create_user(check_dir=True) return PKDict(email=email, jupyterhub_user_name=n)
def test_command_error(capsys): with pytest.raises(argh.CommandError) as e: pkcli.command_error('{abc}', abc='abcdef') assert 'abcdef' in e.value, \ 'When passed a format, command_error should output formatted result'
def default_command(*args): """Run tests one at a time with py.test. Searches in ``tests`` sub-directory if not provided a list of tests. Arguments are directories or files, which are searched for _test.py files. An argument which is ``case=<pattern>``, is passed to pytest as ``-k <pattern>``. Writes failures to ``<base>_test.log`` Args: args (str): test dirs, files, options Returns: str: passed=N if all passed, else raises `pkcli.Error` """ from pykern import pkcli from pykern import pksubprocess from pykern import pkio from pykern import pkunit import os import sys e = PKDict(os.environ) n = 0 f = [] c = [] paths, flags = _args(args) for t in paths: n += 1 o = t.replace('.py', '.log') m = 'pass' try: sys.stdout.write(t) sys.stdout.flush() pksubprocess.check_call_with_signals( ['py.test', '--tb=native', '-v', '-s', '-rs', t] + flags, output=o, env=PKDict(os.environ, ).pkupdate({pkunit.TEST_FILE_ENV: t}), #TODO(robnagler) not necessary # recursive_kill=True, ) except Exception as e: if isinstance(e, RuntimeError) and 'exit(5)' in e.args[0]: # 5 means test was skipped # see http://doc.pytest.org/en/latest/usage.html#possible-exit-codes m = 'skipped' else: m = 'FAIL {}'.format(o) f.append(o) sys.stdout.write(' ' + m + '\n') if len(f) > 5: sys.stdout.write('too many failures aborting\n') break if n == 0: pkcli.command_error('no tests found') if len(f) > 0: # Avoid dumping too many test logs for o in f[:5]: sys.stdout.write(pkio.read_text(o)) sys.stdout.flush() pkcli.command_error('FAILED={} passed={}'.format(len(f), n - len(f))) return 'passed={}'.format(n)