def test_timeout(auth_fc): fc = auth_fc from pykern import pkconfig, pkunit, pkio from pykern import pkjson from pykern.pkdebug import pkdp from pykern.pkunit import pkok, pkre, pkeq, pkexcept import re r = fc.sr_get('authGuestLogin', {'simulation_type': fc.sr_sim_type}, redirect=False) pkeq(200, r.status_code) d = pkjson.load_any(r.data) pkeq(True, d.authState.isLoggedIn) fc.sr_post('listSimulations', {'simulationType': fc.sr_sim_type}) fc.sr_auth_state( isGuestUser=True, isLoggedIn=True, isLoginExpired=False, ) fc.sr_get_json('adjustTime', params={'days': '2'}) fc.sr_auth_state( isGuestUser=True, isLoggedIn=True, isLoginExpired=True, ) with pkexcept('SRException.*guest-expired'): fc.sr_post('listSimulations', {'simulationType': fc.sr_sim_type})
def test_uniquify_beamline(): from pykern import pkio from pykern import pkunit from pykern.pkunit import pkeq from pykern import pkjson from sirepo.template import madx d = pkjson.load_any(pkunit.data_dir().join('in.json')) madx.uniquify_elements(d) pkeq(1, len(d.models.beamlines), 'expecting one beamline={}', d.models.beamlines) l = d.models.beamlines[0]['items'] pkeq(len(list(set(l))), len(l), 'expecting all unique items={}', l) e = {e._id: e.original_id for e in d.models.elements} r = [e[i] for i in d.models.beamlines[0]['items']] pkeq( [ 2, 2, 5, 5, 5, 2, 2, 5, 5, 5, 2, ], r, 'expecting proper reflection of sub-lines. ids of original elements: {}', r)
def _request(**kwargs): def get_api_name(): if 'api_name' in kwargs: return kwargs['api_name'] f = inspect.currentframe() for _ in range(_MAX_FRAME_SEARCH_DEPTH): m = re.search(r'^api_.*$', f.f_code.co_name) if m: return m.group() f = f.f_back else: raise AssertionError( '{}: max frame search depth reached'.format(f.f_code) ) k = PKDict(kwargs) u = k.pkdel('_request_uri') or cfg.supervisor_uri + sirepo.job.SERVER_URI c = k.pkdel('_request_content') if '_request_content' in k else _request_content(k) c.pkupdate( api=get_api_name(), serverSecret=sirepo.job.cfg.server_secret, ) pkdlog('api={} runDir={}', c.api, c.get('runDir')) r = requests.post( u, data=pkjson.dump_bytes(c), headers=PKDict({'Content-type': 'application/json'}), verify=sirepo.job.cfg.verify_tls, ) r.raise_for_status() return pkjson.load_any(r.content)
def parse_response(self, resp, expect_binary_body=False): assert resp.code == 200, 'resp={}'.format(resp) if 'Set-Cookie' in resp.headers: self._headers.Cookie = resp.headers['Set-Cookie'] if 'json' in resp.headers['content-type']: return pkjson.load_any(resp.body) try: b = pkcompat.from_bytes(resp.body) assert not expect_binary_body, \ 'expecting binary body resp={} body={}'.format( resp, b[:1000], ) except UnicodeDecodeError: assert expect_binary_body, \ 'unexpected binary body resp={}'.format(resp) # Binary data files can't be decoded return if 'html' in resp.headers['content-type']: m = re.search('location = "(/[^"]+)', b) if m: if 'error' in m.group(1): return PKDict(state='error', error='server error') return PKDict(state='redirect', uri=m.group(1)) return b
def host_init(j2_ctx, host): from rsconf import db jf = db.secret_path(j2_ctx, _PASSWD_SECRET_JSON_F, visibility=db.VISIBILITY_GLOBAL) if jf.check(): with jf.open() as f: y = pkjson.load_any(f) else: y = pkcollections.Dict() if not host in y: y[host] = _passwd_entry(j2_ctx, host) pkjson.dump_pretty(y, filename=jf) return """install -m 600 /dev/stdin /root/.netrc <<'EOF' machine {} login {} password {} EOF curl {} | install_server={} bash -s {} # On {}: ssh {} true""".format( _vhost(j2_ctx), host, y[host], j2_ctx.rsconf_db.http_host, j2_ctx.rsconf_db.http_host, host, j2_ctx.bkp.primary, host, )
def _rpc(request): """Send an RPC message to the runner daemon, and get the response. Args: request: the request, as a json-encodeable object Returns: response: the server response """ request_bytes = pkjson.dump_bytes(request) with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: sock.connect(str(srdb.runner_socket_path())) # send the request sock.sendall(request_bytes) # send EOF, so the other side knows we've sent the whole thing sock.shutdown(socket.SHUT_WR) # read the response response_bytes = bytearray() while True: chunk = sock.recv(_CHUNK_SIZE) if not chunk: break response_bytes += chunk if response_bytes == b'': raise AssertionError('runner daemon had an unknown error') return pkjson.load_any(bytes(response_bytes))
def _pop_users_flattened(self): jc = self.j2_ctx z = jc.dovecot pw_f = db.secret_path( jc, _PASSWORD_SECRET_JSON_F, visibility=_PASSWORD_VISIBILITY, ) pw_modified = False if pw_f.check(): with pw_f.open() as f: pw_db = pkjson.load_any(f) else: pw_db = pkcollections.Dict() res = [] for u, v in z.pop_users.items(): if not isinstance(v, dict): v = pkcollections.Dict(password=v, username=u) if not v.username in pw_db: pw_modified = True pw_db[v.username] \ = '{' + z.passdb_scheme + '}' + _sha512_crypt(v.password) i = z.base_users.user_spec(u) i.pw_hash = pw_db[v.username] i.username = v.username i.home_d = db.user_home_path(jc, u) res.append(i) self._setup_procmail(i) self.install_access(mode='700', owner=i.uid, group=i.gid) self.install_directory(i.home_d.join(z.user_mail_d)) if pw_modified: pkjson.dump_pretty(pw_db, filename=pw_f) return sorted(res, key=lambda x: x.username)
def test_read_all(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code( 'code1', version='1.1', uri='http://x.com', source_d='/tmp', pyenv='py2', ) v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version pkjson.dump_pretty( {'version': v, 'image': {'type': 'docker'}}, filename=rsmanifest.CONTAINER_FILE, ) m = rsmanifest.read_all() pkeq(v, m.version) pkeq('docker', m.image.type) pkeq('1.1', m.codes.py2.code1.version)
def install_crt_and_login(compt, j2_ctx): from rsconf.pkcli import tls if not update_j2_ctx(j2_ctx): return jf = db.secret_path(j2_ctx, _PASSWD_SECRET_JSON_F, visibility=_PASSWD_VISIBILITY) with jf.open() as f: y = pkjson.load_any(jf) u = j2_ctx.rsconf_db.host p = y.get(u, None) if not p: return j2_ctx.docker.auths[j2_ctx.docker_registry.http_addr] = dict( auth=pkcompat.from_bytes( base64.b64encode( pkcompat.to_bytes(u + ':' + pkcompat.from_bytes(p))), ), ) compt.install_access(mode='700', owner=j2_ctx.docker_registry.run_u) crt = component.tls_key_and_crt(j2_ctx, j2_ctx.docker_registry.host).crt if not tls.is_self_signed_crt(crt): return compt.install_directory(_CERTS_D) d = _CERTS_D.join(j2_ctx.docker_registry.http_addr) compt.install_directory(d) compt.install_access(mode='400', owner=j2_ctx.docker_registry.run_u) compt.install_abspath(crt, d.join('ca.crt')) # need in /etc/pki as well (now) # https://success.docker.com/article/i-get-x509-certificate-signed-by-unknown-authority-error-when-i-try-to-login-to-my-dtr-with-default-certificates compt.install_abspath( crt, _CA_TRUST_D.join(j2_ctx.docker_registry.host + '.crt')) compt.append_root_bash('update-ca-trust')
def _classification_metrics_report(frame_args, filename): def _get_lables(): l = [] for k in d: if not isinstance(d[k], PKDict): continue for x in d[k]: if x not in l: l.append(x) return l def _get_matrix(): r = [] for k in d: if not isinstance(d[k], PKDict): continue try: x = [e[k]] except KeyError: x = [k] x.extend(d[k].values()) r.append(x) return r e = _get_classification_output_col_encoding(frame_args) d = pkjson.load_any(frame_args.run_dir.join(filename)) return PKDict( labels=_get_lables(), matrix=_get_matrix(), )
def run_dir_status(self, run_dir): """Get the current status of whatever's happening in run_dir. Returns: Tuple of (jhash or None, status of that job) """ disk_in_path = run_dir.join('in.json') disk_status_path = run_dir.join('status') if disk_in_path.exists() and disk_status_path.exists(): # status should be recorded on disk XOR in memory assert run_dir not in self.report_jobs disk_in_text = pkio.read_text(disk_in_path) disk_jhash = pkjson.load_any(disk_in_text).reportParametersHash disk_status = pkio.read_text(disk_status_path) if disk_status == 'pending': # We never write this, so it must be stale, in which case # the job is no longer pending... pkdlog( 'found "pending" status, treating as "error" ({})', disk_status_path, ) disk_status = runner_client.JobStatus.ERROR return disk_jhash, runner_client.JobStatus(disk_status) elif run_dir in self.report_jobs: job_info = self.report_jobs[run_dir] return job_info.jhash, job_info.status else: return None, runner_client.JobStatus.MISSING
def default_command(in_file): """Reads `in_file` passes to `msg.jobCmd` Must be called in run_dir Writes its output on stdout. Args: in_file (str): json parsed to msg Returns: str: json output of command, e.g. status msg """ try: job.init() f = pkio.py_path(in_file) msg = pkjson.load_any(f) #TODO(e-carlin): find common place to serialize/deserialize paths msg.runDir = pkio.py_path(msg.runDir) f.remove() res = globals()['_do_' + msg.jobCmd](msg, sirepo.template.import_module( msg.simulationType)) if res is None: return r = PKDict(res).pksetdefault(state=job.COMPLETED) except Exception as e: r = PKDict( state=job.ERROR, error=e.sr_args.error if isinstance(e, sirepo.util.UserAlert) else str(e), stack=pkdexc(), ) return pkjson.dump_pretty(r, pretty=False)
def _write_sasl(self, jc, z): if not z.have_sasl: return self.install_access(mode='400', owner=jc.rsconf_db.root_u) r = [] for domain, u in z.sasl_users.items(): for user, password in u.items(): r.append( pkcollections.Dict( domain=domain, user=user, password=password, ), ) jf = _sasl_password_path(jc) if jf.check(): with jf.open() as f: y = pkjson.load_any(f) for u, p in y.items(): x = u.split('@') r.append( pkcollections.Dict( domain=x[1], user=x[0], password=p, ), ) z.sasl_users_flattened = sorted(r, key=lambda x: x.user + x.domain) self.install_resource( 'postfix/smtpd-sasldb.conf', jc, '/etc/sasl2/smtpd-sasldb.conf', )
def test_read_all(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code( 'code1', version='1.1', uri='http://x.com', source_d='/tmp', pyenv='py2', ) v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version pkjson.dump_pretty( { 'version': v, 'image': { 'type': 'docker' } }, filename=rsmanifest.CONTAINER_FILE, ) m = rsmanifest.read_all() pkeq(v, m.version) pkeq('docker', m.image.type) pkeq('1.1', m.codes.py2.code1.version)
async def run_extract_job(self, run_dir, jhash, subcmd, arg): pkdc('{} {}: {} {}', run_dir, jhash, subcmd, arg) status = self.report_job_status(run_dir, jhash) if status is runner_client.JobStatus.MISSING: pkdlog('{} {}: report is missing; skipping extract job', run_dir, jhash) return {} # figure out which backend and any backend-specific info runner_info_file = run_dir.join(_RUNNER_INFO_BASENAME) if runner_info_file.exists(): runner_info = pkjson.load_any(runner_info_file) else: # Legacy run_dir runner_info = pkcollections.Dict( version=1, backend='local', backend_info={}, ) assert runner_info.version == 1 # run the job cmd = ['sirepo', 'extract', subcmd, arg] result = await _BACKENDS[runner_info.backend].run_extract_job( run_dir, cmd, runner_info.backend_info, ) if result.stderr: pkdlog( 'got output on stderr ({} {}):\n{}', run_dir, jhash, result.stderr.decode('utf-8', errors='ignore'), ) if result.returncode != 0: pkdlog( 'failed with return code {} ({} {}), stdout:\n{}', result.returncode, run_dir, subcmd, result.stdout.decode('utf-8', errors='ignore'), ) raise AssertionError return pkjson.load_any(result.stdout)
def _parse_par(fn): data_file = fn.basename.replace('-flash.par', '') return flash_parser.ParameterParser().parse( pkjson.load_any( pkio.read_text( pkunit.data_dir().join(f'{data_file}-sirepo-data.json'))), pkio.read_text(fn), )
def parse_par(sim_id, par_path): """Returns parsed flash.par values. """ sim_path = _sim_path_from_id(sim_id) return flash_parser.ParameterParser().parse( pkjson.load_any(pkio.read_text(sim_path)), pkio.read_text(par_path), )
def test_load_any(): import json from pykern import pkjson from pykern.pkunit import pkeq j = json.dumps(['a', 'b']) j2 = pkjson.load_any(j) pkeq('a', j2[0])
def __slots_from_dump(cls, pool_name): p = pkio.py_path(_POOLS_DUMP_FILE) if not p.exists(): return PKDict() slots = PKDict() for s in pkjson.load_any(p).pkunchecked_nested_get( f'{pool_name}.slots', ) or []: slots[s.cname] = s return slots
def test_srw_create_predefined(): from pykern import pkunit from pykern import pkjson import sirepo.pkcli.srw d = pkunit.empty_work_dir() sirepo.pkcli.srw.create_predefined(d) j = pkjson.load_any(d.listdir()[0]) pkunit.pkeq(22, len(j.beams))
def _get_default_drift(): """The function parses srw.js file to find the default values for drift propagation parameters, which can be sometimes missed in the exported .py files (when distance = 0), but should be presented in .json files. Returns: str: default drift propagation paramters """ c = pkio.read_text(_JS_DIR.join('srw.js')) m = re.search(r'function defaultDriftPropagationParams.*?return\s*(\[[^\]]+\])', c, re.DOTALL) return pkjson.load_any(m.group(1))
def read_all(): """Merge all manifests Returns: dict: merged data """ from pykern import pkio from pykern import pkjson fn = pkio.py_path(USER_FILE) # Both must exist or error u = pkjson.load_any(fn) c = pkjson.load_any(pkio.py_path(CONTAINER_FILE)) assert u.version == c.version, \ '(user.version) {} != {} (container.version)'.format(u.version, c.version) # There are "guaranteed" to be no collisions, but if there are # we override user. c.update(u) return c
def test_dump_bytes(): import json from pykern import pkjson from pykern.pkunit import pkeq v = ['a', 'b'] expect = json.dumps(v).encode(pkjson.ENCODING).replace(' ', '') actual = pkjson.dump_bytes(v) pkeq(expect, actual) actual = pkjson.load_any(actual) pkeq(v, actual)
def sr_post(self, route_or_uri, data, headers=None): from pykern import pkjson import requests r = requests.post( (f'http://{self._ip}:{self._nginx_proxy_port}' + f'{self._server_route(route_or_uri)}'), json=data, headers=headers, ) r.raise_for_status() return pkjson.load_any(r.text)
def test_add_code(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code('A', 'b', 'c', 'd', pyenv='v') j = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE).read()) pkok(20170101.0 < float(j.version), 'version must be after 2017') pkeq('A', j.codes.v.a.name) pkeq('b', j.codes.v.a.version) rsmanifest.add_code('a', 'bb', 'cc', 'dd') j = pkjson.load_any(pkio.expand_user_path(rsmanifest.USER_FILE).read()) pkeq('A', j.codes.v.a.name) pkeq('a', j.codes[''].a.name) pkeq('bb', j.codes[''].a.version) pkre('20.*T.*Z', j.codes[''].a.installed)
def test_parse_madx_file(): from pykern import pkio, pkjson from pykern.pkunit import pkeq from sirepo.template import madx, madx_parser for name in ('particle_track', ): actual = madx_parser.parse_file( pkio.read_text(pkunit.data_dir().join(f'{name}.madx'))) madx._fixup_madx(actual) del actual['version'] expect = pkjson.load_any(pkunit.data_dir().join(f'{name}.json')) pkeq(expect, actual)
def test_add_code(): from pykern import pkio from pykern import pkjson from pykern import pkunit from pykern.pkunit import pkok, pkeq, pkre from pykern.pkdebug import pkdp from pykern.pkcli import rsmanifest import re with pkunit.save_chdir_work(is_pkunit_prefix=True) as d: rsmanifest.add_code('A', 'b', 'c', 'd', pyenv='v') j = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE).read()) pkok(20170101.0 < float(j.version), 'version must be after 2017') pkeq('A', j.codes.v.a.name) pkeq('b', j.codes.v.a.version) rsmanifest.add_code('a', 'bb', 'cc', 'dd') j = pkjson.load_any(pkio.expand_user_path(rsmanifest.USER_FILE).read()) pkeq('A', j.codes.v.a.name) pkeq('a', j.codes[''].a.name) pkeq('bb', j.codes[''].a.version) pkre('20.*T.*Z', j.codes[''].a.installed)
def background_percent_complete(report, run_dir, is_running): def _grid_columns(): c = _grid_evolution_columns(run_dir) return [x for x in c if x[0] != '#'] if c \ else None def _plot_filenames(): return [ PKDict( time=_time_and_units(yt.load(str(f)).parameters['time']), filename=f.basename, ) for f in files ] def _plot_vars(): names = [] if len(files): io = simulation_db.read_json( run_dir.join(template_common.INPUT_BASE_NAME), ).models.IO_IOMain idx = 1 while io.get(f'plot_var_{idx}', ''): n = io[f'plot_var_{idx}'] if n != 'none': names.append(n) idx += 1 return names res = PKDict( percentComplete=0 if is_running else 100, ) if report == 'setupAnimation': f = run_dir.join(_SIM_DATA.SETUP_PARAMS_SCHEMA_FILE) if f.exists(): res.pkupdate( frameCount=1, flashSchema=pkjson.load_any(pkio.read_text(f)) ) else: _init_yt() files = _h5_file_list(run_dir) if is_running and len(files): # the last file may be unfinished if the simulation is running files.pop() res.pkupdate( frameCount=len(files), plotVars=_plot_vars(), plotFiles=_plot_filenames(), gridEvolutionColumns=_grid_columns(), ) return res
def update_sim_from_par(sim_id, par_path): sim_path = _sim_path_from_id(sim_id) data = pkjson.load_any(pkio.read_text(sim_path)) parser = flash_parser.ParameterParser() values = parser.parse(data, pkio.read_text(par_path)) # reset all model fields to default and override with new par values for (m, fields) in parser.schema.model.items(): for f in fields: if f in values: data.models[m][f] = values[f] else: data.models[m][f] = fields[f][2] pkjson.dump_pretty(data, sim_path)
async def on_stdout_read(self, text): if self._terminating or not self.msg.opId: return try: await self.dispatcher.send( self.dispatcher.format_op( self.msg, job.OP_RUN if self._is_compute else job.OP_ANALYSIS, reply=pkjson.load_any(text), ) ) except Exception as exc: pkdlog('text={} error={} stack={}', text, exc, pkdexc())
def test_parse_madx_file(): from pykern import pkio, pkjson from pykern.pkunit import pkeq from sirepo.template import madx, madx_parser with pkunit.save_chdir_work(): for name in ('particle_track', 'alba'): actual = madx_parser.parse_file(pkio.read_text( pkunit.data_dir().join(f'{name}.madx'))) del actual['version'] outfile = f'{name}.json' pkjson.dump_pretty(actual, outfile) expect = pkjson.load_any(pkunit.data_dir().join(outfile)) pkeq(expect, actual)
def host_init(j2_ctx, host): from rsconf import db jf = _sasl_password_path(j2_ctx) if jf.check(): with jf.open() as f: y = pkjson.load_any(f) else: y = pkcollections.Dict() u = _SASL_PASSWORD_PREFIX + host if not u in y: y[u] = db.random_string() pkjson.dump_pretty(y, filename=jf) return u, y[u]
def _cfg_json(value): from pykern import pkjson if isinstance(value, pkcollections.Dict): return value return pkjson.load_any(value)