def test_follow_email_auth_link_twice(auth_fc):
fc = auth_fc
from pykern import pkconfig, pkunit, pkio, pkcompat
from pykern.pkunit import pkok, pkre
from pykern.pkdebug import pkdp
import json
r = fc.sr_post(
'authEmailLogin',
{
'email': '[email protected]',
'simulationType': fc.sr_sim_type
},
)
# The link comes back in dev mode so we don't have to check email
s = fc.sr_auth_state(isLoggedIn=False)
fc.get(r.uri)
# get the url twice - should still be logged in
d = fc.sr_get(r.uri)
assert not re.search(r'login-fail', pkcompat.from_bytes(d.data))
fc.sr_email_confirm(fc, r)
fc.sr_get('authLogout', {'simulation_type': fc.sr_sim_type})
# now logged out, should see login fail for bad link
pkre('login-fail', pkcompat.from_bytes(fc.get(r.uri).data))
def test_login():
from pykern import pkunit, pkcompat
from pykern.pkunit import pkeq, pkok, pkre, pkfail, pkexcept
from sirepo import auth
import flask
import sirepo.auth.guest
import sirepo.cookie
import sirepo.http_request
import sirepo.util
r = auth.api_authState()
pkre('LoggedIn": false.*Registration": false', pkcompat.from_bytes(r.data))
delattr(flask.g, 'sirepo_cookie')
auth.process_request()
with pkunit.pkexcept('SRException.*routeName=login'):
auth.logged_in_user()
with pkexcept('SRException.*routeName=login'):
auth.require_user()
sirepo.cookie.set_sentinel()
# copying examples for new user takes time
try:
r = auth.login(sirepo.auth.guest, sim_type='myapp')
pkfail('expecting sirepo.util.Response')
except sirepo.util.Response as e:
r = e.sr_args.response
pkre(r'LoggedIn":\s*true.*Registration":\s*false',
pkcompat.from_bytes(r.data))
u = auth.logged_in_user()
pkok(u, 'user should exist')
# guests do not require completeRegistration
auth.require_user()
def install_crt_and_login(compt, j2_ctx):
from rsconf.pkcli import tls
if not update_j2_ctx(j2_ctx):
return
jf = db.secret_path(j2_ctx,
_PASSWD_SECRET_JSON_F,
visibility=_PASSWD_VISIBILITY)
with jf.open() as f:
y = pkjson.load_any(jf)
u = j2_ctx.rsconf_db.host
p = y.get(u, None)
if not p:
return
j2_ctx.docker.auths[j2_ctx.docker_registry.http_addr] = dict(
auth=pkcompat.from_bytes(
base64.b64encode(
pkcompat.to_bytes(u + ':' + pkcompat.from_bytes(p))), ), )
compt.install_access(mode='700', owner=j2_ctx.docker_registry.run_u)
crt = component.tls_key_and_crt(j2_ctx, j2_ctx.docker_registry.host).crt
if not tls.is_self_signed_crt(crt):
return
compt.install_directory(_CERTS_D)
d = _CERTS_D.join(j2_ctx.docker_registry.http_addr)
compt.install_directory(d)
compt.install_access(mode='400', owner=j2_ctx.docker_registry.run_u)
compt.install_abspath(crt, d.join('ca.crt'))
# need in /etc/pki as well (now)
# https://success.docker.com/article/i-get-x509-certificate-signed-by-unknown-authority-error-when-i-try-to-login-to-my-dtr-with-default-certificates
compt.install_abspath(
crt, _CA_TRUST_D.join(j2_ctx.docker_registry.host + '.crt'))
compt.append_root_bash('update-ca-trust')
def test_myapp_basic(fc):
from pykern import pkunit, pkcompat
r = fc.get('/old')
assert 'LandingPageController' in pkcompat.from_bytes(r.data), \
'Top level document is the landing page'
r = fc.get('/robots.txt')
pkunit.pkre('elegant.*myapp.*srw', pkcompat.from_bytes(r.data))
def test_myapp_basic(fc):
from pykern import pkunit, pkcompat
from pykern.pkunit import pkok, pkeq
r = fc.sr_get('/robots.txt')
pkunit.pkre('elegant.*myapp.*srw', pkcompat.from_bytes(r.data))
r = fc.sr_get('/')
pkok(not re.search(r'googletag', pkcompat.from_bytes(r.data)),
'Unexpected injection of googletag data={}', r.data)
def _run(cmd):
try:
pkdc('{}', ' '.join(cmd))
return pkcompat.from_bytes(
subprocess.check_output(cmd, stderr=subprocess.STDOUT), )
except Exception as e:
o = ''
if hasattr(e, 'output'):
o = pkcompat.from_bytes(e.output)
pkdlog('command error: cmd={} error={} out={}', cmd, e, o)
raise
def test_injection(fc):
from pykern import pkcompat, pkunit
from pykern.pkdebug import pkdc, pkdp, pkdlog
from pykern.pkunit import pkeq, pkok, pkre
import re
# test non-static page
r = fc.get('myapp')
pkok(not re.search(r'googletag', pkcompat.from_bytes(r.data)),
'Unexpected injection of googletag data={}', r.data)
# test successful injection
r = fc.get('/en/landing.html')
pkre(_TEST_ID, pkcompat.from_bytes(r.data))
def test_set_get():
from pykern import pkunit, pkcompat
from pykern.pkunit import pkeq
from pykern.pkdebug import pkdp
from pykern import pkcollections
from sirepo import cookie
class _Response(pkcollections.Dict):
def set_cookie(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
cookie.process_header('x')
with pkunit.pkexcept('KeyError'):
cookie.get_value('hi1')
with pkunit.pkexcept('AssertionError'):
cookie.set_value('hi2', 'hello')
pkeq(None, cookie.unchecked_get_value('hi3'))
cookie.set_cookie_for_utils()
cookie.set_value('hi4', 'hello')
r = _Response(status_code=200)
cookie.save_to_cookie(r)
pkeq('sirepo_dev', r.args[0])
pkeq(False, r.kwargs['secure'])
pkeq('hello', cookie.get_value('hi4'))
cookie.unchecked_remove('hi4')
pkeq(None, cookie.unchecked_get_value('hi4'))
cookie.process_header(
'sirepo_dev={}'.format(pkcompat.from_bytes(r.args[1])), )
pkeq('hello', cookie.get_value('hi4'))
def gen_private_key():
"""Generate 32 byte random private key"""
import base64
import os
from pykern import pkcompat
return pkcompat.from_bytes(base64.urlsafe_b64encode(os.urandom(32)))
def import_file(req, test_data=None, **kwargs):
# input_data is passed by test cases only
input_data = test_data
text = pkcompat.from_bytes(req.file_stream.read())
if 'simulationId' in req.req_data:
input_data = simulation_db.read_simulation_json(
SIM_TYPE, sid=req.req_data.simulationId)
if re.search(r'\.ele$', req.filename, re.IGNORECASE):
data = elegant_command_importer.import_file(text)
elif re.search(r'\.lte$', req.filename, re.IGNORECASE):
data = elegant_lattice_importer.import_file(text, input_data)
if input_data:
_map_commands_to_lattice(data)
elif re.search(r'\.madx$', req.filename, re.IGNORECASE):
from sirepo.template import madx_converter, madx_parser
data = madx_converter.from_madx(
SIM_TYPE, madx_parser.parse_file(text, downcase_variables=True))
else:
raise IOError('invalid file extension, expecting .ele or .lte')
data.models.simulation.name = re.sub(r'\.(lte|ele|madx)$',
'',
req.filename,
flags=re.IGNORECASE)
if input_data and not test_data:
simulation_db.delete_simulation(
SIM_TYPE, input_data.models.simulation.simulationId)
return data
def sr_auth_state(self, **kwargs):
"""Gets authState and prases
Returns:
dict: parsed auth_state
"""
from pykern import pkunit
import pykern.pkcollections
m = re.search(
r'(\{.*\})',
pkcompat.from_bytes(self.sr_get('authState').data),
)
s = pykern.pkcollections.json_load_any(m.group(1))
for k, v in kwargs.items():
pkunit.pkeq(
v,
s[k],
'key={} expected={} != actual={}: auth_state={}',
k,
v,
s[k],
s,
)
return s
def _parameters(f):
res = {}
for name in ('integer scalars', 'integer runtime parameters',
'real scalars', 'real runtime parameters'):
for v in f[name]:
res[pkcompat.from_bytes(v[0].strip())] = v[1]
return res
def subprocess_output(cmd, env):
"""Run cmd and return output or None, logging errors.
Args:
cmd (list): what to run
Returns:
str: output is None on error else a stripped string
"""
err = None
out = None
try:
p = subprocess.Popen(
cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = p.communicate()
if p.wait() != 0:
raise subprocess.CalledProcessError(returncode=p.returncode,
cmd=cmd)
except subprocess.CalledProcessError as e:
pkdlog('{}: exit={} err={}', cmd, e.returncode, err)
return None
if out:
out = pkcompat.from_bytes(out)
return out.strip()
return ''
def import_file(req, unit_test_mode=False, **kwargs):
from sirepo.template import opal_parser
text = pkcompat.from_bytes(req.file_stream.read())
if re.search(r'\.in$', req.filename, re.IGNORECASE):
data, input_files = opal_parser.parse_file(text, filename=req.filename)
missing_files = []
for infile in input_files:
if not _SIM_DATA.lib_file_exists(infile.lib_filename):
missing_files.append(infile)
if len(missing_files):
return PKDict(
error='Missing data files',
missingFiles=missing_files,
)
elif re.search(r'\.madx$', req.filename, re.IGNORECASE):
from sirepo.template import madx_converter, madx_parser
madx = madx_parser.parse_file(text)
data = madx_converter.from_madx(SIM_TYPE, madx)
data.models.simulation.name = re.sub(r'\.madx$',
'',
req.filename,
flags=re.IGNORECASE)
_fixup_madx(madx, data)
else:
raise IOError('invalid file extension, expecting .in or .madx')
return data
def parse_response(self, resp, expect_binary_body=False):
assert resp.code == 200, 'resp={}'.format(resp)
if 'Set-Cookie' in resp.headers:
self._headers.Cookie = resp.headers['Set-Cookie']
if 'json' in resp.headers['content-type']:
return pkjson.load_any(resp.body)
try:
b = pkcompat.from_bytes(resp.body)
assert not expect_binary_body, \
'expecting binary body resp={} body={}'.format(
resp,
b[:1000],
)
except UnicodeDecodeError:
assert expect_binary_body, \
'unexpected binary body resp={}'.format(resp)
# Binary data files can't be decoded
return
if 'html' in resp.headers['content-type']:
m = re.search('location = "(/[^"]+)', b)
if m:
if 'error' in m.group(1):
return PKDict(state='error', error='server error')
return PKDict(state='redirect', uri=m.group(1))
return b
def _do(fc, file_ext, parse):
from pykern.pkcollections import PKDict
from pykern import pkio, pkcompat
from pykern import pkunit
from pykern import pkcollections
from pykern.pkdebug import pkdp, pkdlog
from pykern.pkunit import pkeq, pkfail, pkok, pkre
import re
for suffix in (('',) if file_ext == 'py' else ('', ' 2', ' 3')):
for f in pkio.sorted_glob(pkunit.data_dir().join('*.' + file_ext)):
pkdlog('file={}', f)
json = pkcompat.from_bytes(parse(f))
sim_type = re.search(r'^([a-z]+)_', f.basename).group(1)
fc.sr_get_root(sim_type)
is_dev = 'deviance' in f.basename
res = fc.sr_post_form(
'importFile',
PKDict(folder='/importer_test'),
PKDict(simulation_type=sim_type),
file=f,
)
if is_dev:
m = re.search(r'Error: (.+)', json)
if m:
expect = m.group(1)
pkre(expect, res.error)
continue
elif file_ext == 'py':
sim_name = f.purebasename
else:
sim_name = pkcollections.json_load_any(json).models.simulation.name
assert 'models' in res, \
f'file={f} res={res}'
pkeq(sim_name + suffix, res.models.simulation.name)
def create_token(value):
if pkconfig.channel_in_internal_test() and cfg.create_token_secret:
v = base64.b32encode(
hashlib.sha256(
pkcompat.to_bytes(value + cfg.create_token_secret)).digest())
return pkcompat.from_bytes(v[:TOKEN_SIZE])
return random_base62(TOKEN_SIZE)
def test_srw_upload(fc):
from pykern import pkunit, pkcompat
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp
import sirepo.sim_data
d = fc.sr_sim_data('NSLS-II CHX beamline')
s = sirepo.sim_data.get_class(fc.sr_sim_type)
f = s.lib_file_resource_dir().join('mirror_1d.dat')
t = 'mirror'
r = fc.sr_post_form(
'uploadFile',
params=PKDict(
simulation_type=fc.sr_sim_type,
simulation_id=d.models.simulation.simulationId,
file_type=t,
),
data=PKDict(),
file=f,
)
pkunit.pkre('in use in other', r.get('error', ''))
r = fc.sr_post_form(
'uploadFile',
params=PKDict(
simulation_type=fc.sr_sim_type,
simulation_id=d.models.simulation.simulationId,
file_type=t,
),
data=PKDict(confirm='1'),
file=f,
)
e = r.get('error', '')
pkunit.pkok(not e, 'unexpected error={}', e)
r = fc.sr_post_form(
'uploadFile',
params=PKDict(
simulation_type=fc.sr_sim_type,
simulation_id=d.models.simulation.simulationId,
file_type='invalid file type',
),
data=PKDict(),
file=f,
)
pkunit.pkre('invalid file type', r.get('error', ''))
# the above used to delete the file
r = fc.sr_get(
'downloadFile',
params=PKDict(
simulation_type=fc.sr_sim_type,
simulation_id=d.models.simulation.simulationId,
filename=f.basename,
),
data=PKDict(),
)
pkunit.pkre(r'^\s*-1.39500', pkcompat.from_bytes(r.data))
def secret_path_value(self, filename, gen_secret=None, visibility=None):
from rsconf import db
src = db.secret_path(self.hdb, filename, visibility=visibility)
if src.check():
return pkio.read_text(src), src
assert gen_secret, \
'unable to generate secret: path={}'.format(src)
res = gen_secret()
res = pkcompat.from_bytes(self._write_binary(src, res))
return res, src
def import_file(req, **kwargs):
text = pkcompat.from_bytes(req.file_stream.read())
assert re.search(r'\.madx$', req.filename, re.IGNORECASE), \
'invalid file extension, expecting .madx'
data = madx_parser.parse_file(text, downcase_variables=True)
# TODO(e-carlin): need to clean this up. copied from elegant
data.models.simulation.name = re.sub(r'\.madx$',
'',
req.filename,
flags=re.IGNORECASE)
return data
def test_from_bytes():
from pykern import pkcompat
from pykern.pkunit import pkeq
b = pkcompat.to_bytes('你好')
s = pkcompat.from_bytes(b)
pkeq(s, '你好')
pkeq(b, b'\xe4\xbd\xa0\xe5\xa5\xbd')
if six.PY2:
pkeq(b, s)
else:
pkeq(False, b == s)
def _read_last_line(path):
# for performance, don't read whole file if only last line is needed
try:
with open(str(path), 'rb') as f:
f.readline()
f.seek(-2, os.SEEK_END)
while f.read(1) != b'\n':
f.seek(-2, os.SEEK_CUR)
return pkcompat.from_bytes(f.readline())
except IOError:
return ''
def test_srw(fc):
from pykern import pkio, pkcompat
from pykern.pkdebug import pkdpretty
from pykern.pkunit import pkeq, pkre
import json
r = fc.sr_get_root()
pkre('<!DOCTYPE html', pkcompat.from_bytes(r.data))
d = fc.sr_post('listSimulations', {'simulationType': fc.sr_sim_type})
pkeq(fc.get('/find-by-name-auth/srw/default/UndulatorRadiation').status_code, 404)
for sep in (' ', '%20', '+'):
pkeq(fc.get('/find-by-name-auth/srw/default/Undulator{}Radiation'.format(sep)).status_code, 200)
def test_deprecated():
fc, sim_type = _fc(guest_deprecated=True)
from pykern import pkconfig, pkunit, pkio, pkcompat
from pykern.pkunit import pkok, pkre, pkeq
from pykern.pkdebug import pkdp
import re
r = fc.sr_get('authGuestLogin', {'simulation_type': sim_type},
redirect=False)
pkeq(302, r.status_code)
pkre('guest/deprecated', pkcompat.from_bytes(r.data))
def _run_tests():
"""Runs the SRW "Undulator Radiation" simulation's initialIntensityReport"""
_validate_auth_state()
simulation_type = _SIM_TYPE
res = uri_router.call_api(
'findByNameWithAuth',
dict(
simulation_type=simulation_type,
application_mode='default',
simulation_name=_SIM_NAME,
),
)
m = re.search(r'\/source\/(\w+)"', pkcompat.from_bytes(res.data))
if not m:
raise RuntimeError('failed to find sid in resp={}'.format(res.data))
i = m.group(1)
d = simulation_db.read_simulation_json(simulation_type, sid=i)
try:
d.models.electronBeam.current = d.models.electronBeam.current + (
random.random() / 10)
except AttributeError:
assert _SIM_TYPE == 'myapp', \
f'{_SIM_TYPE} should be myapp or have models.electronBeam.current'
pass
d.simulationId = i
d.report = _SIM_REPORT
r = None
try:
resp = uri_router.call_api('runSimulation', data=d)
for _ in range(_MAX_CALLS):
r = simulation_db.json_load(resp.data)
pkdlog('resp={}', r)
if r.state == 'error':
raise RuntimeError('simulation error: resp={}'.format(r))
if r.state == 'completed':
if 'initialIntensityReport' == d.report:
min_size = 50
if len(r.z_matrix) < min_size or len(
r.z_matrix[0]) < min_size:
raise RuntimeError(
'received bad report output: resp={}', r)
return
d = r.nextRequest
resp = uri_router.call_api('runStatus', data=d)
time.sleep(_SLEEP)
raise RuntimeError(
'simulation timed out: seconds={} resp='.format(
_MAX_CALLS * _SLEEP, r), )
finally:
try:
uri_router.call_api('runCancel', data=d)
except Exception:
pass
def sr_run_sim(self,
data,
model,
expect_completed=True,
timeout=10,
**post_args):
from pykern import pkunit
from pykern.pkdebug import pkdlog, pkdexc
import time
if self.sr_job_run_mode:
data.models[model].jobRunMode = self.sr_job_run_mode
cancel = None
try:
r = self.sr_post(
'runSimulation',
PKDict(
models=data.models,
report=model,
simulationId=data.models.simulation.simulationId,
simulationType=data.simulationType,
).pkupdate(**post_args),
)
if r.state == 'completed':
return r
cancel = r.get('nextRequest')
for _ in range(timeout):
if r.state in ('completed', 'error'):
pkdlog(r.state)
cancel = None
break
r = self.sr_post('runStatus', r.nextRequest)
time.sleep(1)
else:
pkunit.pkok(not expect_completed,
'did not complete: runStatus={}', r)
if expect_completed:
pkunit.pkeq('completed', r.state)
return r
finally:
if cancel:
pkdlog('runCancel')
self.sr_post('runCancel', cancel)
import subprocess
o = pkcompat.from_bytes(
subprocess.check_output(['ps', 'axww'],
stderr=subprocess.STDOUT), )
o = list(filter(lambda x: 'mpiexec' in x, o.split('\n')))
if o:
pkdlog('found "mpiexec" after cancel in ps={}', '\n'.join(o))
# this exception won't be seen because in finally
raise AssertionError('cancel failed')
def install_perl_rpm(self, j2_ctx, rpm_base, channel=None):
src = self.rpm_file(j2_ctx, rpm_base, channel)
r = src.basename
if r in self.hdb.component.setdefault('_installed_rpms', set()):
return r
self.hdb.component._installed_rpms.add(r)
dst = j2_ctx.build.dst_d.join(r)
dst.mksymlinkto(src, absolute=False)
self.append_root_bash("rsconf_install_perl_rpm '{}' '{}' '{}'".format(
rpm_base,
r,
pkcompat.from_bytes(
subprocess.check_output(['rpm', '-qp', str(src)]), ).strip(),
))
return r
def import_file(req, unit_test_mode=False, **kwargs):
from sirepo.template import opal_parser
data, input_files = opal_parser.parse_file(pkcompat.from_bytes(
req.file_stream.read()),
filename=req.filename)
missing_files = []
for infile in input_files:
if not _SIM_DATA.lib_file_exists(infile.lib_filename):
missing_files.append(infile)
if len(missing_files):
return PKDict(
error='Missing data files',
missingFiles=missing_files,
)
return data
def _passwd_entry(j2_ctx, host):
from rsconf import db
import subprocess
pw = db.random_string()
pf = passwd_secret_f(j2_ctx)
p = subprocess.Popen(
['openssl', 'passwd', '-stdin', '-apr1'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
out, err = p.communicate(input=pkcompat.to_bytes(pw))
with pf.open(mode='at') as f:
f.write('{}:{}\n'.format(host, pkcompat.from_bytes(out).rstrip()))
return pw
def test_create_zip(fc):
from pykern import pkio
from pykern import pkunit
from pykern import pkcompat
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp, pkdpretty
from pykern.pkunit import pkeq
from sirepo import srunit
import base64
import re
import zipfile
imported = _import(fc)
for sim_type, sim_name, expect in imported + [
('elegant', 'bunchComp - fourDipoleCSR',
['WAKE-inputfile.knsl45.liwake.sdds', 'run.py', 'sirepo-data.json']),
('srw', 'Tabulated Undulator Example',
['magnetic_measurements.zip', 'run.py', 'sirepo-data.json']),
('warppba', 'Laser Pulse', ['run.py', 'sirepo-data.json']),
]:
sim_id = fc.sr_sim_data(
sim_name, sim_type)['models']['simulation']['simulationId']
with pkio.save_chdir(pkunit.work_dir()) as d:
for t in 'zip', 'html':
r = fc.sr_get(
'exportArchive',
PKDict(
simulation_type=sim_type,
simulation_id=sim_id,
filename='anything.' + t,
))
p = d.join(sim_name + '.' + t)
x = r.data
if t == 'html':
x = pkcompat.from_bytes(x)
m = re.search(r'name="zip" \S+ value="([^"]+)"',
x,
flags=re.DOTALL)
x = base64.b64decode(pkcompat.to_bytes(m.group(1)))
p.write_binary(x)
e = expect
if t == 'html':
e.remove('run.py')
pkeq(
e,
sorted(zipfile.ZipFile(str(p)).namelist()),
)
