def test_read_all():
from pykern import pkio
from pykern import pkjson
from pykern import pkunit
from pykern.pkunit import pkok, pkeq, pkre
from pykern.pkdebug import pkdp
from pykern.pkcli import rsmanifest
import re
with pkunit.save_chdir_work(is_pkunit_prefix=True) as d:
rsmanifest.add_code(
'code1',
version='1.1',
uri='http://x.com',
source_d='/tmp',
pyenv='py2',
)
v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version
pkjson.dump_pretty(
{'version': v, 'image': {'type': 'docker'}},
filename=rsmanifest.CONTAINER_FILE,
)
m = rsmanifest.read_all()
pkeq(v, m.version)
pkeq('docker', m.image.type)
pkeq('1.1', m.codes.py2.code1.version)
def test_importer(import_req):
from pykern import pkcollections
from pykern import pkjson
from pykern.pkunit import pkeq
from sirepo.template import zgoubi
import sirepo.sim_data
with pkunit.save_chdir_work() as w:
for fn in pkio.sorted_glob(pkunit.data_dir().join('*.dat')):
error = None
try:
data = zgoubi.import_file(import_req(fn), unit_test_mode=True)
sirepo.sim_data.get_class('zgoubi').fixup_old_data(data)
#TODO(pjm): easier way to convert nested dict to pkcollections.Dict?
data = pkcollections.json_load_any(pkjson.dump_pretty(data))
except Exception as e:
pkdlog(pkdexc())
error = e.message
if error:
actual = error
else:
actual = zgoubi.python_source_for_model(data)
outfile = fn.basename + '.txt'
pkio.write_text(outfile, actual)
e = pkunit.data_dir().join(outfile)
expect = pkio.read_text(e)
pkeq(expect, actual, 'diff {} {}', e, w.join(outfile))
def test_checked_call():
from pykern import pkunit
from pykern.pkunit import pkeq
import sys
import subprocess
with pkunit.save_chdir_work():
cmd = [sys.executable, str(pkunit.data_dir().join('p1.py'))]
for i, a in enumerate((
('normal', 0),
('exit-1', 1),
('divide-zero', 1),
('normal-rank-all', 0),
('divide-zero-rank-2', 86),
('exit-13-rank-0', 13),
)):
f = '{}.out'.format(i)
with open(f, 'w') as o:
c = cmd + [a[0]]
print(a[0])
if 'rank' in a[0]:
c = ['mpiexec', '-n', '4'] + c
actual = subprocess.call(
c,
stdout=o,
stderr=subprocess.STDOUT,
)
pkeq(a[1], actual, '{}: exit({})\n{}', ' '.join(c), actual,
open(f).read())
def test_import():
from pykern import pkjson
from pykern.pkunit import pkeq
from sirepo.template import flash_parser
import re
def _parse_config(fn):
return flash_parser.ConfigParser().parse(pkio.read_text(fn))
def _parse_par(fn):
data_file = fn.basename.replace('-flash.par', '')
return flash_parser.ParameterParser().parse(
pkjson.load_any(
pkio.read_text(
pkunit.data_dir().join(f'{data_file}-sirepo-data.json'))),
pkio.read_text(fn),
)
with pkunit.save_chdir_work():
for fn in pkio.sorted_glob(pkunit.data_dir().join('*')):
if re.search(r'-Config$', fn.basename):
parser = _parse_config
elif re.search(r'flash.par$', fn.basename):
parser = _parse_par
else:
continue
try:
actual = pkjson.dump_pretty(parser(fn))
except Exception as e:
pkdlog(pkdexc())
actual = str(e)
outfile = f'{fn.basename}.out'
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
pkeq(expect, actual)
def test_unchecked_remove():
"""Also tests mkdir_parent"""
from pykern import pkunit
from pykern import pkio
with pkunit.save_chdir_work():
fn = 'f1'
# Should not throw an exception
pkio.unchecked_remove(fn)
pkio.write_text(fn, 'hello')
pkio.unchecked_remove(fn)
assert not os.path.exists(fn), \
'When file removed, should be gone'
for f in ('d1', 'd2/d3'):
assert py.path.local(f) == pkio.mkdir_parent(f), \
'When mkdir_parent is called, returns path passed in'
assert os.path.exists('d1'), \
'When single directory, should exist'
assert os.path.exists('d2/d3'), \
'When nested directory, should exist'
with pytest.raises(AssertionError):
pkio.unchecked_remove('.')
with pytest.raises(AssertionError):
pkio.unchecked_remove(os.getcwd())
with pytest.raises(AssertionError):
pkio.unchecked_remove('/')
def test_read_all():
from pykern import pkio
from pykern import pkjson
from pykern import pkunit
from pykern.pkunit import pkok, pkeq, pkre
from pykern.pkdebug import pkdp
from pykern.pkcli import rsmanifest
import re
with pkunit.save_chdir_work(is_pkunit_prefix=True) as d:
rsmanifest.add_code(
'code1',
version='1.1',
uri='http://x.com',
source_d='/tmp',
pyenv='py2',
)
v = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE)).version
pkjson.dump_pretty(
{
'version': v,
'image': {
'type': 'docker'
}
},
filename=rsmanifest.CONTAINER_FILE,
)
m = rsmanifest.read_all()
pkeq(v, m.version)
pkeq('docker', m.image.type)
pkeq('1.1', m.codes.py2.code1.version)
def test_from_elegant_to_madx_and_back():
from pykern import pkio
from pykern.pkunit import pkeq
from sirepo.template import elegant, madx, madx_converter, madx_parser
with pkunit.save_chdir_work() as d:
for name in ('SPEAR3', 'Compact Storage Ring',
'Los Alamos Proton Storage Ring'):
data = _example_data(name)
mad = madx_parser.parse_file(
elegant.python_source_for_model(data, 'madx'))
madx._fixup_madx(mad)
outfile = name.lower().replace(' ', '-') + '.madx'
actual = madx.python_source_for_model(mad, None)
pkio.write_text(outfile, actual)
e = pkunit.data_dir().join(outfile)
expect = pkio.read_text(e)
pkeq(expect, actual, 'diff {} {}', e, d.join(outfile))
data = madx_parser.parse_file(actual)
lattice = madx_converter.from_madx(elegant.SIM_TYPE, data)
outfile = name.lower().replace(' ', '-') + '.lte'
actual = elegant.python_source_for_model(lattice, None)
pkio.write_text(outfile, actual)
e = pkunit.data_dir().join(outfile)
expect = pkio.read_text(e)
pkeq(expect, actual, 'diff {} {}', e, d.join(outfile))
def test_simple(capsys):
from pykern import pkunit
import pykern.pkcli.test
with pkunit.save_chdir_work() as d:
t = d.join('tests')
pkunit.data_dir().join('tests').copy(t)
with pkunit.pkexcept('FAILED=1 passed=1'):
pykern.pkcli.test.default_command()
o, e = capsys.readouterr()
pkunit.pkre('1_test.py pass', o)
pkunit.pkre('2_test.py FAIL', o)
t.join('2_test.py').rename(t.join('2_test.py-'))
pkunit.pkre('passed=1', pykern.pkcli.test.default_command())
o, e = capsys.readouterr()
pkunit.pkre('1_test.py pass', o)
pkunit.pkre('passed=1',
pykern.pkcli.test.default_command('tests/1_test.py'))
o, e = capsys.readouterr()
pkunit.pkre('1_test.py pass', o)
t.join('2_test.py-').rename(t.join('2_test.py'))
t.join('1_test.py').rename(t.join('1_test.py-'))
with pkunit.pkexcept('FAILED=1 passed=0'):
pykern.pkcli.test.default_command()
o, e = capsys.readouterr()
pkunit.pkre('2_test.py FAIL', o)
pkunit.pkre('x = 1 / 0', o)
def test_checked_call():
from pykern import pkunit
from pykern.pkunit import pkeq
import sys
import subprocess
with pkunit.save_chdir_work():
cmd = [sys.executable, str(pkunit.data_dir().join('p1.py'))]
for i, a in enumerate((
('normal', 0),
('exit-1', 1),
('divide-zero', 1),
('normal-rank-all', 0),
('divide-zero-rank-2', 86),
('exit-13-rank-0', 13),
)):
f = '{}.out'.format(i)
with open(f, 'w') as o:
c = cmd + [a[0]]
print(a[0])
if 'rank' in a[0]:
c = ['mpiexec', '-n', '4'] + c
actual = subprocess.call(
c,
stdout=o,
stderr=subprocess.STDOUT,
)
pkeq(a[1], actual, '{}: exit({})\n{}', ' '.join(c), actual, open(f).read())
def test_py_path():
from pykern import pkunit
from pykern import pkio
from pykern.pkunit import pkeq
with pkunit.save_chdir_work():
d = pkunit.data_dir()
pkeq(d, pkio.py_path(d))
def test_check_call_with_signals():
from pykern import pksubprocess
from pykern import pkunit
import os
import signal
messages = []
def msg(*args):
s = args[0]
messages.append(s.format(*args[1:]))
signals = []
def signal_handler(sig, frame):
signals.append(sig)
with pkunit.save_chdir_work():
with open('true.out', 'w+') as o:
pksubprocess.check_call_with_signals(['true'], output=o)
o.seek(0)
actual = o.read()
assert '' == actual, \
'Expecting empty output "{}"'.format(actual)
with open('echo.out', 'w+') as o:
messages = []
tag = 'xyzzy'
pksubprocess.check_call_with_signals(['echo', tag],
output=o,
msg=msg)
o.seek(0)
actual = o.read()
assert tag in actual, \
'"{}" not in output "{}"'.format(tag, actual)
assert 'started' in messages[0], \
'"started" not in messages[0] "{}"'.format(messages[0])
assert 'normal exit' in messages[1], \
'"normal exit" not in messages[1] "{}"'.format(messages[1])
with open('kill.out', 'w+') as o:
messages = []
signals = []
signal.signal(signal.SIGTERM, signal_handler)
with open('kill.sh', 'w') as f:
f.write('kill -TERM {}\nsleep 10'.format(os.getpid()))
cmd = ['sh', 'kill.sh']
with pytest.raises(RuntimeError):
pksubprocess.check_call_with_signals(cmd, output=o, msg=msg)
o.seek(0)
actual = o.read()
assert '' == actual, \
'Expecting empty output "{}"'.format(actual)
assert signal.SIGTERM in signals, \
'"SIGTERM" not in signals "{}"'.format(signals)
assert 'error exit' in messages[1], \
'"error exit" not in messages[1] "{}"'.format(messages[1])
def test_run_lcls():
"""Run Genesis on an LCLS input file"""
with pkunit.save_chdir_work():
print ' about to call GENESIS'
subprocess.call(["cp", "../../../use_cases/genesis/lcls/lcls.in", "."])
subprocess.call(["genesis", "-i", "lcls.in"])
print ' GENESIS subprocess has returned'
out_file_exists = os.path.exists("./lcls.out.h5")
assert out_file_exists == True
def test_check_call_with_signals():
from pykern import pksubprocess
from pykern import pkunit
import os
import signal
messages = []
def msg(*args):
s = args[0]
messages.append(s.format(*args[1:]))
signals = []
def signal_handler(sig, frame):
signals.append(sig)
with pkunit.save_chdir_work():
with open('true.out', 'w+') as o:
pksubprocess.check_call_with_signals(['true'], output=o)
o.seek(0)
actual = o.read()
assert '' == actual, \
'Expecting empty output "{}"'.format(actual)
with open('echo.out', 'w+') as o:
messages = []
tag = 'xyzzy'
pksubprocess.check_call_with_signals(['echo', tag], output=o, msg=msg)
o.seek(0)
actual = o.read()
assert tag in actual, \
'"{}" not in output "{}"'.format(tag, actual)
assert 'started' in messages[0], \
'"started" not in messages[0] "{}"'.format(messages[0])
assert 'normal exit' in messages[1], \
'"normal exit" not in messages[1] "{}"'.format(messages[1])
with open('kill.out', 'w+') as o:
messages = []
signals = []
signal.signal(signal.SIGTERM, signal_handler)
with open('kill.sh', 'w') as f:
f.write('kill -TERM {}\nsleep 10'.format(os.getpid()))
cmd = ['sh', 'kill.sh']
with pytest.raises(RuntimeError):
pksubprocess.check_call_with_signals(cmd, output=o, msg=msg)
o.seek(0)
actual = o.read()
assert '' == actual, \
'Expecting empty output "{}"'.format(actual)
assert signal.SIGTERM in signals, \
'"SIGTERM" not in signals "{}"'.format(signals)
assert 'error exit' in messages[1], \
'"error exit" not in messages[1] "{}"'.format(messages[1])
def test_render_resource():
t1 = pkunit.import_module_from_data_dir('t1')
with pkunit.save_chdir_work():
out = 'out'
expect = '\n!v1!\n'
assert expect == t1.render(None), \
'render_resource should return rendered template'
assert not glob.glob('*'), \
'render_resource should not create any files'
assert expect == t1.render(out), \
'render_resource should return string even when writing to file'
assert expect == pkio.read_text(out), \
'With out, render_resource should write file'
def test_convert_srw_to_shadow():
from pykern import pkio, pkjson
from pykern.pkunit import pkeq
from sirepo.template.srw_shadow_converter import SRWShadowConverter
with pkunit.save_chdir_work():
for name in ('crl', 'gaussian', 'grating'):
srw = _read_json_from_data_dir(f'{name}-srw.json')
actual = SRWShadowConverter().srw_to_shadow(srw.models)
del actual['version']
pkjson.dump_pretty(actual, f'{name}-shadow.json')
expect = _read_json_from_data_dir(f'{name}-shadow.json')
pkeq(expect, actual)
def test_sirepo_parser():
with pkunit.save_chdir_work():
for b in ['SRWLIB_VirtBL_LCLS_SXR_01']:
base_py = '{}.py'.format(b)
code = pkio.read_text(pkunit.data_dir().join(base_py))
error, actual = import_python(
code,
tmp_dir='.',
lib_dir='.',
user_filename=r'c:\x\{}.y'.format('SRWLIB_VirtBL_LCLS_SXR_01'),
)
assert not error, \
'{}: should be valid input'.format(base_py)
pkunit.assert_object_with_json(b, actual)
def test_generate_python():
from pykern import pkio
from pykern.pkunit import pkeq
from sirepo.template import opal
with pkunit.save_chdir_work():
for name in ('CSR Bend Drift', 'CTF3 RF Photoinjector'):
data = _example_data(name)
data['report'] = 'animation'
actual = opal.python_source_for_model(data, None)
outfile = name.lower().replace(' ', '-') + '.txt'
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
pkeq(expect, actual)
def test_backup():
from pykern import pkconfig
pkconfig.reset_state_for_testing({
'PYKERN_PKCLI_GITHUB_TEST_MODE': '1',
'PYKERN_PKCLI_GITHUB_API_PAUSE_SECONDS': '0',
})
from pykern.pkcli import github
from pykern import pkunit
from pykern import pkio
with pkunit.save_chdir_work():
github.backup()
github.backup()
def test_generate_python():
from pykern import pkio
from pykern.pkunit import pkeq
from sirepo.template import shadow
with pkunit.save_chdir_work():
for name in ('Complete Beamline', 'Wiggler'):
data = _example_data(name)
data['report'] = 'watchpointReport{}'.format(data.models.beamline[-1].id)
actual = shadow._generate_parameters_file(data)
outfile = data.models.simulation.simulationId + '.txt'
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
pkeq(expect, actual)
def test_importer():
from sirepo.template.srw_importer import import_python
from pykern import pkio
from pykern import pkresource
from pykern import pkunit
from pykern.pkdebug import pkdc, pkdp
import glob
import py
_TESTS = { # Values are optional arguments:
'amx': ('amx', None),
'amx_bl2': ('amx', '--op_BL=2'),
'amx_bl3': ('amx', '--op_BL=3'),
'amx_bl4': ('amx', '--op_BL=4'),
'chx': ('chx', None),
'chx_fiber': ('chx_fiber', None),
'exported_chx': ('exported_chx', None),
'exported_gaussian_beam': ('exported_gaussian_beam', None),
'exported_undulator_radiation': ('exported_undulator_radiation', None),
'lcls_simplified': ('lcls_simplified', None),
'lcls_sxr': ('lcls_sxr', None),
'sample_from_image': ('sample_from_image', None),
'smi_es1_bump_norm': ('smi', '--beamline ES1 --bump --BMmode Norm'),
'smi_es1_nobump': ('smi', '--beamline ES1'),
'smi_es2_bump_lowdiv': ('smi', '--beamline ES2 --bump --BMmode LowDiv'),
'smi_es2_bump_norm': ('smi', '--beamline ES2 --bump --BMmode Norm'),
'srx': ('srx', None),
'srx_bl2': ('srx', '--op_BL=2'),
'srx_bl3': ('srx', '--op_BL=3'),
'srx_bl4': ('srx', '--op_BL=4'),
}
dat_dir = py.path.local(pkresource.filename('template/srw/',
import_python))
with pkunit.save_chdir_work():
work_dir = py.path.local('.')
for f in glob.glob(str(dat_dir.join('mirror_*d.dat'))):
py.path.local(f).copy(work_dir)
py.path.local(str(dat_dir.join('sample.tif'))).copy(work_dir)
for b in sorted(_TESTS.keys()):
base_py = '{}.py'.format(_TESTS[b][0])
code = pkio.read_text(pkunit.data_dir().join(base_py))
actual = import_python(
code,
tmp_dir=str(work_dir),
lib_dir=str(work_dir),
user_filename=r'c:\anything\{}.anysuffix'.format(_TESTS[b][0]),
arguments=_TESTS[b][1],
)
actual['version'] = 'IGNORE-VALUE'
pkunit.assert_object_with_json(b, actual)
def test_parse_madx_file():
from pykern import pkio, pkjson
from pykern.pkunit import pkeq
from sirepo.template import madx, madx_parser
with pkunit.save_chdir_work():
for name in ('particle_track', 'alba'):
actual = madx_parser.parse_file(pkio.read_text(
pkunit.data_dir().join(f'{name}.madx')))
del actual['version']
outfile = f'{name}.json'
pkjson.dump_pretty(actual, outfile)
expect = pkjson.load_any(pkunit.data_dir().join(outfile))
pkeq(expect, actual)
def test_init_tree():
"""Normal case"""
from pykern import pkio
from pykern.pkcli import projex
from pykern import pkunit
with pkunit.save_chdir_work():
name = 'proj1'
pkio.mkdir_parent(name)
with pkio.save_chdir(name):
subprocess.check_call(['git', 'init', '.'])
subprocess.check_call(
['git', 'config', 'user.email', '*****@*****.**'])
subprocess.check_call(['git', 'config', 'user.name', 'pykern'])
projex.init_tree(
name=name,
author='zauthor',
author_email='*****@*****.**',
description='some python project',
license='MIT',
url='http://example.com',
)
pkio.write_text('tests/test_1.py', 'def test_1(): pass')
for expect_fn, expect_re in (
('.gitignore', 'MANIFEST.in'),
('LICENSE', 'The MIT License'),
('README.md', 'licenses/MIT'),
('docs/_static/.gitignore', ''),
('docs/_templates/.gitignore', ''),
('docs/index.rst', name),
('setup.py', "author='zauthor'"),
('setup.py', r':copyright:.*zauthor\.'),
('tests/.gitignore', '_work'),
(name + '/__init__.py', ''),
(name + '/package_data/.gitignore', ''),
(
'{}/{}_console.py'.format(name, name),
r"main\('{}'\)".format(name),
),
):
assert re.search(expect_re, pkio.read_text(expect_fn)), \
'{} should exist and match "{}"'.format(expect_fn, expect_re)
subprocess.check_call(['git', 'commit', '-m', 'initial'])
# Do not install from PyPI
pykern_path = py.path.local(__file__).dirpath().dirpath().dirpath()
# pykern must be installed for setup.py to be able to be called
subprocess.check_call(['pip', 'install', '-e', str(pykern_path)])
subprocess.check_call(['python', 'setup.py', 'test'])
subprocess.check_call(['python', 'setup.py', 'tox'])
def test_importer():
from sirepo.template.srw_importer import import_python
from pykern import pkio
from pykern import pkresource
from pykern import pkunit
from pykern.pkdebug import pkdc, pkdp
import glob
import py
_TESTS = { # Values are optional arguments:
'amx': ('amx', None),
'amx_bl2': ('amx', '--op_BL=2'),
'amx_bl3': ('amx', '--op_BL=3'),
'amx_bl4': ('amx', '--op_BL=4'),
'chx': ('chx', None),
'chx_fiber': ('chx_fiber', None),
'exported_chx': ('exported_chx', None),
'exported_gaussian_beam': ('exported_gaussian_beam', None),
'exported_undulator_radiation': ('exported_undulator_radiation', None),
'lcls_simplified': ('lcls_simplified', None),
'lcls_sxr': ('lcls_sxr', None),
'sample_from_image': ('sample_from_image', None),
'smi_es1_bump_norm': ('smi', '--beamline ES1 --bump --BMmode Norm'),
'smi_es1_nobump': ('smi', '--beamline ES1'),
'smi_es2_bump_lowdiv': ('smi', '--beamline ES2 --bump --BMmode LowDiv'),
'smi_es2_bump_norm': ('smi', '--beamline ES2 --bump --BMmode Norm'),
'srx': ('srx', None),
'srx_bl2': ('srx', '--op_BL=2'),
'srx_bl3': ('srx', '--op_BL=3'),
'srx_bl4': ('srx', '--op_BL=4'),
}
dat_dir = py.path.local(pkresource.filename('template/srw/', import_python))
with pkunit.save_chdir_work():
work_dir = py.path.local('.')
for f in glob.glob(str(dat_dir.join('mirror_*d.dat'))):
py.path.local(f).copy(work_dir)
py.path.local(str(dat_dir.join('sample.tif'))).copy(work_dir)
for b in sorted(_TESTS.keys()):
base_py = '{}.py'.format(_TESTS[b][0])
code = pkio.read_text(pkunit.data_dir().join(base_py))
actual = import_python(
code,
tmp_dir=str(work_dir),
lib_dir=str(work_dir),
user_filename=r'c:\anything\{}.anysuffix'.format(_TESTS[b][0]),
arguments=_TESTS[b][1],
)
actual['version'] = 'IGNORE-VALUE'
pkunit.assert_object_with_json(b, actual)
def test_generate_python():
from pykern import pkio
from pykern import pkunit
from sirepo.template import synergia
import re
with pkunit.save_chdir_work():
for f in pkio.sorted_glob(pkunit.data_dir().join('*.txt')):
e = f.read()
m = re.search(r'^#\s*(.*\S)\s*$', e, flags=re.MULTILINE)
assert m
name = m.group(1)
a = synergia._generate_parameters_file(_example_data(name))
pkio.write_text(f.basename, a)
pkunit.pkeq(e, a)
def test_run_beam_solver():
"""Ensure pyhellweg.run_beam_solver produces output and does not crash"""
from pykern import pkio
from pykern.pkunit import pkeq
from rslinac.pkcli import beam_solver
f = _files()
with pkunit.save_chdir_work():
pkio.write_text('Solenoid.txt',
pkio.read_text(pkunit.data_dir().join('Solenoid.txt')))
beam_solver.run(f['ini'], f['input'], f['output'])
assert f['output'].exists()
for outfile in ('PARSED.TXT', 'test1.pid'):
expect = pkio.read_text(pkunit.data_dir().join(outfile))
actual = pkio.read_text(pkunit.work_dir().join(outfile))
pkeq(expect, actual)
def test_walk_tree():
"""Creates looks in data_dir"""
with pkunit.save_chdir_work() as pwd:
for f in ('d1/d7', 'd2/d3', 'd4/d5/d6'):
pkio.mkdir_parent(f)
expect = []
for f in ['d1/d7/f1', 'd4/d5/f2', 'd2/d3/f3']:
pkio.write_text(f, '')
expect.append(py.path.local(f))
assert sorted(expect) == list(pkio.walk_tree('.')), \
'When walking tree, should only return files'
assert [expect[2]] == list(pkio.walk_tree('.', 'f3')), \
'When walking tree with file_re, should only return matching files'
assert [expect[0]] == list(pkio.walk_tree('.', '^d1')), \
'When walking tree with file_re, file to match does not include dir being searched'
def test_generate_python():
from pykern import pkio
from pykern import pkunit
from sirepo.template import synergia
import re
with pkunit.save_chdir_work() as d:
for f in pkio.sorted_glob(pkunit.data_dir().join('*.txt')):
e = pkio.read_text(f)
m = re.search(r'^#\s*(.*\S)\s*$', e, flags=re.MULTILINE)
assert m
pkunit.file_eq(
f,
synergia._generate_parameters_file(_example_data(
m.group(1)), ),
)
def test_importer():
from pykern import pkcollections
from pykern import pkio
from pykern.pkunit import pkeq
from sirepo.template import elegant
with pkunit.save_chdir_work():
for fn in pkio.sorted_glob(pkunit.data_dir().join('*')):
if not pkio.has_file_extension(fn, ('ele', 'lte')) \
or fn.basename.endswith('ele.lte'):
continue
error = None
try:
data = elegant.import_file(FlaskRequest(fn))
except Exception as e:
pkdlog(pkdexc())
error = e.message
if error:
actual = error
else:
if pkio.has_file_extension(fn, 'lte'):
data['models']['commands'] = []
actual = '{}{}'.format(
elegant._generate_variables(data),
elegant.generate_lattice(
data,
elegant._build_filename_map(data),
elegant._build_beamline_map(data),
pkcollections.Dict(),
),
)
else:
data2 = elegant.import_file(FlaskRequest(
'{}.lte'.format(fn)),
test_data=data)
actual = elegant._generate_commands(
data2,
elegant._build_filename_map(data2),
elegant._build_beamline_map(data2),
pkcollections.Dict(),
)
outfile = fn.basename + '.txt'
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
#TODO(pjm): this takes too long if there are a lot of diffs
#assert expect == actual
pkeq(expect, actual)
def test_init_rs_tree():
"""Normal case"""
with pkunit.save_chdir_work():
name = 'rs_proj1'
pkio.mkdir_parent(name)
with pkio.save_chdir(name):
subprocess.check_call(['git', 'init', '.'])
subprocess.check_call(
['git', 'config', 'user.email', '*****@*****.**'])
subprocess.check_call(['git', 'config', 'user.name', 'pykern'])
projex.init_rs_tree(description='some radiasoftee project', )
for expect_fn, expect_re in (
('LICENSE', 'Apache License'),
('setup.py', "author='RadiaSoft LLC'"),
):
assert re.search(expect_re, pkio.read_text(expect_fn)), \
'{} should exist and match "{}"'.format(expect_fn, expect_re)
def test_init_tree():
"""Normal case"""
with pkunit.save_chdir_work():
name = 'proj1'
pkio.mkdir_parent(name)
with pkio.save_chdir(name):
subprocess.check_call(['git', 'init', '.'])
subprocess.check_call(['git', 'config', 'user.email', '*****@*****.**'])
subprocess.check_call(['git', 'config', 'user.name', 'pykern'])
projex.init_tree(
name=name,
author='zauthor',
author_email='*****@*****.**',
description='some python project',
license='MIT',
url='http://example.com',
)
pkio.write_text('tests/test_1.py', 'def test_1(): pass')
for expect_fn, expect_re in (
('.gitignore', 'MANIFEST.in'),
('LICENSE', 'The MIT License'),
('README.md', 'licenses/MIT'),
('docs/_static/.gitignore', ''),
('docs/_templates/.gitignore', ''),
('docs/index.rst', name),
('requirements.txt', 'pykern'),
('setup.py', "author='zauthor'"),
('setup.py', r':copyright:.*zauthor\.'),
('tests/.gitignore', '_work'),
(name + '/__init__.py', ''),
(name + '/package_data/.gitignore', ''),
(
'{}/{}_console.py'.format(name, name),
r"main\('{}'\)".format(name),
),
):
assert re.search(expect_re, pkio.read_text(expect_fn)), \
'{} should exist and match "{}"'.format(expect_fn, expect_re)
subprocess.check_call(['git', 'commit', '-m', 'initial'])
# Do not install from PyPI
pkio.write_text(
'requirements.txt',
'-e ' + str(py.path.local(__file__).dirpath().dirpath().dirpath()),
);
subprocess.check_call(['python', 'setup.py', 'test'])
subprocess.check_call(['python', 'setup.py', 'tox'])
def test_importer():
from pykern import pkcollections
from pykern import pkio
from pykern.pkunit import pkeq
from sirepo.template import elegant
with pkunit.save_chdir_work():
for fn in pkio.sorted_glob(pkunit.data_dir().join('*')):
if not pkio.has_file_extension(fn, ('ele', 'lte')) \
or fn.basename.endswith('ele.lte'):
continue
error = None
try:
data = elegant.import_file(FlaskRequest(fn))
except Exception as e:
pkdlog(pkdexc())
error = e.message
if error:
actual = error
else:
if pkio.has_file_extension(fn, 'lte'):
data['models']['commands'] = []
actual = '{}{}'.format(
elegant._generate_variables(data),
elegant.generate_lattice(
data,
elegant._build_filename_map(data),
elegant._build_beamline_map(data),
pkcollections.Dict(),
),
)
else:
data2 = elegant.import_file(FlaskRequest('{}.lte'.format(fn)), test_data=data)
actual = elegant._generate_commands(
data2,
elegant._build_filename_map(data2),
elegant._build_beamline_map(data2),
pkcollections.Dict(),
)
outfile = fn.basename + '.txt'
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
#TODO(pjm): this takes too long if there are a lot of diffs
#assert expect == actual
pkeq(expect, actual)
def test_init_rs_tree():
"""Normal case"""
with pkunit.save_chdir_work():
name = 'rs_proj1'
pkio.mkdir_parent(name)
with pkio.save_chdir(name):
subprocess.check_call(['git', 'init', '.'])
subprocess.check_call(['git', 'config', 'user.email', '*****@*****.**'])
subprocess.check_call(['git', 'config', 'user.name', 'pykern'])
projex.init_rs_tree(
description='some radiasoftee project',
)
for expect_fn, expect_re in (
('LICENSE', 'Apache License'),
('setup.py', "author='RadiaSoft LLC'"),
):
assert re.search(expect_re, pkio.read_text(expect_fn)), \
'{} should exist and match "{}"'.format(expect_fn, expect_re)
def test_zgoubi_output(test_file):
"""
Runs zgoubi for a selection of simulation results and compares the
new results agains the old results.
Comaparison is done by looking at differences in the output format, ignoring
non-essential information like dates.
"""
from rszgoubi.output import compare_results_format
from rszgoubi.run import zgoubi
with pkunit.save_chdir_work(), open('diff.txt', 'w+') as df:
fn = pkunit.data_dir().join(test_file)
pkio.write_text(ZGOUBI_DAT, pkio.read_text(fn))
zgoubi()
compare_results_format(test_file, ZGOUBI_RES, df)
df.seek(0)
assert df.read() == ''
def flask_client(cfg=None, sim_types=None, job_run_mode=None):
"""Return FlaskClient with easy access methods.
Creates a new run directory every test file so can assume
sharing of state on the server within a file (module).
Two methods of interest: `sr_post` and `sr_get`.
Args:
cfg (dict): extra configuration for reset_state_for_testing
sim_types (str): value for SIREPO_FEATURE_CONFIG_SIM_TYPES [CONFTEST_DEFAULT_CODES]
Returns:
FlaskClient: for local requests to Flask server
"""
global server, app
a = 'srunit_flask_client'
if not cfg:
cfg = PKDict()
t = sim_types or CONFTEST_DEFAULT_CODES
if t:
if isinstance(t, (tuple, list)):
t = ':'.join(t)
cfg['SIREPO_FEATURE_CONFIG_SIM_TYPES'] = t
if not (server and hasattr(app, a)):
from pykern import pkconfig
# initialize pkdebug with correct values
pkconfig.reset_state_for_testing(cfg)
from pykern import pkunit
with pkunit.save_chdir_work() as wd:
from pykern import pkio
setup_srdb_root(cfg=cfg)
pkconfig.reset_state_for_testing(cfg)
from sirepo import server as s
server = s
app = server.init(is_server=True)
app.config['TESTING'] = True
app.test_client_class = _TestClient
setattr(app, a, app.test_client(job_run_mode=job_run_mode))
return getattr(app, a)
def _move_all(monkeypatch, hook=None):
from pykern import pkunit
with pkunit.save_chdir_work() as d:
photos = d.ensure('Dropbox', 'Photos', '2017', '05-11', dir=True)
uploads = d.ensure('Dropbox', 'Uploads', dir=True)
monkeypatch.setenv('HOME', d)
if hook:
hook(uploads)
files = set()
for i in range(3):
f = '2017_05_11_p{}.jpg'.format(i)
files.add(f)
uploads.ensure(f)
from rnpix.pkcli import dropbox_uploads
dropbox_uploads.default_command(str(uploads.join('any-file')))
for f in files:
assert photos.join(f).check(), \
'{}: does not exist'.join(f)
def test_walk_tree_and_sorted_glob():
"""Looks in work_dir"""
from pykern import pkunit
from pykern import pkio
with pkunit.save_chdir_work() as pwd:
for f in ('d1/d7', 'd2/d3', 'd4/d5/d6'):
pkio.mkdir_parent(f)
expect = []
for f in ['d1/d7/f1', 'd4/d5/f2', 'd2/d3/f3']:
pkio.write_text(f, '')
expect.append(py.path.local(f))
assert sorted(expect) == list(pkio.walk_tree('.')), \
'When walking tree, should only return files'
assert [expect[2]] == list(pkio.walk_tree('.', 'f3')), \
'When walking tree with file_re, should only return matching files'
assert [expect[0]] == list(pkio.walk_tree('.', '^d1')), \
'When walking tree with file_re, file to match does not include dir being searched'
assert pkio.sorted_glob('*[42]') == [py.path.local(f) for f in ('d2', 'd4')]
def test_importer(import_req):
from pykern.pkcollections import PKDict
from pykern.pkunit import pkeq
from sirepo.template import lattice
from sirepo.template import elegant
import sirepo.util
import flask
with pkunit.save_chdir_work():
for fn in pkio.sorted_glob(pkunit.data_dir().join('*')):
if not pkio.has_file_extension(fn, ('ele', 'lte')) \
or fn.basename.endswith('ele.lte'):
continue
error = None
try:
data = elegant.import_file(import_req(fn))
except Exception as e:
pkdlog(pkdexc())
error = str(e)
if error:
actual = error
else:
if pkio.has_file_extension(fn, 'lte'):
data['models']['commands'] = []
actual = '{}{}'.format(
elegant._generate_variables(data),
elegant._generate_lattice(
elegant._build_filename_map(data),
lattice.LatticeUtil(data, elegant._SCHEMA),
),
)
else:
#TODO(robnagler) test simulationId
data2 = elegant.import_file(import_req(fn.new(ext='ele.lte')), test_data=data)
actual = elegant._generate_commands(
elegant._build_filename_map(data2),
lattice.LatticeUtil(data2, elegant._SCHEMA),
)
outfile = fn.basename + '.txt'
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
pkeq(expect, actual)
def test_from_elegant_to_madx_and_back():
from pykern.pkunit import pkeq, file_eq
from sirepo.template import elegant
from sirepo.template.elegant import ElegantMadxConverter
with pkunit.save_chdir_work() as d:
for name in ('SPEAR3', 'Compact Storage Ring',
'Los Alamos Proton Storage Ring'):
data = _example_data(name)
actual = ElegantMadxConverter().to_madx_text(data)
file_eq(
name.lower().replace(' ', '-') + '.madx',
actual=actual,
)
file_eq(
name.lower().replace(' ', '-') + '.lte',
actual=elegant.python_source_for_model(
ElegantMadxConverter().from_madx_text(actual),
None,
),
)
def test_add_code():
from pykern import pkio
from pykern import pkjson
from pykern import pkunit
from pykern.pkunit import pkok, pkeq, pkre
from pykern.pkdebug import pkdp
from pykern.pkcli import rsmanifest
import re
with pkunit.save_chdir_work(is_pkunit_prefix=True) as d:
rsmanifest.add_code('A', 'b', 'c', 'd', pyenv='v')
j = pkjson.load_any(pkio.py_path(rsmanifest.USER_FILE).read())
pkok(20170101.0 < float(j.version), 'version must be after 2017')
pkeq('A', j.codes.v.a.name)
pkeq('b', j.codes.v.a.version)
rsmanifest.add_code('a', 'bb', 'cc', 'dd')
j = pkjson.load_any(pkio.expand_user_path(rsmanifest.USER_FILE).read())
pkeq('A', j.codes.v.a.name)
pkeq('a', j.codes[''].a.name)
pkeq('bb', j.codes[''].a.version)
pkre('20.*T.*Z', j.codes[''].a.installed)
def test_importer():
from sirepo.importer import import_python
with pkunit.save_chdir_work():
for b in sorted(_TESTS.keys()):
base_py = '{}.py'.format(_TESTS[b][0])
code = pkio.read_text(pkunit.data_dir().join(base_py))
tmp_dir = _create_tmp_dir()
error, actual = import_python(
code,
tmp_dir='.',
lib_dir=str(tmp_dir),
user_filename=r'c:\anything\{}.anysuffix'.format(_TESTS[b][0]),
arguments=_TESTS[b][1],
)
_remove_dir(tmp_dir)
assert not error, \
'{}: should import with an error: {}'.format(base_py, error)
actual['version'] = 'IGNORE-VALUE'
assert not error, \
'{}: should be valid input'.format(base_py)
pkunit.assert_object_with_json(b, actual)
def test_init_and_run(monkeypatch):
from pykern import pkio
from pykern import pkunit
from pykern.pkcli import sim
from pykern.pkcli import rsmanifest
import netrc
import os
import os.path
import re
import subprocess
cfg = pkunit.cfg.aux.get('sim_test', None)
if not cfg:
# No testing if there's no auth config
return
u, p = cfg.split(' ')
monkeypatch.setattr(netrc, 'netrc', _netrc)
_netrc.result = (u, None, p)
with pkunit.save_chdir_work(is_pkunit_prefix=True):
f = 'out/log'
expect_code = pkunit.random_alpha()
pkio.write_text('run.sh', 'echo {}>{}'.format(expect_code, f))
rsmanifest.pkunit_setup()
sim._cmd_init()
sim._cmd_run()
x = subprocess.check_output(['git', 'remote', '-v']),
m = re.search(r'/(sim-sim_work-\d+-\d+)\.git', x[0])
repo = m.group(1)
pkunit.pkok(m, 'git remote: failed: {}', x)
pkunit.pkeq(expect_code, pkio.read_text('out/log').rstrip())
os.remove('run.sh')
sim._cmd_pip('djson')
pkio.write_text('run.py', 'import djson'.format(expect_code, f))
sim._cmd_run()
sim._git_api_request(
'delete',
'repositories/{user}/{repo}',
dict(repo=repo),
)
def test_importer():
from sirepo.template import elegant
with pkunit.save_chdir_work():
for filename in _FILES:
error, data = elegant.import_file(TestFlaskRequest(filename))
outfile = '{}.txt'.format(filename)
if error:
actual = error
else:
if '.lte' in filename:
data['models']['commands'] = []
actual = '{}{}'.format(
elegant._generate_variables(data),
elegant.generate_lattice(data, elegant._build_filename_map(data), elegant._build_beamline_map(data), {}))
else:
err2, data2 = elegant.import_file(TestFlaskRequest('{}.lte'.format(filename)), test_data=data)
actual = elegant._generate_commands(data2, elegant._build_filename_map(data2), elegant._build_beamline_map(data2), {})
pkio.write_text(outfile, actual)
expect = pkio.read_text(pkunit.data_dir().join(outfile))
#TODO(pjm): this takes too long if there are a lot of diffs
#assert expect == actual
if expect != actual:
assert False
def test_importer():
from sirepo.importer import import_python
dat_dir = py.path.local(pkresource.filename('static/dat/', import_python))
with pkunit.save_chdir_work():
work_dir = py.path.local('.')
for f in glob.glob(str(dat_dir.join('mirror_*d.dat'))):
py.path.local(f).copy(work_dir)
for b in sorted(_TESTS.keys()):
base_py = '{}.py'.format(_TESTS[b][0])
code = pkio.read_text(pkunit.data_dir().join(base_py))
error, actual = import_python(
code,
tmp_dir=str(work_dir),
lib_dir=str(work_dir),
user_filename=r'c:\anything\{}.anysuffix'.format(_TESTS[b][0]),
arguments=_TESTS[b][1],
)
assert not error, \
'{}: should import with an error: {}'.format(base_py, error)
actual['version'] = 'IGNORE-VALUE'
assert not error, \
'{}: should be valid input'.format(base_py)
pkunit.assert_object_with_json(b, actual)
def test_get_data_file():
from sirepo import srunit
from pykern import pkunit
from pykern import pkio
import sdds
fc = srunit.flask_client()
fc.get('/elegant')
data = fc.sr_post(
'listSimulations',
{'simulationType': 'elegant', 'search': {'simulationName': 'fourDipoleCSR'}},
)
data = data[0].simulation
data = fc.sr_get(
'simulationData',
params=dict(
pretty='1',
simulation_id=data.simulationId,
simulation_type='elegant',
),
)
run = fc.sr_post(
'runSimulation',
dict(
forceRun=False,
models=data.models,
report='bunchReport1',
simulationId=data.models.simulation.simulationId,
simulationType=data.simulationType,
),
)
for _ in range(10):
run = fc.sr_post(
'runStatus',
run.nextRequest
)
if run.state == 'completed':
break
time.sleep(1)
else:
pkunit.pkfail('runStatus: failed to complete: {}', run)
resp = fc.sr_get(
'downloadDataFile',
dict(
simulation_type=data.simulationType,
simulation_id=data.models.simulation.simulationId,
model='bunchReport1',
frame='-1',
suffix='csv',
),
raw_response=True,
)
rows = csv.reader(StringIO.StringIO(resp.get_data()))
assert len(list(rows)) == 5001
resp = fc.sr_get(
'downloadDataFile',
dict(
simulation_type=data.simulationType,
simulation_id=data.models.simulation.simulationId,
model='bunchReport1',
frame='-1',
),
raw_response=True,
)
m = re.search(r'attachment; filename="([^"]+)"', resp.headers['Content-Disposition'])
with pkunit.save_chdir_work():
path = pkio.py_path(m.group(1))
with open(str(path), 'w') as f:
f.write(resp.get_data())
assert sdds.sddsdata.InitializeInput(0, str(path)) == 1, \
'{}: sdds failed to open'.format(path)
# Verify we can read something
assert 0 <= len(sdds.sddsdata.GetColumnNames(0))
sdds.sddsdata.Terminate(0)
def test_1():
with pkunit.save_chdir_work():
pkio.write_text('do_not_include_in_sdist.py', 'some text')
