def test_map_items():
n = OrderedMapping(a=1)
n.b = 2
res = pkcollections.map_items(n, lambda k, v: (v + 1, k))
assert [(2, 'a'), (3, 'b')] == res, \
'map_items should call op in order'
assert [('a', 1), ('b', 2)] == pkcollections.map_items(n), \
'map_items should return items with no op'
def test_map_items():
n = OrderedMapping(a=1)
n.b = 2
res = pkcollections.map_items(n, lambda k, v: (v + 1, k))
assert [(2, 'a'), (3, 'b')] == res, \
'map_items should call op in order'
assert [('a', 1), ('b', 2)] == pkcollections.map_items(n), \
'map_items should return items with no op'
def uwsgi():
"""Starts UWSGI server"""
db_dir =_db_dir()
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
# uwsgi.py must be first, because referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
subprocess.check_call(cmd)
def uwsgi():
"""Starts UWSGI server"""
db_dir = _db_dir()
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
# uwsgi.py must be first, because referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
subprocess.check_call(cmd)
def uwsgi():
"""Starts UWSGI server"""
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
values['logto'] = None if pkconfig.channel_in('dev') else str(run_dir.join('uwsgi.log'))
# uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
pksubprocess.check_call_with_signals(cmd)
def uwsgi():
"""Starts UWSGI server"""
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
values['logto'] = None if pkconfig.channel_in('dev') else str(run_dir.join('uwsgi.log'))
# uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
pksubprocess.check_call_with_signals(cmd)
def run_background(cfg_dir):
with pkio.save_chdir(cfg_dir):
fn = 'run_background.py'
cmd = [sys.executable or 'python', fn]
script = pkio.read_text('srw_parameters.py')
p = dict(pkcollections.map_items(cfg))
if cfg.slave_processes > 1:
cmd[0:0] = [
'mpiexec',
'-n',
# SRW includes a master process so 2 really needs 3 processes
str(cfg.slave_processes + 1),
]
script += '''
from mpi4py import MPI
if MPI.COMM_WORLD.Get_rank():
import signal
signal.signal(signal.SIGTERM, lambda x, y: MPI.COMM_WORLD.Abort(1))
'''
else:
# In interactive (dev) mode, output as frequently as possible
p['particles_per_slave'] = 1
script += '''
import srwl_bl
v = srwl_bl.srwl_uti_parse_options(get_srw_params(), use_sys_argv=False)
source_type, mag = setup_source(v)
v.wm = True
v.wm_nm = {total_particles}
v.wm_na = {particles_per_slave}
# Number of "iterations" per save is best set to num processes
v.wm_ns = {slave_processes}
op = get_beamline_optics()
srwl_bl.SRWLBeamline(_name=v.name).calc_all(v, op)
'''.format(**p)
pkio.write_text(fn, script)
try:
p = subprocess.Popen(
cmd,
stdin=open(os.devnull),
stdout=open('run_background.out', 'w'),
stderr=subprocess.STDOUT,
)
signal.signal(signal.SIGTERM, lambda x, y: p.terminate())
rc = p.wait()
if rc != 0:
p = None
raise RuntimeError('child terminated: retcode={}'.format(rc))
finally:
if not p is None:
p.terminate()
def run_background(cfg_dir):
with pkio.save_chdir(cfg_dir):
fn = 'run_background.py'
cmd = [sys.executable or 'python', fn]
script = pkio.read_text('srw_parameters.py')
p = dict(pkcollections.map_items(cfg))
if cfg.slave_processes > 1:
cmd[0:0] = [
'mpiexec',
'-n',
# SRW includes a master process so 2 really needs 3 processes
str(cfg.slave_processes + 1),
]
script += '''
from mpi4py import MPI
if MPI.COMM_WORLD.Get_rank():
import signal
signal.signal(signal.SIGTERM, lambda x, y: MPI.COMM_WORLD.Abort(1))
'''
else:
# In interactive (dev) mode, output as frequently as possible
p['particles_per_slave'] = 1
script += '''
import srwl_bl
v = srwl_bl.srwl_uti_parse_options(get_srw_params(), use_sys_argv=False)
source_type, mag = setup_source(v)
v.wm = True
v.wm_nm = {total_particles}
v.wm_na = {particles_per_slave}
# Number of "iterations" per save is best set to num processes
v.wm_ns = {slave_processes}
op = get_beamline_optics()
srwl_bl.SRWLBeamline(_name=v.name).calc_all(v, op)
'''.format(**p)
pkio.write_text(fn, script)
try:
p = subprocess.Popen(
cmd,
stdin=open(os.devnull),
stdout=open('run_background.out', 'w'),
stderr=subprocess.STDOUT,
)
signal.signal(signal.SIGTERM, lambda x, y: p.terminate())
rc = p.wait()
if rc != 0:
p = None
raise RuntimeError('child terminated: retcode={}'.format(rc))
finally:
if not p is None:
p.terminate()
def nginx_proxy():
"""Starts nginx in container.
Used for development only.
"""
assert pkconfig.channel_in('dev')
run_dir = _run_dir().join('nginx_proxy').ensure(dir=True)
with pkio.save_chdir(run_dir):
f = run_dir.join('default.conf')
values = dict(pkcollections.map_items(cfg))
pkjinja.render_resource('nginx_proxy.conf', values, output=f)
cmd = [
'docker',
'run',
'--net=host',
'--rm',
'--volume={}:/etc/nginx/conf.d/default.conf'.format(f),
'nginx',
]
pksubprocess.check_call_with_signals(cmd)
def uwsgi():
"""Starts UWSGI server"""
in_dev = pkconfig.channel_in('dev')
if in_dev:
from sirepo import server, runner
# uwsgi doesn't pass signals right so can't use _Background
if not issubclass(server.cfg.job_queue, runner.Celery):
pkcli.command_error(
'uwsgi only works if sirepo.server.cfg.job_queue=_Celery')
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
values['logto'] = None if in_dev else str(run_dir.join('uwsgi.log'))
# uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
pksubprocess.check_call_with_signals(cmd)
def nginx_proxy():
"""Starts nginx in container.
Used for development only.
"""
assert pkconfig.channel_in('dev')
run_dir = _run_dir().join('nginx_proxy').ensure(dir=True)
with pkio.save_chdir(run_dir):
f = run_dir.join('default.conf')
values = dict(pkcollections.map_items(cfg))
pkjinja.render_resource('nginx_proxy.conf', values, output=f)
cmd = [
'docker',
'run',
'--net=host',
'--rm',
'--volume={}:/etc/nginx/conf.d/default.conf'.format(f),
'nginx',
]
pksubprocess.check_call_with_signals(cmd)
def uwsgi():
"""Starts UWSGI server"""
in_dev = pkconfig.channel_in('dev')
if in_dev:
from sirepo import server, runner
# uwsgi doesn't pass signals right so can't use _Background
if not issubclass(server.cfg.job_queue, runner.Celery):
pkcli.command_error('uwsgi only works if sirepo.server.cfg.job_queue=_Celery')
db_dir =_db_dir()
run_dir = _run_dir()
with pkio.save_chdir(run_dir):
values = dict(pkcollections.map_items(cfg))
values['logto'] = None if in_dev else str(run_dir.join('uwsgi.log'))
# uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
for f in ('uwsgi.py', 'uwsgi.yml'):
output = run_dir.join(f)
values[f.replace('.', '_')] = str(output)
pkjinja.render_resource(f, values, output=output)
cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
pksubprocess.check_call_with_signals(cmd)
def run_background(cfg_dir):
"""Run srw with mpi in ``cfg_dir``
Args:
cfg_dir (str): directory to run srw in
"""
with pkio.save_chdir(cfg_dir):
script = pkio.read_text(template_common.PARAMETERS_PYTHON_FILE)
p = dict(pkcollections.map_items(cfg))
if pkconfig.channel_in('dev'):
p['particles_per_core'] = 5
p['cores'] = mpi.cfg.cores
script += '''
v.wm_na = v.sm_na = {particles_per_core}
# Number of "iterations" per save is best set to num processes
v.wm_ns = v.sm_ns = {cores}
srwl_bl.SRWLBeamline(_name=v.name).calc_all(v, op)
main()
'''.format(**p)
mpi.run_script(script)
simulation_db.write_result({})
def run_background(cfg_dir):
"""Run srw with mpi in ``cfg_dir``
Args:
cfg_dir (str): directory to run srw in
"""
with pkio.save_chdir(cfg_dir):
script = pkio.read_text(template_common.PARAMETERS_PYTHON_FILE)
p = dict(pkcollections.map_items(cfg))
if pkconfig.channel_in('dev'):
p['particles_per_core'] = 5
p['cores'] = mpi.cfg.cores
script += '''
v.wm_na = v.sm_na = {particles_per_core}
# Number of "iterations" per save is best set to num processes
v.wm_ns = v.sm_ns = {cores}
srwl_bl.SRWLBeamline(_name=v.name).calc_all(v, op)
main()
'''.format(**p)
mpi.run_script(script)
simulation_db.write_result({})
from pykern.pkdebug import pkdc, pkdexc, pkdp, pkdlog
from sirepo.template import template_common
import py.path
import os
celery = Celery('sirepo')
cfg = pkconfig.init(
broker_url=('amqp://guest@localhost//', str, 'Celery: queue broker url'),
celery_result_backend=('rpc://', str, 'configure db other than default'),
celeryd_concurrency=(1, int, 'how many worker processes to start'),
celeryd_task_time_limit=(3600, int, 'max run time for a task in seconds'),
)
celery.conf.update(
pkcollections.map_items(cfg, op=lambda k, v: (k.upper(), v)), )
_SERIALIZER = 'json'
celery.conf.update(
CELERYD_LOG_COLOR=False,
CELERYD_MAX_TASKS_PER_CHILD=1,
CELERYD_PREFETCH_MULTIPLIER=1,
CELERYD_TASK_SOFT_TIME_LIMIT=celery.conf['CELERYD_TASK_TIME_LIMIT'] - 10,
CELERY_ACCEPT_CONTENT=[_SERIALIZER],
CELERY_ACKS_LATE=True,
CELERY_REDIRECT_STDOUTS=not pkconfig.channel_in('dev'),
CELERY_RESULT_PERSISTENT=True,
CELERY_RESULT_SERIALIZER=_SERIALIZER,
CELERY_TASK_PUBLISH_RETRY=False,
CELERY_TASK_RESULT_EXPIRES=None,
from pykern.pkdebug import pkdc, pkdexc, pkdp, pkdlog
from sirepo.template import template_common
import py.path
celery = Celery('sirepo')
cfg = pkconfig.init(
broker_url=('amqp://guest@localhost//', str, 'Celery: queue broker url'),
celery_result_backend=('rpc://', str, 'configure db other than default'),
celeryd_concurrency=(1, int, 'how many worker processes to start'),
celeryd_task_time_limit=(3600, int, 'max run time for a task in seconds'),
)
celery.conf.update(
pkcollections.map_items(cfg, op=lambda k, v: (k.upper(), v)),
)
_SERIALIZER = 'json'
celery.conf.update(
CELERYD_LOG_COLOR=False,
CELERYD_MAX_TASKS_PER_CHILD=1,
CELERYD_PREFETCH_MULTIPLIER=1,
CELERYD_TASK_SOFT_TIME_LIMIT=celery.conf['CELERYD_TASK_TIME_LIMIT'] - 10,
CELERY_ACCEPT_CONTENT=[_SERIALIZER],
CELERY_ACKS_LATE=True,
CELERY_REDIRECT_STDOUTS=not pkconfig.channel_in('dev'),
CELERY_RESULT_PERSISTENT=True,
CELERY_RESULT_SERIALIZER=_SERIALIZER,
CELERY_TASK_PUBLISH_RETRY=False,
