def test_sql_column():
x = sql_column(key='foo', sqltype='integer', lst=[1, 2, 3])
for num, xx in zip([1, 2, 3], x):
assert xx.sqlval == num
assert xx.fileval == num
x = sql_column(key='foo',
sqltype='integer',
lst=[1, 2, 3],
fileval_func=lambda z: "k=%i" % z)
for num, xx in zip([1, 2, 3], x):
assert xx.sqlval == num
assert xx.fileval == "k=%i" % num
x = sql_column(key='foo',
sqltype='integer',
lst=[1, 2, 3],
sqlval_func=lambda z: z**2,
fileval_func=lambda z: "k=%i" % z)
for num, xx in zip([1, 2, 3], x):
assert xx.sqlval == num**2
assert xx.fileval == "k=%i" % num
for xx in sql_column('foo', [1, 2]):
assert xx.sqltype == 'INTEGER'
for xx in sql_column('foo', [1.0, 2.0]):
assert xx.sqltype == 'REAL'
def test_sql_column():
x = sql_column(key="foo", sqltype="integer", lst=[1, 2, 3])
for num, xx in zip([1, 2, 3], x):
assert xx.sqlval == num
assert xx.fileval == num
x = sql_column(key="foo", sqltype="integer", lst=[1, 2, 3], fileval_func=lambda z: "k=%i" % z)
for num, xx in zip([1, 2, 3], x):
assert xx.sqlval == num
assert xx.fileval == "k=%i" % num
x = sql_column(
key="foo", sqltype="integer", lst=[1, 2, 3], sqlval_func=lambda z: z ** 2, fileval_func=lambda z: "k=%i" % z
)
for num, xx in zip([1, 2, 3], x):
assert xx.sqlval == num ** 2
assert xx.fileval == "k=%i" % num
for xx in sql_column("foo", [1, 2]):
assert xx.sqltype == "INTEGER"
for xx in sql_column("foo", [1.0, 2.0]):
assert xx.sqltype == "REAL"
def test_parameter_study():
templ_dir = 'files/calc.templ'
calc_root = pj(testdir, 'calc_test_param_study')
# filename: FileTemplate built from that internally
host0 = batch.Machine(hostname='host0',
subcmd='qsub_host0',
home='/home/host0/user',
scratch='/tmp/host0',
filename='files/calc.templ/job.host0')
# template: provide FileTemplate directly
host1 = batch.Machine(hostname='host1',
subcmd='qsub_host1',
home='/home/host1/user',
scratch='/tmp/host1',
template=batch.FileTemplate(basename='job.host1',
templ_dir=templ_dir))
# use template text here instead of a file
host2_txt = """
subcmd=XXXSUBCMD
scratch=XXXSCRATCH
home=XXXHOME
calc_name=XXXCALC_NAME
idx=XXXIDX
revision=XXXREVISION
study_name=XXXSTUDY_NAME
"""
host2 = batch.Machine(hostname='host2',
subcmd='qsub_host2',
home='/home/host2/user',
scratch='/tmp/host2',
template=batch.FileTemplate(basename='job.host2',
txt=host2_txt))
study_name = 'convergence'
templates = [batch.FileTemplate(basename='pw.in', templ_dir=templ_dir)]
param0 = sql.sql_column(key='param0', lst=[25.0, 50.0])
param1 = sql.sql_column(key='param1', lst=['2x2x2','3x3x3','4x4x4'])
param2 = sql.sql_column(key='param2', lst=[77,88,99,111])
# only needed for this test
machine_dct = {'host0': host0,
'host1': host1,
'host2': host2,
}
nparam0 = len(param0)
nparam1 = len(param1)
nparam2 = len(param2)
#------------------------------------------------------------------------
# revision=0
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
# same as mode='w' + backup=True
calc.write_input(mode='a', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
#------------------------------------------------------------------------
# revision=0, no backup, erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=False)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert not os.path.exists(pj(calc_root, 'calc_host0.0'))
assert not os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' %ii))
for jj in range(1,5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' %(ii+jj,)))
#------------------------------------------------------------------------
# revision=0, backup, then erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert os.path.exists(pj(calc_root, 'calc_host0.0'))
assert os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' %ii))
for jj in range(1,5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' %(ii+jj,)))
#------------------------------------------------------------------------
# revision=1, backup and extend
#------------------------------------------------------------------------
params_lst1 = comb.nested_loops([param1,param2])
calc = batch.ParameterStudy(machines=[host0,host1,host2],
templates=templates,
params_lst=params_lst1,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='a', backup=True)
assert os.path.exists(pj(calc_root, 'calc_host0.1'))
assert os.path.exists(pj(calc_root, 'calc.db.1'))
for ii in range(nparam0 + nparam1*nparam2):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' %ii))
for ii in range(nparam0):
assert not os.path.exists(pj(calc_root, 'calc_host1/%i' %ii))
assert not os.path.exists(pj(calc_root, 'calc_host2/%i' %ii))
for ii in range(nparam0+1, nparam1*nparam2):
assert os.path.exists(pj(calc_root, 'calc_host1/%i' %ii))
assert os.path.exists(pj(calc_root, 'calc_host2/%i' %ii))
# excl_push
excl_fn = pj(calc_root, 'excl_push')
# ['0', '1', '2', ...]
assert common.file_read(excl_fn).split() == \
[str(x) for x in range(len(params_lst0))]
# sum params_lstm b/c we use `idx` from calc.db and that counts params_lst0
# + params_lst1, i.e. all paramseter sets from revision=0 up to now
check_generated(calc_root, machine_dct, params_lst0+params_lst1, revision=1)
def test_sql_column_fail_for_mixed_types():
s = sql_column("foo", [1, 2.0])
def test_sql_column_fail_for_mixed_types():
s = sql_column('foo', [1, 2.0])
def test_parameter_study():
templ_dir = 'files/calc.templ'
calc_root = pj(testdir, 'calc_test_param_study')
# filename: FileTemplate built from that internally
host0 = batch.Machine(hostname='host0',
subcmd='qsub_host0',
home='/home/host0/user',
scratch='/tmp/host0',
filename='files/calc.templ/job.host0')
# template: provide FileTemplate directly
host1 = batch.Machine(hostname='host1',
subcmd='qsub_host1',
home='/home/host1/user',
scratch='/tmp/host1',
template=batch.FileTemplate(basename='job.host1',
templ_dir=templ_dir))
# use template text here instead of a file
host2_txt = """
subcmd=XXXSUBCMD
scratch=XXXSCRATCH
home=XXXHOME
calc_name=XXXCALC_NAME
idx=XXXIDX
revision=XXXREVISION
study_name=XXXSTUDY_NAME
"""
host2 = batch.Machine(hostname='host2',
subcmd='qsub_host2',
home='/home/host2/user',
scratch='/tmp/host2',
template=batch.FileTemplate(basename='job.host2',
txt=host2_txt))
study_name = 'convergence'
templates = [batch.FileTemplate(basename='pw.in', templ_dir=templ_dir)]
param0 = sql.sql_column(key='param0', lst=[25.0, 50.0])
param1 = sql.sql_column(key='param1', lst=['2x2x2', '3x3x3', '4x4x4'])
param2 = sql.sql_column(key='param2', lst=[77, 88, 99, 111])
# only needed for this test
machine_dct = {
'host0': host0,
'host1': host1,
'host2': host2,
}
nparam0 = len(param0)
nparam1 = len(param1)
nparam2 = len(param2)
#------------------------------------------------------------------------
# revision=0
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
# same as mode='w' + backup=True
calc.write_input(mode='a', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
#------------------------------------------------------------------------
# revision=0, no backup, erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=False)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert not os.path.exists(pj(calc_root, 'calc_host0.0'))
assert not os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' % ii))
for jj in range(1, 5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' % (ii + jj, )))
#------------------------------------------------------------------------
# revision=0, backup, then erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert os.path.exists(pj(calc_root, 'calc_host0.0'))
assert os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' % ii))
for jj in range(1, 5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' % (ii + jj, )))
#------------------------------------------------------------------------
# revision=1, backup and extend
#------------------------------------------------------------------------
params_lst1 = comb.nested_loops([param1, param2])
calc = batch.ParameterStudy(machines=[host0, host1, host2],
templates=templates,
params_lst=params_lst1,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='a', backup=True)
assert os.path.exists(pj(calc_root, 'calc_host0.1'))
assert os.path.exists(pj(calc_root, 'calc.db.1'))
for ii in range(nparam0 + nparam1 * nparam2):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' % ii))
for ii in range(nparam0):
assert not os.path.exists(pj(calc_root, 'calc_host1/%i' % ii))
assert not os.path.exists(pj(calc_root, 'calc_host2/%i' % ii))
for ii in range(nparam0 + 1, nparam1 * nparam2):
assert os.path.exists(pj(calc_root, 'calc_host1/%i' % ii))
assert os.path.exists(pj(calc_root, 'calc_host2/%i' % ii))
# excl_push
excl_fn = pj(calc_root, 'excl_push')
# ['0', '1', '2', ...]
assert common.file_read(excl_fn).split() == \
[str(x) for x in range(len(params_lst0))]
# sum params_lstm b/c we use `idx` from calc.db and that counts params_lst0
# + params_lst1, i.e. all paramseter sets from revision=0 up to now
check_generated(calc_root,
machine_dct,
params_lst0 + params_lst1,
revision=1)