def test_backup():
# file
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# dir
name = tempfile.mktemp(prefix='testdir', dir=testdir)
create_full_dir(name)
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# link to file
filename = tempfile.mktemp(prefix='testfile', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
file_write(filename, 'foo')
os.symlink(filename, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isfile(linkname + '.0')
assert file_read(linkname + '.0') == file_read(filename)
# link to dir
dirname = tempfile.mktemp(prefix='testdir', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
create_full_dir(dirname)
os.symlink(dirname, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isdir(linkname + '.0')
for name in ['a', 'b', 'c']:
assert file_read(pj(dirname, name)) == \
file_read(pj(linkname + '.0', name))
# prefix
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name, prefix="-bak")
assert os.path.exists(name + '-bak0')
# nonexisting src, should silently pass
filename = str(uuid.uuid4())
assert not os.path.exists(filename)
backup(filename)
def test_placeholders():
templ_dir = pj(testdir, 'calc.templ')
templ_fn = pj(templ_dir, 'foo.in')
tgt_dir = pj(testdir, 'calc')
tgt_fn = pj(tgt_dir, 'foo.in')
for dr in [templ_dir, tgt_dir]:
if not os.path.exists(dr):
os.makedirs(dr)
templ_txt = "XXXFOO XXXBAR XXXBAZ"
file_write(templ_fn, templ_txt)
# specify keys
templ = FileTemplate(basename='foo.in',
keys=['foo', 'bar'],
templ_dir=templ_dir)
rules = {'foo': 1, 'bar': 'lala', 'baz': 3}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala XXXBAZ"
# no keys
templ = FileTemplate(basename='foo.in', templ_dir=templ_dir)
rules = {'foo': 1, 'bar': 'lala', 'baz': 3}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala 3"
# sql
rules = {
'foo': sql.SQLEntry(sqltype='integer', sqlval=1),
'bar': sql.SQLEntry(sqltype='text', sqlval='lala'),
'baz': sql.SQLEntry(sqltype='integer', sqlval=3)
}
templ.writesql(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala 3"
# non-default placefolders
templ_txt = "@foo@ @bar@"
file_write(templ_fn, templ_txt)
templ = FileTemplate(basename='foo.in',
templ_dir=templ_dir,
func=lambda x: "@%s@" % x)
rules = {'foo': 1, 'bar': 'lala'}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala"
# pass txt directly
templ_txt = "XXXFOO XXXBAR XXXBAZ"
templ = FileTemplate(basename='foo.in', txt=templ_txt)
rules = {'foo': 1, 'bar': 'lala', 'baz': 3}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala 3"
def test_backup():
# file
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# dir
name = tempfile.mktemp(prefix='testdir', dir=testdir)
create_full_dir(name)
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# link to file
filename = tempfile.mktemp(prefix='testfile', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
file_write(filename, 'foo')
os.symlink(filename, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isfile(linkname + '.0')
assert file_read(linkname + '.0') == file_read(filename)
# link to dir
dirname = tempfile.mktemp(prefix='testdir', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
create_full_dir(dirname)
os.symlink(dirname, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isdir(linkname + '.0')
for name in ['a', 'b', 'c']:
assert file_read(pj(dirname, name)) == \
file_read(pj(linkname + '.0', name))
# prefix
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name, prefix="-bak")
assert os.path.exists(name + '-bak0')
def test_placeholders():
templ_dir = pj(testdir, "calc.templ")
templ_fn = pj(templ_dir, "foo.in")
tgt_dir = pj(testdir, "calc")
tgt_fn = pj(tgt_dir, "foo.in")
for dr in [templ_dir, tgt_dir]:
if not os.path.exists(dr):
os.makedirs(dr)
templ_txt = "XXXFOO XXXBAR XXXBAZ"
file_write(templ_fn, templ_txt)
# specify keys
templ = FileTemplate(basename="foo.in", keys=["foo", "bar"], templ_dir=templ_dir)
rules = {"foo": 1, "bar": "lala", "baz": 3}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala XXXBAZ"
# no keys
templ = FileTemplate(basename="foo.in", templ_dir=templ_dir)
rules = {"foo": 1, "bar": "lala", "baz": 3}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala 3"
# sql
rules = {
"foo": sql.SQLEntry(sqltype="integer", sqlval=1),
"bar": sql.SQLEntry(sqltype="text", sqlval="lala"),
"baz": sql.SQLEntry(sqltype="integer", sqlval=3),
}
templ.writesql(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala 3"
# non-default placefolders
templ_txt = "@foo@ @bar@"
file_write(templ_fn, templ_txt)
templ = FileTemplate(basename="foo.in", templ_dir=templ_dir, func=lambda x: "@%s@" % x)
rules = {"foo": 1, "bar": "lala"}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala"
# pass txt directly
templ_txt = "XXXFOO XXXBAR XXXBAZ"
templ = FileTemplate(basename="foo.in", txt=templ_txt)
rules = {"foo": 1, "bar": "lala", "baz": 3}
templ.write(rules, calc_dir=tgt_dir)
assert file_read(tgt_fn).strip() == "1 lala 3"
def test_write_lammps():
st = crys.Structure(coords_frac=rand(20, 3),
cell=rand(3, 3),
symbols=['Al'] * 10 + ['N'] * 10)
# align cell to lammps standard [[x,0,0],...]
st.coords = None
st.cell = None
st.set_all()
st_fn = common.pj(testdir, 'lmp.struct')
io.write_lammps(st_fn, st)
symbols = common.file_read(st_fn + '.symbols').split()
assert st.symbols == symbols
cmd = r"grep -A22 Atoms %s | grep '^[0-9]'" % st_fn
arr = parse.arr2d_from_txt(common.backtick(cmd))
assert arr.shape == (20, 5)
assert np.allclose(st.coords, arr[:, 2:])
def test_write_lammps():
st = crys.Structure(coords_frac=rand(20,3),
cell=rand(3,3),
symbols=['Al']*10+['N']*10)
# align cell to lammps standard [[x,0,0],...]
st.coords = None
st.cell = None
st.set_all()
st_fn = common.pj(testdir, 'lmp.struct')
io.write_lammps(st_fn, st)
symbols = common.file_read(st_fn + '.symbols').split()
assert st.symbols == symbols
cmd = r"grep -A22 Atoms %s | grep '^[0-9]'" %st_fn
arr = parse.arr2d_from_txt(common.backtick(cmd))
assert arr.shape == (20,5)
assert np.allclose(st.coords, arr[:,2:])
def write(self, dct, calc_dir='calc', mode='dct'):
"""Write file self.filename (e.g. calc/0/pw.in) by replacing
placeholders in the template (e.g. calc.templ/pw.in).
Parameters
----------
dct : dict
key-value pairs, dct.keys() are converted to placeholders with
self.func()
calc_dir : str
the dir where to write the target file to
mode : str, {'dct', 'sql'}
| mode='dct': replacement values are dct[<key>]
| mode='sql': replacement values are dct[<key>].fileval and every
| dct[<key>] is an SQLEntry instance
"""
assert mode in ['dct', 'sql'], ("Wrong 'mode' kwarg, use 'dct' "
"or 'sql'")
# copy_only : bypass reading the file and passing the text thru the
# replacement machinery and getting the text back, unchanged. While
# this works, it is slower and useless.
if self.keys == []:
_keys = None
txt = None
copy_only = True
else:
if self.keys is None:
_keys = dct.iterkeys()
warn_not_found = False
else:
_keys = self.keys
warn_not_found = True
if self.txt is None:
txt = common.file_read(self.filename)
else:
txt = self.txt
copy_only = False
tgt = pj(calc_dir, self.basename)
verbose("write: %s" %tgt)
if copy_only:
verbose("write: ignoring input, just copying file to %s"
%(self.filename, tgt))
shutil.copy(self.filename, tgt)
else:
rules = {}
for key in _keys:
if mode == 'dct':
rules[self.func(key)] = dct[key]
elif mode == 'sql':
# dct = sql_record, a list of SQLEntry's
rules[self.func(key)] = dct[key].fileval
else:
raise StandardError("'mode' must be wrong")
new_txt = common.template_replace(txt,
rules,
mode='txt',
conv=True,
warn_not_found=warn_not_found,
warn_mult_found=False,
disp=False)
common.file_write(tgt, new_txt)
def test_parameter_study():
templ_dir = 'files/calc.templ'
calc_root = pj(testdir, 'calc_test_param_study')
# filename: FileTemplate built from that internally
host0 = batch.Machine(hostname='host0',
subcmd='qsub_host0',
home='/home/host0/user',
scratch='/tmp/host0',
filename='files/calc.templ/job.host0')
# template: provide FileTemplate directly
host1 = batch.Machine(hostname='host1',
subcmd='qsub_host1',
home='/home/host1/user',
scratch='/tmp/host1',
template=batch.FileTemplate(basename='job.host1',
templ_dir=templ_dir))
# use template text here instead of a file
host2_txt = """
subcmd=XXXSUBCMD
scratch=XXXSCRATCH
home=XXXHOME
calc_name=XXXCALC_NAME
idx=XXXIDX
revision=XXXREVISION
study_name=XXXSTUDY_NAME
"""
host2 = batch.Machine(hostname='host2',
subcmd='qsub_host2',
home='/home/host2/user',
scratch='/tmp/host2',
template=batch.FileTemplate(basename='job.host2',
txt=host2_txt))
study_name = 'convergence'
templates = [batch.FileTemplate(basename='pw.in', templ_dir=templ_dir)]
param0 = sql.sql_column(key='param0', lst=[25.0, 50.0])
param1 = sql.sql_column(key='param1', lst=['2x2x2','3x3x3','4x4x4'])
param2 = sql.sql_column(key='param2', lst=[77,88,99,111])
# only needed for this test
machine_dct = {'host0': host0,
'host1': host1,
'host2': host2,
}
nparam0 = len(param0)
nparam1 = len(param1)
nparam2 = len(param2)
#------------------------------------------------------------------------
# revision=0
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
# same as mode='w' + backup=True
calc.write_input(mode='a', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
#------------------------------------------------------------------------
# revision=0, no backup, erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=False)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert not os.path.exists(pj(calc_root, 'calc_host0.0'))
assert not os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' %ii))
for jj in range(1,5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' %(ii+jj,)))
#------------------------------------------------------------------------
# revision=0, backup, then erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert os.path.exists(pj(calc_root, 'calc_host0.0'))
assert os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' %ii))
for jj in range(1,5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' %(ii+jj,)))
#------------------------------------------------------------------------
# revision=1, backup and extend
#------------------------------------------------------------------------
params_lst1 = comb.nested_loops([param1,param2])
calc = batch.ParameterStudy(machines=[host0,host1,host2],
templates=templates,
params_lst=params_lst1,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='a', backup=True)
assert os.path.exists(pj(calc_root, 'calc_host0.1'))
assert os.path.exists(pj(calc_root, 'calc.db.1'))
for ii in range(nparam0 + nparam1*nparam2):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' %ii))
for ii in range(nparam0):
assert not os.path.exists(pj(calc_root, 'calc_host1/%i' %ii))
assert not os.path.exists(pj(calc_root, 'calc_host2/%i' %ii))
for ii in range(nparam0+1, nparam1*nparam2):
assert os.path.exists(pj(calc_root, 'calc_host1/%i' %ii))
assert os.path.exists(pj(calc_root, 'calc_host2/%i' %ii))
# excl_push
excl_fn = pj(calc_root, 'excl_push')
# ['0', '1', '2', ...]
assert common.file_read(excl_fn).split() == \
[str(x) for x in range(len(params_lst0))]
# sum params_lstm b/c we use `idx` from calc.db and that counts params_lst0
# + params_lst1, i.e. all paramseter sets from revision=0 up to now
check_generated(calc_root, machine_dct, params_lst0+params_lst1, revision=1)
def write(self, dct, calc_dir='calc', mode='dct'):
"""Write file self.filename (e.g. calc/0/pw.in) by replacing
placeholders in the template (e.g. calc.templ/pw.in).
Parameters
----------
dct : dict
key-value pairs, dct.keys() are converted to placeholders with
self.func()
calc_dir : str
the dir where to write the target file to
mode : str, {'dct', 'sql'}
| mode='dct': replacement values are dct[<key>]
| mode='sql': replacement values are dct[<key>].fileval and every
| dct[<key>] is an SQLEntry instance
"""
assert mode in ['dct', 'sql'], ("Wrong 'mode' kwarg, use 'dct' "
"or 'sql'")
# copy_only : bypass reading the file and passing the text thru the
# replacement machinery and getting the text back, unchanged. While
# this works, it is slower and useless.
if self.keys == []:
_keys = None
txt = None
copy_only = True
else:
if self.keys is None:
_keys = dct.keys()
warn_not_found = False
else:
_keys = self.keys
warn_not_found = True
if self.txt is None:
txt = common.file_read(self.filename)
else:
txt = self.txt
copy_only = False
tgt = pj(calc_dir, self.basename)
verbose("write: %s" % tgt)
if copy_only:
verbose("write: ignoring input, just copying file to %s" %
(self.filename, tgt))
shutil.copy(self.filename, tgt)
else:
rules = {}
for key in _keys:
if mode == 'dct':
rules[self.func(key)] = dct[key]
elif mode == 'sql':
# dct = sql_record, a list of SQLEntry's
rules[self.func(key)] = dct[key].fileval
else:
raise Exception("'mode' must be wrong")
new_txt = common.template_replace(txt,
rules,
mode='txt',
conv=True,
warn_not_found=warn_not_found,
warn_mult_found=False,
disp=False)
common.file_write(tgt, new_txt)
def test_parameter_study():
templ_dir = 'files/calc.templ'
calc_root = pj(testdir, 'calc_test_param_study')
# filename: FileTemplate built from that internally
host0 = batch.Machine(hostname='host0',
subcmd='qsub_host0',
home='/home/host0/user',
scratch='/tmp/host0',
filename='files/calc.templ/job.host0')
# template: provide FileTemplate directly
host1 = batch.Machine(hostname='host1',
subcmd='qsub_host1',
home='/home/host1/user',
scratch='/tmp/host1',
template=batch.FileTemplate(basename='job.host1',
templ_dir=templ_dir))
# use template text here instead of a file
host2_txt = """
subcmd=XXXSUBCMD
scratch=XXXSCRATCH
home=XXXHOME
calc_name=XXXCALC_NAME
idx=XXXIDX
revision=XXXREVISION
study_name=XXXSTUDY_NAME
"""
host2 = batch.Machine(hostname='host2',
subcmd='qsub_host2',
home='/home/host2/user',
scratch='/tmp/host2',
template=batch.FileTemplate(basename='job.host2',
txt=host2_txt))
study_name = 'convergence'
templates = [batch.FileTemplate(basename='pw.in', templ_dir=templ_dir)]
param0 = sql.sql_column(key='param0', lst=[25.0, 50.0])
param1 = sql.sql_column(key='param1', lst=['2x2x2', '3x3x3', '4x4x4'])
param2 = sql.sql_column(key='param2', lst=[77, 88, 99, 111])
# only needed for this test
machine_dct = {
'host0': host0,
'host1': host1,
'host2': host2,
}
nparam0 = len(param0)
nparam1 = len(param1)
nparam2 = len(param2)
#------------------------------------------------------------------------
# revision=0
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
# same as mode='w' + backup=True
calc.write_input(mode='a', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
#------------------------------------------------------------------------
# revision=0, no backup, erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=False)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert not os.path.exists(pj(calc_root, 'calc_host0.0'))
assert not os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' % ii))
for jj in range(1, 5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' % (ii + jj, )))
#------------------------------------------------------------------------
# revision=0, backup, then erase all
#------------------------------------------------------------------------
params_lst0 = comb.nested_loops([param0])
calc = batch.ParameterStudy(machines=host0,
templates=templates,
params_lst=params_lst0,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='w', backup=True)
check_generated(calc_root, machine_dct, params_lst0, revision=0)
assert os.path.exists(pj(calc_root, 'calc_host0.0'))
assert os.path.exists(pj(calc_root, 'calc.db.0'))
# only calc_foo/0 ... calc_foo/{N-1}
for ii in range(nparam0):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' % ii))
for jj in range(1, 5):
assert not os.path.exists(pj(calc_root, 'calc_host0/%i' % (ii + jj, )))
#------------------------------------------------------------------------
# revision=1, backup and extend
#------------------------------------------------------------------------
params_lst1 = comb.nested_loops([param1, param2])
calc = batch.ParameterStudy(machines=[host0, host1, host2],
templates=templates,
params_lst=params_lst1,
study_name=study_name,
calc_root=calc_root)
calc.write_input(mode='a', backup=True)
assert os.path.exists(pj(calc_root, 'calc_host0.1'))
assert os.path.exists(pj(calc_root, 'calc.db.1'))
for ii in range(nparam0 + nparam1 * nparam2):
assert os.path.exists(pj(calc_root, 'calc_host0/%i' % ii))
for ii in range(nparam0):
assert not os.path.exists(pj(calc_root, 'calc_host1/%i' % ii))
assert not os.path.exists(pj(calc_root, 'calc_host2/%i' % ii))
for ii in range(nparam0 + 1, nparam1 * nparam2):
assert os.path.exists(pj(calc_root, 'calc_host1/%i' % ii))
assert os.path.exists(pj(calc_root, 'calc_host2/%i' % ii))
# excl_push
excl_fn = pj(calc_root, 'excl_push')
# ['0', '1', '2', ...]
assert common.file_read(excl_fn).split() == \
[str(x) for x in range(len(params_lst0))]
# sum params_lstm b/c we use `idx` from calc.db and that counts params_lst0
# + params_lst1, i.e. all paramseter sets from revision=0 up to now
check_generated(calc_root,
machine_dct,
params_lst0 + params_lst1,
revision=1)