def write_input(self, atoms, properties=None, system_changes=None):
FileIOCalculator.write_input(self, atoms, properties,
system_changes)
struct = crys.atoms2struct(atoms)
self.cell = common.str_arr(struct.cell)
self.kpoints = pwscf.kpoints_str_pwin(kpts2mp(atoms, self.kpts))
if isinstance(self.pp, types.StringType):
pseudos = ["%s.%s" %(sym, self.pp) for sym in struct.symbols_unique]
else:
assert len(self.pp) == struct.ntypat
pseudos = []
for sym in struct.symbols_unique:
for ppi in self.pp:
if ppi.startswith(sym):
pseudos.append(ppi)
break
assert len(pseudos) == struct.ntypat
self.atspec = pwscf.atspec_str(symbols=struct.symbols_unique,
masses=struct.mass_unique,
pseudos=pseudos)
self.atpos = pwscf.atpos_str(symbols=struct.symbols,
coords=struct.coords_frac)
self.natoms = struct.natoms
self.ntyp = struct.ntypat
if self.backup:
for fn in [self.infile, self.outfile]:
if os.path.exists(fn):
common.backup(fn)
common.file_write(self.infile, self.fill_infile_templ())
def write_input(self, atoms, properties=None, system_changes=None):
FileIOCalculator.write_input(self, atoms, properties, system_changes)
struct = crys.atoms2struct(atoms)
self.cell = common.str_arr(struct.cell)
self.kpoints = pwscf.kpoints_str_pwin(kpts2mp(atoms, self.kpts))
if isinstance(self.pp, bytes):
pseudos = [
"%s.%s" % (sym, self.pp) for sym in struct.symbols_unique
]
else:
assert len(self.pp) == struct.ntypat
pseudos = []
for sym in struct.symbols_unique:
for ppi in self.pp:
if ppi.startswith(sym):
pseudos.append(ppi)
break
assert len(pseudos) == struct.ntypat
self.atspec = pwscf.atspec_str(symbols=struct.symbols_unique,
masses=struct.mass_unique,
pseudos=pseudos)
self.atpos = pwscf.atpos_str(symbols=struct.symbols,
coords=struct.coords_frac)
self.natoms = struct.natoms
self.ntyp = struct.ntypat
if self.backup:
for fn in [self.infile, self.outfile]:
if os.path.exists(fn):
common.backup(fn)
common.file_write(self.infile, self.fill_infile_templ())
def write_input(self, atoms, properties=None, system_changes=None):
FileIOCalculator.write_input(self, atoms, properties,
system_changes)
if self.backup:
for fn in [self.infile, self.outfile, self.dumpfile,
self.structfile, self.logfile]:
if os.path.exists(fn):
common.backup(fn)
common.file_write(self.infile, self.fill_infile_templ())
io.write_lammps(self.structfile,
crys.atoms2struct(atoms),
symbolsbasename=os.path.basename(self.structfile) + \
'.symbols')
def write_input(self, atoms, properties=None, system_changes=None):
FileIOCalculator.write_input(self, atoms, properties, system_changes)
if self.backup:
for fn in [
self.infile, self.outfile, self.dumpfile, self.structfile,
self.logfile
]:
if os.path.exists(fn):
common.backup(fn)
common.file_write(self.infile, self.fill_infile_templ())
io.write_lammps(self.structfile,
crys.atoms2struct(atoms),
symbolsbasename=os.path.basename(self.structfile) + \
'.symbols')
def write_input(self, mode='a', backup=True, sleep=0, excl=True):
"""
Create calculation dir(s) for each parameter set and write input files
based on ``templates``. Write sqlite database storing all relevant
parameters. Write (bash) shell script to start all calculations (run
locally or submitt batch job file, depending on ``machine.subcmd``).
Parameters
----------
mode : str, optional
Fine tune how to write input files (based on ``templates``) to calc
dirs calc_foo/0/, calc_foo/1/, ... . Note that this doesn't change
the base dir calc_foo at all, only the subdirs for each calc.
{'a', 'w'}
| 'a': Append mode (default). If a previous database is found, then
| subsequent calculations are numbered based on the last 'idx'.
| calc_foo/0 # old
| calc_foo/1 # old
| calc_foo/2 # new
| calc_foo/3 # new
| 'w': Write mode. The target dirs are purged and overwritten. Also,
| the database (self.dbfn) is overwritten. Use this to
| iteratively tune your inputs, NOT for working on already
| present results!
| calc_foo/0 # new
| calc_foo/1 # new
backup : bool, optional
Before writing anything, do a backup of self.calc_dir if it already
exists.
sleep : int, optional
For the script to start (submitt) all jobs: time in seconds for the
shell sleep(1) commmand.
excl : bool
If in append mode, a file <calc_root>/excl_push with all indices of
calculations from old revisions is written. Can be used with
``rsync --exclude-from=excl_push`` when pushing appended new
calculations to a cluster.
"""
assert mode in ['a', 'w'], "Unknown mode: '%s'" %mode
if os.path.exists(self.dbfn):
if backup:
common.backup(self.dbfn)
if mode == 'w':
os.remove(self.dbfn)
have_new_db = not os.path.exists(self.dbfn)
common.makedirs(self.calc_root)
# this call creates a file ``self.dbfn`` if it doesn't exist
sqldb = SQLiteDB(self.dbfn, table=self.db_table)
# max_idx: counter for calc dir numbering
revision = 0
if have_new_db:
max_idx = -1
else:
if mode == 'a':
if sqldb.has_column('idx'):
max_idx = sqldb.execute("select max(idx) from %s" \
%self.db_table).fetchone()[0]
else:
raise StandardError("database '%s': table '%s' has no "
"column 'idx', don't know how to number calcs"
%(self.dbfn, self.db_table))
if sqldb.has_column('revision'):
revision = int(sqldb.get_single("select max(revision) \
from %s" %self.db_table)) + 1
elif mode == 'w':
max_idx = -1
sql_records = []
hostnames = []
for imach, machine in enumerate(self.machines):
hostnames.append(machine.hostname)
calc_dir = pj(self.calc_root, self.calc_dir_prefix + \
'_%s' %machine.hostname)
if os.path.exists(calc_dir):
if backup:
common.backup(calc_dir)
if mode == 'w':
common.system("rm -r %s" %calc_dir, wait=True)
run_txt = "here=$(pwd)\n"
for _idx, params in enumerate(self.params_lst):
params = common.flatten(params)
idx = max_idx + _idx + 1
calc_subdir = pj(calc_dir, str(idx))
extra_dct = \
{'revision': revision,
'study_name': self.study_name,
'idx': idx,
'calc_name' : self.study_name + "_run%i" %idx,
}
extra_params = [SQLEntry(key=key, sqlval=val) for key,val in \
extra_dct.iteritems()]
# templates[:] to copy b/c they may be modified in Calculation
calc = Calculation(machine=machine,
templates=self.templates[:],
params=params + extra_params,
calc_dir=calc_subdir,
)
if mode == 'w' and os.path.exists(calc_subdir):
shutil.rmtree(calc_subdir)
calc.write_input()
run_txt += "cd %i && %s %s && cd $here && sleep %i\n" %(idx,\
machine.subcmd, machine.get_jobfile_basename(), sleep)
if imach == 0:
sql_records.append(calc.get_sql_record())
common.file_write(pj(calc_dir, 'run.sh'), run_txt)
for record in sql_records:
record['hostname'] = SQLEntry(sqlval=','.join(hostnames))
# for incomplete parameters: collect header parts from all records and
# make a set = unique entries
raw_header = [(key, entry.sqltype.upper()) for record in sql_records \
for key, entry in record.iteritems()]
header = list(set(raw_header))
if have_new_db:
sqldb.create_table(header)
else:
for record in sql_records:
for key, entry in record.iteritems():
if not sqldb.has_column(key):
sqldb.add_column(key, entry.sqltype.upper())
for record in sql_records:
cmd = "insert into %s (%s) values (%s)"\
%(self.db_table,
",".join(record.keys()),
",".join(['?']*len(record.keys())))
sqldb.execute(cmd, tuple(entry.sqlval for entry in record.itervalues()))
if excl and revision > 0 and sqldb.has_column('revision'):
old_idx_lst = [str(x) for x, in sqldb.execute("select idx from calc where \
revision < ?", (revision,))]
common.file_write(pj(self.calc_root, 'excl_push'),
'\n'.join(old_idx_lst))
sqldb.finish()
def test_backup():
# file
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# dir
name = tempfile.mktemp(prefix='testdir', dir=testdir)
create_full_dir(name)
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# link to file
filename = tempfile.mktemp(prefix='testfile', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
file_write(filename, 'foo')
os.symlink(filename, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isfile(linkname + '.0')
assert file_read(linkname + '.0') == file_read(filename)
# link to dir
dirname = tempfile.mktemp(prefix='testdir', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
create_full_dir(dirname)
os.symlink(dirname, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isdir(linkname + '.0')
for name in ['a', 'b', 'c']:
assert file_read(pj(dirname, name)) == \
file_read(pj(linkname + '.0', name))
# prefix
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name, prefix="-bak")
assert os.path.exists(name + '-bak0')
# nonexisting src, should silently pass
filename = str(uuid.uuid4())
assert not os.path.exists(filename)
backup(filename)
def write_input(self, mode='a', backup=True, sleep=0, excl=True):
"""
Create calculation dir(s) for each parameter set and write input files
based on ``templates``. Write sqlite database storing all relevant
parameters. Write (bash) shell script to start all calculations (run
locally or submitt batch job file, depending on ``machine.subcmd``).
Parameters
----------
mode : str, optional
Fine tune how to write input files (based on ``templates``) to calc
dirs calc_foo/0/, calc_foo/1/, ... . Note that this doesn't change
the base dir calc_foo at all, only the subdirs for each calc.
{'a', 'w'}
| 'a': Append mode (default). If a previous database is found, then
| subsequent calculations are numbered based on the last 'idx'.
| calc_foo/0 # old
| calc_foo/1 # old
| calc_foo/2 # new
| calc_foo/3 # new
| 'w': Write mode. The target dirs are purged and overwritten. Also,
| the database (self.dbfn) is overwritten. Use this to
| iteratively tune your inputs, NOT for working on already
| present results!
| calc_foo/0 # new
| calc_foo/1 # new
backup : bool, optional
Before writing anything, do a backup of self.calc_dir if it already
exists.
sleep : int, optional
For the script to start (submitt) all jobs: time in seconds for the
shell sleep(1) commmand.
excl : bool
If in append mode, a file <calc_root>/excl_push with all indices of
calculations from old revisions is written. Can be used with
``rsync --exclude-from=excl_push`` when pushing appended new
calculations to a cluster.
"""
assert mode in ['a', 'w'], "Unknown mode: '%s'" % mode
if os.path.exists(self.dbfn):
if backup:
common.backup(self.dbfn)
if mode == 'w':
os.remove(self.dbfn)
have_new_db = not os.path.exists(self.dbfn)
common.makedirs(self.calc_root)
# this call creates a file ``self.dbfn`` if it doesn't exist
sqldb = SQLiteDB(self.dbfn, table=self.db_table)
# max_idx: counter for calc dir numbering
revision = 0
if have_new_db:
max_idx = -1
else:
if mode == 'a':
if sqldb.has_column('idx'):
max_idx = sqldb.execute("select max(idx) from %s" \
%self.db_table).fetchone()[0]
else:
raise Exception(
"database '%s': table '%s' has no "
"column 'idx', don't know how to number calcs" %
(self.dbfn, self.db_table))
if sqldb.has_column('revision'):
revision = int(
sqldb.get_single("select max(revision) \
from %s" % self.db_table)) + 1
elif mode == 'w':
max_idx = -1
sql_records = []
hostnames = []
for imach, machine in enumerate(self.machines):
hostnames.append(machine.hostname)
calc_dir = pj(self.calc_root, self.calc_dir_prefix + \
'_%s' %machine.hostname)
if os.path.exists(calc_dir):
if backup:
common.backup(calc_dir)
if mode == 'w':
common.system("rm -r %s" % calc_dir, wait=True)
run_txt = "here=$(pwd)\n"
for _idx, params in enumerate(self.params_lst):
params = common.flatten(params)
idx = max_idx + _idx + 1
calc_subdir = pj(calc_dir, str(idx))
extra_dct = \
{'revision': revision,
'study_name': self.study_name,
'idx': idx,
'calc_name' : self.study_name + "_run%i" %idx,
}
extra_params = [SQLEntry(key=key, sqlval=val) for key,val in \
extra_dct.items()]
# templates[:] to copy b/c they may be modified in Calculation
calc = Calculation(
machine=machine,
templates=self.templates[:],
params=params + extra_params,
calc_dir=calc_subdir,
)
if mode == 'w' and os.path.exists(calc_subdir):
shutil.rmtree(calc_subdir)
calc.write_input()
run_txt += "cd %i && %s %s && cd $here && sleep %i\n" %(idx,\
machine.subcmd, machine.get_jobfile_basename(), sleep)
if imach == 0:
sql_records.append(calc.get_sql_record())
common.file_write(pj(calc_dir, 'run.sh'), run_txt)
for record in sql_records:
record['hostname'] = SQLEntry(sqlval=','.join(hostnames))
# for incomplete parameters: collect header parts from all records and
# make a set = unique entries
raw_header = [(key, entry.sqltype.upper()) for record in sql_records \
for key, entry in record.items()]
header = list(set(raw_header))
if have_new_db:
sqldb.create_table(header)
else:
for record in sql_records:
for key, entry in record.items():
if not sqldb.has_column(key):
sqldb.add_column(key, entry.sqltype.upper())
for record in sql_records:
cmd = "insert into %s (%s) values (%s)"\
%(self.db_table,
",".join(list(record.keys())),
",".join(['?']*len(list(record.keys()))))
sqldb.execute(cmd,
tuple(entry.sqlval for entry in record.values()))
if excl and revision > 0 and sqldb.has_column('revision'):
old_idx_lst = [
str(x) for x, in sqldb.execute(
"select idx from calc where \
revision < ?", (
revision, ))
]
common.file_write(pj(self.calc_root, 'excl_push'),
'\n'.join(old_idx_lst))
sqldb.finish()
def test_backup():
# file
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# dir
name = tempfile.mktemp(prefix='testdir', dir=testdir)
create_full_dir(name)
backup(name)
assert os.path.exists(name + '.0')
backup(name)
assert os.path.exists(name + '.1')
backup(name)
assert os.path.exists(name + '.2')
# link to file
filename = tempfile.mktemp(prefix='testfile', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
file_write(filename, 'foo')
os.symlink(filename, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isfile(linkname + '.0')
assert file_read(linkname + '.0') == file_read(filename)
# link to dir
dirname = tempfile.mktemp(prefix='testdir', dir=testdir)
linkname = tempfile.mktemp(prefix='testlink', dir=testdir)
create_full_dir(dirname)
os.symlink(dirname, linkname)
backup(linkname)
assert os.path.exists(linkname + '.0')
assert os.path.isdir(linkname + '.0')
for name in ['a', 'b', 'c']:
assert file_read(pj(dirname, name)) == \
file_read(pj(linkname + '.0', name))
# prefix
name = tempfile.mktemp(prefix='testfile', dir=testdir)
file_write(name, 'foo')
backup(name, prefix="-bak")
assert os.path.exists(name + '-bak0')
