def extra_folder(functional):
from pylada.jobfolder import JobFolder
jobfolder = JobFolder()
jobfolder.functional = functional
jobfolder.params['indiv'] = 25
jobfolder.params['value'] = 5
jobfolder.params['another'] = 6
return jobfolder
def save_job(path, inputpath="input.py", **kwargs):
""" Jobs to explore possible ground-states.
:Parameters:
path
Path where the job-folder will be saved. Calculations will be
performed in the parent directory of this file. Calculations will be
performed in same directory as this file.
inputpath
Path to an input file. Defaults to input.py.
kwargs
Any keyword/value pair to take precedence over anything in the input
file.
Creates a high-throughput job-folder to compute the non-magnetic
ground-state of a host-material. The new job-folder is loaded into
memory automatically. No need to call explore. It is save to the path
provided on input.
"""
import re
from IPython.core.interactiveshell import InteractiveShell
from copy import deepcopy
from pylada.vasp import read_input
from pylada.jobfolder import JobFolder
from pylada import interactive
from pylada.misc import setBugLev
setBugLev(0) # set global debug level
from pylada.misc import bugLev
# reads input.
input = read_input(inputpath)
input.update(kwargs)
# Job dictionary.
jobfolder = JobFolder()
# loop over material-lattice pairs.
for (material, lattice) in input.matLatPairs:
structure = deepcopy(lattice)
# job folder for this lattice.
lat_jobfolder = jobfolder / material
job = lat_jobfolder / lattice.name / "basin_hoping"
job.functional = input.calc
job.params["structure"] = structure
job.params["ispin"] = 1
job.params["vasp"] = input.vasp
job.params["programs"] = input.programs
job.params["func_param"] = input.func_param
# saves some stuff for future reference.
job.material = material
job.lattice = lattice
#print ' test/hi/test: job: ', job
#print ' test/hi/test: === job.functional ===\n%s\n=== end functional === ' % (job.functional,)
interactive.jobfolder = jobfolder
InteractiveShell.instance().magic("savefolders " + path)
def jobfolder(functional):
from pylada.jobfolder import JobFolder
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15),
('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / str(trial)
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that':
jobfolder.params['value'] = True
return root
def root():
from pylada.jobfolder import JobFolder
from pylada.jobfolder.tests.dummy import functional
sizes = [10, 15, 20, 25]
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15),
('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / trial
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that':
jobfolder.params['value'] = True
return root
def extra_folder(functional):
from pylada.jobfolder import JobFolder
jobfolder = JobFolder()
jobfolder.functional = functional
jobfolder.params['indiv'] = 25
jobfolder.params['value'] = 5
jobfolder.params['another'] = 6
return jobfolder
def create_jobs():
""" Simple job-folders. """
from pylada.jobfolder import JobFolder
from pylada.crystal.binary import zinc_blende
root = JobFolder()
for name, value, species in zip(['diamond', 'diamond/alloy', 'GaAs'],
[0, 1, 2], [('Si', 'Si'), ('Si', 'Ge'),
('Ga', 'As')]):
job = root / name
job.functional = functional
job.params['value'] = value
job.params['structure'] = zinc_blende()
for atom, specie in zip(job.structure, species):
atom.type = specie
return root
def jobfolder(functional):
from pylada.jobfolder import JobFolder
root = JobFolder()
for type, trial, size in [
("this", 0, 10),
("this", 1, 15),
("that", 2, 20),
("that", 1, 20),
]:
job = root / type / str(trial)
job.functional = functional
job.params["indiv"] = size
if type == "that":
job.params["value"] = True
job = root / "this" / "0" / "another"
job.functional = functional
job.params["indiv"] = 25
job.params["value"] = 5
job.params["another"] = 6
return root
def test_jobparams():
from random import randint
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from copy import deepcopy
from pylada.jobfolder import JobFolder, JobParams
from dummy import functional
# create jodictionary to explore
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15), ('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / str(trial)
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that': jobfolder.params['value'] = True
# checks that attributes are forwarded correctly
jobparams = JobParams(root)
assert len(jobparams.functional.values()) == 4
for i, (name, value) in enumerate(jobparams.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 3
for i, (name, value) in enumerate(jobparams.indiv.iteritems()):
if name == '/this/0/': assert value == 10
elif name == '/this/1/': assert value == 15
elif name == '/that/1/': assert value == 20
elif name == '/that/2/': assert value == 20
else: raise RuntimeError()
assert i == 3
# checks that attributes only a few posses are forwarded
for i, (name, value) in enumerate(jobparams.value.iteritems()):
if name == '/that/1/': assert value == True
elif name == '/that/2/': assert value == True
else: raise RuntimeError()
assert i == 1
# check that an AttributeError is raised on an unknown attribute
try: jobparams.that
except AttributeError: pass
else: raise RuntimeError()
# check wildcard indexing.
jobparams.unix_re = True
def check_items(regex, keys, d):
i = 0
for name in d[regex].iterkeys():
i += 1
assert name in keys, KeyError((regex, name))
assert i == len(keys), RuntimeError(regex)
check_items('*/1', set(['/this/1/', '/that/1/']), jobparams)
check_items('this', set(['/this/1/', '/this/0/']), jobparams)
check_items('that/2/', set(['/that/2/']),jobparams)
# check adding an item
jobfolder = JobFolder()
jobfolder.functional = functional
jobfolder.params['indiv'] = 25
jobfolder.params['value'] = 5
jobfolder.params['another'] = 6
jobparams['/this/0/another'] = jobfolder
# continue wildcard indexing.
check_items('*/*/another', ['/this/0/another/'], jobparams)
check_items('*/*/another/', ['/this/0/another/'], jobparams)
check_items('*/another', [], jobparams)
check_items('../that', ['/that/2/', '/that/1/'], jobparams['this'])
check_items('../that', [], jobparams['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'], jobparams['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'], jobparams['this/*'])
# check regex indexing.
jobparams.unix_re = False
check_items('.*/1', set(['/this/1/', '/that/1/']), jobparams)
check_items('this', set(['/this/1/', '/this/0/', '/this/0/another/']), jobparams)
check_items('that/2/', set(['/that/2/']), jobparams)
check_items('.*/.*/another', ['/this/0/another/'], jobparams)
check_items('.*/another', ['/this/0/another/'], jobparams)
jobparams.unix_re = True
jobparams.naked_end = True
assert isinstance(jobparams.another, int)
for i, (key, value) in enumerate(jobparams['*/1'].indiv.iteritems()):
assert {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20,
'/that/2/': 20, '/this/0/another/': 25}[key] == value
assert i == 1
# check setting attributes.
jobparams.indiv = 5
for i, (key, value) in enumerate(jobparams.indiv.iteritems()): assert value == 5
assert i == 4
jobparams['*/0'].indiv = 6
assert len(jobparams.indiv.values()) == 5
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert value == (6 if '0' in key else 5)
assert i == 4
# resets attributes for later tests.
for key in jobparams.keys():
jobparams[key].indiv = {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20,\
'/that/2/': 20, '/this/0/another/': 25}[key]
jobfolder = deepcopy(jobparams.jobfolder)
# check deleting.
del jobparams['/*/1']
assert '/this/1/' not in jobparams and '/that/1/' not in jobparams\
and '/that/2/' in jobparams and '/this/0/' in jobparams\
and '/this/0/another/' in jobparams
# check concatenate with job-folder.
jobparams.indiv = 2
jobparams.concatenate(jobfolder)
for i, (name, value) in enumerate(jobparams.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 4
i = 0
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20,
'/that/2/': 20, '/this/0/another/': 25}[key] == value,\
Exception((key, value))
assert i == 4
# check concatenate with Jobparams and indexing.
del jobparams['/*/1']
jobparams.indiv = 2
jobparams.concatenate(JobParams(jobfolder)['/*/1'])
for i, (name, value) in enumerate(jobparams.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 4
i = 0
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert {'/this/0/': 2, '/this/1/': 15, '/that/1/': 20,
'/that/2/': 2, '/this/0/another/': 2}[key] == value,\
Exception((key, value))
assert i == 4
def rootJob():
"""create a root Job"""
job = JobFolder()
job.params["call"] = _blank
return job
def test(shell):
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs, getcwd, chdir
from os.path import exists, join
from pylada.jobfolder import JobFolder
from pylada import interactive
from dummy import functional
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15),
('that', 2, 20), ('that', 1, 20)]:
job = root / type / str(trial)
job.functional = functional
job.params['indiv'] = size
if type == 'that':
job.params['value'] = True
origdir = getcwd()
directory = mkdtemp()
if exists(directory) and directory == '/tmp/test':
rmtree(directory)
if not exists(directory):
makedirs(directory)
try:
shell.user_ns['jobfolder'] = root
shell.magic("explore jobfolder")
shell.magic("savefolders {0}/dict".format(directory))
for name, job in root.items():
result = job.compute(outdir=join(directory, name))
assert result.success
assert {
'this/0': 10,
'this/1': 15,
'that/1': 20,
'that/2': 20,
'this/0/another': 25
}[name] == result.indiv
shell.magic("explore {0}/dict".format(directory))
shell.magic("goto this/0")
assert getcwd() == '{0}/this/0'.format(directory)
assert interactive.jobfolder.name == '/this/0/'
shell.magic("goto ../1")
assert getcwd() == '{0}/this/1'.format(directory)
assert interactive.jobfolder.name == '/this/1/'
shell.magic("goto /that")
assert getcwd() == '{0}/that'.format(directory)
assert interactive.jobfolder.name == '/that/'
shell.magic("goto 2")
assert getcwd() == '{0}/that/2'.format(directory)
assert interactive.jobfolder.name == '/that/2/'
shell.magic("goto /")
shell.magic("goto next")
assert getcwd() == '{0}/that/1'.format(directory)
assert interactive.jobfolder.name == '/that/1/'
shell.magic("goto next")
assert getcwd() == '{0}/that/2'.format(directory)
assert interactive.jobfolder.name == '/that/2/'
shell.magic("goto previous")
assert getcwd() == '{0}/that/1'.format(directory)
assert interactive.jobfolder.name == '/that/1/'
shell.magic("goto next")
assert getcwd() == '{0}/this/0'.format(directory)
assert interactive.jobfolder.name == '/this/0/'
shell.magic("goto next")
assert getcwd() == '{0}/this/1'.format(directory)
assert interactive.jobfolder.name == '/this/1/'
shell.magic("goto next") # no further jobs
assert getcwd() == '{0}/this/1'.format(directory)
assert interactive.jobfolder.name == '/this/1/'
shell.magic("goto /") # go back to root to avoid errors
finally:
chdir(origdir)
try:
if directory != '/tmp/test':
rmtree(directory)
except:
pass
for ii in find_ngh_indices(vacancy_indices[k], structure2):
structure2[ii].pos = structure2[ii].pos + 0.1*np.array([2*random()-1, 2*random()-1, 2*random()-1])
structure2[ii].spin = 1.
structure2[vacancy_indices[k]].type = 'O'
key = 'OIn'+'_'+str(k)
structures[key]=structure2
############### setting up the jobfolder
from IPython.core.interactiveshell import InteractiveShell
from pylada.jobfolder import JobFolder
from pylada import interactive
from copy import deepcopy
# Job dictionary.
jobfolder = JobFolder()
# loop over material-lattice pairs.
for name in structures:
if name=='epsilon':
structure=structures[name]
# job folder for this lattice.
job = jobfolder / name
vasp_individual = deepcopy(vasp)
vasp_individual.add_keyword('lepsilon',True)
vasp_individual.add_keyword('lrpa',False)
vasp_individual.ibrion = 7
vasp_individual.kpoints=gen_kpts(structure,4000)
def test_subfolder_creation_nonstring():
from pylada.jobfolder import JobFolder
root = JobFolder()
jobfolder = root / 'this' / 0
assert jobfolder.name == "/this/0/"
def test(shell, tmpdir, functional):
from os.path import join
from os import getcwd
from pylada.jobfolder import JobFolder
from pylada import interactive
root = JobFolder()
for type, trial, size in [
("this", 0, 10),
("this", 1, 15),
("that", 2, 20),
("that", 1, 20),
]:
job = root / type / str(trial)
job.functional = functional
job.params["indiv"] = size
if type == "that":
job.params["value"] = True
shell.user_ns["jobfolder"] = root
shell.magic("explore jobfolder")
shell.magic("savefolders {0}/dict".format(tmpdir))
for name, job in root.items():
result = job.compute(outdir=str(tmpdir.join(name)))
assert result.success
assert {
"this/0": 10,
"this/1": 15,
"that/1": 20,
"that/2": 20,
"this/0/another": 25,
}[name] == result.indiv
shell.magic("explore {0}/dict".format(tmpdir))
shell.magic("goto this/0")
assert getcwd() == "{0}/this/0".format(tmpdir)
assert interactive.jobfolder.name == "/this/0/"
shell.magic("goto ../1")
assert getcwd() == "{0}/this/1".format(tmpdir)
assert interactive.jobfolder.name == "/this/1/"
shell.magic("goto /that")
assert getcwd() == "{0}/that".format(tmpdir)
assert interactive.jobfolder.name == "/that/"
shell.magic("goto 2")
assert getcwd() == "{0}/that/2".format(tmpdir)
assert interactive.jobfolder.name == "/that/2/"
shell.magic("goto /")
shell.magic("goto next")
assert getcwd() == "{0}/that/1".format(tmpdir)
assert interactive.jobfolder.name == "/that/1/"
shell.magic("goto next")
assert getcwd() == "{0}/that/2".format(tmpdir)
assert interactive.jobfolder.name == "/that/2/"
shell.magic("goto previous")
assert getcwd() == "{0}/that/1".format(tmpdir)
assert interactive.jobfolder.name == "/that/1/"
shell.magic("goto next")
assert getcwd() == "{0}/this/0".format(tmpdir)
assert interactive.jobfolder.name == "/this/0/"
shell.magic("goto next")
assert getcwd() == "{0}/this/1".format(tmpdir)
assert interactive.jobfolder.name == "/this/1/"
shell.magic("goto next") # no further jobs
assert getcwd() == "{0}/this/1".format(tmpdir)
assert interactive.jobfolder.name == "/this/1/"
shell.magic("goto /") # go back to root to avoid errors
def test(shell, tmpdir, functional):
from os.path import join
from pylada.jobfolder import JobFolder
import pylada
with patch("pylada.misc.cmdl_input", return_value="y"):
root = JobFolder()
for type, trial, size in [
("this", 0, 10),
("this", 1, 15),
("that", 2, 20),
("that", 1, 20),
]:
jobfolder = root / type / str(trial)
jobfolder.functional = functional
jobfolder.params["indiv"] = size
if type == "that":
jobfolder.params["value"] = True
shell.user_ns["jobfolder"] = root
shell.magic("explore jobfolder")
jobfolder = pylada.interactive.jobfolder
assert (
"this/0" in jobfolder
and "this/1" in jobfolder
and "that/2" in jobfolder
and "that/1"
)
assert "0" in jobfolder["this"] and "1" in jobfolder["this"]
assert "1" in jobfolder["that"] and "2" in jobfolder["that"]
assert "other" not in jobfolder
for job in jobfolder.values():
assert repr(job.functional) == repr(functional)
assert getattr(jobfolder["this/0"], "indiv", 0) == 10
assert getattr(jobfolder["this/1"], "indiv", 0) == 15
assert getattr(jobfolder["that/1"], "indiv", 0) == 20
assert getattr(jobfolder["that/2"], "indiv", 0) == 20
assert not hasattr(jobfolder["this/0"], "value")
assert not hasattr(jobfolder["this/1"], "value")
assert getattr(jobfolder["that/1"], "value", False)
assert getattr(jobfolder["that/2"], "value", False)
assert pylada.interactive.jobfolder_path is None
assert "jobparams" in shell.user_ns
assert jobfolder is shell.user_ns["jobparams"].jobfolder
shell.magic("savefolders {0}/dict".format(tmpdir))
pylada.interactive.jobfolder = None
pylada.interactive.jobfolder_path = None
shell.magic("explore {0}/dict".format(tmpdir))
jobfolder = pylada.interactive.jobfolder
assert (
"this/0" in jobfolder
and "this/1" in jobfolder
and "that/2" in jobfolder
and "that/1"
)
assert "0" in jobfolder["this"] and "1" in jobfolder["this"]
assert "1" in jobfolder["that"] and "2" in jobfolder["that"]
assert "other" not in jobfolder
for job in jobfolder.values():
assert repr(job.functional) == repr(functional)
assert getattr(jobfolder["this/0"], "indiv", 0) == 10
assert getattr(jobfolder["this/1"], "indiv", 0) == 15
assert getattr(jobfolder["that/1"], "indiv", 0) == 20
assert getattr(jobfolder["that/2"], "indiv", 0) == 20
assert not hasattr(jobfolder["this/0"], "value")
assert not hasattr(jobfolder["this/1"], "value")
assert getattr(jobfolder["that/1"], "value", False)
assert getattr(jobfolder["that/2"], "value", False)
assert pylada.interactive.jobfolder_path is not None
assert "jobparams" in shell.user_ns
assert jobfolder is shell.user_ns["jobparams"].jobfolder
assert jobfolder is shell.user_ns["collect"].jobfolder
for name, job in root.items():
if name == "this/1":
continue
job.compute(outdir=str(tmpdir.join(name)))
shell.magic("explore results".format(tmpdir))
assert {"/this/0/", "/that/1/", "/that/2/"} == set(
shell.user_ns["collect"].keys()
)
shell.magic("explore errors".format(tmpdir))
assert len(shell.user_ns["collect"]) == 0
shell.magic("explore all".format(tmpdir))
shell.magic("explore errors".format(tmpdir))
assert set(shell.user_ns["collect"].keys()) == {"/this/1/"}
def test(shell):
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from pylada.jobfolder import JobFolder
import pylada
from dummy import functional
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15),
('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / str(trial)
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that':
jobfolder.params['value'] = True
directory = mkdtemp()
if exists(directory) and directory == '/tmp/test':
rmtree(directory)
if not exists(directory):
makedirs(directory)
try:
shell.user_ns['jobfolder'] = root
shell.magic("explore jobfolder")
jobfolder = pylada.interactive.jobfolder
assert 'this/0' in jobfolder and 'this/1' in jobfolder and 'that/2' in jobfolder and 'that/1'
assert '0' in jobfolder['this'] and '1' in jobfolder['this']
assert '1' in jobfolder['that'] and '2' in jobfolder['that']
assert 'other' not in jobfolder
for job in jobfolder.values():
assert repr(job.functional) == repr(functional)
assert getattr(jobfolder['this/0'], 'indiv', 0) == 10
assert getattr(jobfolder['this/1'], 'indiv', 0) == 15
assert getattr(jobfolder['that/1'], 'indiv', 0) == 20
assert getattr(jobfolder['that/2'], 'indiv', 0) == 20
assert not hasattr(jobfolder['this/0'], 'value')
assert not hasattr(jobfolder['this/1'], 'value')
assert getattr(jobfolder['that/1'], 'value', False)
assert getattr(jobfolder['that/2'], 'value', False)
assert pylada.interactive.jobfolder_path is None
assert 'jobparams' in shell.user_ns
assert jobfolder is shell.user_ns['jobparams'].jobfolder
shell.magic("savefolders {0}/dict".format(directory))
pylada.interactive.jobfolder = None
pylada.interactive.jobfolder_path = None
shell.magic("explore {0}/dict".format(directory))
jobfolder = pylada.interactive.jobfolder
assert 'this/0' in jobfolder and 'this/1' in jobfolder and 'that/2' in jobfolder and 'that/1'
assert '0' in jobfolder['this'] and '1' in jobfolder['this']
assert '1' in jobfolder['that'] and '2' in jobfolder['that']
assert 'other' not in jobfolder
for job in jobfolder.values():
assert repr(job.functional) == repr(functional)
assert getattr(jobfolder['this/0'], 'indiv', 0) == 10
assert getattr(jobfolder['this/1'], 'indiv', 0) == 15
assert getattr(jobfolder['that/1'], 'indiv', 0) == 20
assert getattr(jobfolder['that/2'], 'indiv', 0) == 20
assert not hasattr(jobfolder['this/0'], 'value')
assert not hasattr(jobfolder['this/1'], 'value')
assert getattr(jobfolder['that/1'], 'value', False)
assert getattr(jobfolder['that/2'], 'value', False)
assert pylada.interactive.jobfolder_path is not None
assert 'jobparams' in shell.user_ns
assert jobfolder is shell.user_ns['jobparams'].jobfolder
assert jobfolder is shell.user_ns['collect'].jobfolder
for name, job in root.items():
if name == 'this/1':
continue
job.compute(outdir=join(directory, name))
shell.magic("explore results".format(directory))
assert {'/this/0/', '/that/1/', '/that/2/'} \
== set(shell.user_ns['collect'].keys())
shell.magic("explore errors".format(directory))
assert len(shell.user_ns['collect']) == 0
shell.magic("explore all".format(directory))
shell.magic("explore errors".format(directory))
assert set(shell.user_ns['collect'].keys()) == {'/this/1/'}
finally:
if directory != '/tmp/test':
rmtree(directory)
pass
def test():
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from pickle import dump
from pylada.jobfolder import JobFolder, MassExtract
from dummy import functional
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15), ('that', 2, 20), ('that', 1, 20)]:
job = root / type / str(trial)
job.functional = functional
job.params['indiv'] = size
if type == 'that': job.params['value'] = True
job = root / 'this' / '0' / 'another'
job.functional = functional
job.params['indiv'] = 25
job.params['value'] = 5
job.params['another'] = 6
directory = mkdtemp() # '/tmp/test' #
if exists(directory) and directory == '/tmp/test': rmtree(directory)
if not exists(directory): makedirs(directory)
try:
for name, job in root.iteritems():
result = job.compute(outdir=join(directory, name))
assert result.success
assert {'this/0': 10, 'this/1': 15, 'that/1': 20, \
'that/2': 20, 'this/0/another': 25 }[name] == result.indiv
with open(join(directory, 'dict'), 'w') as file: dump(root, file)
collect = MassExtract(path=join(directory, 'dict'))
for i, (name, value) in enumerate(collect.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 4
for i, (name, value) in enumerate(collect.indiv.iteritems()):
assert {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20, \
'/that/2/': 20, '/this/0/another/': 25 }[name] == value,\
Exception((name, value))
assert i == 4
try: collect.that
except: pass
else: raise RuntimeError()
collect.unix_re = True
def check_items(regex, keys, d):
i = 0
for name in d[regex].iterkeys():
i += 1
assert name in keys, KeyError((regex, name))
assert i == len(keys), RuntimeError(regex)
check_items('*/1', set(['/this/1/', '/that/1/']), collect)
check_items('this', set(['/this/1/', '/this/0/', '/this/0/another/']), collect)
check_items('that/2/', set(['/that/2/']),collect)
job = root / 'this' / '0' / 'another'
check_items('*/*/another', ['/this/0/another/'], collect)
check_items('*/*/another/', ['/this/0/another/'], collect)
check_items('*/another', [], collect)
check_items('../that', ['/that/2/', '/that/1/'], collect['this'])
check_items('../that', [], collect['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'], collect['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'], collect['this/*'])
collect.unix_re = False
check_items('.*/1', set(['/this/1/', '/that/1/']), collect)
check_items('this', set(['/this/1/', '/this/0/', '/this/0/another/']), collect)
check_items('that/2/', set(['/that/2/']), collect)
check_items('.*/.*/another', ['/this/0/another/'], collect)
check_items('.*/another', ['/this/0/another/'], collect)
collect.unix_re = True
collect.naked_end = True
for i, (key, value) in enumerate(collect['*/1'].indiv.iteritems()):
assert {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20,
'/that/2/': 20, '/this/0/another/': 25}[key] == value
assert i == 1
finally:
if directory != '/tmp/test': rmtree(directory)
def test(shell):
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs, getcwd, chdir
from os.path import exists, join
from pylada.jobfolder import JobFolder
from pylada import interactive
from dummy import functional
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15), ('that', 2, 20), ('that', 1, 20)]:
job = root / type / str(trial)
job.functional = functional
job.params['indiv'] = size
if type == 'that':
job.params['value'] = True
origdir = getcwd()
directory = mkdtemp()
if exists(directory) and directory == '/tmp/test':
rmtree(directory)
if not exists(directory):
makedirs(directory)
try:
shell.user_ns['jobfolder'] = root
shell.magic("explore jobfolder")
shell.magic("savefolders {0}/dict".format(directory))
for name, job in root.items():
result = job.compute(outdir=join(directory, name))
assert result.success
assert {'this/0': 10, 'this/1': 15, 'that/1': 20,
'that/2': 20, 'this/0/another': 25}[name] == result.indiv
shell.magic("explore {0}/dict".format(directory))
shell.magic("goto this/0")
assert getcwd() == '{0}/this/0'.format(directory)
assert interactive.jobfolder.name == '/this/0/'
shell.magic("goto ../1")
assert getcwd() == '{0}/this/1'.format(directory)
assert interactive.jobfolder.name == '/this/1/'
shell.magic("goto /that")
assert getcwd() == '{0}/that'.format(directory)
assert interactive.jobfolder.name == '/that/'
shell.magic("goto 2")
assert getcwd() == '{0}/that/2'.format(directory)
assert interactive.jobfolder.name == '/that/2/'
shell.magic("goto /")
shell.magic("goto next")
assert getcwd() == '{0}/that/1'.format(directory)
assert interactive.jobfolder.name == '/that/1/'
shell.magic("goto next")
assert getcwd() == '{0}/that/2'.format(directory)
assert interactive.jobfolder.name == '/that/2/'
shell.magic("goto previous")
assert getcwd() == '{0}/that/1'.format(directory)
assert interactive.jobfolder.name == '/that/1/'
shell.magic("goto next")
assert getcwd() == '{0}/this/0'.format(directory)
assert interactive.jobfolder.name == '/this/0/'
shell.magic("goto next")
assert getcwd() == '{0}/this/1'.format(directory)
assert interactive.jobfolder.name == '/this/1/'
shell.magic("goto next") # no further jobs
assert getcwd() == '{0}/this/1'.format(directory)
assert interactive.jobfolder.name == '/this/1/'
shell.magic("goto /") # go back to root to avoid errors
finally:
chdir(origdir)
try:
if directory != '/tmp/test':
rmtree(directory)
except:
pass
def test():
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from pickle import loads, dumps
from pylada.jobfolder import JobFolder
from copy import deepcopy
from dummy import functional
sizes = [10, 15, 20, 25]
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15), ('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / str(trial)
assert jobfolder.name == "/" + type + "/" + str(trial) + "/"
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that': jobfolder.params['value'] = True
assert 'this/0' in root and 'this/1' in root and 'that/2' in root and 'that/1'
assert '0' in root['this'] and '1' in root['this']
assert '1' in root['that'] and '2' in root['that']
assert 'other' not in root
for jobfolder in root.values():
assert repr(jobfolder.functional) == repr(functional)
assert getattr(root['this/0'], 'indiv', 0) == 10
assert getattr(root['this/1'], 'indiv', 0) == 15
assert getattr(root['that/1'], 'indiv', 0) == 20
assert getattr(root['that/2'], 'indiv', 0) == 20
assert not hasattr(root['this/0'], 'value')
assert not hasattr(root['this/1'], 'value')
assert getattr(root['that/1'], 'value', False)
assert getattr(root['that/2'], 'value', False)
for key, test in zip(root, ['that/1', 'that/2', 'this/0', 'this/1']):
assert key == test
for key, test in zip(root['that/1'].root, ['that/1', 'that/2', 'this/0', 'this/1']):
assert key == test
for key, test in zip(root['that'], ['1', '2']): assert key == test
for key, test in zip(root['this'], ['0', '1']): assert key == test
del root['that/2']
assert 'that/2' not in root
directory = mkdtemp() # '/tmp/test'
if exists(directory) and directory == '/tmp/test': rmtree(directory)
if not exists(directory): makedirs(directory)
try:
for name, jobfolder in root.iteritems():
result = jobfolder.compute(outdir=join(directory, name))
assert result.success
assert exists(join(directory, name))
assert exists(join(join(directory, name), 'OUTCAR'))
if name == 'that/1': assert result.indiv == 20
elif name == 'that/2': assert result.indiv == 15
elif name == 'this/1': assert result.indiv == 15
elif name == 'this/0': assert result.indiv == 10
finally:
if directory != '/tmp/test': rmtree(directory)
if exists(directory) and directory == '/tmp/test': rmtree(directory)
if not exists(directory): makedirs(directory)
try:
for name, jobfolder in loads(dumps(root)).iteritems():
result = jobfolder.compute(outdir=join(directory, name))
assert result.success
assert exists(join(directory, name))
assert exists(join(join(directory, name), 'OUTCAR'))
if name == 'that/1': assert result.indiv == 20
elif name == 'that/2': assert result.indiv == 15
elif name == 'this/1': assert result.indiv == 15
elif name == 'this/0': assert result.indiv == 10
finally:
if directory != '/tmp/test': rmtree(directory)
try:
for name, jobfolder in deepcopy(root).iteritems():
result = jobfolder.compute(outdir=join(directory, name))
assert result.success
assert exists(join(directory, name))
assert exists(join(join(directory, name), 'OUTCAR'))
if name == 'that/1': assert result.indiv == 20
elif name == 'that/2': assert result.indiv == 15
elif name == 'this/1': assert result.indiv == 15
elif name == 'this/0': assert result.indiv == 10
finally:
if directory != '/tmp/test': rmtree(directory)
# makes sure that parent are correctly deepcopied.
jobfolder = deepcopy(root)
for value in jobfolder.itervalues(): value.name
def test():
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from pickle import dump
from pylada.jobfolder import JobFolder, MassExtract
from dummy import functional
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15),
('that', 2, 20), ('that', 1, 20)]:
job = root / type / str(trial)
job.functional = functional
job.params['indiv'] = size
if type == 'that': job.params['value'] = True
job = root / 'this' / '0' / 'another'
job.functional = functional
job.params['indiv'] = 25
job.params['value'] = 5
job.params['another'] = 6
directory = mkdtemp() # '/tmp/test' #
if exists(directory) and directory == '/tmp/test': rmtree(directory)
if not exists(directory): makedirs(directory)
try:
for name, job in root.iteritems():
result = job.compute(outdir=join(directory, name))
assert result.success
assert {'this/0': 10, 'this/1': 15, 'that/1': 20, \
'that/2': 20, 'this/0/another': 25 }[name] == result.indiv
with open(join(directory, 'dict'), 'w') as file:
dump(root, file)
collect = MassExtract(path=join(directory, 'dict'))
for i, (name, value) in enumerate(collect.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 4
for i, (name, value) in enumerate(collect.indiv.iteritems()):
assert {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20, \
'/that/2/': 20, '/this/0/another/': 25 }[name] == value,\
Exception((name, value))
assert i == 4
try:
collect.that
except:
pass
else:
raise RuntimeError()
collect.unix_re = True
def check_items(regex, keys, d):
i = 0
for name in d[regex].iterkeys():
i += 1
assert name in keys, KeyError((regex, name))
assert i == len(keys), RuntimeError(regex)
check_items('*/1', set(['/this/1/', '/that/1/']), collect)
check_items('this', set(['/this/1/', '/this/0/', '/this/0/another/']),
collect)
check_items('that/2/', set(['/that/2/']), collect)
job = root / 'this' / '0' / 'another'
check_items('*/*/another', ['/this/0/another/'], collect)
check_items('*/*/another/', ['/this/0/another/'], collect)
check_items('*/another', [], collect)
check_items('../that', ['/that/2/', '/that/1/'], collect['this'])
check_items('../that', [], collect['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'],
collect['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'],
collect['this/*'])
collect.unix_re = False
check_items('.*/1', set(['/this/1/', '/that/1/']), collect)
check_items('this', set(['/this/1/', '/this/0/', '/this/0/another/']),
collect)
check_items('that/2/', set(['/that/2/']), collect)
check_items('.*/.*/another', ['/this/0/another/'], collect)
check_items('.*/another', ['/this/0/another/'], collect)
collect.unix_re = True
collect.naked_end = True
for i, (key, value) in enumerate(collect['*/1'].indiv.iteritems()):
assert {
'/this/0/': 10,
'/this/1/': 15,
'/that/1/': 20,
'/that/2/': 20,
'/this/0/another/': 25
}[key] == value
assert i == 1
finally:
if directory != '/tmp/test': rmtree(directory)
def test():
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from IPython.core.interactiveshell import InteractiveShell
from pylada.jobfolder import JobFolder
import pylada
from dummy import functional
import __builtin__
__builtin__.raw_input = raw_input
self = InteractiveShell.instance()
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15), ('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / str(trial)
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that': jobfolder.params['value'] = True
directory = '/tmp/test' # mkdtemp() #
if exists(directory) and directory == '/tmp/test': rmtree(directory)
if not exists(directory): makedirs(directory)
try:
self.user_ns['jobfolder'] = root
self.magic("explore jobfolder")
jobfolder = pylada.interactive.jobfolder
assert 'this/0' in jobfolder and 'this/1' in jobfolder and 'that/2' in jobfolder and 'that/1'
assert '0' in jobfolder['this'] and '1' in jobfolder['this']
assert '1' in jobfolder['that'] and '2' in jobfolder['that']
assert 'other' not in jobfolder
for job in jobfolder.values():
assert repr(job.functional) == repr(functional)
assert getattr(jobfolder['this/0'], 'indiv', 0) == 10
assert getattr(jobfolder['this/1'], 'indiv', 0) == 15
assert getattr(jobfolder['that/1'], 'indiv', 0) == 20
assert getattr(jobfolder['that/2'], 'indiv', 0) == 20
assert not hasattr(jobfolder['this/0'], 'value')
assert not hasattr(jobfolder['this/1'], 'value')
assert getattr(jobfolder['that/1'], 'value', False)
assert getattr(jobfolder['that/2'], 'value', False)
assert pylada.interactive.jobfolder_path is None
assert 'jobparams' in self.user_ns
assert jobfolder is self.user_ns['jobparams'].jobfolder
self.magic("savefolders {0}/dict".format(directory))
pylada.interactive.jobfolder = None
pylada.interactive.jobfolder_path = None
self.magic("explore {0}/dict".format(directory))
jobfolder = pylada.interactive.jobfolder
assert 'this/0' in jobfolder and 'this/1' in jobfolder and 'that/2' in jobfolder and 'that/1'
assert '0' in jobfolder['this'] and '1' in jobfolder['this']
assert '1' in jobfolder['that'] and '2' in jobfolder['that']
assert 'other' not in jobfolder
for job in jobfolder.values():
assert repr(job.functional) == repr(functional)
assert getattr(jobfolder['this/0'], 'indiv', 0) == 10
assert getattr(jobfolder['this/1'], 'indiv', 0) == 15
assert getattr(jobfolder['that/1'], 'indiv', 0) == 20
assert getattr(jobfolder['that/2'], 'indiv', 0) == 20
assert not hasattr(jobfolder['this/0'], 'value')
assert not hasattr(jobfolder['this/1'], 'value')
assert getattr(jobfolder['that/1'], 'value', False)
assert getattr(jobfolder['that/2'], 'value', False)
assert pylada.interactive.jobfolder_path is not None
assert 'jobparams' in self.user_ns
assert jobfolder is self.user_ns['jobparams'].jobfolder
assert jobfolder is self.user_ns['collect'].jobfolder
for name, job in root.iteritems():
if name == 'this/1': continue
job.compute(outdir=join(directory, name))
self.magic("explore results".format(directory))
assert set(['/this/0/', '/that/1/', '/that/2/']) == set(self.user_ns['collect'].iterkeys())
self.magic("explore errors".format(directory))
assert len(self.user_ns['collect']) == 0
self.magic("explore all".format(directory))
self.magic("explore errors".format(directory))
assert set(self.user_ns['collect'].keys()) == set(['/this/1/'])
finally:
if directory != '/tmp/test': rmtree(directory)
pass
def test():
from random import randint
from tempfile import mkdtemp
from shutil import rmtree
from os import makedirs
from os.path import exists, join
from copy import deepcopy
from pylada.jobfolder import JobFolder, JobParams
from dummy import functional
# create jodictionary to explore
root = JobFolder()
for type, trial, size in [('this', 0, 10), ('this', 1, 15),
('that', 2, 20), ('that', 1, 20)]:
jobfolder = root / type / str(trial)
jobfolder.functional = functional
jobfolder.params['indiv'] = size
if type == 'that': jobfolder.params['value'] = True
# checks that attributes are forwarded correctly
jobparams = JobParams(root)
assert len(jobparams.functional.values()) == 4
for i, (name, value) in enumerate(jobparams.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 3
for i, (name, value) in enumerate(jobparams.indiv.iteritems()):
if name == '/this/0/': assert value == 10
elif name == '/this/1/': assert value == 15
elif name == '/that/1/': assert value == 20
elif name == '/that/2/': assert value == 20
else: raise RuntimeError()
assert i == 3
# checks that attributes only a few posses are forwarded
for i, (name, value) in enumerate(jobparams.value.iteritems()):
if name == '/that/1/': assert value == True
elif name == '/that/2/': assert value == True
else: raise RuntimeError()
assert i == 1
# check that an AttributeError is raised on an unknown attribute
try:
jobparams.that
except AttributeError:
pass
else:
raise RuntimeError()
# check wildcard indexing.
jobparams.unix_re = True
def check_items(regex, keys, d):
i = 0
for name in d[regex].iterkeys():
i += 1
assert name in keys, KeyError((regex, name))
assert i == len(keys), RuntimeError(regex)
check_items('*/1', set(['/this/1/', '/that/1/']), jobparams)
check_items('this', set(['/this/1/', '/this/0/']), jobparams)
check_items('that/2/', set(['/that/2/']), jobparams)
# check adding an item
jobfolder = JobFolder()
jobfolder.functional = functional
jobfolder.params['indiv'] = 25
jobfolder.params['value'] = 5
jobfolder.params['another'] = 6
jobparams['/this/0/another'] = jobfolder
# continue wildcard indexing.
check_items('*/*/another', ['/this/0/another/'], jobparams)
check_items('*/*/another/', ['/this/0/another/'], jobparams)
check_items('*/another', [], jobparams)
check_items('../that', ['/that/2/', '/that/1/'], jobparams['this'])
check_items('../that', [], jobparams['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'],
jobparams['this/0'])
check_items('../*', ['/this/0/', '/this/1/', '/this/0/another/'],
jobparams['this/*'])
# check regex indexing.
jobparams.unix_re = False
check_items('.*/1', set(['/this/1/', '/that/1/']), jobparams)
check_items('this', set(['/this/1/', '/this/0/', '/this/0/another/']),
jobparams)
check_items('that/2/', set(['/that/2/']), jobparams)
check_items('.*/.*/another', ['/this/0/another/'], jobparams)
check_items('.*/another', ['/this/0/another/'], jobparams)
jobparams.unix_re = True
jobparams.naked_end = True
assert isinstance(jobparams.another, int)
for i, (key, value) in enumerate(jobparams['*/1'].indiv.iteritems()):
assert {
'/this/0/': 10,
'/this/1/': 15,
'/that/1/': 20,
'/that/2/': 20,
'/this/0/another/': 25
}[key] == value
assert i == 1
# check setting attributes.
jobparams.indiv = 5
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert value == 5
assert i == 4
jobparams['*/0'].indiv = 6
assert len(jobparams.indiv.values()) == 5
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert value == (6 if '0' in key else 5)
assert i == 4
# resets attributes for later tests.
for key in jobparams.keys():
jobparams[key].indiv = {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20,\
'/that/2/': 20, '/this/0/another/': 25}[key]
jobfolder = deepcopy(jobparams.jobfolder)
# check deleting.
del jobparams['/*/1']
assert '/this/1/' not in jobparams and '/that/1/' not in jobparams\
and '/that/2/' in jobparams and '/this/0/' in jobparams\
and '/this/0/another/' in jobparams
# check concatenate with job-folder.
jobparams.indiv = 2
jobparams.concatenate(jobfolder)
for i, (name, value) in enumerate(jobparams.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 4
i = 0
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert {'/this/0/': 10, '/this/1/': 15, '/that/1/': 20,
'/that/2/': 20, '/this/0/another/': 25}[key] == value,\
Exception((key, value))
assert i == 4
# check concatenate with Jobparams and indexing.
del jobparams['/*/1']
jobparams.indiv = 2
jobparams.concatenate(JobParams(jobfolder)['/*/1'])
for i, (name, value) in enumerate(jobparams.functional.iteritems()):
assert repr(value) == repr(functional)
assert i == 4
i = 0
for i, (key, value) in enumerate(jobparams.indiv.iteritems()):
assert {'/this/0/': 2, '/this/1/': 15, '/that/1/': 20,
'/that/2/': 2, '/this/0/another/': 2}[key] == value,\
Exception((key, value))
assert i == 4
