def test_find_exts(self):
self.assertTrue(len(find_exts(os.path.dirname(__file__), "py")) >= 18)
self.assertEqual(len(find_exts(os.path.dirname(__file__), "bz2")), 1)
self.assertEqual(len(find_exts(os.path.dirname(__file__), "bz2",
exclude_dirs="test_files")), 0)
self.assertEqual(len(find_exts(os.path.dirname(__file__), "bz2",
include_dirs="test_files")), 1)
def test_find_exts(self):
self.assertTrue(len(find_exts(os.path.dirname(__file__), "py")) >= 18)
self.assertEqual(len(find_exts(os.path.dirname(__file__), "bz2")), 2)
self.assertEqual(
len(find_exts(os.path.dirname(__file__), "bz2", exclude_dirs="test_files")),
0,
)
self.assertEqual(
len(find_exts(os.path.dirname(__file__), "bz2", include_dirs="test_files")),
2,
)
def main():
parser = argparse.ArgumentParser(add_help=False)
path_parser = argparse.ArgumentParser(add_help=False)
path_parser.add_argument('top', default=".", help="Top-level directory")
# Create the parsers for the sub-commands
subparsers = parser.add_subparsers(dest='command', help='sub-command help', description="Valid subcommands")
p_generate = subparsers.add_parser('generate', parents=[path_parser], help="Generate ipython notebooks")
p_runipy = subparsers.add_parser('runipy', parents=[path_parser], help="Execute ipython ipython with runipy")
options = parser.parse_args()
from monty.os.path import find_exts
if options.command == "generate":
# Generate ipython notebooks.
exts=("psp8",)
#for path in find_paths(options.top, exts):
for path in find_exts(options.top, exts, exclude_dirs="_*|.*"):
write_notebook(path)
elif options.command == "runipy":
# Use runipy to execute and update the notebook.
# Warning: this does not work in the sense that plots are produced!
from subprocess import check_output, CalledProcessError
#for path in find_paths(options.top, exts="ipynb"):
for path in find_exts(options.top, exts="ipynb", exclude_dirs="_*|.*"):
try:
check_output(["runipy", "-o", path])
#check_output(["ipython", "notebook", path])
#print("here")
except CalledProcessError as exc:
print("returncode:", exc.returncode)
print("output:\n%s", exc.output)
#from runipy.notebook_runner import NotebookRunner
#from IPython.nbformat.current import read
#os.chdir(os.path.dirname(path))
#with open(path, "r") as fh:
# notebook = read(fh, 'json')
# r = NotebookRunner(notebook)
# r.run_notebook()
# from IPython.nbformat.current import write
# write(r.nb, open("MyOtherNotebook.ipynb", 'w'), 'json')
else:
raise ValueError("Don't know how to handle command %s" % options.command)
return 0
def get_pseudos(options):
"""
Find pseudos in paths, return :class:`PseudoTable` object sorted by atomic number Z.
Accepts filepaths or directory.
"""
exts=("psp8",)
paths = options.pseudos
if len(paths) == 1 and os.path.isdir(paths[0]):
top = paths[0]
paths = find_exts(top, exts, exclude_dirs="_*")
#table = PseudoTable.from_dir(paths[0])
pseudos = []
for p in paths:
try:
pseudos.append(Pseudo.from_file(p))
except Exception as exc:
warn("Error in %s:\n%s" % (p, exc))
table = PseudoTable(pseudos)
# Here we select a subset of pseudos according to family or rows
if options.rows:
table = table.select_rows(options.rows)
elif options.family:
table = table.select_families(options.family)
if options.symbols:
table = table.select_symbols(options.symbols)
return table.sort_by_z()
def make_html(ctx):
# Get all ipynb files
top = os.path.join(ABIPY_ROOTDIR, "abitutorials")
from monty.os.path import find_exts
nbpaths = find_exts(top,
".ipynb",
exclude_dirs=".ipynb_checkpoints|old_notebooks")
# Basenames must be unique.
assert len(nbpaths) == len(set([os.path.basename(p) for p in nbpaths]))
#nb_paths = [path for path in nbpaths
# if not any([path.startswith(e) for e in EXCLUDE_NBS])]
#self.nb_paths = sorted(nbpaths)
#print("top", top, "\nnbpaths", nbpaths)
retcode = 0
for path in nbpaths:
print("Building notebook:", path)
try:
nb, errors = notebook_run(path, with_html=True)
if errors:
retcode += 1
for e in errors:
print(e)
except subprocess.CalledProcessError as exc:
retcode += 1
print(exc)
return retcode
def setUp(self):
# Get all ipynb files
top = os.path.dirname(os.path.abspath(__file__))
top = os.path.abspath(os.path.join(top, ".."))
nbpaths = find_exts(top,
".ipynb",
exclude_dirs=".ipynb_checkpoints|old_notebooks")
# Basenames must be unique.
assert len(nbpaths) == len(set([os.path.basename(p) for p in nbpaths]))
#nb_paths = [path for path in nbpaths
# if not any([path.startswith(e) for e in EXCLUDE_NBS])]
self.nb_paths = sorted(nbpaths)
def pyscript(basename):
"""Return the absolute name of one of the scripts in the `examples` directory from its basename."""
global _SCRIPTS
if _SCRIPTS is None:
# Build mapping basename --> path.
from monty.os.path import find_exts
pypaths = find_exts(_SCRIPTS_DIRPATH, ".py", exclude_dirs="_*|.*")
_SCRIPTS = {}
for p in pypaths:
k = os.path.basename(p)
assert k not in _SCRIPTS
_SCRIPTS[k] = p
return _SCRIPTS[basename]
def pyscript(basename):
"""Return the absolute name of one of the scripts in the `examples` directory from its basename."""
global _SCRIPTS
if _SCRIPTS is None:
# Build mapping basename --> path.
from monty.os.path import find_exts
pypaths = find_exts(_SCRIPTS_DIRPATH, ".py", exclude_dirs="_*|.*")
_SCRIPTS = {}
for p in pypaths:
k = os.path.basename(p)
assert k not in _SCRIPTS
_SCRIPTS[k] = p
return _SCRIPTS[basename]
def get_pseudos(top):
"""
Find pseudos within top, return :class:`PseudoTable` object sorted by atomic number Z.
"""
from monty.os.path import find_exts
from pymatgen.io.abinit.pseudos import PseudoTable, Pseudo
exts = ("psp8",)
pseudos = []
for p in find_exts(top, exts, exclude_dirs="_*"):
try:
pseudos.append(Pseudo.from_file(p))
except Exception as exc:
from warnings import warn
warn("Exception in pseudo %s:\n%s" % (p.filepath, exc))
return PseudoTable(pseudos).sort_by_z()
def get_pseudos(top):
"""
Find pseudos within top, return :class:`PseudoTable` object sorted by atomic number Z.
"""
from monty.os.path import find_exts
from pymatgen.io.abinit.pseudos import PseudoTable, Pseudo
exts = ("psp8",)
pseudos = []
for p in find_exts(top, exts, exclude_dirs="_*"):
try:
pseudos.append(Pseudo.from_file(p))
except Exception as exc:
from warnings import warn
warn("Exception in pseudo %s:\n%s" % (p.filepath, exc))
return PseudoTable(pseudos).sort_by_z()
def get_pseudos(options):
"""
Find pseudos in paths, return :class:`DojoTable` object sorted by atomic number Z.
Accepts filepaths or directory.
"""
exts = ("psp8", "xml")
paths = options.pseudos
if len(paths) == 1:
# Handle directory argument
if os.path.isdir(paths[0]):
top = os.path.abspath(paths[0])
paths = find_exts(top, exts, exclude_dirs="_*")
# Handle glob syntax e.g. "./*.psp8"
elif "*" in paths[0]:
paths = glob.glob(paths[0])
if options.verbose > 1: print("Will read pseudo from: %s" % paths)
pseudos = []
for p in paths:
try:
pseudo = dojopseudo_from_file(p)
if pseudo is None:
cprint("[%s] Pseudo.from_file returned None. Something wrong in file!" % p, "red")
continue
pseudos.append(pseudo)
except Exception as exc:
cprint("[%s] Python exception. This pseudo will be ignored" % p, "red")
if options.verbose: print(exc)
table = DojoTable(pseudos)
# Here we select a subset of pseudos according to family or rows
if options.rows:
table = table.select_rows(options.rows)
elif options.family:
table = table.select_families(options.family)
# here we select chemical symbols.
if options.symbols:
table = table.select_symbols(options.symbols)
return table.sort_by_z()
def get_pseudos(options):
"""
Find pseudos in paths, return :class:`DojoTable` object sorted by atomic number Z.
Accepts filepaths or directory.
"""
exts = ("psp8", "xml")
paths = options.pseudos
if len(paths) == 1:
# Handle directory argument
if os.path.isdir(paths[0]):
top = os.path.abspath(paths[0])
paths = find_exts(top, exts, exclude_dirs="_*")
# Handle glob syntax e.g. "./*.psp8"
elif "*" in paths[0]:
paths = glob.glob(paths[0])
if options.verbose > 1: print("Will read pseudo from: %s" % paths)
pseudos = []
for p in paths:
try:
pseudo = dojopseudo_from_file(p)
if pseudo is None:
cprint("[%s] Pseudo.from_file returned None. Something wrong in file!" % p, "red")
continue
pseudos.append(pseudo)
except Exception as exc:
cprint("[%s] Python exception. This pseudo will be ignored" % p, "red")
if options.verbose: print(exc)
table = DojoTable(pseudos)
# Here we select a subset of pseudos according to family or rows
if options.rows:
table = table.select_rows(options.rows)
elif options.family:
table = table.select_families(options.family)
# here we select chemical symbols.
if options.symbols:
table = table.select_symbols(options.symbols)
return table.sort_by_z()
def pyscript(basename):
"""
Return the absolute name of one of the scripts in the `examples` directory from its basename.
"""
global _SCRIPTS
if _SCRIPTS is None:
# Build mapping basename --> path.
from monty.os.path import find_exts
pypaths = find_exts(_SCRIPTS_DIRPATH, ".py", exclude_dirs="_*|.*|develop", match_mode="basename")
_SCRIPTS = {}
for p in pypaths:
k = os.path.basename(p)
# Ignore e.g. __init__.py and private scripts.
if k.startswith("_"): continue
if k in _SCRIPTS:
raise ValueError("Fond duplicated basenames with name %s\nPrevious %s" % (k, _SCRIPTS[k]))
_SCRIPTS[k] = p
return _SCRIPTS[basename]
def get_djrepo_paths(options):
"""
Find and retur list of djson paths from options
Accepts filepaths or directory.
"""
paths = options.paths
if len(paths) == 1:
# Handle directory argument
if os.path.isdir(paths[0]):
top = os.path.abspath(paths[0])
paths = find_exts(top, ["djrepo"], exclude_dirs="_*")
# Handle glob syntax e.g. "./*.psp8"
elif "*" in paths[0]:
paths = [f for f in glob.glob(paths[0]) if f.endswith(".djrepo")]
else:
paths = [p for p in paths if p.endswith(".djrepo")]
return paths
def test_all_pseudo_files(self):
"""Testing all pseudopotential files located in the stable directories."""
def ppath_from_djpath(djpath):
"""Return the path of the pseudo from djrepo path."""
root, ext = os.path.splitext(djpath)
found = []
for ext in KNOWN_PPEXTENSIONS:
if os.path.exists(root + ext): found.append(root + ext)
if not found:
raise ValueError("Cannot find pseudo file associated to %s" %
djpath)
if len(found) > 1:
raise ValueError("Found multiple files associated to %s" %
djpath)
return found[0]
retcode, count = 0, 0
for top in all_dojotable_absdirs():
# Find all djrepo files
djrepo_paths = find_exts(top, exts=("djrepo"), exclude_dirs="_*")
#assert djrepo_paths # TODO
for djp in djrepo_paths:
count += 1
try:
pp_path = ppath_from_djpath(djp)
except ValueError as exc:
print(
"Exception %s\n while trying to find pp_file from djrepo: %s"
% (exc, djp))
retcode += 1
continue
try:
retcode += check_pseudo_path(pp_path, verbose=1)
except Exception as exc:
print("Exception %s\n while testing: %s" % (exc, pp_path))
retcode += 1
print("Found %s/%s (errors/totnum_files)" % (retcode, count))
assert count > 0
def get_djrepo_paths(options):
"""
Find and retur list of djson paths from options
Accepts filepaths or directory.
"""
paths = options.paths
if len(paths) == 1:
# Handle directory argument
if os.path.isdir(paths[0]):
top = os.path.abspath(paths[0])
paths = find_exts(top, ["djrepo"], exclude_dirs="_*")
# Handle glob syntax e.g. "./*.psp8"
elif "*" in paths[0]:
paths = [
f for f in glob.glob(paths[0]) if f.endswith(".djrepo")
]
else:
paths = [p for p in paths if p.endswith(".djrepo")]
return paths
def main():
try:
top = sys.argv[1]
except IndexError:
print("Usage: extract_djreport TOPDIR")
# Find all .psp8 files starting from top.
paths = find_exts(top, ["psp8"], exclude_dirs="_*")
#print(paths)
for path in paths:
try:
pseudo = Pseudo.from_file(path)
except Exception as exc:
print(path, exc)
raise
if pseudo is None:
print("Parser error in %s" % path)
continue
report_file = path.replace(".psp8", ".djrepo")
if os.path.exists(report_file):
#print("New DOJO file already exists. Ignoring", pseudo.filepath)
continue
print("Moving DOJOREPORT to", report_file)
report = remove_dojo_report(pseudo.filepath)
# Change md5 and pseudo_type
report["md5"] = pseudo.compute_md5()
if report["pseudo_type"] == "norm-conserving":
report["pseudo_type"] = "NC"
with open(report_file, "wt") as fh:
json.dump(report, fh, indent=-1, sort_keys=True)
def test_all_pseudo_files(self):
"""Testing all pseudopotential files located in the stable directories."""
def ppath_from_djpath(djpath):
"""Return the path of the pseudo from djrepo path."""
root, ext = os.path.splitext(djpath)
found = []
for ext in KNOWN_PPEXTENSIONS:
if os.path.exists(root + ext): found.append(root + ext)
if not found:
raise ValueError("Cannot find pseudo file associated to %s" % djpath)
if len(found) > 1:
raise ValueError("Found multiple files associated to %s" % djpath)
return found[0]
retcode, count = 0, 0
for top in all_dojotable_absdirs():
# Find all djrepo files
djrepo_paths = find_exts(top, exts=("djrepo"), exclude_dirs="_*")
#assert djrepo_paths # TODO
for djp in djrepo_paths:
count += 1
try:
pp_path = ppath_from_djpath(djp)
except ValueError as exc:
print("Exception %s\n while trying to find pp_file from djrepo: %s" % (exc, djp))
retcode += 1
continue
try:
retcode += check_pseudo_path(pp_path, verbose=1)
except Exception as exc:
print("Exception %s\n while testing: %s" % (exc, pp_path))
retcode += 1
print("Found %s/%s (errors/totnum_files)" % (retcode, count))
assert count > 0
def update_directory(directory):
"""
Modifies all objects in a directory such that
any incompatible module names (e. g. beep_ep)
are renamed to beep
Args:
directory (str): directory
Returns:
None
"""
fnames = find_exts(directory, "json")
for fname in tqdm(fnames):
with open(fname, "r") as f:
data = json.loads(f.read())
# For now just skip lists
if isinstance(data, dict):
module = data.get("@module", "")
if "beep_ep" in module:
data["@module"] = module.replace("beep_ep", "beep")
with open(fname, "w") as f:
f.write(json.dumps(data))
def pyscript(basename):
"""
Return the absolute name of one of the scripts in the `examples` directory from its basename.
"""
global _SCRIPTS
if _SCRIPTS is None:
# Build mapping basename --> path.
from monty.os.path import find_exts
pypaths = find_exts(_SCRIPTS_DIRPATH,
".py",
exclude_dirs="_*|.*|develop",
match_mode="basename")
_SCRIPTS = {}
for p in pypaths:
k = os.path.basename(p)
# Ignore e.g. __init__.py and private scripts.
if k.startswith("_"): continue
if k in _SCRIPTS:
raise ValueError(
"Fond duplicated basenames with name %s\nPrevious %s" %
(k, _SCRIPTS[k]))
_SCRIPTS[k] = p
return _SCRIPTS[basename]