def _safe_run(cmd_arg_list,
descr="External command",
require_empty_stderr=False,
require_zero_exit=True):
process = Popen(cmd_arg_list, stdout=PIPE, stderr=PIPE)
stdoutstr, stderrstr = process.communicate()
return_code = process.returncode
if (require_zero_exit
and return_code != 0) or (require_empty_stderr
and len(stderrstr) > 0):
raise RuntimeError(
"{} run failed. Command was:\n{}\nstdout was:\n{}\nstderr was:\n{}\nexit code was: {}"
.format(descr, indent(" ".join(cmd_arg_list)),
indent(stdoutstr), indent(stderrstr), return_code))
return stdoutstr, stderrstr
def run(self):
while True:
if Queue.verbose:
#TODO write this kind of stuff to something from the python standard library `logging` module
print("Thread {} awaiting task...".format(self))
next_task = self.parent_fifo.get()
if Queue.verbose:
print("Thread {} recieved task {}".format(self, next_task))
if next_task is TerminateTask:
self.parent_fifo.task_done()
break
else:
index, c, lock = next_task
try:
already_run = c.already_run()
if not already_run and not Queue.quiet:
# TODO acquire global output lock? (and release after output). Or use the `logging` module
print("Running computation #{}:\n{}".format(index + 1, indent(c.task_description())))
elif Queue.verbose and already_run:
print("Computation #{} already run in {}, not re-running".format(index + 1, c.directory))
c.run(lock=lock)
if not already_run and not Queue.quiet:
print("Computation #{} completed.".format(index + 1))
except Exception, e:
# TODO figure out how to handle exceptions here
print(e)
self.parent_fifo.task_done()
if Queue.verbose:
print("Thread {} finished task {}".format(self, next_task))
def run(self):
while True:
if Queue.verbose:
#TODO write this kind of stuff to something from the python standard library `logging` module
print("Thread {} awaiting task...".format(self))
next_task = self.parent_fifo.get()
if Queue.verbose:
print("Thread {} recieved task {}".format(self, next_task))
if next_task is TerminateTask:
self.parent_fifo.task_done()
break
else:
index, c, lock = next_task
try:
already_run = c.already_run()
if not already_run and not Queue.quiet:
# TODO acquire global output lock? (and release after output). Or use the `logging` module
print("Running computation #{}:\n{}".format(
index + 1, indent(c.task_description())))
elif Queue.verbose and already_run:
print(
"Computation #{} already run in {}, not re-running"
.format(index + 1, c.directory))
c.run(lock=lock)
if not already_run and not Queue.quiet:
print("Computation #{} completed.".format(index + 1))
except Exception, e:
# TODO figure out how to handle exceptions here
print(e)
self.parent_fifo.task_done()
if Queue.verbose:
print("Thread {} finished task {}".format(self, next_task))
def parse_geometry_string(string, filetype=None):
possible_parsers = []
for parser in known_parsers:
if filetype is None:
if parser.string_is_filetype(string):
possible_parsers.append(parser)
else:
if parser._matches_filetype_name(filetype):
possible_parsers.append(parser)
if len(possible_parsers) == 0:
if filetype is None:
raise ValueError("Can't automatically determine file type of file with data:\n{}".format(
string
))
else:
raise ValueError("No known parsers for filetype '{}'".format(
filetype
))
elif len(possible_parsers) > 1:
raise ValueError("File type is ambiguous. File could be interpreted as:"
" include:\n{}\nfor file with data:\n{}".format(
indent("\n".join(p.format_description for p in possible_parsers)), string
))
else:
return possible_parsers[0].parse_string(string)
def parse_geometry_file(filename, filetype=None):
# TODO check file extension
possible_parsers = []
for parser in known_parsers:
if filetype is None:
if parser.is_this_filetype(filename):
possible_parsers.append(parser)
else:
if parser._matches_filetype_name(filetype):
possible_parsers.append(parser)
if len(possible_parsers) == 0:
if filetype is None:
raise ValueError("Can't automatically determine file type of file {}".format(
filename
))
else:
raise ValueError("No known parsers for filetype '{}'".format(
filetype
))
elif len(possible_parsers) > 1:
raise ValueError("File type of {} is ambiguous. File could be interpreted as:"
" include:\n{}".format(
filename,
indent("\n".join(p.format_description for p in possible_parsers))
))
else:
return possible_parsers[0].parse_file(filename)
def _safe_run(cmd_arg_list, descr="External command",
require_empty_stderr=False, require_zero_exit=True
):
process = Popen(cmd_arg_list,
stdout=PIPE,
stderr=PIPE
)
stdoutstr, stderrstr = process.communicate()
return_code = process.returncode
if (require_zero_exit and return_code != 0) or (require_empty_stderr and len(stderrstr) > 0):
raise RuntimeError("{} run failed. Command was:\n{}\nstdout was:\n{}\nstderr was:\n{}\nexit code was: {}".format(
descr,
indent(" ".join(cmd_arg_list)),
indent(stdoutstr),
indent(stderrstr),
return_code
))
return stdoutstr, stderrstr
def __str__(self):
ret_val = "CartesianRepresentation of molecule {}, in {}:\n".format(
shortstr(self.molecule), self.units)
for i, (x, y, z) in enumerate(grouper(3, self)):
ret_val += indent(
'{:>9} {:12.8f} {:12.8f} {:12.8f}\n'.format(
'[{}-{}]'.format(3 * i, 3 * i + 2), x.value, y.value,
z.value), 2)
return ret_val
def __str__(self):
ret_val = "CartesianRepresentation of molecule {}, in {}:\n".format(shortstr(self.molecule), self.units)
for i, (x, y, z) in enumerate(grouper(3, self)):
ret_val += indent(
"{:>9} {:12.8f} {:12.8f} {:12.8f}\n".format(
"[{}-{}]".format(3 * i, 3 * i + 2), x.value, y.value, z.value
),
2,
)
return ret_val
def __str__(self):
namestrs = []
valstrs = []
it = np.nditer(self, flags=['multi_index'])
for x in it:
namestrs.append(", ".join(shortstr(self.representation[i]) for i in it.multi_index))
valstrs.append("{: .8f}{}".format(float(x), (" " + str(self.units) if hasunits(self) else '')))
width=max(len(s) for s in namestrs) + 4
ret_val = self.__class__.__name__ + ":\n"
for name, val in zip(namestrs, valstrs):
ret_val += ("{:>"+str(width)+"}: {}\n").format(name, val)
ret_val += " for representation:\n{}".format(indent(str(self.representation)))
return ret_val
def __str__(self):
namestrs = []
valstrs = []
it = np.nditer(self, flags=['multi_index'])
for x in it:
namestrs.append(", ".join(
shortstr(self.representation[i]) for i in it.multi_index))
valstrs.append("{: .8f}{}".format(
float(x), (" " + str(self.units) if hasunits(self) else '')))
width = max(len(s) for s in namestrs) + 4
ret_val = self.__class__.__name__ + ":\n"
for name, val in zip(namestrs, valstrs):
ret_val += ("{:>" + str(width) + "}: {}\n").format(name, val)
ret_val += " for representation:\n{}".format(
indent(str(self.representation)))
return ret_val
def write_3d_pdf(self,
filename,
verbose=False,
extra_latex_before="",
extra_latex_after="",
latex_file_handler=None,
toolbar=False,
quiet=False,
**kwargs):
self._ensure_tmp_xyz()
#----------------------------------------#
if not _have_meshlab:
raise RuntimeError(
"You need to have meshlab.app installed (currently it must be installed in /Applications/ to work...) to do make 3D pdfs. Get it from Sourceforge"
)
#----------------------------------------#
new_kw = JmolRenderer.default_keywords.copy()
# 3d keywords are higher priority
new_kw.update(JmolRenderer.default_3d_keywords)
# user preferences always have veto power
new_kw.update(kwargs)
extra_lines = self._get_kwarg_script_lines(**new_kw)
#----------------------------------------#
if verbose: print("Generating idtf file with Jmol...")
jscript = """
load "{molfile}"
{extra_lines}
write "{tempdir}/output.idtf"
""".format(molfile=self._geom_path,
tempdir=self.tempdir,
extra_lines=extra_lines)
self._run_jmol_script(jscript)
idtf_path = path_join(self.tempdir, "output.idtf")
if not exists(idtf_path):
raise RuntimeError("Jmol command failed to produce "
"file '{}'. Script was:\n{}".format(
idtf_path, indent(jscript)))
#----------------------------------------#
# now convert to u3d format
# TODO make this portable
if verbose: print("Converting idtf file to u3d file...")
idtf_cmd = _idtf_to_u3d_template.format(
input=path_join(self.tempdir, "output.idtf"),
output=path_join(self.tempdir, "output.u3d"))
JmolRenderer._safe_run(shlex.split(idtf_cmd),
descr="IDTF to u3d conversion")
#shlex.os.system(idtf_cmd)
#----------------------------------------#
# Now embed in a latex document and run PDFLatex
if verbose:
print("Constructing latex document...".format(
pdflatex_path, self.tempdir))
with open(path_join(self.tempdir, "output.idtf.tex")) as f:
jmol_latex_data = f.read()
stuff3d = jmol_latex_data.split("toolbar=false,")[1].split(
"inline=true,")[0].strip()
if latex_file_handler is None:
f = open(path_join(self.tempdir, "output.tex"), "w+")
run_pdflatex = True
else:
f = latex_file_handler
run_pdflatex = False
if run_pdflatex:
f.write(pdf_3d_header)
pdf3d_section = pdf_3d_entry_template\
.replace("@@@LABEL@@@", "molecule")\
.replace("@@@3DSTUFF@@@", stuff3d)\
.replace("@@@FNAME@@@", path_join(self.tempdir, "output.u3d"))
if toolbar:
pdf3d_section = pdf3d_section.replace("toolbar=false",
"toolbar=true")
f.write(extra_latex_before)
f.write(pdf3d_section)
f.write(extra_latex_after)
if run_pdflatex:
f.write(pdf_3d_footer)
#- - - - - - - - - - - - - - - - - - - - #
if run_pdflatex:
if verbose:
print("Running '{} {} output' in directory {}...".format(
pdflatex_path, " ".join(pdflatex_args), self.tempdir))
pdflatex_cmd_list = [pdflatex_path] + pdflatex_args + ["output"]
with working_directory(self.tempdir):
# Run 3 times to get references right
JmolRenderer._safe_run(pdflatex_cmd_list,
"pdflatex",
require_empty_stderr=True,
require_zero_exit=False)
if verbose: print(" ...run 1 complete")
JmolRenderer._safe_run(pdflatex_cmd_list,
"output",
require_empty_stderr=True,
require_zero_exit=False)
if verbose: print(" ...run 2 complete")
JmolRenderer._safe_run(pdflatex_cmd_list,
"output",
require_empty_stderr=True,
require_zero_exit=False)
if verbose: print(" ...run 3 complete")
#----------------------------------------#
# Finally, copy back the temporary output to the requested file path
# but only if we wrote the file
move(path_join(self.tempdir, "output.pdf"), filename)
def write_3d_pdf(self, filename,
verbose=False,
extra_latex_before="",
extra_latex_after="",
latex_file_handler=None,
toolbar=False,
quiet=False,
**kwargs
):
self._ensure_tmp_xyz()
#----------------------------------------#
if not _have_meshlab:
raise RuntimeError("You need to have meshlab.app installed (currently it must be installed in /Applications/ to work...) to do make 3D pdfs. Get it from Sourceforge")
#----------------------------------------#
new_kw = JmolRenderer.default_keywords.copy()
# 3d keywords are higher priority
new_kw.update(JmolRenderer.default_3d_keywords)
# user preferences always have veto power
new_kw.update(kwargs)
extra_lines = self._get_kwarg_script_lines(**new_kw)
#----------------------------------------#
if verbose: print("Generating idtf file with Jmol...")
jscript = """
load "{molfile}"
{extra_lines}
write "{tempdir}/output.idtf"
""".format(
molfile=self._geom_path,
tempdir=self.tempdir,
extra_lines=extra_lines
)
self._run_jmol_script(jscript)
idtf_path = path_join(self.tempdir, "output.idtf")
if not exists(idtf_path):
raise RuntimeError("Jmol command failed to produce "
"file '{}'. Script was:\n{}".format(
idtf_path,
indent(jscript)
))
#----------------------------------------#
# now convert to u3d format
# TODO make this portable
if verbose: print("Converting idtf file to u3d file...")
idtf_cmd = _idtf_to_u3d_template.format(
input=path_join(self.tempdir, "output.idtf"),
output=path_join(self.tempdir, "output.u3d")
)
JmolRenderer._safe_run(shlex.split(idtf_cmd), descr="IDTF to u3d conversion")
#shlex.os.system(idtf_cmd)
#----------------------------------------#
# Now embed in a latex document and run PDFLatex
if verbose: print("Constructing latex document...".format(pdflatex_path, self.tempdir))
with open(path_join(self.tempdir, "output.idtf.tex")) as f:
jmol_latex_data = f.read()
stuff3d = jmol_latex_data.split("toolbar=false,")[1].split("inline=true,")[0].strip()
if latex_file_handler is None:
f = open(path_join(self.tempdir, "output.tex"), "w+")
run_pdflatex = True
else:
f = latex_file_handler
run_pdflatex = False
if run_pdflatex:
f.write(pdf_3d_header)
pdf3d_section = pdf_3d_entry_template\
.replace("@@@LABEL@@@", "molecule")\
.replace("@@@3DSTUFF@@@", stuff3d)\
.replace("@@@FNAME@@@", path_join(self.tempdir, "output.u3d"))
if toolbar:
pdf3d_section = pdf3d_section.replace("toolbar=false", "toolbar=true")
f.write(extra_latex_before)
f.write(pdf3d_section)
f.write(extra_latex_after)
if run_pdflatex:
f.write(pdf_3d_footer)
#- - - - - - - - - - - - - - - - - - - - #
if run_pdflatex:
if verbose: print("Running '{} {} output' in directory {}...".format(pdflatex_path, " ".join(pdflatex_args), self.tempdir))
pdflatex_cmd_list = [pdflatex_path] + pdflatex_args + ["output"]
with working_directory(self.tempdir):
# Run 3 times to get references right
JmolRenderer._safe_run(pdflatex_cmd_list, "pdflatex",
require_empty_stderr=True, require_zero_exit=False)
if verbose: print(" ...run 1 complete")
JmolRenderer._safe_run(pdflatex_cmd_list, "output",
require_empty_stderr=True, require_zero_exit=False)
if verbose: print(" ...run 2 complete")
JmolRenderer._safe_run(pdflatex_cmd_list, "output",
require_empty_stderr=True, require_zero_exit=False)
if verbose: print(" ...run 3 complete")
#----------------------------------------#
# Finally, copy back the temporary output to the requested file path
# but only if we wrote the file
move(path_join(self.tempdir, "output.pdf"), filename)
def __new__(cls, **kwargs):
"""
"""
def get_kwarg(arg, cls, subclass=False):
gotten = pop_kwarg(kwargs, arg)
if not subclass:
if gotten is not None and not isinstance(gotten, cls):
if not isinstance(cls, tuple):
raise TypeError("#{0} argument given to Computation() "\
"must be an instance of the #{1} class.".format(arg, classname(cls)))
else:
raise TypeError("#{0} argument given to Computation() "\
"must be one of (#{1}) .".format(arg, ', '.join(classname(cl) for cl in cls)))
else:
if gotten is not None and not issubclass(gotten, cls):
if not isinstance(cls, tuple):
raise TypeError("#{0} argument given to Computation() "\
"must be an subclass of the #{1} class.".format(arg, classname(cls)))
else:
raise TypeError("#{0} argument given to Computation() "\
"must be subclass of one of (#{1}) .".format(arg, ', '.join(classname(cl) for cl in cls)))
return gotten
#--------------------------------------------------------------------------------#
# Handle molecule, make a ResultGetter
molecule = get_kwarg('molecule', Molecule)
res_getter = get_kwarg('result_getter', ComputationResultGetter)
if res_getter is None:
res_getter = ComputationResultGetter(**kwargs)
molecule.result_getters.append(res_getter)
#--------------------------------------------------------------------------------#
# Now see what package we're using
package = get_kwarg('package', (basestring, PackageInterface))
comp_class = cls
interface = None
if isinstance(package, PackageInterface):
interface = package
elif isinstance(package, basestring):
interface = PackageInterface.load_package(
package,
input_generator=get_kwarg('input_generator', InputGenerator, True),
runner=get_kwarg('runner', Runner, True),
output_parser=get_kwarg('output_parser', OutputParser, True),
details_class=get_kwarg('details_class', ComputationDetails, True),
computation_class=get_kwarg('computation_class', cls, True)
)
if interface.computation_class is not None:
comp_class = cls
ret_val = object.__new__(comp_class)
#--------------------------------------------------------------------------------#
ret_val.molecule = molecule
res_getter.add_computation(ret_val)
ret_val.input_generator = interface.input_generator(ret_val)
ret_val.output_parser = interface.output_parser(ret_val)
#TODO accept option for directory to put input/output files in
#TODO accept option for input/output filenames
#TODO support for runners that don't follow the input/output pattern
ret_val.runner = interface.runner(input_file='input.dat', output_file='output.dat')
#--------------------------------------------------------------------------------#
ret_val.details = get_kwarg('details', interface.details_class)
ret_val.properties = pop_kwarg(kwargs, 'property')
if isinstance(ret_val.properties, Iterable):
ret_val.properties = list(ret_val.properties)
else:
ret_val.properties = [ret_val.properties]
if not all(issubclass(prop, MolecularProperty) or isinstance(prop, MolecularProperty) for prop in ret_val.properties):
raise TypeError("property argument given to Computation() must be a list of MolecularProperty subclasses or instances.".format(classname(cls)))
fixed_properties = []
for prop in ret_val.properties:
if isinstance(prop, MolecularProperty):
fixed_properties.append(prop)
prop.details = ret_val.details
elif issubclass(prop, MolecularProperty):
fixed_properties.append(prop(molecule=ret_val.molecule, details=ret_val.details))
ret_val.properties = fixed_properties
#--------------------------------------------------------------------------------#
# Get input_filename and output_filename, if specified. Make sure that neither
# is a path.
# first get input_filename
input_filename = pop_kwarg(kwargs, 'input_filename')
if input_filename:
if os.path.split(input_filename)[0] != '':
raise ValueError("File name '{0}' given for argument 'input_filename' of Computation constructor "
"must be a single file, not a path. Use the 'directory' argument to specify "
"a directory.".format(input_filename))
ret_val.runner.input_file = input_filename
else:
ret_val.runner.input_file = Runner.default_input_filename
#----------------------------------------#
# now get output_filename
output_filename = pop_kwarg(kwargs, 'output_filename')
if output_filename:
if os.path.split(output_filename)[0] != '':
raise ValueError("File name '{0}' given for argument 'output_filename' of Computation constructor "
"must be a single file, not a path. Use the 'directory' argument to specify "
"a directory.".format(output_filename))
ret_val.runner.output_file = output_filename
else:
ret_val.runner.output_file = Runner.default_output_filename
#--------------------------------------------------------------------------------#
# Find out the desired directory and make new ones if necessary and allowed.
ret_val.directory = pop_kwarg(kwargs, 'directory')
used_template = False
if ret_val.directory is not None:
ret_val.directory = full_path(ret_val.directory)
elif cls.directory_template is not None:
ret_val.directory = cls.directory_template
used_template = True
else:
ret_val.directory = full_path(os.curdir)
#----------------------------------------#
# Do mako-template style replacement, if the user has mako
try:
from mako.template import Template
from mako import exceptions
# Start with some obvious standard things to add...
variables = dict(
mol = ret_val.molecule, molecule = ret_val.molecule,
calc = ret_val, calculation = ret_val,
rg = res_getter, result_getter = res_getter,
details = ret_val.details
)
# Get all of the known details, as strings
det_strs = dict()
for k, v in ret_val.details._known_details.items():
det_strs[str(k)] = str(v)
det_strs[str(k).lower()] = str(v)
det_strs[str(k).upper()] = str(v)
if k in Computation.attribute_aliases:
for alias in Computation.attribute_aliases[k]:
alias = make_safe_identifier(alias)
det_strs[str(alias)] = str(v)
det_strs[str(alias).lower()] = str(v)
det_strs[str(alias).upper()] = str(v)
variables.update(det_strs)
# Get any additional keywords specified by the user (advanced feature)
additional = kwargs.pop('directory_creation_vars', dict())
variables.update(additional)
# Now run mako on the template...
try:
ret_val.directory = Template(
text=ret_val.directory,
strict_undefined=True
).render(
**variables
)
except:
raise ValueError(
"invalid MAKO template:\n{}\nMako came back with error:\n{}".format(
indent(ret_val.directory),
indent(exceptions.text_error_template().render())
)
)
# and make it into a "url safe string":
ret_val.directory = urllib.quote_plus(
ret_val.directory,
safe='/' + cls.additional_safe_directory_characters
)
except ImportError:
# Can't do mako parsing of path. Oh well...
# But if the user specified additional args, they probably thought they did have
# mako installed...
additional = kwargs.pop('directory_creation_vars', None)
if additional is not None:
raise
# and if they have a ${ in the directory name or a <% in the directory name,
# they probably also thought that they had mako.
# (Note: this could be an issue in the unlikely scenario that the user is using
# e.g. zsh and specifying environment variables from within Python as ${ }.
# If someone is having that problem, they probably have enough intelligence
# also to figure out how to get around it.)
if '${' in ret_val.directory or '<%' in ret_val.directory:
raise
# if they're using a class-level template, they also need to have mako installed
if used_template:
raise
#----------------------------------------#
ret_val.directory = full_path(ret_val.directory)
#----------------------------------------#
# Now create the directory if we're allowed to and need to
create_dirs = kwargs.pop('create_directories', cls.always_create_missing)
if create_dirs:
if not os.path.isdir(ret_val.directory):
if os.path.exists(ret_val.directory):
raise OSError(
"path '{}' exists and is not a directory".format(
ret_val.directory
)
)
os.makedirs(ret_val.directory)
else:
if not os.path.isdir(ret_val.directory):
raise OSError(
"'{0}' is not a valid directory, and will not be created. Set the "
"'create_directories' argument of computation to True to do directory "
"creation automatically.".format(
ret_val.directory
))
#----------------------------------------#
# now set the input_file and output_file values
ret_val.runner.input_file = os.path.join(ret_val.directory, ret_val.runner.input_file)
ret_val.runner.output_file = os.path.join(ret_val.directory, ret_val.runner.output_file)
ret_val.runner.working_directory = ret_val.directory
#--------------------------------------------------------------------------------#
# Pass the remaining kwargs on to the initialization, which may have been customized for the particular package.
ret_val.__init__(**kwargs)
return ret_val