def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
TODO: in the case of job arrays, decide what to do (i.e., if we want
to pass the -t options to list each subjob).
"""
from aiida.common.exceptions import FeatureNotAvailable
command = [
'qstat', '-f', '-w',
'@arien-pro.ics.muni.cz @wagap-pro.cerit-sc.cz @pbs.elixir-czech.cz '
]
if jobs and user:
raise FeatureNotAvailable('Cannot query by user and job(s) in PBS')
user = '******'
if user:
command.append('-u{}'.format(user))
if jobs:
if isinstance(jobs, six.string_types):
command.append('{}'.format(escape_for_bash(jobs)))
else:
try:
command.append('{}'.format(' '.join(
escape_for_bash(j) for j in jobs)))
except TypeError:
raise TypeError(
"If provided, the 'jobs' variable must be a string or an iterable of strings"
)
comm = ' '.join(command)
_LOGGER.debug('qstat command: {}'.format(comm))
return comm
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
TODO: in the case of job arrays, decide what to do (i.e., if we want
to pass the -t options to list each subjob).
"""
from aiida.common.exceptions import FeatureNotAvailable
command = ['qstat', '-f']
if jobs and user:
raise FeatureNotAvailable("Cannot query by user and job(s) in PBS")
if user:
command.append('-u{}'.format(user))
if jobs:
if isinstance(jobs, basestring):
command.append('{}'.format(escape_for_bash(jobs)))
else:
try:
command.append('{}'.format(' '.join(escape_for_bash(j) for j in jobs)))
except TypeError:
raise TypeError(
"If provided, the 'jobs' variable must be a string or an iterable of strings")
comm = ' '.join(command)
self.logger.debug("qstat command: {}".format(comm))
return comm
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
Separate the fields with the _field_separator string order:
jobnum, state, walltime, queue[=partition], user, numnodes, numcores, title
"""
from aiida.common.exceptions import FeatureNotAvailable
# I add the environment variable SLURM_TIME_FORMAT in front to be
# sure to get the times in 'standard' format
command = [
"SLURM_TIME_FORMAT='standard'", 'squeue', '--noheader', "-o '{}'".format(
_FIELD_SEPARATOR.join(_[0] for _ in self.fields))
]
if user and jobs:
raise FeatureNotAvailable('Cannot query by user and job(s) in SLURM')
if user:
command.append('-u{}'.format(user))
if jobs:
joblist = []
if isinstance(jobs, six.string_types):
joblist.append(jobs)
else:
if not isinstance(jobs, (tuple, list)):
raise TypeError("If provided, the 'jobs' variable must be a string or a list of strings")
joblist = jobs
command.append('--jobs={}'.format(','.join(joblist)))
comm = ' '.join(command)
self.logger.debug('squeue command: {}'.format(comm))
return comm
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
TODO: in the case of job arrays, decide what to do (i.e., if we want
to pass the -t options to list each subjob).
!!!ALL COPIED FROM PBSPRO!!!
TODO: understand if it is worth escaping the username,
or rather leave it unescaped to allow to pass $USER
"""
from aiida.common.exceptions import FeatureNotAvailable
if jobs:
raise FeatureNotAvailable('Cannot query by jobid in SGE')
command = 'qstat -ext -urg -xml '
if user:
command += f'-u {str(user)}'
else:
# All users if no user is specified
command += "-u '*'"
self.logger.debug(f'qstat command: {command}')
return command
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
Separates the fields with the _field_separator string order:
jobnum, state, walltime, queue[=partition], user, numnodes, numcores, title
"""
from aiida.common.exceptions import FeatureNotAvailable
# I add the environment variable SLURM_TIME_FORMAT in front to be
# sure to get the times in 'standard' format
command = ['bjobs', '-noheader', f"-o '{' '.join(self._joblist_fields)} delimiter=\"{_FIELD_SEPARATOR}\"'"]
if user and jobs:
raise FeatureNotAvailable('Cannot query by user and job(s) in LSF')
if user:
command.append(f'-u{user}')
if jobs:
joblist = []
if isinstance(jobs, str):
joblist.append(jobs)
else:
if not isinstance(jobs, (tuple, list)):
raise TypeError("If provided, the 'jobs' variable must be a string or a list of strings")
joblist = jobs
command.append(' '.join(joblist))
comm = ' '.join(command)
self.logger.debug(f'bjobs command: {comm}')
return comm
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
Separate the fields with the _field_separator string order:
jobnum, state, walltime, queue[=partition], user, numnodes, numcores, title
"""
from aiida.common.exceptions import FeatureNotAvailable
# I add the environment variable SLURM_TIME_FORMAT in front to be
# sure to get the times in 'standard' format
command = [
"SLURM_TIME_FORMAT='standard'", 'squeue', '--noheader',
f"-o '{_FIELD_SEPARATOR.join(_[0] for _ in self.fields)}'"
]
if user and jobs:
raise FeatureNotAvailable(
'Cannot query by user and job(s) in SLURM')
if user:
command.append(f'-u{user}')
if jobs:
joblist = []
if isinstance(jobs, str):
joblist.append(jobs)
else:
if not isinstance(jobs, (tuple, list)):
raise TypeError(
"If provided, the 'jobs' variable must be a string or a list of strings"
)
joblist = jobs
# Trick: When asking for a single job, append the same job once more.
# This helps provide a reliable way of knowing whether the squeue command failed (if its exit code is
# non-zero, _parse_joblist_output assumes that an error has occurred and raises an exception).
# When asking for a single job, squeue also returns a non-zero exit code if the corresponding job is
# no longer in the queue (stderr: "slurm_load_jobs error: Invalid job id specified"), which typically
# happens once in the life time of an AiiDA job,
# However, when providing two or more jobids via `squeue --jobs=123,234`, squeue stops caring whether
# the jobs are still in the queue and returns exit code zero irrespectively (allowing AiiDA to rely on the
# exit code for detection of real issues).
# Duplicating job ids has no other effect on the output.
# Verified on slurm versions 17.11.2, 19.05.3-2 and 20.02.2.
# See also https://github.com/aiidateam/aiida-core/issues/4326
if len(joblist) == 1:
joblist += [joblist[0]]
command.append(f"--jobs={','.join(joblist)}")
comm = ' '.join(command)
self.logger.debug(f'squeue command: {comm}')
return comm
def _validate_resources(self, **kwargs):
from aiida.common.exceptions import FeatureNotAvailable
for key in [
'num_machines', 'num_mpiprocs_per_machine', 'tot_num_mpiprocs'
]:
if key in kwargs and kwargs[key] != 1:
raise FeatureNotAvailable(
"Cannot set resouce '{}' to value '{}' for {}: "
"parallelization is not supported, only value of "
"'1' is accepted.".format(key, kwargs[key],
self.__class__.__name__))
def _get_detailed_jobinfo_command(self, jobid):
"""
Return the command to run to get the detailed information on a job.
This is typically called after the job has finished, to retrieve
the most detailed information possible about the job. This is done
because most schedulers just make finished jobs disappear from the
'qstat' command, and instead sometimes it is useful to know some
more detailed information about the job exit status, etc.
:raises: :class:`aiida.common.exceptions.FeatureNotAvailable`
"""
# pylint: disable=no-self-use, not-callable, unused-argument
raise FeatureNotAvailable("Cannot get detailed job info")
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
"""
from aiida.common.exceptions import FeatureNotAvailable
if user:
raise FeatureNotAvailable("Cannot query by user in Yascheduler")
command = ["yastatus"]
# make list from job ids (taken from slurm scheduler)
if jobs:
joblist = []
if isinstance(jobs, str):
joblist.append(jobs)
else:
if not isinstance(jobs, (tuple, list)):
raise TypeError(
"If provided, the 'jobs' variable must be a string or a list of strings"
)
joblist = jobs
command.append('--jobs {}'.format(' '.join(joblist)))
return ' '.join(command)
def _get_joblist_command(self, jobs=None, user=None):
"""
The command to report full information on existing jobs.
I separate the fields with the _field_separator string
order: jobnum, state, walltime, queue[=partition],
user, numnodes, numcores, title
"""
from aiida.common.exceptions import FeatureNotAvailable
# I add the environment variable SLURM_TIME_FORMAT in front to be
# sure to get the times in 'standard' format
command = ["bjobs", "-noheader",
"-o '{} delimiter=\"{}\"'".format(' '.join(self._joblist_fields),
_field_separator)]
if user and jobs:
raise FeatureNotAvailable("Cannot query by user and job(s) in LSF")
if user:
command.append('-u{}'.format(user))
if jobs:
joblist = []
if isinstance(jobs, basestring):
joblist.append(jobs)
else:
if not isinstance(jobs, (tuple, list)):
raise TypeError(
"If provided, the 'jobs' variable must be a string or "
"a list of strings")
joblist = jobs
command.append(' '.join(joblist))
comm = ' '.join(command)
self.logger.debug("bjobs command: {}".format(comm))
return comm
def create_input_nodes(self, open_transport, input_file_name=None,
output_file_name=None, remote_workdir=None):
"""
Create calculation input nodes based on the job's files.
:param open_transport: An open instance of the transport class of the
calculation's computer. See the tutorial for more information.
:type open_transport: :py:class:`aiida.transport.plugins.local.LocalTransport`
| :py:class:`aiida.transport.plugins.ssh.SshTransport`
This method parses the files in the job's remote working directory to
create the input nodes that would exist if the calculation were
submitted using AiiDa. These nodes are
* a ``'parameters'`` ParameterData node, based on the namelists and
their variable-value pairs;
* a ``'kpoints'`` KpointsData node, based on the *K_POINTS* card;
* a ``'structure'`` StructureData node, based on the
*ATOMIC_POSITIONS* and *CELL_PARAMETERS* cards;
* one ``'pseudo_X'`` UpfData node for the pseudopotential used for
the atomic species with name ``X``, as specified in the
*ATOMIC_SPECIES* card;
* a ``'settings'`` ParameterData node, if there are any fixed
coordinates, or if the gamma kpoint is used;
and can be retrieved as a dictionary using the ``get_inputs_dict()``
method. *These input links are cached-links; nothing is stored by this
method (including the calculation node itself).*
.. note:: QE stores the calculation's pseudopotential files in the
``<outdir>/<prefix>.save/`` subfolder of the job's working
directory, where ``outdir`` and ``prefix`` are QE *CONTROL*
variables (see
`pw input file description <http://www.quantum-espresso.org/wp-content/uploads/Doc/INPUT_PW.html>`_).
This method uses these files to either get--if the a node already
exists for the pseudo--or create a UpfData node for each
pseudopotential.
**Keyword arguments**
.. note:: These keyword arguments can also be set when instantiating the
class or using the ``set_`` methods (e.g. ``set_remote_workdir``).
Offering to set them here simply offers the user an additional
place to set their values. *Only the values that have not yet been
set need to be specified.*
:param input_file_name: The file name of the job's input file.
:type input_file_name: str
:param output_file_name: The file name of the job's output file (i.e.
the file containing the stdout of QE).
:type output_file_name: str
:param remote_workdir: Absolute path to the directory where the job
was run. The transport of the computer you link ask input to the
calculation is the transport that will be used to retrieve the
calculation's files. Therefore, ``remote_workdir`` should be the
absolute path to the job's directory on that computer.
:type remote_workdir: str
:raises aiida.common.exceptions.InputValidationError: if
``open_transport`` is a different type of transport than the
computer's.
:raises aiida.common.exceptions.InvalidOperation: if
``open_transport`` is not open.
:raises aiida.common.exceptions.InputValidationError: if
``remote_workdir``, ``input_file_name``, and/or ``output_file_name``
are not set prior to or during the call of this method.
:raises aiida.common.exceptions.FeatureNotAvailable: if the input file
uses anything other than ``ibrav = 0``, which is not currently
implimented in aiida.
:raises aiida.common.exceptions.ParsingError: if there are issues
parsing the input file.
:raises IOError: if there are issues reading the input file.
"""
import re
# Make sure the remote workdir and input + output file names were
# provided either before or during the call to this method. If they
# were just provided during this method call, store the values.
if remote_workdir is not None:
self.set_remote_workdir(remote_workdir)
elif self.get_attr('remote_workdir', None) is None:
raise InputValidationError(
'The remote working directory has not been specified.\n'
'Please specify it using one of the following...\n '
'(a) pass as a keyword argument to create_input_nodes\n'
' [create_input_nodes(remote_workdir=your_remote_workdir)]\n'
'(b) pass as a keyword argument when instantiating\n '
' [calc = PwCalculationImport(remote_workdir='
'your_remote_workdir)]\n'
'(c) use the set_remote_workdir method\n'
' [calc.set_remote_workdir(your_remote_workdir)]'
)
if input_file_name is not None:
self._INPUT_FILE_NAME = input_file_name
elif self._INPUT_FILE_NAME is None:
raise InputValidationError(
'The input file_name has not been specified.\n'
'Please specify it using one of the following...\n '
'(a) pass as a keyword argument to create_input_nodes\n'
' [create_input_nodes(input_file_name=your_file_name)]\n'
'(b) pass as a keyword argument when instantiating\n '
' [calc = PwCalculationImport(input_file_name='
'your_file_name)]\n'
'(c) use the set_input_file_name method\n'
' [calc.set_input_file_name(your_file_name)]'
)
if output_file_name is not None:
self._OUTPUT_FILE_NAME = output_file_name
elif self._OUTPUT_FILE_NAME is None:
raise InputValidationError(
'The input file_name has not been specified.\n'
'Please specify it using one of the following...\n '
'(a) pass as a keyword argument to create_input_nodes\n'
' [create_input_nodes(output_file_name=your_file_name)]\n'
'(b) pass as a keyword argument when instantiating\n '
' [calc = PwCalculationImport(output_file_name='
'your_file_name)]\n'
'(c) use the set_output_file_name method\n'
' [calc.set_output_file_name(your_file_name)]'
)
# Check that open_transport is the correct transport type.
if type(open_transport) is not self.get_computer().get_transport_class():
raise InputValidationError(
"The transport passed as the `open_transport` parameter is "
"not the same transport type linked to the computer. Please "
"obtain the correct transport class using the "
"`get_transport_class` method of the calculation's computer. "
"See the tutorial for more information."
)
# Check that open_transport is actually open.
if not open_transport._is_open:
raise InvalidOperation(
"The transport passed as the `open_transport` parameter is "
"not open. Please execute the open the transport using it's "
"`open` method, or execute the call to this method within a "
"`with` statement context guard. See the tutorial for more "
"information."
)
# Copy the input file and psuedo files to a temp folder for parsing.
with SandboxFolder() as folder:
# Copy the input file to the temp folder.
remote_path = os.path.join(self._get_remote_workdir(),
self._INPUT_FILE_NAME)
open_transport.get(remote_path, folder.abspath)
# Parse the input file.
local_path = os.path.join(folder.abspath, self._INPUT_FILE_NAME)
with open(local_path) as fin:
pwinputfile = pwinputparser.PwInputFile(fin)
# Determine PREFIX, if it hasn't already been set by the user.
if self._PREFIX is None:
control_dict = pwinputfile.namelists['CONTROL']
# If prefix is not set in input file, use the default,
# 'pwscf'.
self._PREFIX = control_dict.get('prefix', 'pwscf')
# Determine _OUTPUT_SUBFOLDER, if it hasn't already been set by
# the user.
# TODO: Prompt user before using the environment variable???
if self._OUTPUT_SUBFOLDER is None:
# See if it's specified in the CONTROL namelist.
control_dict = pwinputfile.namelists['CONTROL']
self._OUTPUT_SUBFOLDER = control_dict.get('outdir', None)
if self._OUTPUT_SUBFOLDER is None:
# See if the $ESPRESSO_TMPDIR is set.
envar = open_transport.exec_command_wait(
'echo $ESPRESSO_TMPDIR'
)[1]
if len(envar.strip()) > 0:
self._OUTPUT_SUBFOLDER = envar.strip()
else:
# Use the default dir--the dir job was submitted in.
self._OUTPUT_SUBFOLDER = self._get_remote_workdir()
# Copy the pseudo files to the temp folder.
for fnm in pwinputfile.atomic_species['pseudo_file_names']:
remote_path = os.path.join(self._get_remote_workdir(),
self._OUTPUT_SUBFOLDER,
'{}.save/'.format(self._PREFIX),
fnm)
open_transport.get(remote_path, folder.abspath)
# Make sure that ibrav = 0, since aiida doesn't support anything
# else.
if pwinputfile.namelists['SYSTEM']['ibrav'] != 0:
raise FeatureNotAvailable(
'Found ibrav !=0 while parsing the input file. '
'Currently, AiiDa only supports ibrav = 0.'
)
# Create ParameterData node based on the namelist and link as input.
# First, strip the namelist items that aiida doesn't allow or sets
# later.
# NOTE: ibrav = 0 is checked above.
# NOTE: If any of the position or cell units are in alat or crystal
# units, that will be taken care of by the input parsing tools, and
# we are safe to fake that they were never there in the first place.
parameters_dict = deepcopy(pwinputfile.namelists)
for namelist, blocked_key in self._blocked_keywords:
keys = parameters_dict[namelist].keys()
for this_key in parameters_dict[namelist].keys():
# take into account that celldm and celldm(*) must be blocked
if re.sub("[(0-9)]", "", this_key) == blocked_key:
parameters_dict[namelist].pop(this_key, None)
parameters = ParameterData(dict=parameters_dict)
self.use_parameters(parameters)
# Initialize the dictionary for settings parameter data for possible
# use later for gamma kpoint and fixed coordinates.
settings_dict = {}
# Create a KpointsData node based on the K_POINTS card block
# and link as input.
kpointsdata = pwinputfile.get_kpointsdata()
self.use_kpoints(kpointsdata)
# If only the gamma kpoint is used, add to the settings dictionary.
if pwinputfile.k_points['type'] == 'gamma':
settings_dict['gamma_only'] = True
# Create a StructureData node based on the ATOMIC_POSITIONS,
# CELL_PARAMETERS, and ATOMIC_SPECIES card blocks, and link as
# input.
structuredata = pwinputfile.get_structuredata()
self.use_structure(structuredata)
# Get or create a UpfData node for the pseudopotentials used for
# the calculation.
names = pwinputfile.atomic_species['names']
pseudo_file_names = pwinputfile.atomic_species['pseudo_file_names']
for name, fnm in zip(names, pseudo_file_names):
local_path = os.path.join(folder.abspath, fnm)
pseudo, created = UpfData.get_or_create(local_path)
self.use_pseudo(pseudo, kind=name)
# If there are any fixed coordinates (i.e. force modification
# present in the input file, create a ParameterData node for these
# special settings.
fixed_coords = pwinputfile.atomic_positions['fixed_coords']
# NOTE: any() only works for 1-dimensional lists.
if any((any(fc_xyz) for fc_xyz in fixed_coords)):
settings_dict['FIXED_COORDS'] = fixed_coords
# If the settings_dict has been filled in, create a ParameterData
# node from it and link as input.
if settings_dict:
self.use_settings(ParameterData(dict=settings_dict))
self._set_attr('input_nodes_created', True)