def _handle_time_limits(self, calculation):
"""
If calculation fails due to time limits, we simply resubmit it.
"""
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_TIME_LIMIT']):
self.report(
'FleurCalculation failed due to time limits, I restart it from where it ended'
)
remote = calculation.get_outgoing().get_node_by_label('remote_folder')
# if previous calculation failed for the same reason, do not restart
prev_calculation_status = remote.get_incoming().all()[-1].exit_status
if prev_calculation_status in FleurProcess.get_exit_statuses(
['ERROR_TIME_LIMIT']):
self.ctx.is_finished = True
return ErrorHandlerReport(True, True)
# however, if it is the first time, resubmit profiding inp.xml and cdn from the remote folder
self.ctx.is_finished = False
self.ctx.inputs.parent_folder = remote
if 'fleurinpdata' in self.ctx.inputs:
del self.ctx.inputs.fleurinpdata
return ErrorHandlerReport(True, True)
def check_failure(self):
"""
Throws an exit code if scf failed
"""
try:
scf_wc = self.ctx.scf_res
except AttributeError:
message = 'ERROR: Something went wrong I do not have new atom positions calculation'
self.control_end_wc(message)
return self.exit_codes.ERROR_NO_SCF_OUTPUT
if not scf_wc.is_finished_ok:
exit_statuses = FleurScfWorkChain.get_exit_statuses(
['ERROR_FLEUR_CALCULATION_FAILED'])
if scf_wc.exit_status == exit_statuses[0]:
fleur_calc = load_node(
scf_wc.outputs.output_scf_wc_para.get_dict()
['last_calc_uuid'])
if fleur_calc.exit_status == FleurCalc.get_exit_statuses(
['ERROR_VACUUM_SPILL_RELAX'])[0]:
self.control_end_wc(
'ERROR: Failed due to atom and vacuum overlap')
return self.exit_codes.ERROR_VACUUM_SPILL_RELAX
elif fleur_calc.exit_status == FleurCalc.get_exit_statuses(
['ERROR_MT_RADII_RELAX'])[0]:
self.control_end_wc('ERROR: Failed due to MT overlap')
return self.exit_codes.ERROR_MT_RADII_RELAX
return self.exit_codes.ERROR_SCF_FAILED
def _handle_time_limits(self, calculation):
"""
If calculation fails due to time limits, we simply resubmit it.
"""
from aiida.common.exceptions import NotExistent
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_TIME_LIMIT']):
# if previous calculation failed for the same reason, do not restart
try:
prev_calculation_remote = calculation.get_incoming(
).get_node_by_label('parent_folder')
prev_calculation_status = prev_calculation_remote.get_incoming(
).all()[-1].node.exit_status
if prev_calculation_status in FleurProcess.get_exit_statuses(
['ERROR_TIME_LIMIT']):
self.ctx.is_finished = True
return ErrorHandlerReport(True, True)
except NotExistent:
pass
self.report(
'FleurCalculation failed due to time limits, I restart it from where it ended'
)
# increase wallclock time
propose_wallclock = self.ctx.inputs.metadata.options[
'max_wallclock_seconds'] * 2
if propose_wallclock > self.ctx.max_queue_wallclock_sec:
propose_wallclock = self.ctx.max_queue_wallclock_sec
self.ctx.inputs.metadata.options[
'max_wallclock_seconds'] = propose_wallclock
# increase number of nodes
propose_nodes = self.ctx.num_machines * 2
if propose_nodes > self.ctx.max_queue_nodes:
propose_nodes = self.ctx.max_queue_nodes
self.ctx.num_machines = propose_nodes
remote = calculation.get_outgoing().get_node_by_label('remote_folder')
# resubmit providing inp.xml and cdn from the remote folder
self.ctx.is_finished = False
if 'fleurinpdata' in self.ctx.inputs:
modes = self.ctx.inputs.fleurinpdata.get_fleur_modes()
if not (modes['force_theorem'] or modes['dos'] or modes['band']):
# in modes listed above it makes no sense copying cdn.hdf
self.ctx.inputs.parent_folder = remote
del self.ctx.inputs.fleurinpdata
else:
# it is harder to extract modes in this case - simply try to reuse cdn.hdf and hope it works
self.ctx.inputs.parent_folder = remote
return ErrorHandlerReport(True, True)
def _handle_not_enough_memory(self, calculation):
"""
Calculation failed due to lack of memory.
Probably works for JURECA only, has to be tested for other systems.
"""
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_NOT_ENOUGH_MEMORY']):
if self.ctx.can_be_optimised:
self.ctx.restart_calc = None
self.ctx.is_finished = False
self.report(
'Calculation failed due to lack of memory, I resubmit it with twice larger'
' amount of computational nodes and smaller MPI/OMP ratio')
self.ctx.num_machines = self.ctx.num_machines * 2
self.ctx.suggest_mpi_omp_ratio = self.ctx.suggest_mpi_omp_ratio / 2
self.check_kpts()
if 'settings' not in self.ctx.inputs:
self.ctx.inputs.settings = {}
else:
self.ctx.inputs.settings = self.inputs.settings.get_dict()
self.ctx.inputs.settings.setdefault('remove_from_remotecopy_list',
[]).append('mixing_history*')
return ErrorHandlerReport(True, True)
else:
self.ctx.restart_calc = calculation
self.ctx.is_finished = True
self.report(
'I am not allowed to optimize your settings. Consider providing at least'
'num_machines and num_mpiprocs_per_machine')
self.results()
return ErrorHandlerReport(
True, True, self.exit_codes.ERROR_MEMORY_ISSUE_NO_SOLUTION)
def _handle_dirac_equation(self, calculation):
"""
Sometimes relaxation calculation fails with Diraq problem which is usually caused by
problems with reusing charge density. In this case we resubmit the calculation, dropping the input cdn.
"""
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_DROP_CDN']):
# try to drop remote folder and see if it helps
is_fleurinp_from_relax = False
if 'fleurinpdata' in self.ctx.inputs:
if 'relax.xml' in self.ctx.inputs.fleurinpdata.files:
is_fleurinp_from_relax = True
if 'parent_folder' in self.ctx.inputs and is_fleurinp_from_relax:
del self.ctx.inputs.parent_folder
self.ctx.restart_calc = None
self.ctx.is_finished = False
self.report(
'Calculation seems to fail due to corrupted charge density (can happen'
'during relaxation). I drop cdn from previous step')
return ErrorHandlerReport(True, True)
self.ctx.restart_calc = calculation
self.ctx.is_finished = True
self.report(
'Can not drop charge density. If I drop the remote folder, there will be'
'no inp.xml')
self.results()
return ErrorHandlerReport(True, True,
self.exit_codes.ERROR_SOMETHING_WENT_WRONG)
def _handle_mt_relax_error(self, calculation):
"""
Calculation failed for unknown reason.
"""
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_MT_RADII_RELAX']):
self.ctx.restart_calc = calculation
self.ctx.is_finished = True
self.report('FLEUR calculation failed due to MT overlap.'
' Can be fixed via RelaxBaseWorkChain')
self.results()
return ErrorHandlerReport(True, True,
self.exit_codes.ERROR_MT_RADII_RELAX)
def _handle_vacuum_spill_error(self, calculation):
"""
Calculation failed for unknown reason.
"""
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_VACUUM_SPILL_RELAX']):
self.ctx.restart_calc = calculation
self.ctx.is_finished = True
self.report(
'FLEUR calculation failed because an atom spilled to the vacuum during'
'relaxation. Can be fixed via RelaxBaseWorkChain.')
self.results()
return ErrorHandlerReport(True, True,
self.exit_codes.ERROR_VACUUM_SPILL_RELAX)
def _handle_invalid_elements_mmpmat(self, calculation):
"""
Calculation failed due to invalid elements in the LDA+U density matrix.
Mixing history is reset.
TODO: HOw to handle consecutive errors
"""
if calculation.exit_status in FleurProcess.get_exit_statuses(
['ERROR_INVALID_ELEMENTS_MMPMAT']):
self.ctx.restart_calc = None
self.ctx.is_finished = False
self.report(
'FLEUR calculation failed due to invalid elements in mmpmat. Resetting mixing_history'
)
if 'settings' not in self.ctx.inputs:
self.ctx.inputs.settings = {}
else:
self.ctx.inputs.settings = self.inputs.settings.get_dict()
self.ctx.inputs.settings.setdefault('remove_from_remotecopy_list',
[]).append('mixing_history*')
return ErrorHandlerReport(True, True)
def _handle_general_error(self, calculation):
"""
Calculation failed for unknown reason.
"""
if calculation.exit_status in FleurProcess.get_exit_statuses([
'ERROR_FLEUR_CALC_FAILED', 'ERROR_MT_RADII',
'ERROR_NO_RETRIEVED_FOLDER', 'ERROR_OPENING_OUTPUTS',
'ERROR_NO_OUTXML', 'ERROR_XMLOUT_PARSING_FAILED',
'ERROR_RELAX_PARSING_FAILED'
]):
self.ctx.restart_calc = calculation
self.ctx.is_finished = True
self.report(
'Calculation failed for a reason that can not be resolved automatically'
)
self.results()
return ErrorHandlerReport(True, True,
self.exit_codes.ERROR_SOMETHING_WENT_WRONG)
else:
raise ValueError(
'Calculation failed for unknown reason, please register the '
'corresponding exit code in this error handler')