def load_documentation_profile():
"""Load a dummy profile just for the purposes of being able to build the documentation.
The building of the documentation will require importing the `aiida` package and some code will try to access the
loaded configuration and profile, which if not done will except. On top of that, Django will raise an exception if
the database models are loaded before its settings are loaded. This also is taken care of by loading a Django
profile and loading the corresponding backend. Calling this function will perform all these requirements allowing
the documentation to be built without having to install and configure AiiDA nor having an actual database present.
"""
import tempfile
from aiida.manage.manager import get_manager
from .config import Config
from .profile import Profile
global PROFILE
global CONFIG
with tempfile.NamedTemporaryFile() as handle:
profile_name = 'readthedocs'
profile = {
'AIIDADB_ENGINE': 'postgresql_psycopg2',
'AIIDADB_BACKEND': 'django',
'AIIDADB_PORT': 5432,
'AIIDADB_HOST': 'localhost',
'AIIDADB_NAME': 'aiidadb',
'AIIDADB_PASS': '******',
'AIIDADB_USER': '******',
'AIIDADB_REPOSITORY_URI': 'file:///dev/null',
}
config = {'default_profile': profile_name, 'profiles': {profile_name: profile}}
PROFILE = Profile(profile_name, profile, from_config=True)
CONFIG = Config(handle.name, config)
get_manager()._load_backend(schema_check=False) # pylint: disable=protected-access
def load_backend_if_not_loaded():
"""Load the current profile if necessary while running the spinner to show command hasn't crashed."""
from aiida.manage.configuration import load_profile
from aiida.manage.manager import get_manager
with spinner():
load_profile()
get_manager().get_backend()
def store_attributes(self):
"""Store new attributes in Node extras and reset self._node"""
if self.new_attributes:
optimade = self._get_optimade_extras()
if optimade:
optimade.update(self.new_attributes)
else:
optimade = self.new_attributes
extras = (self._get_unique_node_property("extras")
if self._get_unique_node_property("extras") else {})
extras["optimade"] = optimade
profile = get_manager().get_profile()
if profile.database_backend == "django":
from aiida.backends.djsite.db.models import DbNode
with get_manager().get_backend().transaction():
DbNode.objects.filter(pk=self._pk).update(extras=extras)
elif profile.database_backend == "sqlalchemy":
from aiida.backends.sqlalchemy.models.node import DbNode
with get_manager().get_backend().transaction() as session:
session.query(DbNode).filter(DbNode.id == self._pk).update(
values={"extras": extras})
else:
raise AiidaError(
f'Unknown AiiDA backend "{profile.database_backend}" for profile'
f"{profile}")
# For posterity, this is how to do the same, going through AiiDA's API:
# self._node.set_extra(self.EXTRAS_KEY, optimade)
# Lastly, reset NODE in an attempt to remove it from memory
self._node = None
def wrapper(*args, **kwargs):
try:
value = func(*args, **kwargs)
finally:
from aiida.manage.manager import get_manager
get_manager().get_backend().get_session().close()
return value
def submit(process, **inputs):
"""Submit the process with the supplied inputs to the daemon immediately returning control to the interpreter.
.. warning: this should not be used within another process. Instead, there one should use the `submit` method of
the wrapping process itself, i.e. use `self.submit`.
.. warning: submission of processes requires `store_provenance=True`
:param process: the process class to submit
:type process: :class:`aiida.engine.Process`
:param inputs: the inputs to be passed to the process
:type inputs: dict
:return: the calculation node of the process
:rtype: :class:`aiida.orm.ProcessNode`
"""
assert not is_process_function(process), 'Cannot submit a process function'
# Submitting from within another process requires `self.submit` unless it is a work function, in which case the
# current process in the scope should be an instance of `FunctionProcess`
if is_process_scoped() and not isinstance(Process.current(),
FunctionProcess):
raise InvalidOperation(
'Cannot use top-level `submit` from within another process, use `self.submit` instead'
)
runner = manager.get_manager().get_runner()
controller = manager.get_manager().get_process_controller()
process = instantiate_process(runner, process, **inputs)
# If a dry run is requested, simply forward to `run`, because it is not compatible with `submit`. We choose for this
# instead of raising, because in this way the user does not have to change the launcher when testing.
if process.metadata.get('dry_run', False):
_, node = run_get_node(process)
return node
if not process.metadata.store_provenance:
raise InvalidOperation(
'cannot submit a process with `store_provenance=False`')
runner.persister.save_checkpoint(process)
process.close()
# Do not wait for the future's result, because in the case of a single worker this would c**k-block itself
controller.continue_process(process.pid, nowait=False, no_reply=True)
return process.node
def setUpClass(cls, *args, **kwargs):
# Note: this will raise an exception, that will be seen as a test
# failure. To be safe, you should do the same check also in the tearDownClass
# to avoid that it is run
check_if_tests_can_run()
# Force the loading of the backend which will load the required database environment
get_manager().get_backend()
cls.__backend_instance = cls.get_backend_class()()
cls.__backend_instance.setUpClass_method(*args, **kwargs)
cls.backend = cls.__backend_instance.backend
cls.insert_data()
cls._class_was_setup = True
def test_simple_kill_through_process(self):
"""
Run the workchain for one step and then kill it. This should have the
workchain and its children end up in the KILLED state.
"""
runner = get_manager().get_runner()
process = TestWorkChainAbortChildren.MainWorkChain(inputs={'kill': Bool(True)})
@gen.coroutine
def run_async():
yield run_until_waiting(process)
process.kill()
with self.assertRaises(plumpy.KilledError):
yield process.future()
runner.schedule(process)
runner.loop.run_sync(lambda: run_async())
child = process.node.get_outgoing(link_type=LinkType.CALL_WORK).first().node
self.assertEqual(child.is_finished_ok, False)
self.assertEqual(child.is_excepted, False)
self.assertEqual(child.is_killed, True)
self.assertEqual(process.node.is_finished_ok, False)
self.assertEqual(process.node.is_excepted, False)
self.assertEqual(process.node.is_killed, True)
def test_simple_kill_through_process(self):
"""
Run the workchain for one step and then kill it by calling kill
on the workchain itself. This should have the workchain end up
in the KILLED state.
"""
runner = get_manager().get_runner()
process = TestWorkChainAbort.AbortableWorkChain()
@gen.coroutine
def run_async():
yield run_until_paused(process)
self.assertTrue(process.paused)
process.kill()
with self.assertRaises(plumpy.ClosedError):
launch.run(process)
runner.schedule(process)
runner.loop.run_sync(lambda: run_async())
self.assertEqual(process.node.is_finished_ok, False)
self.assertEqual(process.node.is_excepted, False)
self.assertEqual(process.node.is_killed, True)
def test_if_block_persistence(self):
"""
This test was created to capture issue #902
"""
runner = get_manager().get_runner()
wc = IfTest()
runner.schedule(wc)
@gen.coroutine
def run_async(workchain):
yield run_until_paused(workchain)
self.assertTrue(workchain.ctx.s1)
self.assertFalse(workchain.ctx.s2)
# Now bundle the thing
bundle = plumpy.Bundle(workchain)
# Need to close the process before recreating a new instance
workchain.close()
# Load from saved state
workchain2 = bundle.unbundle()
self.assertTrue(workchain2.ctx.s1)
self.assertFalse(workchain2.ctx.s2)
bundle2 = plumpy.Bundle(workchain2)
self.assertDictEqual(bundle, bundle2)
workchain.play()
yield workchain.future()
self.assertTrue(workchain.ctx.s1)
self.assertTrue(workchain.ctx.s2)
runner.loop.run_sync(lambda: run_async(wc))
def set_process_state_change_timestamp(process):
"""
Set the global setting that reflects the last time a process changed state, for the process type
of the given process, to the current timestamp. The process type will be determined based on
the class of the calculation node it has as its database container.
:param process: the Process instance that changed its state
"""
from aiida.common import timezone
from aiida.common.exceptions import UniquenessError
from aiida.manage.manager import get_manager # pylint: disable=cyclic-import
from aiida.orm import ProcessNode, CalculationNode, WorkflowNode
if isinstance(process.node, CalculationNode):
process_type = 'calculation'
elif isinstance(process.node, WorkflowNode):
process_type = 'work'
elif isinstance(process.node, ProcessNode):
# This will only occur for testing, as in general users cannot launch plain Process classes
return
else:
raise ValueError('unsupported calculation node type {}'.format(type(process.node)))
key = PROCESS_STATE_CHANGE_KEY.format(process_type)
description = PROCESS_STATE_CHANGE_DESCRIPTION.format(process_type)
value = timezone.datetime_to_isoformat(timezone.now())
try:
manager = get_manager()
manager.get_backend_manager().get_settings_manager().set(key, value, description)
except UniquenessError as exception:
process.logger.debug('could not update the {} setting because of a UniquenessError: {}'.format(key, exception))
def load_instance_state(self, saved_state, load_context):
"""Load instance state.
:param saved_state: saved instance state
:param load_context:
:type load_context: :class:`!plumpy.persistence.LoadSaveContext`
"""
from aiida.manage import manager
if 'runner' in load_context:
self._runner = load_context.runner
else:
self._runner = manager.get_manager().get_runner()
load_context = load_context.copyextend(loop=self._runner.loop, communicator=self._runner.communicator)
super().load_instance_state(saved_state, load_context)
if self.SaveKeys.CALC_ID.value in saved_state:
self._node = orm.load_node(saved_state[self.SaveKeys.CALC_ID.value])
self._pid = self.node.pk
else:
self._pid = self._create_and_setup_db_record()
self.node.logger.info(f'Loaded process<{self.node.pk}> from saved state')
def start_daemon() -> None:
"""Start a daemon runner for the currently configured profile."""
daemon_client = get_daemon_client()
configure_logging(daemon=True, daemon_log_file=daemon_client.daemon_log_file)
try:
manager = get_manager()
runner = manager.create_daemon_runner()
manager.set_runner(runner)
except Exception:
LOGGER.exception('daemon runner failed to start')
raise
signals = (signal.SIGTERM, signal.SIGINT)
for s in signals: # pylint: disable=invalid-name
runner.loop.add_signal_handler(s, lambda s=s: asyncio.create_task(shutdown_runner(runner)))
try:
LOGGER.info('Starting a daemon runner')
runner.start()
except SystemError as exception:
LOGGER.info('Received a SystemError: %s', exception)
runner.close()
LOGGER.info('Daemon runner started')
def __init__(
self, name, hostname, description='', transport_type='', scheduler_type='', workdir=None, backend=None
):
"""Construct a new computer
:type name: str
:type hostname: str
:type description: str
:type transport_type: str
:type scheduler_type: str
:type workdir: str
:type backend: :class:`aiida.orm.implementation.Backend`
:rtype: :class:`aiida.orm.Computer`
"""
# pylint: disable=too-many-arguments
backend = backend or get_manager().get_backend()
model = backend.computers.create(
name=name,
hostname=hostname,
description=description,
transport_type=transport_type,
scheduler_type=scheduler_type
)
super(Computer, self).__init__(model)
if workdir is not None:
self.set_workdir(workdir)
def detect_invalid_nodes():
"""Scan the database for invalid nodes."""
from tabulate import tabulate
from aiida.manage.database.integrity.sql.nodes import INVALID_NODE_SELECT_STATEMENTS
from aiida.manage.manager import get_manager
integrity_violated = False
backend = get_manager().get_backend()
for check in INVALID_NODE_SELECT_STATEMENTS:
result = backend.execute_prepared_statement(check.sql,
check.parameters)
if result:
integrity_violated = True
echo.echo_warning(f'{check.message}:\n')
echo.echo(tabulate(result, headers=check.headers))
if not integrity_violated:
echo.echo_success('no integrity violations detected')
else:
echo.echo_critical('one or more integrity violations detected')
def load_instance_state(
self, saved_state: MutableMapping[str, Any],
load_context: plumpy.persistence.LoadSaveContext) -> None:
"""Load instance state.
:param saved_state: saved instance state
:param load_context:
"""
from aiida.manage import manager
if 'runner' in load_context:
self._runner = load_context.runner
else:
self._runner = manager.get_manager().get_runner()
load_context = load_context.copyextend(
loop=self._runner.loop, communicator=self._runner.communicator)
super().load_instance_state(saved_state, load_context)
if self.SaveKeys.CALC_ID.value in saved_state:
self._node = orm.load_node(
saved_state[self.SaveKeys.CALC_ID.value])
self._pid = self.node.pk # pylint: disable=attribute-defined-outside-init
else:
self._pid = self._create_and_setup_db_record() # pylint: disable=attribute-defined-outside-init
self.node.logger.info(
f'Loaded process<{self.node.pk}> from saved state')
def generate_calcinfo(entry_point_name, folder, inputs=None):
"""Generate a `CalcInfo` instance for testing calculation jobs.
A new `CalcJob` process instance is instantiated,
and `prepare_for_submission` is called to populate the supplied folder,
with raw inputs.
Parameters
----------
entry_point_name: str
folder: aiida.common.folders.Folder
inputs: dict or None
"""
from aiida.engine.utils import instantiate_process
from aiida.manage.manager import get_manager
from aiida.plugins import CalculationFactory
manager = get_manager()
runner = manager.get_runner()
process_class = CalculationFactory(entry_point_name)
process = instantiate_process(runner, process_class, **inputs)
calc_info = process.prepare_for_submission(folder)
return calc_info
def generate_context(wkchain_cls, inputs, outline_steps):
"""instantiate a WorkChain,
call a list of methods (that should be part of `spec.outline`),
then return a sanitized version of the workchain context for testing
"""
from aiida.common.extendeddicts import AttributeDict
from aiida.engine import ProcessBuilder
from aiida.engine.utils import instantiate_process
from aiida.manage.manager import get_manager
from aiida.orm import Node
class ContextDumper(yaml.Dumper):
"""Custom yaml dumper for a process context."""
def represent_data(self, data):
if isinstance(data, Node):
data = str(data.__class__)
if isinstance(data, AttributeDict):
data = dict(data)
return super(ContextDumper, self).represent_data(data)
manager = get_manager()
runner = manager.get_runner()
if isinstance(inputs, ProcessBuilder):
wkchain = instantiate_process(runner, inputs)
else:
wkchain = instantiate_process(runner, wkchain_cls, **inputs)
step_outcomes = []
for step in outline_steps:
step_outcomes.append(getattr(wkchain, step)())
context = yaml.dump(wkchain.ctx, Dumper=ContextDumper)
return wkchain, step_outcomes, yaml.safe_load(context)
def detect_duplicate_uuid(table, apply_patch):
"""Detect and fix entities with duplicate UUIDs.
Before aiida-core v1.0.0, there was no uniqueness constraint on the UUID column of the node table in the database
and a few other tables as well. This made it possible to store multiple entities with identical UUIDs in the same
table without the database complaining. This bug was fixed in aiida-core=1.0.0 by putting an explicit uniqueness
constraint on UUIDs on the database level. However, this would leave databases created before this patch with
duplicate UUIDs in an inconsistent state. This command will run an analysis to detect duplicate UUIDs in a given
table and solve it by generating new UUIDs. Note that it will not delete or merge any rows.
"""
from aiida.manage.database.integrity.duplicate_uuid import deduplicate_uuids
from aiida.manage.manager import get_manager
manager = get_manager()
manager._load_backend(schema_check=False) # pylint: disable=protected-access
try:
messages = deduplicate_uuids(table=table, dry_run=not apply_patch)
except Exception as exception: # pylint: disable=broad-except
echo.echo_critical('integrity check failed: {}'.format(str(exception)))
else:
for message in messages:
echo.echo_info(message)
if apply_patch:
echo.echo_success('integrity patch completed')
else:
echo.echo_success('dry-run of integrity patch completed')
def __init__(self, **kwargs):
"""
Initialise the parameters.
Create the basic query_help
"""
# basic initialization
super().__init__(**kwargs)
self._default_projections = [
'id', 'label', 'node_type', 'process_type', 'ctime', 'mtime',
'uuid', 'user_id'
]
# Inspect the subclasses of NodeTranslator, to avoid hard-coding
# (should resemble the following tree)
r"""
/- CodeTranslator
/
/- KpointsTranslator
/
/- DataTranslator -- StructureTranslator
/ \
/ \- BandsTranslator
/
NodeTranslator
\
\- CalculationTranslator
"""
self._subclasses = self._get_subclasses()
self._backend = get_manager().get_backend()
def delete_nodes_and_connections_sqla(pks_to_delete): # pylint: disable=invalid-name
"""
Delete all nodes corresponding to pks in the input.
:param pks_to_delete: A list, tuple or set of pks that should be deleted.
"""
# pylint: disable=no-value-for-parameter
from aiida.backends.sqlalchemy.models.node import DbNode, DbLink
from aiida.backends.sqlalchemy.models.group import table_groups_nodes
from aiida.manage.manager import get_manager
backend = get_manager().get_backend()
with backend.transaction() as session:
# I am first making a statement to delete the membership of these nodes to groups.
# Since table_groups_nodes is a sqlalchemy.schema.Table, I am using expression language to compile
# a stmt to be executed by the session. It works, but it's not nice that two different ways are used!
# Can this be changed?
stmt = table_groups_nodes.delete().where(
table_groups_nodes.c.dbnode_id.in_(list(pks_to_delete)))
session.execute(stmt)
# First delete links, then the Nodes, since we are not cascading deletions.
# Here I delete the links coming out of the nodes marked for deletion.
session.query(DbLink).filter(DbLink.input_id.in_(
list(pks_to_delete))).delete(synchronize_session='fetch')
# Here I delete the links pointing to the nodes marked for deletion.
session.query(DbLink).filter(DbLink.output_id.in_(
list(pks_to_delete))).delete(synchronize_session='fetch')
# Now I am deleting the nodes
session.query(DbNode).filter(DbNode.id.in_(
list(pks_to_delete))).delete(synchronize_session='fetch')
def __init__(self, inputs=None, logger=None, runner=None, parent_pid=None, enable_persistence=True):
""" Process constructor.
:param inputs: process inputs
:type inputs: dict
:param logger: aiida logger
:type logger: :class:`logging.Logger`
:param runner: process runner
:type: :class:`aiida.engine.runners.Runner`
:param parent_pid: id of parent process
:type parent_pid: int
:param enable_persistence: whether to persist this process
:type enable_persistence: bool
"""
from aiida.manage import manager
self._runner = runner if runner is not None else manager.get_manager().get_runner()
super().__init__(
inputs=self.spec().inputs.serialize(inputs),
logger=logger,
loop=self._runner.loop,
communicator=self.runner.communicator)
self._node = None
self._parent_pid = parent_pid
self._enable_persistence = enable_persistence
if self._enable_persistence and self.runner.persister is None:
self.logger.warning('Disabling persistence, runner does not have a persister')
self._enable_persistence = False
def get_num_workers():
"""
Get the number of active daemon workers from the circus client
"""
from aiida.common.exceptions import CircusCallError
from aiida.manage.manager import get_manager
manager = get_manager()
client = manager.get_daemon_client()
if client.is_daemon_running:
response = client.get_numprocesses()
if response['status'] != 'ok':
if response['status'] == client.DAEMON_ERROR_TIMEOUT:
raise CircusCallError(
'verdi thought the daemon was alive, but the call to the daemon timed-out'
)
elif response['status'] == client.DAEMON_ERROR_NOT_RUNNING:
raise CircusCallError(
'verdi thought the daemon was running, but really it is not'
)
else:
raise CircusCallError
try:
return response['numprocesses']
except KeyError:
raise CircusCallError(
'Circus did not return the number of daemon processes')
def get_creation_statistics(self, user_pk=None):
"""
Return a dictionary with the statistics of node creation, summarized by day,
optimized for the Django backend.
:note: Days when no nodes were created are not present in the returned `ctime_by_day` dictionary.
:param user_pk: If None (default), return statistics for all users.
If user pk is specified, return only the statistics for the given user.
:return: a dictionary as
follows::
{
"total": TOTAL_NUM_OF_NODES,
"types": {TYPESTRING1: count, TYPESTRING2: count, ...},
"ctime_by_day": {'YYYY-MMM-DD': count, ...}
where in `ctime_by_day` the key is a string in the format 'YYYY-MM-DD' and the value is
an integer with the number of nodes created that day."""
# pylint: disable=no-member
import sqlalchemy as sa
import aiida.backends.djsite.db.models as djmodels
from aiida.manage.manager import get_manager
backend = get_manager().get_backend()
# Get the session (uses internally aldjemy - so, sqlalchemy) also for the Djsite backend
session = backend.get_session()
retdict = {}
total_query = session.query(djmodels.DbNode.sa)
types_query = session.query(
djmodels.DbNode.sa.node_type.label('typestring'),
sa.func.count(djmodels.DbNode.sa.id))
stat_query = session.query(
sa.func.date_trunc('day', djmodels.DbNode.sa.ctime).label('cday'),
sa.func.count(djmodels.DbNode.sa.id))
if user_pk is not None:
total_query = total_query.filter(
djmodels.DbNode.sa.user_id == user_pk)
types_query = types_query.filter(
djmodels.DbNode.sa.user_id == user_pk)
stat_query = stat_query.filter(
djmodels.DbNode.sa.user_id == user_pk)
# Total number of nodes
retdict['total'] = total_query.count()
# Nodes per type
retdict['types'] = dict(types_query.group_by('typestring').all())
# Nodes created per day
stat = stat_query.group_by('cday').order_by('cday').all()
ctime_by_day = {_[0].strftime('%Y-%m-%d'): _[1] for _ in stat}
retdict['ctime_by_day'] = ctime_by_day
return retdict
def start_daemon():
"""Start a daemon runner for the currently configured profile."""
daemon_client = get_daemon_client()
configure_logging(daemon=True,
daemon_log_file=daemon_client.daemon_log_file)
try:
manager = get_manager()
runner = manager.create_daemon_runner()
manager.set_runner(runner)
except Exception as exception:
LOGGER.exception('daemon runner failed to start')
raise
def shutdown_daemon(_num, _frame):
LOGGER.info('Received signal to shut down the daemon runner')
runner.close()
signal.signal(signal.SIGINT, shutdown_daemon)
signal.signal(signal.SIGTERM, shutdown_daemon)
LOGGER.info('Starting a daemon runner')
try:
runner.start()
except SystemError as exception:
LOGGER.info('Received a SystemError: %s', exception)
runner.close()
LOGGER.info('Daemon runner stopped')
