def retriever(): configure_logging(daemon=True, daemon_log_file=DAEMON_LOG_FILE) from aiida.daemon.execmanager import retrieve_jobs LOGGER.info('Checking for calculations to retrieve') set_daemon_timestamp(task_name='retriever', when='start') retrieve_jobs() set_daemon_timestamp(task_name='retriever', when='stop')
def workflow_stepper(): # daemon for legacy workflow from aiida.daemon.workflowmanager import execute_steps print "aiida.daemon.tasks.workflowmanager: Checking for workflows to manage" # RUDIMENTARY way to check if this task is already running (to avoid acting # again and again on the same workflow steps) try: stepper_is_running = (get_last_daemon_timestamp('workflow',when='stop') -get_last_daemon_timestamp('workflow',when='start'))<=timedelta(0) except TypeError: # when some timestamps are None (undefined) stepper_is_running = (get_last_daemon_timestamp('workflow',when='stop') is None and get_last_daemon_timestamp('workflow',when='start') is not None) if not stepper_is_running: set_daemon_timestamp(task_name='workflow', when='start') # the previous wf manager stopped already -> we can run a new one print "aiida.daemon.tasks.workflowmanager: running execute_steps" execute_steps() set_daemon_timestamp(task_name='workflow', when='stop') else: print "aiida.daemon.tasks.workflowmanager: execute_steps already running"
def workflow_stepper(): # daemon for legacy workflow configure_logging(daemon=True, daemon_log_file=DAEMON_LOG_FILE) from aiida.daemon.workflowmanager import execute_steps LOGGER.info('Checking for workflows to manage') # RUDIMENTARY way to check if this task is already running (to avoid acting # again and again on the same workflow steps) try: stepper_is_running = (get_last_daemon_timestamp( 'workflow', when='stop') - get_last_daemon_timestamp( 'workflow', when='start')) <= timedelta(0) except TypeError: # when some timestamps are None (undefined) stepper_is_running = ( get_last_daemon_timestamp('workflow', when='stop') is None and get_last_daemon_timestamp('workflow', when='start') is not None) if not stepper_is_running: set_daemon_timestamp(task_name='workflow', when='start') # the previous wf manager stopped already -> we can run a new one LOGGER.info('running execute_steps') execute_steps() set_daemon_timestamp(task_name='workflow', when='stop') else: LOGGER.info('execute_steps already running')
def retriever(): from aiida.daemon.execmanager import retrieve_jobs print "aiida.daemon.tasks.retrieve: Checking for calculations to retrieve" set_daemon_timestamp(task_name='retriever', when='start') retrieve_jobs() set_daemon_timestamp(task_name='retriever', when='stop')
def updater(): from aiida.daemon.execmanager import update_jobs print "aiida.daemon.tasks.update: Checking for calculations to update" set_daemon_timestamp(task_name='updater', when='start') update_jobs() set_daemon_timestamp(task_name='updater', when='stop')
def submitter(): from aiida.daemon.execmanager import submit_jobs print "aiida.daemon.tasks.submitter: Checking for calculations to submit" set_daemon_timestamp(task_name='submitter', when='start') submit_jobs() set_daemon_timestamp(task_name='submitter', when='stop')
def workflow_stepper(): # daemon for legacy workflow from aiida.daemon.workflowmanager import execute_steps print "aiida.daemon.tasks.workflowmanager: Checking for workflows to manage" set_daemon_timestamp(task_name='workflow', when='start') execute_steps() set_daemon_timestamp(task_name='workflow', when='stop')
def daemon_start(self, *args): """ Start the daemon """ if not is_dbenv_loaded(): from aiida.backends.utils import load_dbenv load_dbenv(process='daemon') from aiida.daemon.timestamps import get_last_daemon_timestamp, set_daemon_timestamp if args: print >> sys.stderr, ( "No arguments allowed for the '{}' command.".format( self.get_full_command_name())) sys.exit(1) from aiida.backends.utils import get_daemon_user from aiida.common.utils import get_configured_user_email daemon_user = get_daemon_user() this_user = get_configured_user_email() if daemon_user != this_user: print "You are not the daemon user! I will not start the daemon." print "(The daemon user is '{}', you are '{}')".format( daemon_user, this_user) print "" print "** FOR ADVANCED USERS ONLY: **" print "To change the current default user, use 'verdi install --only-config'" print "To change the daemon user, use 'verdi daemon configureuser'" sys.exit(1) pid = self.get_daemon_pid() if pid is not None: print "Daemon already running, try asking for its status" return print "Clearing all locks ..." from aiida.orm.lock import LockManager LockManager().clear_all() # rotate an existing log file out of the way if os.path.isfile(self.logfile): with open(self.logfile, 'rb') as curr_log_fh: with gzip.open(self.logfile + '.-1.gz', 'wb') as old_log_fh: shutil.copyfileobj(curr_log_fh, old_log_fh) os.remove(self.logfile) print "Starting AiiDA Daemon (log file: {})...".format(self.logfile) currenv = _get_env_with_venv_bin() process = subprocess.Popen([ "celery", "worker", "--app", "tasks", "--loglevel", "INFO", "--beat", "--schedule", self.celerybeat_schedule, "--logfile", self.logfile, "--pidfile", self._get_pid_full_path(), ], cwd=self.workdir, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=currenv) # The following lines are needed for the workflow_stepper # (re-initialize the timestamps used to lock the task, in case # it crashed for some reason). # TODO: remove them when the old workflow system will be # taken away. try: if (get_last_daemon_timestamp('workflow', when='stop') - get_last_daemon_timestamp('workflow', when='start')) < timedelta(0): logger.info("Workflow stop timestamp was {}; re-initializing " "it to current time".format( get_last_daemon_timestamp('workflow', when='stop'))) print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') except TypeError: # when some timestamps are None (i.e. not present), we make # sure that at least the stop timestamp is defined print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') print "Daemon started"
def daemon_stop(self, *args, **kwargs): """ Stop the daemon. :param wait_for_death: If True, also verifies that the process was already killed. It attempts at most ``max_retries`` times, with ``sleep_between_retries`` seconds between one attempt and the following one (both variables are for the time being hardcoded in the function). :return: None if ``wait_for_death`` is False. True/False if the process was actually dead or after all the retries it was still alive. """ if not is_dbenv_loaded(): from aiida.backends.utils import load_dbenv load_dbenv(process='daemon') from aiida.daemon.timestamps import get_last_daemon_timestamp, set_daemon_timestamp if args: print >> sys.stderr, ( "No arguments allowed for the '{}' command.".format( self.get_full_command_name())) sys.exit(1) wait_for_death = kwargs.get('wait_for_death', True) import time max_retries = 20 sleep_between_retries = 3 # Note: NO check here on the daemon user: allow the daemon to be shut # down if it was inadvertently left active and the setting was changed. self.kill_daemon() dead = None if wait_for_death: dead = False for _ in range(max_retries): pid = self.get_daemon_pid() if pid is None: dead = True print "AiiDA Daemon shut down correctly." # The following lines are needed for the workflow_stepper # (re-initialize the timestamps used to lock the task, in case # it crashed for some reason). # TODO: remove them when the old workflow system will be # taken away. try: if (get_last_daemon_timestamp('workflow', when='stop') - get_last_daemon_timestamp( 'workflow', when='start')) < timedelta(0): logger.info( "Workflow stop timestamp was {}; re-initializing" " it to current time".format( get_last_daemon_timestamp('workflow', when='stop'))) print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') except TypeError: # when some timestamps are None (i.e. not present), we make # sure that at least the stop timestamp is defined print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') break else: print "Waiting for the AiiDA Daemon to shut down..." # Wait two seconds between retries time.sleep(sleep_between_retries) if not dead: print( "Unable to stop (the daemon took too much time to " "shut down).") print("Probably, it is in the middle of a long operation.") print( "The shut down signal was sent, anyway, so it should " "shut down soon.") return dead
def daemon_start(self, *args): """ Start the daemon """ if not is_dbenv_loaded(): from aiida.backends.utils import load_dbenv load_dbenv(process='daemon') from aiida.daemon.timestamps import get_last_daemon_timestamp, set_daemon_timestamp if args: print >> sys.stderr, ( "No arguments allowed for the '{}' command.".format( self.get_full_command_name())) sys.exit(1) from aiida.backends.utils import get_daemon_user from aiida.common.utils import get_configured_user_email daemon_user = get_daemon_user() this_user = get_configured_user_email() if daemon_user != this_user: print "You are not the daemon user! I will not start the daemon." print "(The daemon user is '{}', you are '{}')".format( daemon_user, this_user) print "" print "** FOR ADVANCED USERS ONLY: **" print "To change the current default user, use 'verdi install --only-config'" print "To change the daemon user, use 'verdi daemon configureuser'" sys.exit(1) pid = self.get_daemon_pid() if pid is not None: print "Daemon already running, try asking for its status" return print "Clearing all locks ..." from aiida.orm.lock import LockManager LockManager().clear_all() print "Starting AiiDA Daemon ..." currenv = _get_env_with_venv_bin() process = subprocess.Popen("supervisord -c {}".format( self.conffile_full_path), shell=True, stdout=subprocess.PIPE, env=currenv) process.wait() # The following lines are needed for the workflow_stepper # (re-initialize the timestamps used to lock the task, in case # it crashed for some reason). # TODO: remove them when the old workflow system will be # taken away. try: if (get_last_daemon_timestamp('workflow', when='stop') - get_last_daemon_timestamp('workflow', when='start')) < timedelta(0): logger.info("Workflow stop timestamp was {}; re-initializing " "it to current time".format( get_last_daemon_timestamp('workflow', when='stop'))) print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') except TypeError: # when some timestamps are None (i.e. not present), we make # sure that at least the stop timestamp is defined print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') if (process.returncode == 0): print "Daemon started"