def workflow_stepper(): # daemon for legacy workflow from aiida.daemon.workflowmanager import execute_steps print "aiida.daemon.tasks.workflowmanager: Checking for workflows to manage" # RUDIMENTARY way to check if this task is already running (to avoid acting # again and again on the same workflow steps) try: stepper_is_running = (get_last_daemon_timestamp('workflow',when='stop') -get_last_daemon_timestamp('workflow',when='start'))<=timedelta(0) except TypeError: # when some timestamps are None (undefined) stepper_is_running = (get_last_daemon_timestamp('workflow',when='stop') is None and get_last_daemon_timestamp('workflow',when='start') is not None) if not stepper_is_running: set_daemon_timestamp(task_name='workflow', when='start') # the previous wf manager stopped already -> we can run a new one print "aiida.daemon.tasks.workflowmanager: running execute_steps" execute_steps() set_daemon_timestamp(task_name='workflow', when='stop') else: print "aiida.daemon.tasks.workflowmanager: execute_steps already running"
def workflow_stepper(): # daemon for legacy workflow configure_logging(daemon=True, daemon_log_file=DAEMON_LOG_FILE) from aiida.daemon.workflowmanager import execute_steps LOGGER.info('Checking for workflows to manage') # RUDIMENTARY way to check if this task is already running (to avoid acting # again and again on the same workflow steps) try: stepper_is_running = (get_last_daemon_timestamp( 'workflow', when='stop') - get_last_daemon_timestamp( 'workflow', when='start')) <= timedelta(0) except TypeError: # when some timestamps are None (undefined) stepper_is_running = ( get_last_daemon_timestamp('workflow', when='stop') is None and get_last_daemon_timestamp('workflow', when='start') is not None) if not stepper_is_running: set_daemon_timestamp(task_name='workflow', when='start') # the previous wf manager stopped already -> we can run a new one LOGGER.info('running execute_steps') execute_steps() set_daemon_timestamp(task_name='workflow', when='stop') else: LOGGER.info('execute_steps already running')
def _list_calculations_old(cls, states=None, past_days=None, group=None, group_pk=None, all_users=False, pks=[], relative_ctime=True): """ Return a string with a description of the AiiDA calculations. .. todo:: does not support the query for the IMPORTED state (since it checks the state in the Attributes, not in the DbCalcState table). Decide which is the correct logic and implement the correct query. :param states: a list of string with states. If set, print only the calculations in the states "states", otherwise shows all. Default = None. :param past_days: If specified, show only calculations that were created in the given number of past days. :param group: If specified, show only calculations belonging to a user-defined group with the given name. Can use colons to separate the group name from the type, as specified in :py:meth:`aiida.orm.group.Group.get_from_string` method. :param group_pk: If specified, show only calculations belonging to a user-defined group with the given PK. :param pks: if specified, must be a list of integers, and only calculations within that list are shown. Otherwise, all calculations are shown. If specified, sets state to None and ignores the value of the ``past_days`` option.") :param relative_ctime: if true, prints the creation time relative from now. (like 2days ago). Default = True :param all_users: if True, list calculation belonging to all users. Default = False :return: a string with description of calculations. """ # I assume that calc_states are strings. If this changes in the future, # update the filter below from dbattributes__tval to the correct field. from aiida.backends.djsite.db.models import DbAuthInfo, DbAttribute from aiida.daemon.timestamps import get_last_daemon_timestamp if states: for state in states: if state not in calc_states: return "Invalid state provided: {}.".format(state) warnings_list = [] now = timezone.now() if pks: q_object = Q(pk__in=pks) else: q_object = Q() if group is not None: g_pk = Group.get_from_string(group).pk q_object.add(Q(dbgroups__pk=g_pk), Q.AND) if group_pk is not None: q_object.add(Q(dbgroups__pk=group_pk), Q.AND) if not all_users: q_object.add(Q(user=get_automatic_user()), Q.AND) if states is not None: q_object.add(Q(dbattributes__key='state', dbattributes__tval__in=states, ), Q.AND) if past_days is not None: now = timezone.now() n_days_ago = now - datetime.timedelta(days=past_days) q_object.add(Q(ctime__gte=n_days_ago), Q.AND) calc_list_pk = list( cls.query(q_object).distinct().values_list('pk', flat=True)) calc_list = cls.query(pk__in=calc_list_pk).order_by('ctime') scheduler_states = dict( DbAttribute.objects.filter(dbnode__pk__in=calc_list_pk, key='scheduler_state').values_list( 'dbnode__pk', 'tval')) # I do the query now, so that the list of pks gets cached calc_list_data = list( calc_list.filter( # dbcomputer__dbauthinfo__aiidauser=F('user') ).distinct().order_by('ctime').values( 'pk', 'dbcomputer__name', 'ctime', 'type', 'dbcomputer__enabled', 'dbcomputer__pk', 'user__pk')) list_comp_pk = [i['dbcomputer__pk'] for i in calc_list_data] list_aiduser_pk = [i['user__pk'] for i in calc_list_data] enabled_data = DbAuthInfo.objects.filter( dbcomputer__pk__in=list_comp_pk, aiidauser__pk__in=list_aiduser_pk ).values_list('dbcomputer__pk', 'aiidauser__pk', 'enabled') enabled_auth_dict = {(i[0], i[1]): i[2] for i in enabled_data} states = {c.pk: c._get_state_string() for c in calc_list} scheduler_lastcheck = dict(DbAttribute.objects.filter( dbnode__in=calc_list, key='scheduler_lastchecktime').values_list('dbnode__pk', 'dval')) ## Get the last daemon check try: last_daemon_check = get_last_daemon_timestamp('updater', when='stop') except ValueError: last_check_string = ("# Last daemon state_updater check: " "(Error while retrieving the information)") else: if last_daemon_check is None: last_check_string = "# Last daemon state_updater check: (Never)" else: last_check_string = ("# Last daemon state_updater check: " "{} ({})".format( str_timedelta(now - last_daemon_check, negative_to_zero=True), timezone.localtime(last_daemon_check).strftime( "at %H:%M:%S on %Y-%m-%d"))) disabled_ignorant_states = [ None, calc_states.FINISHED, calc_states.SUBMISSIONFAILED, calc_states.RETRIEVALFAILED, calc_states.PARSINGFAILED, calc_states.FAILED ] if not calc_list: return last_check_string else: # first save a matrix of results to be printed res_str_list = [last_check_string] str_matrix = [] title = ['# Pk', 'State', 'Creation', 'Sched. state', 'Computer', 'Type'] str_matrix.append(title) len_title = [len(i) for i in title] for calcdata in calc_list_data: remote_state = "None" calc_state = states[calcdata['pk']] remote_computer = calcdata['dbcomputer__name'] try: sched_state = scheduler_states.get(calcdata['pk'], None) if sched_state is None: remote_state = "(unknown)" else: remote_state = '{}'.format(sched_state) if calc_state == calc_states.WITHSCHEDULER: last_check = scheduler_lastcheck.get(calcdata['pk'], None) if last_check is not None: when_string = " {}".format( str_timedelta(now - last_check, short=True, negative_to_zero=True)) verb_string = "was " else: when_string = "" verb_string = "" remote_state = "{}{}{}".format(verb_string, sched_state, when_string) except ValueError: raise calc_module = \ from_type_to_pluginclassname(calcdata['type']).rsplit(".", 1)[0] prefix = 'calculation.job.' prefix_len = len(prefix) if calc_module.startswith(prefix): calc_module = calc_module[prefix_len:].strip() if relative_ctime: calc_ctime = str_timedelta(now - calcdata['ctime'], negative_to_zero=True, max_num_fields=1) else: calc_ctime = " ".join([timezone.localtime( calcdata['ctime']).isoformat().split('T')[0], timezone.localtime(calcdata[ 'ctime']).isoformat().split( 'T')[1].split('.')[ 0].rsplit(":", 1)[0]]) the_state = states[calcdata['pk']] # decide if it is needed to print enabled/disabled information # By default, if the computer is not configured for the # given user, assume it is user_enabled user_enabled = enabled_auth_dict.get( (calcdata['dbcomputer__pk'], calcdata['user__pk']), True) global_enabled = calcdata["dbcomputer__enabled"] enabled = "" if (user_enabled and global_enabled or the_state in disabled_ignorant_states) else " [Disabled]" str_matrix.append([calcdata['pk'], the_state, calc_ctime, remote_state, remote_computer + "{}".format(enabled), calc_module ]) # prepare a formatted text of minimal row length (to fit in terminals!) rows = [] for j in range(len(str_matrix[0])): rows.append([len(str(i[j])) for i in str_matrix]) line_lengths = [str(max(max(rows[i]), len_title[i])) for i in range(len(rows))] fmt_string = "{:<" + "}|{:<".join(line_lengths) + "}" for row in str_matrix: res_str_list.append(fmt_string.format(*[str(i) for i in row])) res_str_list += ["# {}".format(_) for _ in warnings_list] return "\n".join(res_str_list)
def daemon_start(self, *args): """ Start the daemon """ if not is_dbenv_loaded(): from aiida.backends.utils import load_dbenv load_dbenv(process='daemon') from aiida.daemon.timestamps import get_last_daemon_timestamp, set_daemon_timestamp if args: print >> sys.stderr, ( "No arguments allowed for the '{}' command.".format( self.get_full_command_name())) sys.exit(1) from aiida.backends.utils import get_daemon_user from aiida.common.utils import get_configured_user_email daemon_user = get_daemon_user() this_user = get_configured_user_email() if daemon_user != this_user: print "You are not the daemon user! I will not start the daemon." print "(The daemon user is '{}', you are '{}')".format( daemon_user, this_user) print "" print "** FOR ADVANCED USERS ONLY: **" print "To change the current default user, use 'verdi install --only-config'" print "To change the daemon user, use 'verdi daemon configureuser'" sys.exit(1) pid = self.get_daemon_pid() if pid is not None: print "Daemon already running, try asking for its status" return print "Clearing all locks ..." from aiida.orm.lock import LockManager LockManager().clear_all() # rotate an existing log file out of the way if os.path.isfile(self.logfile): with open(self.logfile, 'rb') as curr_log_fh: with gzip.open(self.logfile + '.-1.gz', 'wb') as old_log_fh: shutil.copyfileobj(curr_log_fh, old_log_fh) os.remove(self.logfile) print "Starting AiiDA Daemon (log file: {})...".format(self.logfile) currenv = _get_env_with_venv_bin() process = subprocess.Popen([ "celery", "worker", "--app", "tasks", "--loglevel", "INFO", "--beat", "--schedule", self.celerybeat_schedule, "--logfile", self.logfile, "--pidfile", self._get_pid_full_path(), ], cwd=self.workdir, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=currenv) # The following lines are needed for the workflow_stepper # (re-initialize the timestamps used to lock the task, in case # it crashed for some reason). # TODO: remove them when the old workflow system will be # taken away. try: if (get_last_daemon_timestamp('workflow', when='stop') - get_last_daemon_timestamp('workflow', when='start')) < timedelta(0): logger.info("Workflow stop timestamp was {}; re-initializing " "it to current time".format( get_last_daemon_timestamp('workflow', when='stop'))) print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') except TypeError: # when some timestamps are None (i.e. not present), we make # sure that at least the stop timestamp is defined print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') print "Daemon started"
def daemon_stop(self, *args, **kwargs): """ Stop the daemon. :param wait_for_death: If True, also verifies that the process was already killed. It attempts at most ``max_retries`` times, with ``sleep_between_retries`` seconds between one attempt and the following one (both variables are for the time being hardcoded in the function). :return: None if ``wait_for_death`` is False. True/False if the process was actually dead or after all the retries it was still alive. """ if not is_dbenv_loaded(): from aiida.backends.utils import load_dbenv load_dbenv(process='daemon') from aiida.daemon.timestamps import get_last_daemon_timestamp, set_daemon_timestamp if args: print >> sys.stderr, ( "No arguments allowed for the '{}' command.".format( self.get_full_command_name())) sys.exit(1) wait_for_death = kwargs.get('wait_for_death', True) import time max_retries = 20 sleep_between_retries = 3 # Note: NO check here on the daemon user: allow the daemon to be shut # down if it was inadvertently left active and the setting was changed. self.kill_daemon() dead = None if wait_for_death: dead = False for _ in range(max_retries): pid = self.get_daemon_pid() if pid is None: dead = True print "AiiDA Daemon shut down correctly." # The following lines are needed for the workflow_stepper # (re-initialize the timestamps used to lock the task, in case # it crashed for some reason). # TODO: remove them when the old workflow system will be # taken away. try: if (get_last_daemon_timestamp('workflow', when='stop') - get_last_daemon_timestamp( 'workflow', when='start')) < timedelta(0): logger.info( "Workflow stop timestamp was {}; re-initializing" " it to current time".format( get_last_daemon_timestamp('workflow', when='stop'))) print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') except TypeError: # when some timestamps are None (i.e. not present), we make # sure that at least the stop timestamp is defined print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') break else: print "Waiting for the AiiDA Daemon to shut down..." # Wait two seconds between retries time.sleep(sleep_between_retries) if not dead: print( "Unable to stop (the daemon took too much time to " "shut down).") print("Probably, it is in the middle of a long operation.") print( "The shut down signal was sent, anyway, so it should " "shut down soon.") return dead
def daemon_start(self, *args): """ Start the daemon """ if not is_dbenv_loaded(): from aiida.backends.utils import load_dbenv load_dbenv(process='daemon') from aiida.daemon.timestamps import get_last_daemon_timestamp, set_daemon_timestamp if args: print >> sys.stderr, ( "No arguments allowed for the '{}' command.".format( self.get_full_command_name())) sys.exit(1) from aiida.backends.utils import get_daemon_user from aiida.common.utils import get_configured_user_email daemon_user = get_daemon_user() this_user = get_configured_user_email() if daemon_user != this_user: print "You are not the daemon user! I will not start the daemon." print "(The daemon user is '{}', you are '{}')".format( daemon_user, this_user) print "" print "** FOR ADVANCED USERS ONLY: **" print "To change the current default user, use 'verdi install --only-config'" print "To change the daemon user, use 'verdi daemon configureuser'" sys.exit(1) pid = self.get_daemon_pid() if pid is not None: print "Daemon already running, try asking for its status" return print "Clearing all locks ..." from aiida.orm.lock import LockManager LockManager().clear_all() print "Starting AiiDA Daemon ..." currenv = _get_env_with_venv_bin() process = subprocess.Popen("supervisord -c {}".format( self.conffile_full_path), shell=True, stdout=subprocess.PIPE, env=currenv) process.wait() # The following lines are needed for the workflow_stepper # (re-initialize the timestamps used to lock the task, in case # it crashed for some reason). # TODO: remove them when the old workflow system will be # taken away. try: if (get_last_daemon_timestamp('workflow', when='stop') - get_last_daemon_timestamp('workflow', when='start')) < timedelta(0): logger.info("Workflow stop timestamp was {}; re-initializing " "it to current time".format( get_last_daemon_timestamp('workflow', when='stop'))) print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') except TypeError: # when some timestamps are None (i.e. not present), we make # sure that at least the stop timestamp is defined print "Re-initializing workflow stepper stop timestamp" set_daemon_timestamp(task_name='workflow', when='stop') if (process.returncode == 0): print "Daemon started"