def update(self): """Update tree, if necessary.""" if self.next_scan_time is not None and self.next_scan_time > time(): return # Scan for running suites choices = [] for host, port, suite_identity in scan_many(get_scan_items_from_fs(), timeout=self.timeout): name = suite_identity[KEY_NAME] owner = suite_identity[KEY_OWNER] if is_remote_user(owner): continue # current user only auth = "%s:%s" % (host, port) choices.append((name, auth)) choices.sort() self.next_scan_time = time() + self.SCAN_INTERVAL if choices == self.running_choices: return # Update tree if running suites changed self.running_choices = choices self.construct_newtree() self.update_treestore(self.newtree, self.regd_treestore.get_iter_first())
def update(self): """Update tree, if necessary.""" if self.next_scan_time is not None and self.next_scan_time > time(): return # Scan for running suites choices = [] for host, port, suite_identity in scan_many( get_scan_items_from_fs(), timeout=self.timeout): name = suite_identity[KEY_NAME] owner = suite_identity[KEY_OWNER] if is_remote_user(owner): continue # current user only auth = "%s:%s" % (host, port) choices.append((name, auth)) choices.sort() self.next_scan_time = time() + self.SCAN_INTERVAL if choices == self.running_choices: return # Update tree if running suites changed self.running_choices = choices self.construct_newtree() self.update_treestore( self.newtree, self.regd_treestore.get_iter_first())
def update_suites_info(updater, full_mode=False): """Return mapping of suite info by host, owner and suite name. Args: updater (object): gscan or gpanel updater: Compulsory attributes from updater: hosts: hosts to scan owner_pattern: re to filter results by owners suite_info_map: previous results returned by this function Optional attributes from updater: timeout: communication timeout full_mode (boolean): update in full mode? Return: dict: {(host, owner, name): suite_info, ...} where each "suite_info" is a dict with keys: KEY_GROUP: group name of suite KEY_META: suite metadata (new in 7.6) KEY_OWNER: suite owner name KEY_PORT: suite port, for running suites only KEY_STATES: suite state KEY_TASKS_BY_STATE: tasks by state KEY_TITLE: suite title KEY_UPDATE_TIME: last update time of suite """ # Compulsory attributes from updater # hosts - hosts to scan, or the default set in the site/user global.rc # owner_pattern - return only suites with owners matching this compiled re # suite_info_map - previous results returned by this function # Optional attributes from updater # timeout - communication timeout owner_pattern = updater.owner_pattern timeout = getattr(updater, "comms_timeout", None) # name_pattern - return only suites with names matching this compiled re name_pattern = getattr(updater, "name_pattern", None) # Determine items to scan results = {} items = [] if full_mode and updater.hosts: # Scan full port range on all hosts items.extend(updater.hosts) if owner_pattern is None: owner_pattern = re.compile(r"\A" + get_user() + r"\Z") elif full_mode: # Get (host, port) list from file system items.extend(get_scan_items_from_fs(owner_pattern, updater)) else: # Scan suites in previous results only for (host, owner, name), prev_result in updater.suite_info_map.items(): port = prev_result.get(KEY_PORT) if port: items.append((host, port)) else: results[(host, owner, name)] = prev_result if not items: return results if cylc.flags.debug: sys.stderr.write( 'Scan items:%s%s\n' % (DEBUG_DELIM, DEBUG_DELIM.join(str(item) for item in items))) # Scan for host, port, result in scan_many(items, timeout=timeout, updater=updater): if updater.quit: return if (name_pattern and not name_pattern.match(result[KEY_NAME]) or owner_pattern and not owner_pattern.match(result[KEY_OWNER])): continue try: result[KEY_PORT] = port results[(host, result[KEY_OWNER], result[KEY_NAME])] = result result[KEY_UPDATE_TIME] = int(float(result[KEY_UPDATE_TIME])) except (KeyError, TypeError, ValueError): pass expire_threshold = time() - DURATION_EXPIRE_STOPPED for (host, owner, name), prev_result in updater.suite_info_map.items(): if updater.quit: return if ((host, owner, name) in results or owner_pattern and not owner_pattern.match(owner) or name_pattern and not name_pattern.match(name)): # OK if suite already in current results set. # Don't bother if: # * previous owner does not match current owner pattern # * previous suite name does not match current name pattern continue if prev_result.get(KEY_PORT): # A previously running suite is no longer running. # Get suite info with "ls-checkpoints", if possible, and include in # the results set. try: prev_result.update( _update_stopped_suite_info((host, owner, name))) del prev_result[KEY_PORT] except (IndexError, TypeError, ValueError): continue if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold: results[(host, owner, name)] = prev_result return results
def update_suites_info(updater, full_mode=False): """Return mapping of suite info by host, owner and suite name. Args: updater (object): gscan or gpanel updater: Compulsory attributes from updater: hosts: hosts to scan owner_pattern: re to filter results by owners suite_info_map: previous results returned by this function Optional attributes from updater: timeout: communication timeout full_mode (boolean): update in full mode? Return: dict: {(host, owner, name): suite_info, ...} where each "suite_info" is a dict with keys: KEY_GROUP: group name of suite KEY_META: suite metadata (new in 7.6) KEY_OWNER: suite owner name KEY_PORT: suite port, for running suites only KEY_STATES: suite state KEY_TASKS_BY_STATE: tasks by state KEY_TITLE: suite title KEY_UPDATE_TIME: last update time of suite """ # Compulsory attributes from updater # hosts - hosts to scan, or the default set in the site/user global.rc # owner_pattern - return only suites with owners matching this compiled re # suite_info_map - previous results returned by this function # Optional attributes from updater # timeout - communication timeout owner_pattern = updater.owner_pattern timeout = getattr(updater, "comms_timeout", None) # name_pattern - return only suites with names matching this compiled re name_pattern = getattr(updater, "name_pattern", None) # Determine items to scan results = {} items = [] if full_mode and updater.hosts: # Scan full port range on all hosts items.extend(updater.hosts) if owner_pattern is None: owner_pattern = re.compile(r"\A" + get_user() + r"\Z") elif full_mode: # Get (host, port) list from file system items.extend(get_scan_items_from_fs(owner_pattern, updater)) else: # Scan suites in previous results only for (host, owner, name), prev_result in updater.suite_info_map.items(): port = prev_result.get(KEY_PORT) if port: items.append((host, port)) else: results[(host, owner, name)] = prev_result if not items: return results if cylc.flags.debug: sys.stderr.write('Scan items:%s%s\n' % ( DEBUG_DELIM, DEBUG_DELIM.join(str(item) for item in items))) # Scan for host, port, result in scan_many( items, timeout=timeout, updater=updater): if updater.quit: return if (name_pattern and not name_pattern.match(result[KEY_NAME]) or owner_pattern and not owner_pattern.match(result[KEY_OWNER])): continue try: result[KEY_PORT] = port results[(host, result[KEY_OWNER], result[KEY_NAME])] = result result[KEY_UPDATE_TIME] = int(float(result[KEY_UPDATE_TIME])) except (KeyError, TypeError, ValueError): pass expire_threshold = time() - DURATION_EXPIRE_STOPPED for (host, owner, name), prev_result in updater.suite_info_map.items(): if updater.quit: return if ((host, owner, name) in results or owner_pattern and not owner_pattern.match(owner) or name_pattern and not name_pattern.match(name)): # OK if suite already in current results set. # Don't bother if: # * previous owner does not match current owner pattern # * previous suite name does not match current name pattern continue if prev_result.get(KEY_PORT): # A previously running suite is no longer running. # Get suite info with "ls-checkpoints", if possible, and include in # the results set. try: prev_result.update( _update_stopped_suite_info((host, owner, name))) del prev_result[KEY_PORT] except (IndexError, TypeError, ValueError): continue if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold: results[(host, owner, name)] = prev_result return results
def update_suites_info(updater, full_mode=False): """Return mapping of suite info by host, owner and suite name. Args: updater (object): gscan or gpanel updater: Compulsory attributes from updater: hosts: hosts to scan owner_pattern: re to filter results by owners suite_info_map: previous results returned by this function Optional attributes from updater: timeout: communication timeout full_mode (boolean): update in full mode? Return: dict: {(host, owner, name): suite_info, ...} where each "suite_info" is a dict with keys: KEY_GROUP: group name of suite KEY_OWNER: suite owner name KEY_PORT: suite port, for running suites only KEY_STATES: suite state KEY_TASKS_BY_STATE: tasks by state KEY_TITLE: suite title KEY_UPDATE_TIME: last update time of suite """ # Compulsory attributes from updater # hosts - hosts to scan, or the default set in the site/user global.rc # owner_pattern - return only suites with owners matching this compiled re # suite_info_map - previous results returned by this function # Optional attributes from updater # timeout - communication timeout owner_pattern = updater.owner_pattern timeout = getattr(updater, "comms_timeout", None) # name_pattern - return only suites with names matching this compiled re name_pattern = getattr(updater, "name_pattern", None) # Determine items to scan results = {} items = [] if full_mode and not updater.hosts: # Scan users suites. Walk "~/cylc-run/" to get (host, port) from # ".service/contact" for active suites suite_srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [GLOBAL_CFG.get_host_item('run directory')] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append( GLOBAL_CFG.get_host_item('run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir)) if cylc.flags.debug: sys.stderr.write( 'Listing suites:%s%s\n' % (_UPDATE_DEBUG_DELIM, _UPDATE_DEBUG_DELIM.join(run_dirs))) for run_d in run_dirs: for dirpath, dnames, fnames in os.walk(run_d, followlinks=True): if updater.quit: return # Always descend for top directory, but # don't descend further if it has a: # * .service/ # * cylc-suite.db: (pre-cylc-7 suites don't have ".service/"). if dirpath != run_d and (suite_srv_files_mgr.DIR_BASE_SRV in dnames or 'cylc-suite.db' in fnames): dnames[:] = [] # Choose only suites with .service and matching filter reg = os.path.relpath(dirpath, run_d) try: contact_data = suite_srv_files_mgr.load_contact_file(reg) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: items.append((contact_data[suite_srv_files_mgr.KEY_HOST], contact_data[suite_srv_files_mgr.KEY_PORT])) elif full_mode: # Scan full port range on all hosts items.extend(updater.hosts) else: # Scan suites in previous results only for (host, owner, name), prev_result in updater.suite_info_map.items(): port = prev_result.get(KEY_PORT) if port: items.append((host, port)) else: results[(host, owner, name)] = prev_result if not items: return results if cylc.flags.debug: sys.stderr.write( 'Scan items:%s%s\n' % (_UPDATE_DEBUG_DELIM, _UPDATE_DEBUG_DELIM.join(str(item) for item in items))) # Scan for host, port, result in scan_many(items, timeout=timeout, updater=updater): if updater.quit: return if (name_pattern and not name_pattern.match(result[KEY_NAME]) or owner_pattern and not owner_pattern.match(result[KEY_OWNER])): continue try: result[KEY_PORT] = port results[(host, result[KEY_OWNER], result[KEY_NAME])] = result result[KEY_UPDATE_TIME] = int(float(result[KEY_UPDATE_TIME])) except (KeyError, TypeError, ValueError): pass expire_threshold = time() - DURATION_EXPIRE_STOPPED for (host, owner, name), prev_result in updater.suite_info_map.items(): if updater.quit: return if ((host, owner, name) in results or owner_pattern and not owner_pattern.match(owner) or name_pattern and not name_pattern.match(name)): # OK if suite already in current results set. # Don't bother if: # * previous owner does not match current owner pattern # * previous suite name does not match current name pattern continue if prev_result.get(KEY_PORT): # A previously running suite is no longer running. # Get suite info with "cat-state", if possible, and include in the # results set. try: prev_result = _update_stopped_suite_info((host, owner, name)) except (IndexError, TypeError, ValueError): continue if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold: results[(host, owner, name)] = prev_result return results