def get_user_account_shares(self, user): """Get the list of accounts & fairshare value associated with a user""" fair_share = {} cmd = [ self.sshare, '-U', '-u', '{}'.format(user), '--format=Account,User,Fairshare', '--noheader', '--parsable2' ] getent = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, query=True) (stdout, _) = getent.communicate() for assoc in stdout.decode().splitlines(): if user in assoc: fields = assoc.split('|') if fields[1].strip( ) == user and fields[0].strip() != "default": if fields[0].strip() in self.get_mx_caps(): fair_share[fields[0]] = float(-1) else: fair_share[fields[0]] = fields[2] if fair_share is not None and len(fair_share) > 0: self.log.debug("Found fairshare value(s) for %s", user) return fair_share else: self.log.debug("Fairshare value not found for %s", user) return None
def getusers(self, account): """Get the list of users currently able to use the account""" self.log.debug( "Retrieving list of users under {} account".format(account)) user_set = set() cmd = [ self.sacctmgr, 'show', 'associations', '--parsable2', '--noheader', 'account={}'.format(account), 'format=user' ] sacctmgr = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, query=True) (stdout, _) = sacctmgr.communicate() for user in stdout.splitlines(): user = user.decode('utf-8') if user is not None and user is not "": user_set.add(user) self.log.debug("List of users under {} account: {}".format( account, user_set)) return user_set
def get_cluster_associations(self, cluster): """Get all user/account associations for `cluster` and return them""" self.log.debug( "Getting {} user & account associations".format(cluster)) from collections import defaultdict associations = defaultdict(list) cmd = [ self.sacctmgr, 'show', 'associations', 'where', 'cluster={}'.format(cluster), 'format=account,user', '--noheader', '--parsable2' ] sacctmgr = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, query=True) (stdout, _) = sacctmgr.communicate() # Parse associations into a dictionary for res in stdout.splitlines(): res = res.decode('utf-8').split("|") if res[1] is not '': account, user = res associations[account].append(user) else: account = res[0] associations[account].append('') return associations
def recursive_chown(self, uid=None, gid=None): """Change the ownership of the user's home directory""" if uid is None: uid = self.user_attrs.pw_uid if gid is None: gid = self.user_attrs.pw_gid si = os.stat(self.homepath) if si.st_uid != uid or si.st_gid != gid: self.log.critical( "Permission mismatch: user home dir {} uid {} should be {}, gid {} should be {}" .format(self.homepath, si.st_uid, uid, si.st_gid, gid)) self.log.debug("Recursively chowning path {}".format( self.homepath)) try: cmd = ["chown", "-R", "{}:{}".format(uid, gid), self.homepath] chown = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = chown.communicate() if stdout is not None: self.log.debug("stdout for chown -R is {}:".format(stdout)) if stderr is not None: self.log.debug("stderr for chown -R is {}:".format(stderr)) return True except Exception as e: self.log.critical("Unable to chown -R dir {}".format( self.sshpath)) self.log.exception(e)
def setUp(self): self.process = subprocess.Popen( [sys.executable, join(examples_directory, self.path)], cwd=examples_directory, stderr=subprocess.PIPE) time.sleep(1)
def copy_idrsa_to_authkeys(self): # Check that the authorized_keys file exists, if not, create it if self.check_user_auth(): pass else: self.create_user_auth() # If the id_rsa key is not in authorized_keys, add it if not self.check_user_keys(): self.log.debug("Key not present in {}, adding".format( self.authfile)) cmd = ["cat", self.publickey, ">>", self.authfile] try: add_key = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = add_key.communicate() if stdout is not None: self.log.debug( "stdout for key copy is {}:".format(stdout)) if stderr is not None: self.log.debug( "stderr for key copy is {}:".format(stderr)) return True except Exception as e: self.log.critical( "Adding id_rsa to authorized_keys failed") self.log.exception(e) else: return False
def system(command): p = subprocess.Popen(command, shell=True) try: start = time.time() while time.time() < start + TIMEOUT and p.poll() is None: time.sleep(0.1) if p.poll() is None: p.kill() return 'KILLED' return p.poll() finally: if p.poll() is None: p.kill()
def get_all_shares(self): fields = ['Account', 'NormShares', 'NormUsage', 'LevelFS', 'RawUsage'] cmd = [self.sshare, '-l', '--format={}'.format(','.join(fields)), '-P'] import subprocess rv = {} p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() for l in stdout.splitlines(): values = l.split(b'|') d = dict(zip(fields, values)) rv.update({d['Account']: d}) return rv
def get_project_parent(self, account): import mysubprocess as subprocess cmd = [self.sacctmgr, 'show', 'Account', account, 'withassoc', '-p'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() lines = stdout.decode().splitlines() parentcol = lines[0].split('|').index('Par Name') for l in lines[1:]: parent = l.split('|')[parentcol] if parent is not None: return parent return None
def check_nfs_quota(self): """Check the quota for the user""" cmd = ["quota", "-u", self.username] try: getquota = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = getquota.communicate() self.log.debug("stdout for quota is {}:".format(stdout)) self.log.debug("stderr for quota is {}:".format(stderr)) except Exception as e: self.log.critical("get quota failed") self.log.exception(e) return stdout
def check_account_status(self, account, parent=None): """Check that a slurm account exists, ignoring the parent account""" cmd = [ self.sacctmgr, 'show', 'account', account, 'withassoc', 'format=parentname', '--parsable', '--noheader' ] sacctmgr = subprocess.Popen(cmd, query=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, _) = sacctmgr.communicate() if len(stdout.splitlines()) == 0: return False else: return True
def get_shares(self, user): """Get the list of accounts and norm shares and usage""" fields = ['Account', 'NormShares', 'NormUsage'] cmd = [self.sshare, '-l', '--format={}'.format(','.join(fields)), '-P'] import subprocess rv = {} p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() for l in stdout.splitlines(): values = l.split(b'|') d = dict(zip(fields, values)) rv.update({d['Account']: d}) return rv
def check_sacctmgr_status(self): cmd = [self.sacctmgr, 'show', 'cluster', '-P', '-n'] sacctmgr = subprocess.Popen(cmd, query=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = sacctmgr.communicate() if "slurm_persist_conn_open_without_init" in stdout.decode() or\ "slurm_persist_conn_open_without_init" in stderr.decode(): l.critical("sacctmgr can't communicate with slurmdbd, exiting...") code = 1 else: code = 0 return code
def get_current_default_account(self, user): cmd = [ self.sacctmgr, 'show', 'user', 'Format=User,DefaultAccount', '--noheader', '--parsable2', 'where', 'user={}'.format(user) ] getent = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, query=True) (stdout, _) = getent.communicate() try: fields = stdout.decode().split('|') current_default_account = fields[1] return current_default_account except IndexError: self.log.debug("Default account not found for %s", user)
def get_account_share_quota_usage(self, account): cmd = [ self.sshare, '-l', '-A', account, '--format=Account,NormShares,NormUsage', '-P' ] import subprocess p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() lines = stdout.splitlines() if len(lines) == 1: return (None, None) values = lines[-1].split(b'|') try: rv = (float(values[1]), float(values[2])) except ValueError: rv = (None, None) return rv
def create_dir(self, targetpath): """Generic method to create a folder""" self.log.debug("Creating dir {}".format(targetpath)) try: cmd = ["mkdir", targetpath] mkdir = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = mkdir.communicate() if stdout is not None: self.log.debug( "stdout for .ssh creation is {}:".format(stdout)) if stderr is not None: self.log.debug( "stderr for .ssh creation is {}:".format(stderr)) return True except Exception as e: self.log.critical("Unable to create dir {}".format(targetpath)) self.log.exception(e)
def get_all_parent_projects(self): import mysubprocess as subprocess cmd = [ self.sacctmgr, 'show', 'assoc', 'format=parentname,account', '-P', '-n' ] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, query=True) (stdout, stderr) = p.communicate() lines = stdout.decode('utf-8').splitlines() parent_dict = {} for line in lines: parent, account = line.split("|") # A nice hack to remove duplicates from the list if parent != '': parent_dict[account] = parent return parent_dict
def get_desired_users(account): """Query the OS via getent to determine which users should be added to which accounts (each slurm account also has a linux group)""" l.debug("Getting list of desired users for account {}".format(account)) cmd = ['getent', 'group', account] getent = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, query=True) (stdout, _) = getent.communicate() users = stdout.decode('utf-8').strip().split(':')[3] user_set = set() for user in users.split(','): user_set.add(user) l.debug("List of desired users for account {}: {}".format( account, user_set)) return user_set
def check_slurm_status(self): cmd = [self.scontrol, 'ping'] scontrol = subprocess.Popen(cmd, query=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, __) = scontrol.communicate() if 'DOWN/DOWN' in stdout.decode(): code = 0 l.critical( 'Slurm primary and backup controllers are down: {}'.format( stdout.decode())) else: if 'UP/DOWN' in stdout.decode(): msg = 'Slurm backup controller is down: {}' code = 0 else: msg = 'Slurm controllers are ok: {}' code = 1 logging.debug(msg.format(stdout.decode())) return code
def accountexists(self, account, parent=None): """Create the account for the project assigning it to the correct parent""" cmd = [ self.sacctmgr, 'show', 'account', account, 'withassoc', 'format=parentname', '--parsable', '--noheader' ] sacctmgr = subprocess.Popen(cmd, query=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, _) = sacctmgr.communicate() for assoc in stdout.splitlines(): fields = assoc.split(b'|') if fields[0].decode() == parent: self.log.debug("Parent account {} for {} is correct".format( parent, account)) return True self.log.debug("Parent account for {} is not correct".format(account)) return False
def create_user_sshkey(self): """Create the user's id_rsa key for ssh between nodes""" cmd = "ssh-keygen -t rsa -N \"\" -f {}".format(self.idrsapath) self.log.debug("Creating ssh key {} with command {}".format( self.idrsapath, cmd)) try: keygen = mysubprocess.Popen([cmd], shell=True, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = keygen.communicate() if stdout is not None: self.log.debug( "stdout for key creation is {}:".format(stdout)) if stderr is not None: self.log.debug( "stderr for key creation is {}:".format(stderr)) return True except Exception as e: self.log.critical("ssh-keygen failed") self.log.exception(e)
def all_jobs(self, starttime, endtime): import subprocess fields = [ 'JobID', 'User', 'Account', 'AllocCPUS', 'AllocGRES', 'ReqGRES', 'CPUTimeRAW', 'JobName', 'Submit', 'Start', 'End', 'State' ] result = [] cmd = [ self.sacct, '-X', '-S', "{}".format(starttime), '-E', "{}".format(endtime), '-a', '--format={}'.format(','.join(fields)), '-n', '-p' ] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() if len(stderr) > 0: print(stderr) for l in stdout.decode().splitlines(): values = l.split('|') job = dict(zip(fields, values)) result.append(job) return result
def copy_skeleton(self): """Copies the home directory skeleton into the user's home directory""" cmd = [ 'rsync', '-av', '--ignore-existing', self.skelpath, self.homepath ] try: rsync = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = rsync.communicate() if stdout is not None: self.log.debug( "stdout for skeleton rsync is: {}:".format(stdout)) if stderr is not None: self.log.debug( "stderr for skeleton rsync is: {}:".format(stderr)) return True except Exception as e: self.log.exeption(e)
def set_nfs_quota(self, quotas): """Set block usage and inode quotas for the user homedir""" cmd = [ "/sbin/setquota", "-u", self.username, str(quotas['bsoft']), str(quotas['bhard']), str(quotas['isoft']), str(quotas['ihard']), self.mnt ] self.log.debug("Setting quota for {} with command {}".format( self.username, cmd)) try: setquota = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = setquota.communicate() self.log.debug("stdout for setquota is {}:".format(stdout)) self.log.debug("stderr for setquota is {}:".format(stderr)) except Exception as e: self.log.critical("setquota failed") self.log.exception(e)
def create_user_auth(self): """Create the user's authorized_keys file""" cmd = ["cp", self.publickey, self.authfile] self.log.debug( "Creating authorized_keys file {} with command {}".format( self.authfile, cmd)) try: cp = mysubprocess.Popen(cmd, stdout=mysubprocess.PIPE, stderr=mysubprocess.PIPE) stdout, stderr = cp.communicate() if stdout is not None: self.log.debug( "stdout for authorized_keys creation is {}:".format( stdout)) if stderr is not None: self.log.debug( "stderr for authorized_keys creation is {}:".format( stderr)) except Exception as e: self.log.critical("authorized_keys cp failed") self.log.exception(e)