def get_user_account_shares(self, user):
"""Get the list of accounts & fairshare value associated with a user"""
fair_share = {}
cmd = [
self.sshare, '-U', '-u', '{}'.format(user),
'--format=Account,User,Fairshare', '--noheader', '--parsable2'
]
getent = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
query=True)
(stdout, _) = getent.communicate()
for assoc in stdout.decode().splitlines():
if user in assoc:
fields = assoc.split('|')
if fields[1].strip(
) == user and fields[0].strip() != "default":
if fields[0].strip() in self.get_mx_caps():
fair_share[fields[0]] = float(-1)
else:
fair_share[fields[0]] = fields[2]
if fair_share is not None and len(fair_share) > 0:
self.log.debug("Found fairshare value(s) for %s", user)
return fair_share
else:
self.log.debug("Fairshare value not found for %s", user)
return None
def getusers(self, account):
"""Get the list of users currently able to use the account"""
self.log.debug(
"Retrieving list of users under {} account".format(account))
user_set = set()
cmd = [
self.sacctmgr, 'show', 'associations', '--parsable2', '--noheader',
'account={}'.format(account), 'format=user'
]
sacctmgr = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
query=True)
(stdout, _) = sacctmgr.communicate()
for user in stdout.splitlines():
user = user.decode('utf-8')
if user is not None and user is not "":
user_set.add(user)
self.log.debug("List of users under {} account: {}".format(
account, user_set))
return user_set
def get_cluster_associations(self, cluster):
"""Get all user/account associations for `cluster` and return them"""
self.log.debug(
"Getting {} user & account associations".format(cluster))
from collections import defaultdict
associations = defaultdict(list)
cmd = [
self.sacctmgr, 'show', 'associations', 'where',
'cluster={}'.format(cluster), 'format=account,user', '--noheader',
'--parsable2'
]
sacctmgr = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
query=True)
(stdout, _) = sacctmgr.communicate()
# Parse associations into a dictionary
for res in stdout.splitlines():
res = res.decode('utf-8').split("|")
if res[1] is not '':
account, user = res
associations[account].append(user)
else:
account = res[0]
associations[account].append('')
return associations
def recursive_chown(self, uid=None, gid=None):
"""Change the ownership of the user's home directory"""
if uid is None:
uid = self.user_attrs.pw_uid
if gid is None:
gid = self.user_attrs.pw_gid
si = os.stat(self.homepath)
if si.st_uid != uid or si.st_gid != gid:
self.log.critical(
"Permission mismatch: user home dir {} uid {} should be {}, gid {} should be {}"
.format(self.homepath, si.st_uid, uid, si.st_gid, gid))
self.log.debug("Recursively chowning path {}".format(
self.homepath))
try:
cmd = ["chown", "-R", "{}:{}".format(uid, gid), self.homepath]
chown = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = chown.communicate()
if stdout is not None:
self.log.debug("stdout for chown -R is {}:".format(stdout))
if stderr is not None:
self.log.debug("stderr for chown -R is {}:".format(stderr))
return True
except Exception as e:
self.log.critical("Unable to chown -R dir {}".format(
self.sshpath))
self.log.exception(e)
def setUp(self):
self.process = subprocess.Popen(
[sys.executable,
join(examples_directory, self.path)],
cwd=examples_directory,
stderr=subprocess.PIPE)
time.sleep(1)
def copy_idrsa_to_authkeys(self):
# Check that the authorized_keys file exists, if not, create it
if self.check_user_auth():
pass
else:
self.create_user_auth()
# If the id_rsa key is not in authorized_keys, add it
if not self.check_user_keys():
self.log.debug("Key not present in {}, adding".format(
self.authfile))
cmd = ["cat", self.publickey, ">>", self.authfile]
try:
add_key = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = add_key.communicate()
if stdout is not None:
self.log.debug(
"stdout for key copy is {}:".format(stdout))
if stderr is not None:
self.log.debug(
"stderr for key copy is {}:".format(stderr))
return True
except Exception as e:
self.log.critical(
"Adding id_rsa to authorized_keys failed")
self.log.exception(e)
else:
return False
def system(command):
p = subprocess.Popen(command, shell=True)
try:
start = time.time()
while time.time() < start + TIMEOUT and p.poll() is None:
time.sleep(0.1)
if p.poll() is None:
p.kill()
return 'KILLED'
return p.poll()
finally:
if p.poll() is None:
p.kill()
def get_all_shares(self):
fields = ['Account', 'NormShares', 'NormUsage', 'LevelFS', 'RawUsage']
cmd = [self.sshare, '-l', '--format={}'.format(','.join(fields)), '-P']
import subprocess
rv = {}
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
for l in stdout.splitlines():
values = l.split(b'|')
d = dict(zip(fields, values))
rv.update({d['Account']: d})
return rv
def get_project_parent(self, account):
import mysubprocess as subprocess
cmd = [self.sacctmgr, 'show', 'Account', account, 'withassoc', '-p']
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
lines = stdout.decode().splitlines()
parentcol = lines[0].split('|').index('Par Name')
for l in lines[1:]:
parent = l.split('|')[parentcol]
if parent is not None:
return parent
return None
def check_nfs_quota(self):
"""Check the quota for the user"""
cmd = ["quota", "-u", self.username]
try:
getquota = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = getquota.communicate()
self.log.debug("stdout for quota is {}:".format(stdout))
self.log.debug("stderr for quota is {}:".format(stderr))
except Exception as e:
self.log.critical("get quota failed")
self.log.exception(e)
return stdout
def check_account_status(self, account, parent=None):
"""Check that a slurm account exists, ignoring the parent account"""
cmd = [
self.sacctmgr, 'show', 'account', account, 'withassoc',
'format=parentname', '--parsable', '--noheader'
]
sacctmgr = subprocess.Popen(cmd,
query=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, _) = sacctmgr.communicate()
if len(stdout.splitlines()) == 0:
return False
else:
return True
def get_shares(self, user):
"""Get the list of accounts and norm shares and usage"""
fields = ['Account', 'NormShares', 'NormUsage']
cmd = [self.sshare, '-l', '--format={}'.format(','.join(fields)), '-P']
import subprocess
rv = {}
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
for l in stdout.splitlines():
values = l.split(b'|')
d = dict(zip(fields, values))
rv.update({d['Account']: d})
return rv
def check_sacctmgr_status(self):
cmd = [self.sacctmgr, 'show', 'cluster', '-P', '-n']
sacctmgr = subprocess.Popen(cmd,
query=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = sacctmgr.communicate()
if "slurm_persist_conn_open_without_init" in stdout.decode() or\
"slurm_persist_conn_open_without_init" in stderr.decode():
l.critical("sacctmgr can't communicate with slurmdbd, exiting...")
code = 1
else:
code = 0
return code
def get_current_default_account(self, user):
cmd = [
self.sacctmgr, 'show', 'user', 'Format=User,DefaultAccount',
'--noheader', '--parsable2', 'where', 'user={}'.format(user)
]
getent = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
query=True)
(stdout, _) = getent.communicate()
try:
fields = stdout.decode().split('|')
current_default_account = fields[1]
return current_default_account
except IndexError:
self.log.debug("Default account not found for %s", user)
def get_account_share_quota_usage(self, account):
cmd = [
self.sshare, '-l', '-A', account,
'--format=Account,NormShares,NormUsage', '-P'
]
import subprocess
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
lines = stdout.splitlines()
if len(lines) == 1:
return (None, None)
values = lines[-1].split(b'|')
try:
rv = (float(values[1]), float(values[2]))
except ValueError:
rv = (None, None)
return rv
def create_dir(self, targetpath):
"""Generic method to create a folder"""
self.log.debug("Creating dir {}".format(targetpath))
try:
cmd = ["mkdir", targetpath]
mkdir = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = mkdir.communicate()
if stdout is not None:
self.log.debug(
"stdout for .ssh creation is {}:".format(stdout))
if stderr is not None:
self.log.debug(
"stderr for .ssh creation is {}:".format(stderr))
return True
except Exception as e:
self.log.critical("Unable to create dir {}".format(targetpath))
self.log.exception(e)
def get_all_parent_projects(self):
import mysubprocess as subprocess
cmd = [
self.sacctmgr, 'show', 'assoc', 'format=parentname,account', '-P',
'-n'
]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
query=True)
(stdout, stderr) = p.communicate()
lines = stdout.decode('utf-8').splitlines()
parent_dict = {}
for line in lines:
parent, account = line.split("|")
# A nice hack to remove duplicates from the list
if parent != '':
parent_dict[account] = parent
return parent_dict
def get_desired_users(account):
"""Query the OS via getent to determine which users should be added to
which accounts (each slurm account also has a linux group)"""
l.debug("Getting list of desired users for account {}".format(account))
cmd = ['getent', 'group', account]
getent = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
query=True)
(stdout, _) = getent.communicate()
users = stdout.decode('utf-8').strip().split(':')[3]
user_set = set()
for user in users.split(','):
user_set.add(user)
l.debug("List of desired users for account {}: {}".format(
account, user_set))
return user_set
def check_slurm_status(self):
cmd = [self.scontrol, 'ping']
scontrol = subprocess.Popen(cmd,
query=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, __) = scontrol.communicate()
if 'DOWN/DOWN' in stdout.decode():
code = 0
l.critical(
'Slurm primary and backup controllers are down: {}'.format(
stdout.decode()))
else:
if 'UP/DOWN' in stdout.decode():
msg = 'Slurm backup controller is down: {}'
code = 0
else:
msg = 'Slurm controllers are ok: {}'
code = 1
logging.debug(msg.format(stdout.decode()))
return code
def accountexists(self, account, parent=None):
"""Create the account for the project assigning it to the correct
parent"""
cmd = [
self.sacctmgr, 'show', 'account', account, 'withassoc',
'format=parentname', '--parsable', '--noheader'
]
sacctmgr = subprocess.Popen(cmd,
query=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, _) = sacctmgr.communicate()
for assoc in stdout.splitlines():
fields = assoc.split(b'|')
if fields[0].decode() == parent:
self.log.debug("Parent account {} for {} is correct".format(
parent, account))
return True
self.log.debug("Parent account for {} is not correct".format(account))
return False
def create_user_sshkey(self):
"""Create the user's id_rsa key for ssh between nodes"""
cmd = "ssh-keygen -t rsa -N \"\" -f {}".format(self.idrsapath)
self.log.debug("Creating ssh key {} with command {}".format(
self.idrsapath, cmd))
try:
keygen = mysubprocess.Popen([cmd],
shell=True,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = keygen.communicate()
if stdout is not None:
self.log.debug(
"stdout for key creation is {}:".format(stdout))
if stderr is not None:
self.log.debug(
"stderr for key creation is {}:".format(stderr))
return True
except Exception as e:
self.log.critical("ssh-keygen failed")
self.log.exception(e)
def all_jobs(self, starttime, endtime):
import subprocess
fields = [
'JobID', 'User', 'Account', 'AllocCPUS', 'AllocGRES', 'ReqGRES',
'CPUTimeRAW', 'JobName', 'Submit', 'Start', 'End', 'State'
]
result = []
cmd = [
self.sacct, '-X', '-S', "{}".format(starttime), '-E',
"{}".format(endtime), '-a', '--format={}'.format(','.join(fields)),
'-n', '-p'
]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
if len(stderr) > 0:
print(stderr)
for l in stdout.decode().splitlines():
values = l.split('|')
job = dict(zip(fields, values))
result.append(job)
return result
def copy_skeleton(self):
"""Copies the home directory skeleton into the user's home
directory"""
cmd = [
'rsync', '-av', '--ignore-existing', self.skelpath,
self.homepath
]
try:
rsync = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = rsync.communicate()
if stdout is not None:
self.log.debug(
"stdout for skeleton rsync is: {}:".format(stdout))
if stderr is not None:
self.log.debug(
"stderr for skeleton rsync is: {}:".format(stderr))
return True
except Exception as e:
self.log.exeption(e)
def set_nfs_quota(self, quotas):
"""Set block usage and inode quotas for the user homedir"""
cmd = [
"/sbin/setquota", "-u", self.username,
str(quotas['bsoft']),
str(quotas['bhard']),
str(quotas['isoft']),
str(quotas['ihard']), self.mnt
]
self.log.debug("Setting quota for {} with command {}".format(
self.username, cmd))
try:
setquota = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = setquota.communicate()
self.log.debug("stdout for setquota is {}:".format(stdout))
self.log.debug("stderr for setquota is {}:".format(stderr))
except Exception as e:
self.log.critical("setquota failed")
self.log.exception(e)
def create_user_auth(self):
"""Create the user's authorized_keys file"""
cmd = ["cp", self.publickey, self.authfile]
self.log.debug(
"Creating authorized_keys file {} with command {}".format(
self.authfile, cmd))
try:
cp = mysubprocess.Popen(cmd,
stdout=mysubprocess.PIPE,
stderr=mysubprocess.PIPE)
stdout, stderr = cp.communicate()
if stdout is not None:
self.log.debug(
"stdout for authorized_keys creation is {}:".format(
stdout))
if stderr is not None:
self.log.debug(
"stderr for authorized_keys creation is {}:".format(
stderr))
except Exception as e:
self.log.critical("authorized_keys cp failed")
self.log.exception(e)
