def execute(self): Flag=True modcmd="tests/bash_module_test.bash" output=capture(modcmd) if "not found" in output: self.error_message+=" Error: Module is not defined." return False modcmd="tests/csh_module_test.csh" output=capture(modcmd) if not output: self.error_message+=" Error: Module is not defined." return False unknown = "**UNKNOWN**" value = os.environ.get('LMOD_CMD',unknown) if (value == unknown): self.error_message+=" Error: Module is not defined." return False lmodcmd=os.environ['LMOD_CMD'] mlcmd=lmodcmd + " python list" output=captureErr(mlcmd).split() module_need=["TACC","TACC-paths","Linux","cluster"] for mod1 in module_need: if mod1 not in output: self.error_message+=" Error: Module \"%s\" is not loaded.\n" %mod1 Flag=False return Flag
def execute(self): Flag=True modcmd="tests/bash_module_test.bash" output=capture(modcmd) if "not found" in output: self.error_message+="\tError: Module is not defined." return False modcmd="tests/csh_module_test.csh" output=capture(modcmd) if not output: self.error_message+="\tError: Module is not defined." return False unknown = "**UNKNOWN**" value = os.environ.get('LMOD_CMD',unknown) if (value == unknown): self.error_message+="\tError: Module is not defined." return False lmodcmd=os.environ['LMOD_CMD'] mlcmd=lmodcmd + " python list" output=captureErr(mlcmd).split() module_need=["TACC"] # print(output) for mod1 in module_need: if not any(mod1 in tmpstr for tmpstr in output): self.error_message+="\tError: Necessary module \"%s\" is not loaded.\n" %mod1 Flag=False return Flag
def __init__(self, exec_progA): ignoreT = { 'env' : True, 'time' : True, } cmd = None for prog in exec_progA: bare = os.path.basename(prog) if (not (bare in ignoreT)): cmd = prog break self.__execName = which(cmd) ldd = None if (self.__execName): outStr = capture(["file", self.__execName]) if (outStr.find("ASCII text") > 0): self.__execType = "script" elif (outStr.find("executable") > 0): self.__execType = "binary" ldd = capture(["ldd", self.__execName]) else: self.__execType = None info = os.stat(self.__execName) self.__modify = info.st_mtime self.__libA = self.__parseLDD(ldd) self.__hash = self.__computeHash(self.__execName)
def __init__(self, exec_progA): ignoreT = { 'env' : True, 'time' : True, } cmd = None for prog in exec_progA: bare = os.path.basename(prog) if (not (bare in ignoreT)): cmd = prog break self.__execType = None self.__execName = which(cmd) self.__libA = [] if (self.__execName): outStr = capture(["file", self.__execName]) if (outStr.find("script") > 0 or outStr.find("text") > 0): self.__execType = "script" else: self.__execType = "binary" ldd = capture(["ldd", self.__execName]) self.__libA = self.__parseLDD(ldd) info = os.stat(self.__execName) self.__modify = info.st_mtime self.__hash = self.__computeHash(self.__execName)
def execute(self): result = True # home = os.environ['HOME'] userid=capture("whoami").rstrip() # grepcmd="grep %s /etc/passwd | cut -d ':' -f6" %userid grepcmd="/bin/awk -F: -v user=%s '$1 == user {print $6}' </etc/passwd" %userid home=capture(grepcmd) if not home: self.error_message+="\tError: Your home directory is inaccessible!\n" return False home=home[:-1] sshD = os.path.join(home,".ssh") dirA = [ home, sshD ] if not os.path.isdir(sshD): self.error_message+="\tError: .ssh directory does not exist or is inaccessible!\n" return False keyfile=os.path.join(sshD,"authorized_keys") if not os.path.isfile(keyfile): self.error_message+="\tError: Authorized key file does not exist or is inaccessible!\n" return False # The following part is a little wordy, because we want to make some customized error messages: a = self.__write_group_other_test(home) for entry in a: if (entry['value']): result = False self.error_message+="\tError: "+entry['name']+ " permission on $HOME will cause RSA to fail!\n" #remove entry['name'] in the Error message on July 22, 2015 a = self.__read_user_test(home) for entry in a: if (entry['value']): result = False self.error_message+="\tError: no user read permission on $HOME will cause RSA to fail!\n" # The following part is a little wordy, because we want to make some customized error messages: a = self.__write_group_other_test(sshD) for entry in a: if (entry['value']): result = False self.error_message+="\tError: "+entry['name']+ " permission on $HOME/.ssh will cause RSA to fail!\n" #remove entry['name'] in the Error message on July 22, 2015 a = self.__read_user_test(sshD) for entry in a: if (entry['value']): result = False self.error_message+="\tError: no user read permission on $HOME/.ssh will cause RSA to fail!\n" # test owner and perm on ~/.ssh/authorized_keys r = self.__authorized_keys_test(keyfile) if (r): result = False self.error_message+="\tError: The permission and/or ownership on " + keyfile +" will cause RSA to fail!\n" return result
def execute(self): userid=capture("whoami").rstrip() host=syshost() if (host=="stampede" or host=="ls5"): spaces=["/home1","/work"] elif host=="ls4": spaces=["/home1","/work"] elif (host=="maverick"): spaces=["/home","/work"] else: return True Flag=True for space in spaces: if ( (host=="ls4" and space=="/home1") or (host=="maverick" and space=="/home") or (host=="ls5" and space=="/home1")): quotacmd="quota" rawinfo=capture(quotacmd).split("\n") if len(rawinfo)<3: return False quotainfo=rawinfo[-2].split() ## print(quotainfo[0],quotainfo[2],quotainfo[3],quotainfo[5]) ## print(quotainfo) ## if len(quotainfo) <6: if len(quotainfo) <6 or (not (quotainfo[0]).isdigit()): self.error_message+="\tError: "+"No valid quota report\n" return False if float(quotainfo[0]) >= float(quotainfo[2])*0.95 : Flag=False self.error_message+="\tError: You are over/close to the disk limit under %s.\n" %space if float(quotainfo[3]) >= float(quotainfo[5])*0.95 : Flag=False self.error_message+="\tError: You are over/close to the inode limit under %s.\n" %space else: lfscmd="lfs quota -u %s %s" %(userid,space) ## print(lfscmd) quotainfo=capture(lfscmd).split("\n")[2].split() quotainfo[5]=quotainfo[5].strip("*") quotainfo[1]=quotainfo[1].strip("*") ## print(quotainfo[1], quotainfo[3], quotainfo[5], quotainfo[7] ## print(quotainfo) if len(quotainfo) <8: self.error_message+="\tError: "+"No valid quota report\n" return False if float(quotainfo[1]) >= float(quotainfo[3])*0.95 : Flag=False self.error_message+="\tError: You are over/close to the disk limit under %s.\n" %space if float(quotainfo[5]) >= float(quotainfo[7])*0.95 : Flag=False self.error_message+="\tError: Your are over/close to the inode limit under %s.\n" %space return Flag
def execute(self): userid=capture("whoami").rstrip() host=syshost() if (host=="stampede2" or host=="ls5"): spaces=["/home1","/work"] elif host=="ls4": spaces=["/home1","/work"] elif (host=="maverick"): spaces=["/home","/work"] else: return True Flag=True for space in spaces: if ( (host=="ls4" and space=="/home1") or (host=="maverick" and space=="/home") or (host=="ls5" and space=="/home1")): quotacmd="quota" rawinfo=capture(quotacmd).split("\n") if len(rawinfo)<3: return False quotainfo=rawinfo[-2].split() ## print(quotainfo[0],quotainfo[2],quotainfo[3],quotainfo[5]) ## print(quotainfo) if len(quotainfo) <6 or (not (quotainfo[0]).isdigit()): self.error_message+="\tError: "+"No valid quota report\n" return False if float(quotainfo[0]) >= float(quotainfo[2])*0.9 : Flag=False self.error_message+="\tError: You are over/close to the disk limit under %s.\n" %space if float(quotainfo[3]) >= float(quotainfo[5])*0.9 : Flag=False self.error_message+="\tError: You are over/close to the inode limit under %s.\n" %space else: lfscmd="lfs quota -u %s %s" %(userid,space) ## print(lfscmd) quotainfo=capture(lfscmd).split("\n")[2].split() quotainfo[5]=quotainfo[5].strip("*") quotainfo[1]=quotainfo[1].strip("*") ## print(quotainfo[1], quotainfo[3], quotainfo[5], quotainfo[7] ## print(quotainfo) if len(quotainfo) <8: self.error_message+="\tError: "+"No valid quota report\n" return False if float(quotainfo[1]) >= float(quotainfo[3])*0.85 : Flag=False self.error_message+="\tError: You are over/close to the disk limit under %s.\n" %space if float(quotainfo[5]) >= float(quotainfo[7])*0.85 : Flag=False self.error_message+="\tError: Your are over/close to the inode limit under %s.\n" %space return Flag
def __parseLDD(self, ldd): if (ldd.find("not a dynamic executable") > 0): return [] lineA = ldd.split('\n') libA = [] d = {} for line in lineA: fieldA = line.split() N = len(fieldA) if (N < 1): break elif (N == 4): lib = fieldA[2] else: lib = fieldA[0] lib = os.path.realpath(lib) d[lib] = True libA = d.keys() libA = sorted(libA) libB = [] for lib in libA: hash_line = capture(['sha1sum', lib]) if (hash_line.find("No such file or directory") != -1): v = "unknown" else: v = hash_line.split()[0] libB.append([lib, v]) return libB
def main(): try: uuid = sys.argv[ 1] status = sys.argv[ 2] wd = sys.argv[ 3] syshost = sys.argv[ 4] pstree = sys.argv[ 5] execname = sys.argv[ 6] xaltobj = sys.argv[ 7] build_epoch = sys.argv[ 8] linklineFn = sys.argv[ 9] resultFn = sys.argv[10] if (execname.find("conftest") != -1): return 1 hash_line = capture(['sha1sum', execname]) if (hash_line.find("No such file or directory") != -1): return 1 hash_id = hash_line.split()[0] # Step one clean up linkline data sA = cleanup(xaltobj, linklineFn) resultT = {} resultT['uuid'] = uuid resultT['link_program'] = extract_compiler(pstree) resultT['build_user'] = os.environ['USER'] resultT['exit_code'] = status resultT['build_epoch'] = build_epoch resultT['exec_path'] = os.path.abspath(execname) resultT['hash_id'] = hash_id resultT['wd'] = wd resultT['build_syshost'] = syshost resultT['linkA'] = sA dirname,fn = os.path.split(resultFn) tmpFn = os.path.join(dirname, "." + fn) if (not os.path.isdir(dirname)): os.mkdir(dirname); s = json.dumps(resultT, sort_keys=True, indent=2, separators=(',',': ')) f = open(tmpFn,'w') f.write(s) f.write("\n") if (hash_line.find("sha1sum:") != -1): f.write(hash_line) f.close() os.rename(tmpFn, resultFn) except: logger.exception("XALT_EXCEPTION:xalt_generate_linkdata") return 0
def execute(self): result = True pub_key = capture('cat ~/.ssh/id_rsa.pub').replace('\n','') cmd = "grep -F \"%s\" ~/.ssh/authorized_keys > /dev/null 2>&1" % pub_key status = run_cmd(cmd) if (status != 0): self.error_message+=" Error: ~/.ssh/id_rsa.pub not found in ~/.ssh/authorized_keys.\n" return False cmd2 ="awk '{if ($1!=\"ssh-dss\" && $1!=\"ssh-rsa\" || NF <= 1) print $0}' ~/.ssh/authorized_keys" cmd2_out = capture(cmd2) if cmd2_out: self.error_message+=" Error: ~/.ssh/authorized_keys includes invalid or broken key(s).\n" return False; return result
def execute(self): result = True pub_key = capture('cat ~/.ssh/id_rsa.pub').replace('\n', '') cmd = "grep -F \"%s\" ~/.ssh/authorized_keys > /dev/null 2>&1" % pub_key status = run_cmd(cmd) if (status != 0): self.error_message += "\tError: ~/.ssh/id_rsa.pub not found in ~/.ssh/authorized_keys.\n" return False cmd2 = "awk '{if ($1!=\"ssh-dss\" && $1!=\"ssh-rsa\" || NF <= 1) print $0}' ~/.ssh/authorized_keys" cmd2_out = capture(cmd2) if cmd2_out: self.error_message += "\tError: ~/.ssh/authorized_keys includes invalid or broken key(s).\n" return False return result
def execute(self): result = True # home = os.environ['HOME'] userid=capture("whoami").rstrip() # grepcmd="grep %s /etc/passwd | cut -d ':' -f6" %userid grepcmd="/bin/awk -F: -v user=%s '$1 == user {print $6}' </etc/passwd" %userid home=capture(grepcmd) if not home: self.error_message+=" Error: Can not find home directory!\n" return False home=home[:-1] sshD = os.path.join(home,".ssh") dirA = [ home, sshD ] if not os.path.isdir(sshD): self.error_message+=" Error: .ssh directory does not exist or is inaccessible!\n" return False keyfile=os.path.join(sshD,"authorized_keys") if not os.path.isfile(keyfile): self.error_message+=" Error: Authorized key file does not exist or is inaccessible!\n" return False for d in dirA: a = self.__write_group_other_test(d) for entry in a: if (entry['value']): result = False self.error_message+=" Error: "+entry['name']+ " permission on " + d +" is bad!\n" a = self.__read_user_test(d) for entry in a: if (entry['value']): result = False self.error_message+=" Error: "+entry['name']+ " permission on " + d +" is bad!\n" # test owner and perm on ~/.ssh/authorized_keys r = self.__authorized_keys_test(keyfile) if (r): result = False self.error_message+=" Error: The permission and/or ownership on " + keyfile +" is bad!\n" return result
def execute(self): host = syshost() if host!="stampede" and host!="frontera": return True userid=getpass.getuser() grepcmd1="""awk '/ALL = /,/ALL = /' /etc/slurm/tacc_filter_options | awk '{print substr($0,7,length($0)-7)}' | awk -v user=%s 'BEGIN {FS=" !!"}; {for (i=1; i<=NF; i++) if($i == user) {print $0}}'""" %userid # print(grepcmd1) myrecord1=capture(grepcmd1).split('\n') # print(myrecord1) grepcmd2="""awk '($1=="largemem")' /etc/slurm/tacc_filter_options | awk '{gsub(/ /, "", $0);print substr($0,12, length($0)-12)}' | awk -v user=%s 'BEGIN {FS="!!"}; {for (i=1; i<=NF; i++) if ($i == user) {print $0}}'""" %userid # print(grepcmd2) myrecord2=capture(grepcmd2).split('\n') # print(myrecord2) # grepcmd3="""awk '($1=="normal-mic" || $1=="normal-2mic")' /etc/slurm/tacc_filter_options | awk '{gsub(/ /, "", $0); print substr($0,14, length($0)-14)}' | awk -v user=%s 'BEGIN {FS="!!"}; {for (i=1; i<=NF; i++) if ($i == user) {print $0}}'""" %userid # print(grepcmd3) # myrecord3=capture(grepcmd3).split('\n') # print(myrecord3) myrecord=myrecord1 + myrecord2 # print(myrecord) for ss in myrecord1: if ss and not ss.strip().startswith('#'): self.error_message+=" ERROR: You are blocked from general submission.\n" return False for ss in myrecord2: if ss and not ss.strip().startswith('#'): self.error_message+=" ERROR: You are blocked from largemem queue submission.\n" return False # for ss in myrecord3: # if ss and not ss.strip().startswith('#'): # self.error_message+=" ERROR: You are blocked from mic queue submission.\n" # return False return True
def execute(self): compilers=["gcc","g++","gfortran","icc","icpc","ifort","mpicc","mpicxx","mpif90"] Flag=True for compiler1 in compilers: typecmd="type %s" %compiler1 output=capture(typecmd) if "not found" in output: Flag=False self.error_message+=" ERROR: Compiler %s is not available now!\n" %compiler1 return Flag
def execute(self): host = syshost() if host!="stampede": return True userid=capture("whoami").rstrip() grepcmd1="""awk '/ALL = /,/ALL = /' /etc/slurm/tacc_filter_options | awk '{print substr($0,7,length($0)-7)}' | awk -v user=%s 'BEGIN {FS=" !!"}; {for (i=1; i<=NF; i++) if($i == user) {print $0}}'""" %userid # print(grepcmd1) myrecord1=capture(grepcmd1).split('\n') # print(myrecord1) grepcmd2="""awk '($1=="largemem")' /etc/slurm/tacc_filter_options | awk '{gsub(/ /, "", $0);print substr($0,12, length($0)-12)}' | awk -v user=%s 'BEGIN {FS="!!"}; {for (i=1; i<=NF; i++) if ($i == user) {print $0}}'""" %userid # print(grepcmd2) myrecord2=capture(grepcmd2).split('\n') # print(myrecord2) grepcmd3="""awk '($1=="normal-mic" || $1=="normal-2mic")' /etc/slurm/tacc_filter_options | awk '{gsub(/ /, "", $0); print substr($0,14, length($0)-14)}' | awk -v user=%s 'BEGIN {FS="!!"}; {for (i=1; i<=NF; i++) if ($i == user) {print $0}}'""" %userid # print(grepcmd3) myrecord3=capture(grepcmd3).split('\n') # print(myrecord3) myrecord=myrecord1 + myrecord2 + myrecord3 # print(myrecord) for ss in myrecord1: if ss and not ss.strip().startswith('#'): self.error_message+="\tError: You are blocked from general submission.\n" return False for ss in myrecord2: if ss and not ss.strip().startswith('#'): self.error_message+="\tError: You are blocked from largemem queue submission.\n" return False for ss in myrecord3: if ss and not ss.strip().startswith('#'): self.error_message+="\tError: You are blocked from mic queue submission.\n" return False return True
def execute(self): result = True # home = os.environ['HOME'] userid = capture("whoami").rstrip() # grepcmd="grep %s /etc/passwd | cut -d ':' -f6" %userid grepcmd = "/bin/awk -F: -v user=%s '$1 == user {print $6}' </etc/passwd" % userid home = capture(grepcmd) if not home: self.error_message += " ERROR: Can not find your home directory!\n" return False home = home[:-1] sshD = os.path.join(home, ".ssh") dirA = [home, sshD] if not os.path.isdir(sshD): self.error_message += " ERROR: No .ssh directory is found!\n" return False keyfile = os.path.join(sshD, "authorized_keys") if not os.path.isfile(keyfile): self.error_message += " ERROR: No authorized key files is found!\n" return False for d in dirA: a = self.__write_group_other_test(d) for entry in a: if (entry['value']): result = False self.error_message += " ERROR: " + entry[ 'name'] + " permission on " + d + " is bad!\n" # test owner and perm on ~/.ssh/authorized_keys r = self.__authorized_keys_test(keyfile) if (r): result = False self.error_message += " ERROR: The permission and/or ownership on " + keyfile + " is bad!\n" return result
def execute(self): compilers=["gcc","g++","gfortran","icc","icpc","ifort","mpicc","mpicxx","mpif90"] Flag=True for compiler1 in compilers: typecmd="type %s" %compiler1 output=capture(typecmd) # print(output) if "not found" in output: Flag=False self.error_message+="\tError: Compiler %s is not available at this time!\n" %compiler1 return Flag
def cleanup(xaltobj, fn): f = open(fn,"r") lines = f.readlines() d = {} for s in lines: # remove lines with ':' if (s.find(":") != -1): continue # remove line with the xalt.o file if (s.find(xaltobj) != -1): continue # remove a file that is something like: /tmp/ccT33qQt.o m = tmpObjPat.search(s) if (m): continue # Capture the library name in the parens: # -lgcc_s (/usr/lib/gcc/x86_64-linux-gnu/4.8/libgcc_s.so) m = parenPat.search(s) if (m): s = os.path.realpath(m.group(1)) d[s] = True continue # Save everything else idx = s.find("\n") if (idx != -1): s = s[:idx] s = os.path.realpath(s) d[s] = True # make the list unique sA = d.keys() sA = sorted(sA) sB = [] for lib in sA: hash_line = capture(['sha1sum', lib]) if (hash_line.find("No such file or directory") != -1): continue else: v = hash_line.split()[0] sB.append([lib, v]) return sB
def execute(self): compilers=["gcc","g++","gfortran","icc","icpc","ifort","mpicc","mpicxx","mpif90"] Flag=True for compiler1 in compilers: typecmd="type %s" %compiler1 output=capture(typecmd) # print(output) if "not found" in output: print("\033[1;33m\tWarning: %s is not available right now.\033[0m" %compiler1) # Flag=False # self.error_message+="\tError: Compiler %s is not available at this time!\n" %compiler1 return Flag
def execute(self): host=syshost() if host=="stampede": commands=["sbatch","squeue","scancel"] elif host=="ls4": commands=["qsub","qstat","qdel"] else: return True Flag=True for command1 in commands: typecmd="type %s" %command1 output=capture(typecmd) if "not found" in output: Flag=False self.error_message+=" ERROR: Scheduler command \"%s\" is not available now!\n" %command1 return Flag
def execute(self): host=syshost() if host=="stampede": commands=["sbatch","squeue","scancel"] elif host=="ls4": commands=["qsub","qstat","qdel"] else: return True Flag=True for command1 in commands: typecmd="type %s" %command1 output=capture(typecmd) if "not found" in output: Flag=False self.error_message+=" Error: Scheduler command \"%s\" is not available now!\n" %command1 return Flag
def execute(self): host = syshost() if (host == "stampede" or host == "maverick" or host == "ls5"): commands = ["sbatch", "squeue", "scancel", "scontrol"] elif host == "ls4": commands = ["qsub", "qstat", "qdel"] else: return True Flag = True for command1 in commands: typecmd = "type %s" % command1 output = capture(typecmd) if "not found" in output: Flag = False self.error_message += "\tError: Scheduler command \"%s\" is not available at this time!\n" % command1 return Flag
def main(): args = CmdLineOptions().execute() xalt = XALTdb(ConfigFn) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 for user, hdir in passwd_generator(): xaltDir = os.path.join(hdir, ".xalt.d") if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link.*.json") lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA) if (args.delete): remove_files(linkFnA) remove_files(files_in_tree(xaltDir, "*/.link.*.json")) runFnA = files_in_tree(xaltDir, "*/run.*.json") runCnt += run_json_to_db(xalt, user, rmapT, runFnA) if (args.delete): remove_files(runFnA) remove_files(files_in_tree(xaltDir, "*/.run.*.json")) icnt += 1 pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def main(): args = CmdLineOptions().execute() xalt = XALTdb(ConfigFn) num = int(capture("getent passwd | wc -l")) pbar = ProgressBar(maxVal=num) icnt = 0 t1 = time.time() rmapT = Rmap(args.rmapD).reverseMapT() iuser = 0 lnkCnt = 0 runCnt = 0 for user, hdir in passwd_generator(): xaltDir = os.path.join(hdir,".xalt.d") if (os.path.isdir(xaltDir)): iuser += 1 linkFnA = files_in_tree(xaltDir, "*/link.*.json") lnkCnt += link_json_to_db(xalt, user, rmapT, linkFnA) if (args.delete): remove_files(linkFnA) remove_files(files_in_tree(xaltDir, "*/.link.*.json")) runFnA = files_in_tree(xaltDir, "*/run.*.json") runCnt += run_json_to_db(xalt, user, rmapT, runFnA) if (args.delete): remove_files(runFnA) remove_files(files_in_tree(xaltDir, "*/.run.*.json")) icnt += 1 pbar.update(icnt) xalt.connect().close() pbar.fini() t2 = time.time() rt = t2 - t1 if (args.timer): print("Time: ", time.strftime("%T", time.gmtime(rt))) print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
def __build_fieldT(self, cmd): outStr = capture(["objdump", "-s", "-j", ".xalt", cmd]) fieldT = {} if (not outStr.find("Contents of section .xalt:") != -1): return fieldT outputA = outStr.split('\n') outputA.pop(0) outputA.pop(0) outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): sA.append(split[-1]) s = "".join(sA) xaltA = re.split('%%', s) fieldT = {} N = len(xaltA) idx = -1 while (True): idx = idx + 1 line = xaltA[idx] if (line.find("XALT_Link_Info_End") != -1): break m = keyPat.search(line) if (m): key = m.group(1) idx = idx + 1 value = xaltA[idx].replace("_%_%_"," ") fieldT[key] = value return fieldT
def __parseLDD(self,ldd): if (not ldd or ldd.find("not a dynamic executable") > 0): return [] lineA = ldd.split('\n') libA = [] d = {} for line in lineA: fieldA = line.split() N = len(fieldA) if (N < 1): break elif (N == 4): lib = fieldA[2] else: lib = fieldA[0] lib = os.path.realpath(lib) d[lib] = True libA = d.keys() libA = sorted(libA) libB = [] for lib in libA: hash_line = capture(['sha1sum', lib]) if (hash_line.find("No such file or directory") != -1): v = "unknown" else: v = hash_line.split()[0] libB.append([lib, v]) return libB
def diskquota(): # primitive argument input for userid - no error checking if len(sys.argv) == 2: userid = sys.argv[1] else: userid = capture("whoami").rstrip() #userid="u6006455" host = syshost() grepcmd1 = "curl --data 'unid={0}' https://www.chpc.utah.edu/apps/systems/curl_post/CHPCUserQuota.php -k -s | tail -n +2".format( userid) myfs = capture(grepcmd1).splitlines() goodcolor = "32m" badcolor = "31m" for space in myfs: #print(space) # if ( (host=="ls4" and space=="/home1") or (host=="maverick" and space=="/home") or (host=="ls5" and space=="/home1")): rawinfo = space.split("\t") #print(rawinfo) if (rawinfo[0] == userid) or (userid in rawinfo[0]): usage = rawinfo[1][:-4] files = rawinfo[3] quota = rawinfo[2][:-1] if rawinfo[0] == userid: fsuse = rawinfo[5][:-4] fsmax = rawinfo[6][:-1] #if ((fsmax != 0) and (float(fsuse)/float(fsmax) > 0.9)): if (isinstance(fsuse, int) and isinstance(fsmax, int)): if (float(fsuse) / float(fsmax) > 0.9): fscolor = badcolor else: fscolor = goodcolor if quota == "0": badusage = 100 else: badusage = int(quota) if float(usage) > badusage: ucolor = badcolor umsg = "\033[1;31mare over the quota limit\033[0m" else: ucolor = goodcolor umsg = "have quota limit" if int(files) > 100000: fcolor = badcolor else: fcolor = goodcolor if rawinfo[0] == userid: if (rawinfo[2] == "0G"): if (files == 0): print( "\tIn \033[1;36m{0}\033[0m you have no quota limit, use \033[1;{2}{1} GB\033[0m" .format("General Home Directory", usage, ucolor)) else: print( "\tIn \033[1;36m{0}\033[0m you have no quota limit, use \033[1;{3}{1} GB\033[0m and have \033[1;{4}{2}\033[0m files" .format("General Home Directory", usage, files, ucolor, fcolor)) if (fsuse != 0): print( "\t (total file system usage is \033[1;{2}{0} GB\033[0m out of \033[1;{2}{1} GB\033[0m)" .format(fsuse, fsmax, fscolor)) #print("\tOn \033[1;36m{0}\033[0m you have no quota limit, use \033[1;{3}{1} GB\033[0m (total file system usage is {5} GB out of {6} GB) and have \033[1;{4}{2}\033[0m files".format(rawinfo[0],usage,files,ucolor,fcolor,fsuse,fsmax)) else: if (files == '0'): print( "\tIn \033[1;36m{0}\033[0m you {4} \033[1;{3}{2} GB\033[0m and use \033[1;{3}{1} GB\033[0m" .format("General home directory", usage, quota, ucolor, umsg)) else: print( "\tIn \033[1;36m{0}\033[0m you {6} \033[1;{4}{3} GB\033[0m, use \033[1;{4}{1} GB\033[0m and have \033[1;{5}{2}\033[0m files" .format("General home directory", usage, files, quota, ucolor, fcolor, umsg)) if (fsuse != '0'): print( "\t (total file system usage is \033[1;{2}{0} GB\033[0m out of \033[1;{2}{1} GB\033[0m)" .format(fsuse, fsmax, fscolor)) else: print( "\tIn \033[1;36m{0}\033[0m you {6} \033[1;{7}{3} GB\033[0m, use \033[1;{4}{1} GB\033[0m and have \033[1;{5}{2}\033[0m files" .format("PE Home Directory", usage, files, quota, ucolor, fcolor, umsg, ucolor))
def execute(self): userid=capture("whoami").rstrip() ### userid="jkatzene" #Fake test host=syshost() if (host=="stampede" or host=="maverick" or host=="ls5"): TACC_ACC_DIR="/usr/local/etc/" elif host=="ls4": TACC_ACC_DIR="/sge_common/default/acct/map/" else: return True proj_map=TACC_ACC_DIR+"project.map" projuser_map=TACC_ACC_DIR+"projectuser.map" projbalance_map=TACC_ACC_DIR+"projectbalance.map" usage_map=TACC_ACC_DIR+"usage.map" # grepcmd="grep %s %s" %(userid,projuser_map) grepcmd="""awk -v user=%s '$1 == user {print $0}' %s """ %(userid,projuser_map) # print(grepcmd) myprojects=capture(grepcmd).split() # print(myprojects) if len(myprojects) < 2: self.error_message+="\tError: "+"No valid allocation\n" return False Flag=False for proj in myprojects[1:]: grepcmd="grep %s$ %s" %(proj,proj_map) # print(grepcmd) proj_name_all=capture(grepcmd).split() # print(proj_name_all) if proj_name_all: proj_name=proj_name_all[0] # grepcmd="grep %s %s" %(proj_name,usage_map) grepcmd="awk -F: '{ if ($1==\"%s\") print}' %s" %(proj_name,usage_map) # print(grepcmd) # print(capture(grepcmd)) usage_report=capture(grepcmd).split(":") if len(usage_report) < 6: print("\033[1;33m\tWarning: Valid allocation detected.\033[0m") continue exp_date=usage_report[3].split("-") exp_date_f=date(int(exp_date[0]), int(exp_date[1]), int(exp_date[2])) today = date.today() cur_bal=float(usage_report[6]) # print(" My project '%s'; balance %s." %(proj_name,cur_bal)) if exp_date_f < date.today(): print("\033[1;33m\tWarning: One of your projects '%s' has expired.\033[0m" %proj_name[0]) elif cur_bal < 0: print("\033[1;33m\tWarning: One of your projects '%s' has negative balance %s.\033[0m" %(proj_name,cur_bal)) else: Flag=True if Flag: return True else: self.error_message+="\tError: All your allocations are invalid" return Flag
def _capture(self,cmd): return capture(self._cmd(cmd))
def __computeHash(self, cmd): fieldA = capture(["sha1sum", cmd]).split() return fieldA[0]
def main(): try: uuid = sys.argv[ 1] status = sys.argv[ 2] wd = sys.argv[ 3] syshost = sys.argv[ 4] pstree = sys.argv[ 5] execname = sys.argv[ 6] xaltobj = sys.argv[ 7] build_epoch = sys.argv[ 8] linklineFn = sys.argv[ 9] resultFn = sys.argv[10] if (execname.find("conftest") != -1): return 1 hash_line = capture(['sha1sum', execname]) if (hash_line.find("No such file or directory") != -1): return 1 hash_id = hash_line.split()[0] # Step one clean up linkline data sA = cleanup(xaltobj, linklineFn) resultT = {} resultT['uuid'] = uuid resultT['link_program'] = extract_compiler(pstree) resultT['build_user'] = os.environ['USER'] resultT['exit_code'] = status resultT['build_epoch'] = build_epoch resultT['exec_path'] = os.path.abspath(execname) resultT['hash_id'] = hash_id resultT['wd'] = wd resultT['build_syshost'] = syshost resultT['linkA'] = sA dirname,fn = os.path.split(resultFn) tmpFn = os.path.join(dirname, "." + fn) except: logger.exception("XALT_EXCEPTION:xalt_generate_linkdata") try: if (not os.path.isdir(dirname)): os.mkdir(dirname); s = json.dumps(resultT, sort_keys=True, indent=2, separators=(',',': ')) f = open(tmpFn,'w') f.write(s) f.write("\n") if (hash_line.find("sha1sum:") != -1): f.write(hash_line) f.close() os.rename(tmpFn, resultFn) except (OSError): logger.exception("XALT_EXCEPTION:xalt_generate_linkdata(write json file)") return 0
def allocations(): host = syshost() #print(host) if (host!="kingspeak")and(host!="ember")and(host!="lonepeak")and(host!="notchpeak")and(host!="ash")and(host!="redwood")and(host!="crystalpeak"): print("This command needs to run on one of the CHPC clusters") sys.exit(1) # primitive argument input for userid - no error checking if len(sys.argv)==2: userid=sys.argv[1] else: userid=capture("whoami").rstrip() #userid="u1119546" #userid="u0631741" # redwood tests #userid="u6000771" #userid="u0413537" #userid="u6002243" # MC Jan 20 # potentially cleaner version may be to create a list of account-QOS associations with sacctmgr # and then compare QOS from each of the association to the "scontrol -o show partition" output to get the corresponding partition # the scontrol can be run only once with result stored in an array so that it's not run repeatedly # sacctmgr -p show qos lists what QOSes can this one preempt (Preempt = column 4), can see if preemptable QOS is in this output which would mean that it's preemptable grepcmd1="sacctmgr -n -p show assoc where user={0}".format(userid) #print(grepcmd1) myaccts=capture(grepcmd1).split() #print(myaccts,len(myaccts)) if host=="redwood": clusters=["redwood"] elif host=="crystalpeak": clusters=["crystalpeak"] else: clusters=["kingspeak","notchpeak","lonepeak","ash"] for cluster in clusters: FCFlag=True if cluster=="kingspeak": cl="kp" elif cluster=="notchpeak": cl="np" elif cluster=="ember": cl="em" elif cluster=="lonepeak": cl="lp" elif cluster=="ash": cl="smithp-ash" elif cluster=="redwood": cl="rw" elif cluster=="crystalpeak": cl="cp" matchcl = [s for s in myaccts if cluster in s] #print(matchcl, len(matchcl)) if len(matchcl) > 0: if (len(matchcl) > 1): # first filter out owner accounts # this will be true if there are owner nodes matchstr="^((?!-{0}).)*$".format(cl) #print(matchstr) r=re.compile(matchstr) matchcl = list(filter(r.match, matchcl)) #print(matchcl) #print("Error, more than 1 match: {0}".format(matchcl)) # now filter out the freecycle accounts matchfc = [s for s in matchcl if "freecycle" in s] if len(matchfc) > 0: #print(matchfc) for matchfc0 in matchfc: pnames=matchfc0.split('|') #print(pnames) print("\tYour group \033[1;31m{0}\033[0m does not have a \033[1;36mgeneral\033[0m allocation on \033[1;34m{1}\033[0m".format(pnames[1],cluster)) print("\tYou can use \033[1;33mpreemptable\033[0m mode on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],pnames[17])) print("\tYou can use \033[1;33mpreemptable\033[0m mode on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],cluster+"-shared-freecycle")) # now look at allocated group accounts - so need to exclude owner-guest and freecycle matchg1 = [s for s in matchcl if not "freecycle" in s] #print(matchg1) matchg2 = [s for s in matchg1 if not "guest" in s] matchg3 = [s for s in matchg2 if not "collab" in s] # also filter out gpu accounts matchg4 = [s for s in matchg3 if not "gpu" in s] #matchg = [s for s in matchg2 if not "shared-short" in s] matchg = [s for s in matchg4 if not "notchpeak-shared" in s] if len(matchg)>0: #print(matchg) for matchg1 in matchg: #print(matchg1) myrecord1 = matchg1.split('|') #print(myrecord1) print("\tYou have a \033[1;36mgeneral\033[0m allocation on \033[1;34m{1}\033[0m. Account: \033[1;32m{0}\033[0m, Partition: \033[1;32m{2}\033[0m".format(myrecord1[1],cluster,myrecord1[18])) if (myrecord1[1] != "dtn"): # account dtn that matches here does not have shared partition print("\tYou have a \033[1;36mgeneral\033[0m allocation on \033[1;34m{1}\033[0m. Account: \033[1;32m{0}\033[0m, Partition: \033[1;32m{2}\033[0m".format(myrecord1[1],cluster,cluster+"-shared")) # shared-short matchgrp = [s for s in myaccts if "shared-short" in s] matchcl = [s for s in matchgrp if cluster in s] if len(matchcl) > 0: matchstr="^((?!{0}).)*$".format(cl) r=re.compile(matchstr) matchcl = list(filter(r.match, matchcl)) pnames=matchcl[0].split('|') print("\tYou have a \033[1;36mgeneral\033[0m allocation on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{1}\033[0m".format(cluster,pnames[1])) # owner accounts grepcmd1="sacctmgr -p show assoc where user={0} | grep {1} | grep -w {2} | grep -v guest".format(userid,cluster,cl) # need to grep out guest since for ash cl=smithp-ash #print(grepcmd1) myprojects=capture(grepcmd1).split() #print(myprojects,len(myprojects)) if len(myprojects) > 0: for project in myprojects: pnames=project.split('|') #print(pnames) # MC 1/24/20 - using scontrol to grep for partition that corresponds to the QOS in pnames[18] # example user that had QOS vs partition mix up - u6022494 qosname = pnames[18] # in case "Def QOS" = pnames[18] is not defined, try "QOS" = pnames[17] if len(pnames[18]) == 0: qosname = pnames[17] grepcmd2="scontrol -M {1} -o show partition | grep {0} | grep -v shared".format(qosname,cluster) #print(grepcmd2) myparts=capture(grepcmd2).split() if len(myparts) > 0: #print(myparts,len(myparts)) #print(myparts[0]) mypart=myparts[0].split('=') #print(mypart[1]) pgroup=mypart[1].split('-') print("\tYou have an \033[1;36mowner\033[0m allocation on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],mypart[1])) print("\tYou have an \033[1;36mowner\033[0m allocation on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],pgroup[0]+"-shared-"+pgroup[1])) else: print("\t\033[1;31mError:\033[0m you are in QOS \033[1;34m{0}\033[0m, but partition \033[1;32m{0}\033[0m does not exist. Please contact CHPC to fix this.".format(qosname)) # collab accounts grepcmd1="sacctmgr -p show assoc where user={0} | grep {1} | grep -w {2} | grep -v guest".format(userid,cluster,"collab") # need to grep out guest since for ash cl=smithp-ash #print(grepcmd1) myprojects=capture(grepcmd1).split() #print(myprojects,len(myprojects)) if len(myprojects) > 0: for project in myprojects: pnames=project.split('|') pgroup=pnames[17].split('-') #print(pnames) print("\tYou have an \033[1;36mowner\033[0m allocation on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],pgroup[0]+"-"+cl)) print("\tYou have an \033[1;36mowner\033[0m allocation on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],pgroup[0]+"-shared-"+cl)) # owner guest # have to get matchcl again since we may have changed it above matchcl = [s for s in myaccts if cluster in s] matchstr=".*\\bguest\\.*" #print(matchstr) #print(matchcl, len(matchcl)) r=re.compile(matchstr) myprojects = list(filter(r.match, matchcl)) #print(myprojects) if len(myprojects) > 0: for project in myprojects: if "gpu" in project: gpustr = " GPU" else: gpustr = "" pnames=project.split('|') part=pnames[17].split(',') # #print(pnames) print("\tYou can use \033[1;33mpreemptable{3}\033[0m mode on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],part[0],gpustr)) # GPU accounts grepcmd1="sacctmgr -p show assoc where user={0} | grep {1} | grep -w gpu | grep -v guest".format(userid,cluster) #print(grepcmd1) myprojects=capture(grepcmd1).split() if len(myprojects) > 0: for project in myprojects: pnames=project.split('|') #print(pnames) print("\tYou have a \033[1;36mGPU\033[0m allocation on \033[1;34m{0}\033[0m. Account: \033[1;32m{1}\033[0m, Partition: \033[1;32m{2}\033[0m".format(cluster,pnames[1],pnames[17]))