def __init__(self, exec_progA): ignoreT = { 'env' : True, 'time' : True, } cmd = None for prog in exec_progA: bare = os.path.basename(prog) if (not (bare in ignoreT)): cmd = prog break self.__execType = None self.__execName = which(cmd) self.__libA = [] if (self.__execName): outStr = capture(["file", self.__execName]) if (outStr.find("script") > 0 or outStr.find("text") > 0): self.__execType = "script" else: self.__execType = "binary" ldd = capture(["ldd", self.__execName]) self.__libA = self.__parseLDD(ldd) info = os.stat(self.__execName) self.__modify = info.st_mtime self.__hash = self.__computeHash(self.__execName)
def __init__(self, cmd): """ Find the full path to the executable. Then find the shared libraries if there and get the hash time. @param exec_progA: the command line after the mpirun type arguments have been removed. """ self.__execType = None self.__execName = which(cmd) self.__libA = [] if (self.__execName): outStr = capture(["@file@", self.__execName]) if (outStr.find("script") > 0 or outStr.find("text") > 0): self.__execType = "script" else: self.__execType = "binary" # ldd is a shell script so it must do path resolution! os.environ['PATH'] = '/usr/bin:/bin' ldd = capture(["@ldd@", self.__execName]) self.__libA = self.__parseLDD(ldd) info = os.stat(self.__execName) self.__modify = info.st_mtime self.__hash = self.__computeHash(self.__execName)
def main(): """ Write the environment, XALT info from executable and use the transmission factory to save it. """ try: # parse command line options: args = CmdLineOptions().execute() runA = json.loads(args.exec_progT[0]) if (args.endTime > 0): uuidA = json.loads(args.uuidA) else: dateStr = capture("date +%Y_%m_%d_%H_%M_%S")[0:-1] N = len(runA) uuidA = [] for i in xrange(N): fnA = [] uuid = capture(args.uuidgen)[0:-1] fnA.append(os.environ.get("HOME", "/")) fnA.append("/.xalt.d/run.") fnA.append(args.syshost) fnA.append(".") fnA.append(dateStr) fnA.append(".") fnA.append(uuid) fnA.append(".json") fn = "".join(fnA) uuidA.append({'uuid': uuid, 'fn': fn}) for i, run in enumerate(runA): uuid = uuidA[i]['uuid'] fn = uuidA[i]['fn'] userExec = UserExec(run['exec_prog']) if (not userExec.execName()): return userT = UserEnvT(args, uuid, run['ntasks'], userExec).userT() submitT = {} submitT['userT'] = userT submitT['xaltLinkT'] = ExtractXALT( userExec.execName()).xaltRecordT() submitT['libA'] = userExec.libA() submitT['envT'] = EnvT().envT() submitT['hash_id'] = userExec.hash() xfer = XALT_transmission_factory.build(XALT_TRANSMISSION_STYLE, args.syshost, "run", fn) xfer.save(submitT) if (args.endTime == 0): print(json.dumps(uuidA)) except Exception as e: print("XALT_EXCEPTION(xalt_run_submission.py): ", e, file=sys.stderr) logger.exception("XALT_EXCEPTION:xalt_run_submission.py")
def main(): """ Write the environment, XALT info from executable and use the transmission factory to save it. """ try: # parse command line options: args = CmdLineOptions().execute() runA = json.loads(args.exec_progT[0]) if (args.endTime > 0): uuidA = json.loads(args.uuidA) else: dateStr = capture("date +%Y_%m_%d_%H_%M_%S")[0:-1] N = len(runA) uuidA = [] for i in xrange(N): fnA = [] uuid = capture(args.uuidgen)[0:-1] fnA.append(os.environ.get("HOME","/")) fnA.append("/.xalt.d/run.") fnA.append(args.syshost) fnA.append(".") fnA.append(dateStr) fnA.append(".") fnA.append(uuid) fnA.append(".json") fn = "".join(fnA) uuidA.append({'uuid' : uuid, 'fn' : fn}) for i, run in enumerate(runA): uuid = uuidA[i]['uuid'] fn = uuidA[i]['fn'] userExec = UserExec(run['exec_prog']) if (not userExec.execName()): return userT = UserEnvT(args, uuid, run['ntasks'], userExec).userT() submitT = {} submitT['userT'] = userT submitT['xaltLinkT'] = ExtractXALT(userExec.execName()).xaltRecordT() submitT['libA'] = userExec.libA() submitT['envT'] = EnvT().envT() submitT['hash_id'] = userExec.hash() xfer = XALT_transmission_factory.build(XALT_TRANSMISSION_STYLE, args.syshost, "run", fn) xfer.save(submitT) if (args.endTime == 0): print(json.dumps(uuidA)) except Exception as e: print("XALT_EXCEPTION(xalt_run_submission.py): ",e, file=sys.stderr) logger.exception("XALT_EXCEPTION:xalt_run_submission.py")
def find_exec(ignoreT, argT, npT, cmdArg, argA, *n, **kw): """ Walk the command line and first walk over the command line options. @param ignoreT: Table of names to ignore. @param argT: Table of command line args. Keys are the name of the argument, values are the number of arguments it takes (> 0). @param npT: Options for the number of tasks. @param cmdArg: command (like in ibrun.symm) that points to a user program. @param argA: The command line split into an array. @param kw: If dot=True then add "." to the end of PATH. """ N = len(argA) compute_ntasks = kw.get("compute_ntasks", default_compute_ntasks) if ('dot' in kw): os.environ['PATH'] = os.environ.get('PATH', "") + ":." i = 0 done = False resultA = [] path = None while (not done): t = {} while (i < N): arg = argA[i] i = parse_ntasks(npT, arg, i, argA, t) if (arg == cmdArg): path = which(find_cmd(ignoreT, 0, argA[i + 1].split())) break n = argT.get(arg, -1) if (n > 0): i += n + 1 continue if (arg[0:1] == "-"): i = i + 1 continue break if (not path): path = which(find_cmd(ignoreT, i, argA)) or "unknown" ntasks = compute_ntasks(t) uuid = capture("@uuidgen@").rstrip('\n') resultA.append({'exec_prog': path, 'ntasks': ntasks, 'uuid': uuid}) # Loop for colons done = True while (i < N): arg = argA[i] i = i + 1 if (arg == ":"): done = False path = None break return json.dumps(resultA)
def find_exec(ignoreT, argT, npT, cmdArg, argA, *n, **kw): """ Walk the command line and first walk over the command line options. @param ignoreT: Table of names to ignore. @param argT: Table of command line args. Keys are the name of the argument, values are the number of arguments it takes (> 0). @param npT: Options for the number of tasks. @param cmdArg: command (like in ibrun.symm) that points to a user program. @param argA: The command line split into an array. @param kw: If dot=True then add "." to the end of PATH. """ N = len(argA) compute_ntasks=kw.get("compute_ntasks",default_compute_ntasks) if ('dot' in kw): os.environ['PATH'] = os.environ.get('PATH',"") + ":." i = 0 done = False resultA = [] path = None while (not done): t = {} while (i < N): arg = argA[i] i = parse_ntasks(npT, arg, i, argA, t) if (arg == cmdArg): path = which(find_cmd(ignoreT, 0, argA[i+1].split())) break n = argT.get(arg,-1) if (n > 0): i += n + 1 continue if (arg[0:1] == "-"): i = i + 1 continue break if (not path): path = which(find_cmd(ignoreT, i, argA)) or "unknown" ntasks = compute_ntasks(t) uuid = capture("@uuidgen@").rstrip('\n') resultA.append({'exec_prog':path, 'ntasks':ntasks, 'uuid': uuid}) # Loop for colons done = True while (i < N): arg = argA[i] i = i + 1 if (arg == ":"): done = False path = None break return json.dumps(resultA)
def __init__(self, cmd): """ Find the full path to the executable. Then find the shared libraries if there and get the hash time. @param exec_progA: the command line after the mpirun type arguments have been removed. """ self.__execType = None self.__execName = which(cmd) self.__libA = [] if (self.__execName): outStr = capture(["file", self.__execName]) if (outStr.find("script") > 0 or outStr.find("text") > 0): self.__execType = "script" else: self.__execType = "binary" ldd = capture(["ldd", self.__execName]) self.__libA = self.__parseLDD(ldd) info = os.stat(self.__execName) self.__modify = info.st_mtime self.__hash = self.__computeHash(self.__execName)
def __init__(self, exec_progA): """ Find the user's executable by walking the command line and skipping the executables name in ignoreT. Then find the full path to the executable. Finally find the shared libraries if there and get the hash time. @param exec_progA: the command line after the mpirun type arguments have been removed. """ ignoreT = { 'env' : True, 'time' : True, } cmd = None for prog in exec_progA: bare = os.path.basename(prog) if (not (bare in ignoreT)): cmd = prog break self.__execType = None self.__execName = which(cmd) self.__libA = [] if (self.__execName): outStr = capture(["file", self.__execName]) if (outStr.find("script") > 0 or outStr.find("text") > 0): self.__execType = "script" else: self.__execType = "binary" ldd = capture(["ldd", self.__execName]) self.__libA = self.__parseLDD(ldd) info = os.stat(self.__execName) self.__modify = info.st_mtime self.__hash = self.__computeHash(self.__execName)
def __init__(self, cmd): """ Parse the input path for the xalt record. If it exists extract the key value pairs and store in a table. @param cmd: the path to the program or shared library that has (or could have) an XALT record. """ outStr = capture(["objdump", "-s", "-j", ".xalt", cmd]) self.__fieldT = {} if (not outStr.find("Contents of section .xalt:") != -1): return outputA = outStr.split('\n') outputA.pop(0) outputA.pop(0) outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): sA.append(split[-1]) s = "".join(sA) xaltA = re.split('%%', s) fieldT = {} N = len(xaltA) idx = -1 while (True): idx = idx + 1 line = xaltA[idx] if (line.find("XALT_Link_Info_End") != -1): break m = keyPat.search(line) if (m): key = m.group(1) idx = idx + 1 value = xaltA[idx].replace("_%_%_", " ") fieldT[key] = value self.__fieldT = fieldT
def __init__(self, cmd): """ Parse the input path for the xalt record. If it exists extract the key value pairs and store in a table. @param cmd: the path to the program or shared library that has (or could have) an XALT record. """ outStr = capture(["objdump", "-s", "-j", ".xalt", cmd]) self.__fieldT = {} if (not outStr.find("Contents of section .xalt:") != -1): return outputA = outStr.split('\n') outputA.pop(0) outputA.pop(0) outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): sA.append(split[-1]) s = "".join(sA) xaltA = re.split('%%', s) fieldT = {} N = len(xaltA) idx = -1 while (True): idx = idx + 1 line = xaltA[idx] if (line.find("XALT_Link_Info_End") != -1): break m = keyPat.search(line) if (m): key = m.group(1) idx = idx + 1 value = xaltA[idx].replace("_%_%_"," ") fieldT[key] = value self.__fieldT = fieldT
def __extract_xalt_darwin(self, execPath): """ Use objdump to extract the xalt record in a linux executable. @param execPath: the path to the program or shared library that has (or could have) an XALT record. """ outStr = capture (["@otool@", "-s", ".XALT", ".xalt", execPath]) outputA = outStr.split("\n") outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): for hexStr in split[1:]: sA.append(chr(int(hexStr,16))) return "".join(sA)
def __extract_xalt_darwin(self, execPath): """ Use objdump to extract the xalt record in a linux executable. @param execPath: the path to the program or shared library that has (or could have) an XALT record. """ outStr = capture(["@otool@", "-s", ".XALT", ".xalt", execPath]) outputA = outStr.split("\n") outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): for hexStr in split[1:]: sA.append(chr(int(hexStr, 16))) return "".join(sA)
def __build_fieldT(self, cmd): outStr = capture(["objdump", "-s", "-j", ".xalt", cmd]) fieldT = {} if (not outStr.find("Contents of section .xalt:") != -1): return fieldT outputA = outStr.split('\n') outputA.pop(0) outputA.pop(0) outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): sA.append(split[-1]) s = "".join(sA) xaltA = re.split('%%', s) fieldT = {} N = len(xaltA) idx = -1 while (True): idx = idx + 1 line = xaltA[idx] if (line.find("XALT_Link_Info_End") != -1): break m = keyPat.search(line) if (m): key = m.group(1) idx = idx + 1 value = xaltA[idx].replace("_%_%_"," ") fieldT[key] = value return fieldT
def __extract_xalt_linux(self, execPath): """ Use objdump to extract the xalt record in a linux executable. @param execPath: the path to the program or shared library that has (or could have) an XALT record. """ outStr = capture(["@objdump@", "-s", "-j", ".xalt", execPath ]) self.__fieldT = {} if (not outStr.find("Contents of section .xalt:") != -1): return outputA = outStr.split('\n') outputA.pop(0) outputA.pop(0) outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): sA.append(split[-1]) return "".join(sA)
def __extract_xalt_linux(self, execPath): """ Use objdump to extract the xalt record in a linux executable. @param execPath: the path to the program or shared library that has (or could have) an XALT record. """ outStr = capture(["@objdump@", "-s", "-j", ".xalt", execPath]) self.__fieldT = {} if (not outStr.find("Contents of section .xalt:") != -1): return outputA = outStr.split('\n') outputA.pop(0) outputA.pop(0) outputA.pop(0) outputA.pop(0) sA = [] for line in outputA: split = line.split() if (len(split) > 0): sA.append(split[-1]) return "".join(sA)
def __parseLDD(self,ldd): """ Return the list of shared libraries with their sha1sum. """ if (not ldd or ldd.find("not a dynamic executable") > 0): return [] lineA = ldd.split('\n') libA = [] d = {} for line in lineA: fieldA = line.split() N = len(fieldA) if (N < 1): break elif (N == 4): lib = fieldA[2] else: lib = fieldA[0] lib = os.path.realpath(lib) d[lib] = True libA = d.keys() libA = sorted(libA) libB = [] for lib in libA: hash_line = capture(['@sha1sum@', lib]) if (hash_line.find("No such file or directory") != -1): v = "unknown" else: v = hash_line.split()[0] libB.append([lib, v]) return libB
def __parseLDD(self, ldd): """ Return the list of shared libraries with their sha1sum. """ if (not ldd or ldd.find("not a dynamic executable") > 0): return [] lineA = ldd.split('\n') libA = [] d = {} for line in lineA: fieldA = line.split() N = len(fieldA) if (N < 1): break elif (N == 4): lib = fieldA[2] else: lib = fieldA[0] lib = os.path.realpath(lib) d[lib] = True libA = d.keys() libA = sorted(libA) libB = [] for lib in libA: hash_line = capture(['@sha1sum@', lib]) if (hash_line.find("No such file or directory") != -1): v = "unknown" else: v = hash_line.split()[0] libB.append([lib, v]) return libB
def __computeHash(self, cmd): """ Compute the sha1sum of the executable. """ fieldA = capture(["@sha1sum@", cmd]).split() return fieldA[0]
def __computeHash(self, cmd): fieldA = capture(["sha1sum", cmd]).split() return fieldA[0]