def __init__(self, exec_progA):
ignoreT = {
'env' : True,
'time' : True,
}
cmd = None
for prog in exec_progA:
bare = os.path.basename(prog)
if (not (bare in ignoreT)):
cmd = prog
break
self.__execType = None
self.__execName = which(cmd)
self.__libA = []
if (self.__execName):
outStr = capture(["file", self.__execName])
if (outStr.find("script") > 0 or outStr.find("text") > 0):
self.__execType = "script"
else:
self.__execType = "binary"
ldd = capture(["ldd", self.__execName])
self.__libA = self.__parseLDD(ldd)
info = os.stat(self.__execName)
self.__modify = info.st_mtime
self.__hash = self.__computeHash(self.__execName)
def __init__(self, cmd):
"""
Find the full path to the executable. Then find the shared
libraries if there and get the hash time.
@param exec_progA: the command line after the mpirun type
arguments have been removed.
"""
self.__execType = None
self.__execName = which(cmd)
self.__libA = []
if (self.__execName):
outStr = capture(["@file@", self.__execName])
if (outStr.find("script") > 0 or outStr.find("text") > 0):
self.__execType = "script"
else:
self.__execType = "binary"
# ldd is a shell script so it must do path resolution!
os.environ['PATH'] = '/usr/bin:/bin'
ldd = capture(["@ldd@", self.__execName])
self.__libA = self.__parseLDD(ldd)
info = os.stat(self.__execName)
self.__modify = info.st_mtime
self.__hash = self.__computeHash(self.__execName)
def __init__(self, cmd):
"""
Find the full path to the executable. Then find the shared
libraries if there and get the hash time.
@param exec_progA: the command line after the mpirun type
arguments have been removed.
"""
self.__execType = None
self.__execName = which(cmd)
self.__libA = []
if (self.__execName):
outStr = capture(["@file@", self.__execName])
if (outStr.find("script") > 0 or outStr.find("text") > 0):
self.__execType = "script"
else:
self.__execType = "binary"
# ldd is a shell script so it must do path resolution!
os.environ['PATH'] = '/usr/bin:/bin'
ldd = capture(["@ldd@", self.__execName])
self.__libA = self.__parseLDD(ldd)
info = os.stat(self.__execName)
self.__modify = info.st_mtime
self.__hash = self.__computeHash(self.__execName)
def main():
"""
Write the environment, XALT info from executable and use the
transmission factory to save it.
"""
try:
# parse command line options:
args = CmdLineOptions().execute()
runA = json.loads(args.exec_progT[0])
if (args.endTime > 0):
uuidA = json.loads(args.uuidA)
else:
dateStr = capture("date +%Y_%m_%d_%H_%M_%S")[0:-1]
N = len(runA)
uuidA = []
for i in xrange(N):
fnA = []
uuid = capture(args.uuidgen)[0:-1]
fnA.append(os.environ.get("HOME", "/"))
fnA.append("/.xalt.d/run.")
fnA.append(args.syshost)
fnA.append(".")
fnA.append(dateStr)
fnA.append(".")
fnA.append(uuid)
fnA.append(".json")
fn = "".join(fnA)
uuidA.append({'uuid': uuid, 'fn': fn})
for i, run in enumerate(runA):
uuid = uuidA[i]['uuid']
fn = uuidA[i]['fn']
userExec = UserExec(run['exec_prog'])
if (not userExec.execName()):
return
userT = UserEnvT(args, uuid, run['ntasks'], userExec).userT()
submitT = {}
submitT['userT'] = userT
submitT['xaltLinkT'] = ExtractXALT(
userExec.execName()).xaltRecordT()
submitT['libA'] = userExec.libA()
submitT['envT'] = EnvT().envT()
submitT['hash_id'] = userExec.hash()
xfer = XALT_transmission_factory.build(XALT_TRANSMISSION_STYLE,
args.syshost, "run", fn)
xfer.save(submitT)
if (args.endTime == 0):
print(json.dumps(uuidA))
except Exception as e:
print("XALT_EXCEPTION(xalt_run_submission.py): ", e, file=sys.stderr)
logger.exception("XALT_EXCEPTION:xalt_run_submission.py")
def main():
"""
Write the environment, XALT info from executable and use the
transmission factory to save it.
"""
try:
# parse command line options:
args = CmdLineOptions().execute()
runA = json.loads(args.exec_progT[0])
if (args.endTime > 0):
uuidA = json.loads(args.uuidA)
else:
dateStr = capture("date +%Y_%m_%d_%H_%M_%S")[0:-1]
N = len(runA)
uuidA = []
for i in xrange(N):
fnA = []
uuid = capture(args.uuidgen)[0:-1]
fnA.append(os.environ.get("HOME","/"))
fnA.append("/.xalt.d/run.")
fnA.append(args.syshost)
fnA.append(".")
fnA.append(dateStr)
fnA.append(".")
fnA.append(uuid)
fnA.append(".json")
fn = "".join(fnA)
uuidA.append({'uuid' : uuid, 'fn' : fn})
for i, run in enumerate(runA):
uuid = uuidA[i]['uuid']
fn = uuidA[i]['fn']
userExec = UserExec(run['exec_prog'])
if (not userExec.execName()):
return
userT = UserEnvT(args, uuid, run['ntasks'], userExec).userT()
submitT = {}
submitT['userT'] = userT
submitT['xaltLinkT'] = ExtractXALT(userExec.execName()).xaltRecordT()
submitT['libA'] = userExec.libA()
submitT['envT'] = EnvT().envT()
submitT['hash_id'] = userExec.hash()
xfer = XALT_transmission_factory.build(XALT_TRANSMISSION_STYLE,
args.syshost, "run", fn)
xfer.save(submitT)
if (args.endTime == 0):
print(json.dumps(uuidA))
except Exception as e:
print("XALT_EXCEPTION(xalt_run_submission.py): ",e, file=sys.stderr)
logger.exception("XALT_EXCEPTION:xalt_run_submission.py")
def find_exec(ignoreT, argT, npT, cmdArg, argA, *n, **kw):
"""
Walk the command line and first walk over the command line options.
@param ignoreT: Table of names to ignore.
@param argT: Table of command line args. Keys are the name of the argument, values are the number of arguments it takes (> 0).
@param npT: Options for the number of tasks.
@param cmdArg: command (like in ibrun.symm) that points to a user program.
@param argA: The command line split into an array.
@param kw: If dot=True then add "." to the end of PATH.
"""
N = len(argA)
compute_ntasks = kw.get("compute_ntasks", default_compute_ntasks)
if ('dot' in kw):
os.environ['PATH'] = os.environ.get('PATH', "") + ":."
i = 0
done = False
resultA = []
path = None
while (not done):
t = {}
while (i < N):
arg = argA[i]
i = parse_ntasks(npT, arg, i, argA, t)
if (arg == cmdArg):
path = which(find_cmd(ignoreT, 0, argA[i + 1].split()))
break
n = argT.get(arg, -1)
if (n > 0):
i += n + 1
continue
if (arg[0:1] == "-"):
i = i + 1
continue
break
if (not path):
path = which(find_cmd(ignoreT, i, argA)) or "unknown"
ntasks = compute_ntasks(t)
uuid = capture("@uuidgen@").rstrip('\n')
resultA.append({'exec_prog': path, 'ntasks': ntasks, 'uuid': uuid})
# Loop for colons
done = True
while (i < N):
arg = argA[i]
i = i + 1
if (arg == ":"):
done = False
path = None
break
return json.dumps(resultA)
def find_exec(ignoreT, argT, npT, cmdArg, argA, *n, **kw):
"""
Walk the command line and first walk over the command line options.
@param ignoreT: Table of names to ignore.
@param argT: Table of command line args. Keys are the name of the argument, values are the number of arguments it takes (> 0).
@param npT: Options for the number of tasks.
@param cmdArg: command (like in ibrun.symm) that points to a user program.
@param argA: The command line split into an array.
@param kw: If dot=True then add "." to the end of PATH.
"""
N = len(argA)
compute_ntasks=kw.get("compute_ntasks",default_compute_ntasks)
if ('dot' in kw):
os.environ['PATH'] = os.environ.get('PATH',"") + ":."
i = 0
done = False
resultA = []
path = None
while (not done):
t = {}
while (i < N):
arg = argA[i]
i = parse_ntasks(npT, arg, i, argA, t)
if (arg == cmdArg):
path = which(find_cmd(ignoreT, 0, argA[i+1].split()))
break
n = argT.get(arg,-1)
if (n > 0):
i += n + 1
continue
if (arg[0:1] == "-"):
i = i + 1
continue
break
if (not path):
path = which(find_cmd(ignoreT, i, argA)) or "unknown"
ntasks = compute_ntasks(t)
uuid = capture("@uuidgen@").rstrip('\n')
resultA.append({'exec_prog':path, 'ntasks':ntasks, 'uuid': uuid})
# Loop for colons
done = True
while (i < N):
arg = argA[i]
i = i + 1
if (arg == ":"):
done = False
path = None
break
return json.dumps(resultA)
def __init__(self, cmd):
"""
Find the full path to the executable. Then find the shared
libraries if there and get the hash time.
@param exec_progA: the command line after the mpirun type
arguments have been removed.
"""
self.__execType = None
self.__execName = which(cmd)
self.__libA = []
if (self.__execName):
outStr = capture(["file", self.__execName])
if (outStr.find("script") > 0 or outStr.find("text") > 0):
self.__execType = "script"
else:
self.__execType = "binary"
ldd = capture(["ldd", self.__execName])
self.__libA = self.__parseLDD(ldd)
info = os.stat(self.__execName)
self.__modify = info.st_mtime
self.__hash = self.__computeHash(self.__execName)
def __init__(self, cmd):
"""
Find the full path to the executable. Then find the shared
libraries if there and get the hash time.
@param exec_progA: the command line after the mpirun type
arguments have been removed.
"""
self.__execType = None
self.__execName = which(cmd)
self.__libA = []
if (self.__execName):
outStr = capture(["file", self.__execName])
if (outStr.find("script") > 0 or outStr.find("text") > 0):
self.__execType = "script"
else:
self.__execType = "binary"
ldd = capture(["ldd", self.__execName])
self.__libA = self.__parseLDD(ldd)
info = os.stat(self.__execName)
self.__modify = info.st_mtime
self.__hash = self.__computeHash(self.__execName)
def __init__(self, exec_progA):
"""
Find the user's executable by walking the command line
and skipping the executables name in ignoreT. Then find
the full path to the executable. Finally find the shared
libraries if there and get the hash time.
@param exec_progA: the command line after the mpirun type
arguments have been removed.
"""
ignoreT = {
'env' : True,
'time' : True,
}
cmd = None
for prog in exec_progA:
bare = os.path.basename(prog)
if (not (bare in ignoreT)):
cmd = prog
break
self.__execType = None
self.__execName = which(cmd)
self.__libA = []
if (self.__execName):
outStr = capture(["file", self.__execName])
if (outStr.find("script") > 0 or outStr.find("text") > 0):
self.__execType = "script"
else:
self.__execType = "binary"
ldd = capture(["ldd", self.__execName])
self.__libA = self.__parseLDD(ldd)
info = os.stat(self.__execName)
self.__modify = info.st_mtime
self.__hash = self.__computeHash(self.__execName)
def __init__(self, cmd):
"""
Parse the input path for the xalt record. If it exists extract the
key value pairs and store in a table.
@param cmd: the path to the program or shared library that has (or could have) an XALT record.
"""
outStr = capture(["objdump", "-s", "-j", ".xalt", cmd])
self.__fieldT = {}
if (not outStr.find("Contents of section .xalt:") != -1):
return
outputA = outStr.split('\n')
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
sA.append(split[-1])
s = "".join(sA)
xaltA = re.split('%%', s)
fieldT = {}
N = len(xaltA)
idx = -1
while (True):
idx = idx + 1
line = xaltA[idx]
if (line.find("XALT_Link_Info_End") != -1):
break
m = keyPat.search(line)
if (m):
key = m.group(1)
idx = idx + 1
value = xaltA[idx].replace("_%_%_", " ")
fieldT[key] = value
self.__fieldT = fieldT
def __init__(self, cmd):
"""
Parse the input path for the xalt record. If it exists extract the
key value pairs and store in a table.
@param cmd: the path to the program or shared library that has (or could have) an XALT record.
"""
outStr = capture(["objdump", "-s", "-j", ".xalt", cmd])
self.__fieldT = {}
if (not outStr.find("Contents of section .xalt:") != -1):
return
outputA = outStr.split('\n')
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
sA.append(split[-1])
s = "".join(sA)
xaltA = re.split('%%', s)
fieldT = {}
N = len(xaltA)
idx = -1
while (True):
idx = idx + 1
line = xaltA[idx]
if (line.find("XALT_Link_Info_End") != -1):
break
m = keyPat.search(line)
if (m):
key = m.group(1)
idx = idx + 1
value = xaltA[idx].replace("_%_%_"," ")
fieldT[key] = value
self.__fieldT = fieldT
def __extract_xalt_darwin(self, execPath):
"""
Use objdump to extract the xalt record in a linux executable.
@param execPath: the path to the program or shared library that has (or could have) an XALT record.
"""
outStr = capture (["@otool@", "-s", ".XALT", ".xalt", execPath])
outputA = outStr.split("\n")
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
for hexStr in split[1:]:
sA.append(chr(int(hexStr,16)))
return "".join(sA)
def __extract_xalt_darwin(self, execPath):
"""
Use objdump to extract the xalt record in a linux executable.
@param execPath: the path to the program or shared library that has (or could have) an XALT record.
"""
outStr = capture(["@otool@", "-s", ".XALT", ".xalt", execPath])
outputA = outStr.split("\n")
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
for hexStr in split[1:]:
sA.append(chr(int(hexStr, 16)))
return "".join(sA)
def __build_fieldT(self, cmd):
outStr = capture(["objdump", "-s", "-j", ".xalt", cmd])
fieldT = {}
if (not outStr.find("Contents of section .xalt:") != -1):
return fieldT
outputA = outStr.split('\n')
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
sA.append(split[-1])
s = "".join(sA)
xaltA = re.split('%%', s)
fieldT = {}
N = len(xaltA)
idx = -1
while (True):
idx = idx + 1
line = xaltA[idx]
if (line.find("XALT_Link_Info_End") != -1):
break
m = keyPat.search(line)
if (m):
key = m.group(1)
idx = idx + 1
value = xaltA[idx].replace("_%_%_"," ")
fieldT[key] = value
return fieldT
def __extract_xalt_linux(self, execPath):
"""
Use objdump to extract the xalt record in a linux executable.
@param execPath: the path to the program or shared library that has (or could have) an XALT record.
"""
outStr = capture(["@objdump@", "-s", "-j", ".xalt", execPath ])
self.__fieldT = {}
if (not outStr.find("Contents of section .xalt:") != -1):
return
outputA = outStr.split('\n')
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
sA.append(split[-1])
return "".join(sA)
def __extract_xalt_linux(self, execPath):
"""
Use objdump to extract the xalt record in a linux executable.
@param execPath: the path to the program or shared library that has (or could have) an XALT record.
"""
outStr = capture(["@objdump@", "-s", "-j", ".xalt", execPath])
self.__fieldT = {}
if (not outStr.find("Contents of section .xalt:") != -1):
return
outputA = outStr.split('\n')
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
outputA.pop(0)
sA = []
for line in outputA:
split = line.split()
if (len(split) > 0):
sA.append(split[-1])
return "".join(sA)
def __parseLDD(self,ldd):
""" Return the list of shared libraries with their sha1sum. """
if (not ldd or ldd.find("not a dynamic executable") > 0):
return []
lineA = ldd.split('\n')
libA = []
d = {}
for line in lineA:
fieldA = line.split()
N = len(fieldA)
if (N < 1):
break
elif (N == 4):
lib = fieldA[2]
else:
lib = fieldA[0]
lib = os.path.realpath(lib)
d[lib] = True
libA = d.keys()
libA = sorted(libA)
libB = []
for lib in libA:
hash_line = capture(['@sha1sum@', lib])
if (hash_line.find("No such file or directory") != -1):
v = "unknown"
else:
v = hash_line.split()[0]
libB.append([lib, v])
return libB
def __parseLDD(self, ldd):
""" Return the list of shared libraries with their sha1sum. """
if (not ldd or ldd.find("not a dynamic executable") > 0):
return []
lineA = ldd.split('\n')
libA = []
d = {}
for line in lineA:
fieldA = line.split()
N = len(fieldA)
if (N < 1):
break
elif (N == 4):
lib = fieldA[2]
else:
lib = fieldA[0]
lib = os.path.realpath(lib)
d[lib] = True
libA = d.keys()
libA = sorted(libA)
libB = []
for lib in libA:
hash_line = capture(['@sha1sum@', lib])
if (hash_line.find("No such file or directory") != -1):
v = "unknown"
else:
v = hash_line.split()[0]
libB.append([lib, v])
return libB
def __computeHash(self, cmd): """ Compute the sha1sum of the executable. """ fieldA = capture(["@sha1sum@", cmd]).split() return fieldA[0]
def __computeHash(self, cmd): fieldA = capture(["sha1sum", cmd]).split() return fieldA[0]
def __computeHash(self, cmd):
""" Compute the sha1sum of the executable. """
fieldA = capture(["@sha1sum@", cmd]).split()
return fieldA[0]
