def mpdallexit():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(sys.argv) > 1 and (sys.argv[1] == '-h' or sys.argv[1] == '--help') :
print __doc__
sys.exit(-1)
mpd_set_my_id(myid='mpdallexit')
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os,'getuid') and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = { 'cmd' : 'mpdallexit' }
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=8.0)
if not msg:
mpd_print(1,'no msg recvd from mpd before timeout')
elif msg['cmd'] != 'mpdallexit_ack':
mpd_print(1,'unexpected msg from mpd :%s:' % (msg) )
sys.exit(-1)
conSock.close()
def mpdallexit():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(sys.argv) > 1 and (sys.argv[1] == '-h' or sys.argv[1] == '--help') :
print __doc__
sys.exit(-1)
mpd_set_my_id(myid='mpdallexit')
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os,'getuid') and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = { 'cmd' : 'mpdallexit' }
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=8.0)
if not msg:
mpd_print(1,'no msg recvd from mpd before timeout')
elif msg['cmd'] != 'mpdallexit_ack':
mpd_print(1,'unexpected msg from mpd :%s:' % (msg) )
sys.exit(-1)
conSock.close()
def mpdtrace():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(sys.argv) > 1:
if (sys.argv[1] == '-h'
or sys.argv[1] == '--help') or (sys.argv[1] != '-l'):
usage()
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdtrace')
parmdb = MPDParmDB(
orderedSources=['cmdline', 'xml', 'env', 'rcfile', 'thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD': 0,
'MPD_SECRETWORD': '',
}
for (k, v) in parmsToOverride.items():
parmdb[('thispgm', k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os, 'getuid')
and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(
sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName, 'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,
secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = {'cmd': 'mpdtrace'}
conSock.send_dict_msg(msgToSend)
# Main Loop
done = 0
while not done:
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg: # also get this on ^C
mpd_print(1, 'got eof on console')
sys.exit(-1)
elif not msg.has_key('cmd'):
print 'mpdtrace: unexpected msg from mpd=:%s:' % (msg)
sys.exit(-1)
if msg['cmd'] == 'mpdtrace_info':
if len(sys.argv) > 1 and sys.argv[1] == '-l':
print '%s (%s)' % (msg['id'], msg['ifhn'])
else:
pos = msg['id'].find('.')
if pos < 0:
pos = msg['id'].rfind('_')
print msg['id'][:pos] # strip off domain and port
elif msg['cmd'] == 'mpdtrace_trailer':
done = 1
conSock.close()
def mpdkilljob():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(sys.argv) < 2 or sys.argv[1] == '-h' or sys.argv[1] == '--help':
usage()
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdkilljob')
mpdid = ''
if sys.argv[1] == '-a':
jobalias = sys.argv[2]
jobnum = '0'
else:
jobalias = ''
jobid = sys.argv[1]
sjobid = jobid.split('@')
jobnum = sjobid[0]
if len(sjobid) > 1:
mpdid = sjobid[1]
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os,'getuid') and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = { 'cmd':'mpdkilljob', 'jobnum' : jobnum, 'mpdid' : mpdid,
'jobalias' : jobalias, 'username' : mpd_get_my_username() }
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1,'no msg recvd from mpd before timeout')
sys.exit(-1)
if msg['cmd'] != 'mpdkilljob_ack':
if msg['cmd'] == 'already_have_a_console':
print 'mpd already has a console (e.g. for long ringtest); try later'
else:
print 'unexpected message from mpd: %s' % (msg)
sys.exit(-1)
if not msg['handled']:
print 'job not found'
sys.exit(-1)
conSock.close()
def mpdkilljob():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(sys.argv) < 2 or sys.argv[1] == '-h' or sys.argv[1] == '--help':
usage()
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdkilljob')
mpdid = ''
if sys.argv[1] == '-a':
jobalias = sys.argv[2]
jobnum = '0'
else:
jobalias = ''
jobid = sys.argv[1]
sjobid = jobid.split('@')
jobnum = sjobid[0]
if len(sjobid) > 1:
mpdid = sjobid[1]
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os,'getuid') and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = { 'cmd':'mpdkilljob', 'jobnum' : jobnum, 'mpdid' : mpdid,
'jobalias' : jobalias, 'username' : mpd_get_my_username() }
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1,'no msg recvd from mpd before timeout')
sys.exit(-1)
if msg['cmd'] != 'mpdkilljob_ack':
if msg['cmd'] == 'already_have_a_console':
print 'mpd already has a console (e.g. for long ringtest); try later'
else:
print 'unexpected message from mpd: %s' % (msg)
sys.exit(-1)
if not msg['handled']:
print 'job not found'
sys.exit(-1)
conSock.close()
def mpdtrace():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(sys.argv) > 1:
if (sys.argv[1] == '-h' or sys.argv[1] == '--help') or (sys.argv[1] != '-l'):
usage()
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdtrace')
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os,'getuid') and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = { 'cmd' : 'mpdtrace' }
conSock.send_dict_msg(msgToSend)
# Main Loop
done = 0
while not done:
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg: # also get this on ^C
mpd_print(1, 'got eof on console')
sys.exit(-1)
elif not msg.has_key('cmd'):
print 'mpdtrace: unexpected msg from mpd=:%s:' % (msg)
sys.exit(-1)
if msg['cmd'] == 'mpdtrace_info':
if len(sys.argv) > 1 and sys.argv[1] == '-l':
print '%s (%s)' % (msg['id'],msg['ifhn'])
else:
pos = msg['id'].find('.')
if pos < 0:
pos = msg['id'].rfind('_')
print msg['id'][:pos] # strip off domain and port
elif msg['cmd'] == 'mpdtrace_trailer':
done = 1
conSock.close()
def err_exit(msg):
global myHost, fullDirName, topMPDBoot, user
mpd_print(1, 'mpd failed to start correctly on %s' % (myHost))
mpdPid = 0
if msg != 'RC=MPDBOOT_ERREXIT':
print ' reason: %s' % (msg)
stdout.flush()
try:
logfile = open('/tmp/mpd2.logfile_%s' % (user), 'r')
mpd_print(1, ' contents of mpd logfile in /tmp:')
for line in logfile:
print ' ', line,
stdout.flush()
if line.startswith('logfile for mpd with pid'):
mpdPid = re.findall(r'logfile for mpd with pid (\d+)',
line)
except:
pass
try:
system('%s/mpdallexit.py > /dev/null' %
(fullDirName)) # stop any current mpds
except:
pass
try:
kill(mpdPid, SIGKILL)
except:
pass
if not topMPDBoot:
mpd_print(1, 'RC=MPDBOOT_ERREXIT') # printable rc
exit(-1)
def handle_mpd_output(fd, fd2idx, hostsAndInfo):
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
idx = fd2idx[fd]
host = hostsAndInfo[idx]['host']
# port = fd.readline().strip()
port = 'no_port'
for line in fd.readlines(): # handle output from shells that echo stuff
line = line.strip()
splitLine = line.split('=')
if splitLine[0] == 'mpd_port':
port = splitLine[1]
break
if debug:
print "debug: mpd on %s on port %s" % (host, port)
if port.isdigit():
hostsAndInfo[idx]['list_port'] = int(port)
tempSock = MPDSock(name='temp_to_mpd')
try:
tempSock.connect((host, int(port)))
except:
tempSock.close()
tempSock = 0
if tempSock:
msgToSend = {'cmd': 'ping', 'ifhn': 'dummy', 'port': 0}
tempSock.send_dict_msg(msgToSend)
msg = tempSock.recv_dict_msg() # RMB: WITH TIMEOUT ??
if not msg or not msg.has_key('cmd') or msg['cmd'] != 'challenge':
mpd_print(
1, 'failed to handshake mpd on %s; recvd output=%s' %
(host, msg))
tempOut = tempSock.recv(1000)
print tempOut
try:
getoutput('%s/mpdallexit.py' % (fullDirName))
except:
pass
exit(-1)
tempSock.close()
else:
mpd_print(1, 'failed to connect to mpd on %s' % (host))
try:
getoutput('%s/mpdallexit.py' % (fullDirName))
except:
pass
exit(-1)
else:
mpd_print(1, 'from mpd on %s, invalid port info:' % (host))
print port
print fd.read()
try:
getoutput('%s/mpdallexit.py' % (fullDirName))
except:
pass
exit(-1)
if verbose:
print "RUNNING: mpd on", hostsAndInfo[fd2idx[fd]]['host']
if debug:
print "debug: info for running mpd:", hostsAndInfo[fd2idx[fd]]
def mpdexit():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if (len(sys.argv) > 1 and (sys.argv[1] == '-h' or sys.argv[1] == '--help')) or \
(len(sys.argv) < 2):
print __doc__
sys.exit(-1)
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdexit')
parmdb = MPDParmDB(
orderedSources=['cmdline', 'xml', 'env', 'rcfile', 'thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD': 0,
'MPD_SECRETWORD': '',
}
for (k, v) in parmsToOverride.items():
parmdb[('thispgm', k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os, 'getuid')
and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(
sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName, 'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,
secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = {'cmd': 'mpdexit', 'mpdid': sys.argv[1]}
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1, 'no msg recvd from mpd before timeout')
sys.exit(-1)
elif msg['cmd'] == 'already_have_a_console':
mpd_print(
1, 'mpd already has a console (e.g. for long ringtest); try later')
sys.exit(-1)
if not msg.has_key('cmd'):
mpd_print(1, 'mpdexit: INVALID msg=:%s:' % (msg))
sys.exit(-1)
if msg['cmd'] != 'mpdexit_ack':
mpd_print(1, 'mpdexit failed; may have wrong mpdid')
sys.exit(-1)
def mpdexit():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if (len(sys.argv) > 1 and (sys.argv[1] == '-h' or sys.argv[1] == '--help')) or \
(len(sys.argv) < 2):
print __doc__
sys.exit(-1)
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdexit')
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os,'getuid') and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = { 'cmd' : 'mpdexit', 'mpdid' : sys.argv[1] }
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1,'no msg recvd from mpd before timeout')
sys.exit(-1)
elif msg['cmd'] == 'already_have_a_console':
mpd_print(1,'mpd already has a console (e.g. for long ringtest); try later')
sys.exit(-1)
if not msg.has_key('cmd'):
mpd_print(1,'mpdexit: INVALID msg=:%s:' % (msg))
sys.exit(-1)
if msg['cmd'] != 'mpdexit_ack':
mpd_print(1,'mpdexit failed; may have wrong mpdid')
sys.exit(-1)
def handle_mpd_output(fd,fd2idx,hostsAndInfo):
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
idx = fd2idx[fd]
host = hostsAndInfo[idx]['host']
# port = fd.readline().strip()
port = 'no_port'
for line in fd.readlines(): # handle output from shells that echo stuff
line = line.strip()
splitLine = line.split('=')
if splitLine[0] == 'mpd_port':
port = splitLine[1]
break
if debug:
print "debug: mpd on %s on port %s" % (host,port)
if port.isdigit():
hostsAndInfo[idx]['list_port'] = int(port)
tempSock = MPDSock(name='temp_to_mpd')
try:
tempSock.connect((host,int(port)))
except:
tempSock.close()
tempSock = 0
if tempSock:
msgToSend = { 'cmd' : 'ping', 'ifhn' : 'dummy', 'port' : 0}
tempSock.send_dict_msg(msgToSend)
msg = tempSock.recv_dict_msg() # RMB: WITH TIMEOUT ??
if not msg or not msg.has_key('cmd') or msg['cmd'] != 'challenge':
mpd_print(1,'failed to handshake with mpd on %s; recvd output=%s' % \
(host,msg) )
tempOut = tempSock.recv(1000)
print tempOut
try: getoutput('%s/mpdallexit.py' % (fullDirName))
except: pass
exit(-1)
tempSock.close()
else:
mpd_print(1,'failed to connect to mpd on %s' % (host) )
try: getoutput('%s/mpdallexit.py' % (fullDirName))
except: pass
exit(-1)
else:
mpd_print(1,'from mpd on %s, invalid port info:' % (host) )
print port
print fd.read()
try: getoutput('%s/mpdallexit.py' % (fullDirName))
except: pass
exit(-1)
if verbose:
print "RUNNING: mpd on", hostsAndInfo[fd2idx[fd]]['host']
if debug:
print "debug: info for running mpd:", hostsAndInfo[fd2idx[fd]]
def mpdboot():
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
myHost = gethostname()
mpd_set_my_id('mpdboot_%s' % (myHost))
fullDirName = path.abspath(path.split(argv[0])[0])
rshCmd = 'ssh'
user = mpd_get_my_username()
mpdCmd = path.join(fullDirName, 'mpd.py')
hostsFilename = 'mpd.hosts'
totalnumToStart = 0
debug = 0
verbose = 0
chkupIndicator = 0 # 1 -> chk and start ; 2 -> just chk
maxUnderOneRoot = 4
try:
shell = path.split(environ['SHELL'])[-1]
except:
shell = 'csh'
if environ.has_key('MPD_TMPDIR'):
tmpdir = environ['MPD_TMPDIR']
else:
tmpdir = ''
argidx = 1 # skip arg 0
while argidx < len(argv):
if argv[argidx] == '-h' or argv[argidx] == '--help':
usage()
elif argv[argidx] == '-r': # or --rsh=
rshCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--rsh'):
splitArg = argv[argidx].split('=')
try:
rshCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-u': # or --user=
user = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--user'):
splitArg = argv[argidx].split('=')
try:
user = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-m': # or --mpd=
mpdCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--mpd'):
splitArg = argv[argidx].split('=')
try:
mpdCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-f': # or --file=
hostsFilename = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--file'):
splitArg = argv[argidx].split('=')
try:
hostsFilename = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-n': # or --totalnum=
totalnumToStart = int(argv[argidx + 1])
argidx += 2
elif argv[argidx].startswith('--totalnum'):
splitArg = argv[argidx].split('=')
try:
totalnumToStart = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--maxbranch'):
splitArg = argv[argidx].split('=')
try:
maxUnderOneRoot = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-d' or argv[argidx] == '--debug':
debug = 1
argidx += 1
elif argv[argidx] == '-s' or argv[argidx] == '--shell':
shell = 'bourne'
argidx += 1
elif argv[argidx] == '-v' or argv[argidx] == '--verbose':
verbose = 1
argidx += 1
elif argv[argidx] == '-c' or argv[argidx] == '--chkup':
chkupIndicator = 1
argidx += 1
elif argv[argidx] == '--chkuponly':
chkupIndicator = 2
argidx += 1
else:
print 'mpdboot: unrecognized argument:', argv[argidx]
usage()
if debug:
print 'debug: starting'
lines = []
try:
f = open(hostsFilename, 'r')
for line in f:
if not line or line[0] == '#':
continue
lines.append(line)
except:
print 'unable to open (or read) hostsfile %s' % (hostsFilename)
exit(-1)
if totalnumToStart == 0:
totalnumToStart = len(lines)
numRead = 0
hostsAndInfo = []
for line in lines:
line = line.strip()
splitLine = re.split(r'\s+', line)
host = splitLine[0]
ncpus = 1 # default
ifhn = '' # default
cons = '' # default
for kv in splitLine[1:]:
(k, v) = kv.split('=', 1)
if k == 'ifhn':
ifhn = v
elif k == 'ncpus':
ncpus = int(v)
elif k == 'cons':
cons = v
else:
print "unrecognized key:", k
exit(-1)
hostsAndInfo.append({
'host': host,
'ifhn': ifhn,
'ncpus': ncpus,
'cons': cons
})
numRead += 1
if numRead >= totalnumToStart:
break
if len(hostsAndInfo) < totalnumToStart: # one is local
print 'totalnum=%d numhosts=%d' % (totalnumToStart, len(hostsAndInfo))
print 'there are not enough hosts on which to start all processes'
exit(-1)
if chkupIndicator:
hostsToCheck = [hai['host'] for hai in hostsAndInfo]
(upList, dnList) = chkupdn(hostsToCheck)
if dnList:
print "these hosts are down; exiting"
print dnList
exit(-1)
print "there are %d hosts up" % (len(upList))
if chkupIndicator == 2: # do the chkup and quit
exit(0)
try:
from os import sysconf
maxfds = sysconf('SC_OPEN_MAX')
except:
maxfds = 1024
maxAtOnce = min(128, maxfds -
8) # -8 for stdout, etc. + a few more for padding
fd2idx = {}
hostsSeen = {}
fdsToSelect = []
numStarted = 0
numStarting = 0
numUnderCurrRoot = 0
possRoots = []
currRoot = 0
idxToStart = 0
while numStarted < totalnumToStart:
if numStarting < maxAtOnce and idxToStart < totalnumToStart:
if numUnderCurrRoot < maxUnderOneRoot:
if idxToStart == 0:
entryHost = ''
entryPort = ''
else:
entryHost = hostsAndInfo[currRoot]['host']
entryPort = hostsAndInfo[currRoot]['list_port']
hostsAndInfo[idxToStart]['entry_host'] = entryHost
hostsAndInfo[idxToStart]['entry_port'] = entryPort
if entryHost:
entryHost = '-h ' + entryHost
entryPort = '-p ' + str(entryPort)
ifhn = hostsAndInfo[idxToStart]['ifhn']
ncpus = hostsAndInfo[idxToStart]['ncpus']
cons = hostsAndInfo[idxToStart]['cons']
if ifhn:
ifhn = '--ifhn=%s' % (ifhn)
if ncpus:
ncpus = '--ncpus=%s' % (ncpus)
if cons == 'n':
cons = '-n'
mpdArgs = '%s %s %s %s %s ' % (cons, entryHost, entryPort,
ifhn, ncpus)
if tmpdir:
mpdArgs += ' --tmpdir=%s' % (tmpdir)
(mpdPID, mpdFD) = launch_one_mpd(idxToStart, currRoot, mpdArgs,
hostsAndInfo)
hostsAndInfo[idxToStart]['pid'] = mpdPID
hostsSeen[hostsAndInfo[idxToStart]['host']] = 1
fd2idx[mpdFD] = idxToStart
if idxToStart == 0:
handle_mpd_output(mpdFD, fd2idx, hostsAndInfo)
numStarted += 1
else:
numUnderCurrRoot += 1
fdsToSelect.append(mpdFD)
numStarting += 1
idxToStart += 1
else:
if possRoots:
currRoot = possRoots.pop()
numUnderCurrRoot = 0
selectTime = 0.01
else:
selectTime = 0.1
try:
(readyFDs, unused1, unused2) = select(fdsToSelect, [], [],
selectTime)
except error, errmsg:
mpd_print(1, 'mpdboot: select failed: errmsg=:%s:' % (errmsg))
exit(-1)
for fd in readyFDs:
handle_mpd_output(fd, fd2idx, hostsAndInfo)
numStarted += 1
numStarting -= 1
possRoots.append(fd2idx[fd])
fdsToSelect.remove(fd)
fd.close()
def mpdsigjob():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(argv) < 3 or argv[1] == '-h' or argv[1] == '--help':
usage()
signal(SIGINT, sig_handler)
mpd_set_my_id(myid='mpdsigjob')
sigtype = argv[1]
if sigtype.startswith('-'):
sigtype = sigtype[1:]
if sigtype.startswith('SIG'):
sigtype = sigtype[3:]
import signal as tmpsig # just to get valid SIG's
if sigtype.isdigit():
if int(sigtype) > tmpsig.NSIG:
print 'invalid signum: %s' % (sigtype)
exit(-1)
else:
if not tmpsig.__dict__.has_key('SIG' + sigtype):
print 'invalid sig type: %s' % (sigtype)
exit(-1)
jobalias = ''
jobnum = ''
mpdid = ''
single_or_group = 'g'
i = 2
while i < len(argv):
if argv[i] == '-a':
if jobnum: # should not have both alias and jobid
print '** cannot specify both jobalias and jobid'
usage()
jobalias = argv[i + 1]
i += 1
jobnum = '0'
elif argv[i] == '-j':
if jobalias: # should not have both alias and jobid
print '** cannot specify both jobalias and jobid'
usage()
jobid = argv[i + 1]
i += 1
sjobid = jobid.split('@')
jobnum = sjobid[0]
if len(sjobid) > 1:
mpdid = sjobid[1]
elif argv[i] == '-s':
single_or_group = 's'
elif argv[i] == '-g':
single_or_group = 'g'
else:
print '** unrecognized arg: %s' % (argv[i])
usage()
i += 1
parmdb = MPDParmDB(
orderedSources=['cmdline', 'xml', 'env', 'rcfile', 'thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD': 0,
'MPD_SECRETWORD': '',
}
for (k, v) in parmsToOverride.items():
parmdb[('thispgm', k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if getuid() == 0 or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = path.abspath(path.split(argv[0])[0]) # normalize
mpdroot = path.join(fullDirName, 'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,
secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = {
'cmd': 'mpdsigjob',
'sigtype': sigtype,
'jobnum': jobnum,
'mpdid': mpdid,
'jobalias': jobalias,
's_or_g': single_or_group,
'username': mpd_get_my_username()
}
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1, 'no msg recvd from mpd before timeout')
if msg['cmd'] != 'mpdsigjob_ack':
if msg['cmd'] == 'already_have_a_console':
mpd_print(
1,
'mpd already has a console (e.g. for long ringtest); try later'
)
else:
mpd_print(1, 'unexpected message from mpd: %s' % (msg))
exit(-1)
if not msg['handled']:
print 'job not found'
exit(-1)
conSock.close()
# print "RMB:GDBDRV: ARGS=", argv
if len(argv) > 2:
args = ' '.join(argv[2:])
else:
args = ''
gdb_info = Popen4('gdb -q %s %s' % (argv[1], args), 0)
gdbPid = gdb_info.pid
# print "PID=%d GDBPID=%d" % (getpid(),gdbPid) ; stdout.flush()
gdb_sin = gdb_info.tochild
gdb_sin_fileno = gdb_sin.fileno()
gdb_sout_serr = gdb_info.fromchild
gdb_sout_serr_fileno = gdb_sout_serr.fileno()
write(gdb_sin_fileno, 'set prompt (gdb)\\n\n')
gdb_line = gdb_sout_serr.readline()
mpd_print(0000, "LINE1=|%s|" % (gdb_line.rstrip()))
write(gdb_sin_fileno, 'set confirm off\n')
gdb_line = gdb_sout_serr.readline()
mpd_print(0000, "LINE2=|%s|" % (gdb_line.rstrip()))
write(gdb_sin_fileno, 'handle SIGUSR1 nostop noprint\n')
gdb_line = gdb_sout_serr.readline()
mpd_print(0000, "LINE3=|%s|" % (gdb_line.rstrip()))
write(gdb_sin_fileno, 'handle SIGPIPE nostop noprint\n')
gdb_line = gdb_sout_serr.readline()
mpd_print(0000, "LINE4=|%s|" % (gdb_line.rstrip()))
write(gdb_sin_fileno, 'set confirm on\n')
gdb_line = gdb_sout_serr.readline()
mpd_print(0000, "LINE5=|%s|" % (gdb_line.rstrip()))
write(gdb_sin_fileno, 'echo hi1\n')
gdb_line = gdb_sout_serr.readline()
mpd_print(0000, "LINE6=|%s|" % (gdb_line.rstrip()))
kill(appPid,SIGKILL)
except:
pass
if __name__ == '__main__': # so I can be imported by pydoc
signal(SIGINT,sig_handler)
signal(SIGUSR1,sig_handler)
mpd_set_my_id('mpdgdbdrv')
## mpd_print(1,"RMB:GDBDRV: ARGS=%s" % argv)
if argv[1] == '-attach':
gdb_args = '%s %s' % (argv[2],argv[3]) # userpgm and userpid
else:
if len(argv) > 2:
mpd_print(1, "when using gdb, pass cmd-line args to user pgms via the 'run' cmd")
exit(-1)
gdb_args = argv[1]
gdb_info = Popen4('gdb -q %s' % (gdb_args), 0 )
gdbPid = gdb_info.pid
# print "PID=%d GDBPID=%d" % (getpid(),gdbPid) ; stdout.flush()
gdb_sin = gdb_info.tochild
gdb_sin_fileno = gdb_sin.fileno()
gdb_sout_serr = gdb_info.fromchild
gdb_sout_serr_fileno = gdb_sout_serr.fileno()
write(gdb_sin_fileno,'set prompt (gdb)\\n\n')
gdb_line = gdb_sout_serr.readline()
# check if gdb reports any errors
if findall(r'.*: No such file or directory.',gdb_line) != []:
print gdb_line, ; stdout.flush()
exit(-1)
def mpdlistjobs():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid="mpdlistjobs")
uname = ""
jobid = ""
sjobid = ""
jobalias = ""
sssPrintFormat = 0
if len(sys.argv) > 1:
aidx = 1
while aidx < len(sys.argv):
if sys.argv[aidx] == "-h" or sys.argv[aidx] == "--help":
usage()
if sys.argv[aidx] == "-u": # or --user=
uname = sys.argv[aidx + 1]
aidx += 2
elif sys.argv[aidx].startswith("--user"):
splitArg = sys.argv[aidx].split("=")
try:
uname = splitArg[1]
except:
print "mpdlistjobs: invalid argument:", sys.argv[aidx]
usage()
aidx += 1
elif sys.argv[aidx] == "-j": # or --jobid=
jobid = sys.argv[aidx + 1]
aidx += 2
sjobid = jobid.split("@") # jobnum and originating host
elif sys.argv[aidx].startswith("--jobid"):
splitArg = sys.argv[aidx].split("=")
try:
jobid = splitArg[1]
sjobid = jobid.split("@") # jobnum and originating host
except:
print "mpdlistjobs: invalid argument:", sys.argv[aidx]
usage()
aidx += 1
elif sys.argv[aidx] == "-a": # or --alias=
jobalias = sys.argv[aidx + 1]
aidx += 2
elif sys.argv[aidx].startswith("--alias"):
splitArg = sys.argv[aidx].split("=")
try:
jobalias = splitArg[1]
except:
print "mpdlistjobs: invalid argument:", sys.argv[aidx]
usage()
aidx += 1
elif sys.argv[aidx] == "--sss":
sssPrintFormat = 1
aidx += 1
else:
print "unrecognized arg: %s" % sys.argv[aidx]
sys.exit(-1)
parmdb = MPDParmDB(orderedSources=["cmdline", "xml", "env", "rcfile", "thispgm"])
parmsToOverride = {"MPD_USE_ROOT_MPD": 0, "MPD_SECRETWORD": ""}
for (k, v) in parmsToOverride.items():
parmdb[("thispgm", k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os, "getuid") and os.getuid() == 0) or parmdb["MPD_USE_ROOT_MPD"]:
fullDirName = os.path.abspath(os.path.split(sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName, "mpdroot")
conSock = MPDConClientSock(mpdroot=mpdroot, secretword=parmdb["MPD_SECRETWORD"])
else:
conSock = MPDConClientSock(secretword=parmdb["MPD_SECRETWORD"])
msgToSend = {"cmd": "mpdlistjobs"}
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1, "no msg recvd from mpd before timeout")
if msg["cmd"] != "local_mpdid": # get full id of local mpd for filters later
mpd_print(1, "did not recv local_mpdid msg from local mpd; instead, recvd: %s" % msg)
else:
if len(sjobid) == 1:
sjobid.append(msg["id"])
done = 0
while not done:
msg = conSock.recv_dict_msg()
if not msg.has_key("cmd"):
mpd_print(1, "mpdlistjobs: INVALID msg=:%s:" % (msg))
sys.exit(-1)
if msg["cmd"] == "mpdlistjobs_info":
smjobid = msg["jobid"].split(" ") # jobnum, mpdid, and alias (if present)
if len(smjobid) < 3:
smjobid.append("")
print_based_on_uname = 0 # default
print_based_on_jobid = 0 # default
print_based_on_jobalias = 0 # default
if not uname or uname == msg["username"]:
print_based_on_uname = 1
if not jobid and not jobalias:
print_based_on_jobid = 1
print_based_on_jobalias = 1
else:
if sjobid and sjobid[0] == smjobid[0] and sjobid[1] == smjobid[1]:
print_based_on_jobid = 1
if jobalias and jobalias == smjobid[2]:
print_based_on_jobalias = 1
if not smjobid[2]:
smjobid[2] = " " # just for printing
if print_based_on_uname and (print_based_on_jobid or print_based_on_jobalias):
if sssPrintFormat:
print "%s %s %s" % (msg["host"], msg["clipid"], msg["sid"])
else:
print "jobid = %s@%s" % (smjobid[0], smjobid[1])
print "jobalias = %s" % (smjobid[2])
print "username = %s" % (msg["username"])
print "host = %s" % (msg["host"])
print "pid = %s" % (msg["clipid"])
print "sid = %s" % (msg["sid"])
print "rank = %s" % (msg["rank"])
print "pgm = %s" % (msg["pgm"])
print
else: # mpdlistjobs_trailer
done = 1
conSock.close()
def mpdboot():
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
myHost = gethostname()
mpd_set_my_id('mpdboot_%s' % (myHost) )
fullDirName = path.abspath(path.split(argv[0])[0])
rshCmd = 'ssh'
user = mpd_get_my_username()
mpdCmd = path.join(fullDirName,'mpd.py')
hostsFilename = 'mpd.hosts'
totalnumToStart = 1 # may get chgd below
debug = 0
verbose = 0
localConArg = ''
remoteConArg = ''
oneMPDPerHost = 1
myNcpus = 1
myIfhn = ''
chkupIndicator = 0 # 1 -> chk and start ; 2 -> just chk
maxUnderOneRoot = 4
try:
shell = path.split(environ['SHELL'])[-1]
except:
shell = 'csh'
argidx = 1 # skip arg 0
while argidx < len(argv):
if argv[argidx] == '-h' or argv[argidx] == '--help':
usage()
elif argv[argidx] == '-r': # or --rsh=
rshCmd = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--rsh'):
splitArg = argv[argidx].split('=')
try:
rshCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-u': # or --user=
user = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--user'):
splitArg = argv[argidx].split('=')
try:
user = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-m': # or --mpd=
mpdCmd = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--mpd'):
splitArg = argv[argidx].split('=')
try:
mpdCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-f': # or --file=
hostsFilename = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--file'):
splitArg = argv[argidx].split('=')
try:
hostsFilename = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ncpus'):
splitArg = argv[argidx].split('=')
try:
myNcpus = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ifhn'):
splitArg = argv[argidx].split('=')
myIfhn = splitArg[1]
myHost = splitArg[1]
argidx += 1
elif argv[argidx] == '-n': # or --totalnum=
totalnumToStart = int(argv[argidx+1])
argidx += 2
elif argv[argidx].startswith('--totalnum'):
splitArg = argv[argidx].split('=')
try:
totalnumToStart = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--maxbranch'):
splitArg = argv[argidx].split('=')
try:
maxUnderOneRoot = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-d' or argv[argidx] == '--debug':
debug = 1
argidx += 1
elif argv[argidx] == '-s' or argv[argidx] == '--shell':
shell = 'bourne'
argidx += 1
elif argv[argidx] == '-v' or argv[argidx] == '--verbose':
verbose = 1
argidx += 1
elif argv[argidx] == '-c' or argv[argidx] == '--chkup':
chkupIndicator = 1
argidx += 1
elif argv[argidx] == '--chkuponly':
chkupIndicator = 2
argidx += 1
elif argv[argidx] == '-1':
oneMPDPerHost = 0
argidx += 1
elif argv[argidx] == '--loccons':
localConArg = '-n'
argidx += 1
elif argv[argidx] == '--remcons':
remoteConArg = '-n'
argidx += 1
else:
print 'mpdboot: unrecognized argument:', argv[argidx]
usage()
# Fix for tt#662, make sure the config file is available to avoid some very
# confusing error messages. We don't actually need these values here.
parmdb = MPDParmDB()
parmdb.get_parms_from_rcfile(parmsToOverride={}, errIfMissingFile=1)
if debug:
print 'debug: starting'
lines = []
if totalnumToStart > 1:
try:
f = open(hostsFilename,'r')
for line in f:
lines.append(line)
except:
print 'unable to open (or read) hostsfile %s' % (hostsFilename)
exit(-1)
hostsAndInfo = [ {'host' : myHost, 'ncpus' : myNcpus, 'ifhn' : myIfhn} ]
for line in lines:
line = line.strip()
if not line or line[0] == '#':
continue
splitLine = re.split(r'\s+',line)
host = splitLine[0]
ncpus = 1 # default
if ':' in host:
(host,ncpus) = host.split(':',1)
ncpus = int(ncpus)
ifhn = '' # default
for kv in splitLine[1:]:
(k,v) = kv.split('=',1)
if k == 'ifhn':
ifhn = v
hostsAndInfo.append( {'host' : host, 'ncpus' : ncpus, 'ifhn' : ifhn} )
cachedIPs = {}
if oneMPDPerHost and totalnumToStart > 1:
oldHostsAndInfo = hostsAndInfo[:]
hostsAndInfo = []
for hostAndInfo in oldHostsAndInfo:
oldhost = hostAndInfo['host']
try:
ips = gethostbyname_ex(oldhost)[2] # may fail if invalid host
except:
print 'unable to obtain IP for host:', oldhost
continue
uips = {} # unique ips
for ip in ips:
uips[ip] = 1
keep = 1
for ip in uips.keys():
if cachedIPs.has_key(ip):
keep = 0
break
if keep:
hostsAndInfo.append(hostAndInfo)
cachedIPs.update(uips)
if len(hostsAndInfo) < totalnumToStart: # one is local
print 'totalnum=%d numhosts=%d' % (totalnumToStart,len(hostsAndInfo))
print 'there are not enough hosts on which to start all processes'
exit(-1)
if chkupIndicator:
hostsToCheck = [ hai['host'] for hai in hostsAndInfo[1:totalnumToStart] ]
(upList,dnList) = chkupdn(hostsToCheck)
if dnList:
print "these hosts are down; exiting"
print dnList
exit(-1)
print "there are %d hosts up (counting local)" % (len(upList)+1)
if chkupIndicator == 2: # do the chkup and quit
exit(0)
try:
# stop current (if any) mpds; ignore the output
getoutput('%s/mpdallexit.py' % (fullDirName))
if verbose or debug:
print 'running mpdallexit on %s' % (myHost)
except:
pass
if environ.has_key('MPD_TMPDIR'):
tmpdir = environ['MPD_TMPDIR']
else:
tmpdir = ''
if myIfhn:
ifhn = '--ifhn=%s' % (myIfhn)
else:
ifhn = ''
hostsAndInfo[0]['entry_host'] = ''
hostsAndInfo[0]['entry_port'] = ''
mpdArgs = '%s %s --ncpus=%d' % (localConArg,ifhn,myNcpus)
if tmpdir:
mpdArgs += ' --tmpdir=%s' % (tmpdir)
(mpdPID,mpdFD) = launch_one_mpd(0,0,mpdArgs,hostsAndInfo)
fd2idx = {mpdFD : 0}
handle_mpd_output(mpdFD,fd2idx,hostsAndInfo)
try:
from os import sysconf
maxfds = sysconf('SC_OPEN_MAX')
except:
maxfds = 1024
maxAtOnce = min(128,maxfds-8) # -8 for stdeout, etc. + a few more for padding
hostsSeen = { myHost : 1 }
fdsToSelect = []
numStarted = 1 # local already going
numStarting = 0
numUnderCurrRoot = 0
possRoots = []
currRoot = 0
idxToStart = 1 # local mpd already going
while numStarted < totalnumToStart:
if numStarting < maxAtOnce and idxToStart < totalnumToStart:
if numUnderCurrRoot < maxUnderOneRoot:
entryHost = hostsAndInfo[currRoot]['host']
entryPort = hostsAndInfo[currRoot]['list_port']
hostsAndInfo[idxToStart]['entry_host'] = entryHost
hostsAndInfo[idxToStart]['entry_port'] = entryPort
if hostsSeen.has_key(hostsAndInfo[idxToStart]['host']):
remoteConArg = '-n'
myNcpus = hostsAndInfo[idxToStart]['ncpus']
ifhn = hostsAndInfo[idxToStart]['ifhn']
if ifhn:
ifhn = '--ifhn=%s' % (ifhn)
mpdArgs = '%s -h %s -p %s %s --ncpus=%d' % (remoteConArg,entryHost,entryPort,ifhn,myNcpus)
if tmpdir:
mpdArgs += ' --tmpdir=%s' % (tmpdir)
(mpdPID,mpdFD) = launch_one_mpd(idxToStart,currRoot,mpdArgs,hostsAndInfo)
numStarting += 1
numUnderCurrRoot += 1
hostsAndInfo[idxToStart]['pid'] = mpdPID
hostsSeen[hostsAndInfo[idxToStart]['host']] = 1
fd2idx[mpdFD] = idxToStart
fdsToSelect.append(mpdFD)
idxToStart += 1
else:
if possRoots:
currRoot = possRoots.pop()
numUnderCurrRoot = 0
selectTime = 0.01
else:
selectTime = 0.1
try:
(readyFDs,unused1,unused2) = select(fdsToSelect,[],[],selectTime)
except error, errmsg:
mpd_print(1,'mpdboot: select failed: errmsg=:%s:' % (errmsg) )
exit(-1)
for fd in readyFDs:
handle_mpd_output(fd,fd2idx,hostsAndInfo)
numStarted += 1
numStarting -= 1
possRoots.append(fd2idx[fd])
fdsToSelect.remove(fd)
fd.close()
def mpdboot():
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
myHost = gethostname()
mpd_set_my_id('mpdboot_%s' % (myHost) )
fullDirName = path.abspath(path.split(argv[0])[0])
rshCmd = 'ssh'
user = mpd_get_my_username()
mpdCmd = path.join(fullDirName,'mpd.py')
hostsFilename = 'mpd.hosts'
totalnumToStart = 0
debug = 0
verbose = 0
chkupIndicator = 0 # 1 -> chk and start ; 2 -> just chk
maxUnderOneRoot = 4
try:
shell = path.split(environ['SHELL'])[-1]
except:
shell = 'csh'
if environ.has_key('MPD_TMPDIR'):
tmpdir = environ['MPD_TMPDIR']
else:
tmpdir = ''
argidx = 1 # skip arg 0
while argidx < len(argv):
if argv[argidx] == '-h' or argv[argidx] == '--help':
usage()
elif argv[argidx] == '-r': # or --rsh=
rshCmd = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--rsh'):
splitArg = argv[argidx].split('=')
try:
rshCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-u': # or --user=
user = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--user'):
splitArg = argv[argidx].split('=')
try:
user = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-m': # or --mpd=
mpdCmd = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--mpd'):
splitArg = argv[argidx].split('=')
try:
mpdCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-f': # or --file=
hostsFilename = argv[argidx+1]
argidx += 2
elif argv[argidx].startswith('--file'):
splitArg = argv[argidx].split('=')
try:
hostsFilename = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-n': # or --totalnum=
totalnumToStart = int(argv[argidx+1])
argidx += 2
elif argv[argidx].startswith('--totalnum'):
splitArg = argv[argidx].split('=')
try:
totalnumToStart = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--maxbranch'):
splitArg = argv[argidx].split('=')
try:
maxUnderOneRoot = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-d' or argv[argidx] == '--debug':
debug = 1
argidx += 1
elif argv[argidx] == '-s' or argv[argidx] == '--shell':
shell = 'bourne'
argidx += 1
elif argv[argidx] == '-v' or argv[argidx] == '--verbose':
verbose = 1
argidx += 1
elif argv[argidx] == '-c' or argv[argidx] == '--chkup':
chkupIndicator = 1
argidx += 1
elif argv[argidx] == '--chkuponly':
chkupIndicator = 2
argidx += 1
else:
print 'mpdboot: unrecognized argument:', argv[argidx]
usage()
if debug:
print 'debug: starting'
lines = []
try:
f = open(hostsFilename,'r')
for line in f:
if not line or line[0] == '#':
continue
lines.append(line)
except:
print 'unable to open (or read) hostsfile %s' % (hostsFilename)
exit(-1)
if totalnumToStart == 0:
totalnumToStart = len(lines)
numRead = 0
hostsAndInfo = []
for line in lines:
line = line.strip()
splitLine = re.split(r'\s+',line)
host = splitLine[0]
ncpus = 1 # default
ifhn = '' # default
cons = '' # default
for kv in splitLine[1:]:
(k,v) = kv.split('=',1)
if k == 'ifhn':
ifhn = v
elif k == 'ncpus':
ncpus = int(v)
elif k == 'cons':
cons = v
else:
print "unrecognized key:", k
exit(-1)
hostsAndInfo.append( {'host' : host, 'ifhn' : ifhn, 'ncpus' : ncpus, 'cons' : cons} )
numRead += 1
if numRead >= totalnumToStart:
break
if len(hostsAndInfo) < totalnumToStart: # one is local
print 'totalnum=%d numhosts=%d' % (totalnumToStart,len(hostsAndInfo))
print 'there are not enough hosts on which to start all processes'
exit(-1)
if chkupIndicator:
hostsToCheck = [ hai['host'] for hai in hostsAndInfo ]
(upList,dnList) = chkupdn(hostsToCheck)
if dnList:
print "these hosts are down; exiting"
print dnList
exit(-1)
print "there are %d hosts up" % (len(upList))
if chkupIndicator == 2: # do the chkup and quit
exit(0)
try:
from os import sysconf
maxfds = sysconf('SC_OPEN_MAX')
except:
maxfds = 1024
maxAtOnce = min(128,maxfds-8) # -8 for stdout, etc. + a few more for padding
fd2idx = {}
hostsSeen = {}
fdsToSelect = []
numStarted = 0
numStarting = 0
numUnderCurrRoot = 0
possRoots = []
currRoot = 0
idxToStart = 0
while numStarted < totalnumToStart:
if numStarting < maxAtOnce and idxToStart < totalnumToStart:
if numUnderCurrRoot < maxUnderOneRoot:
if idxToStart == 0:
entryHost = ''
entryPort = ''
else:
entryHost = hostsAndInfo[currRoot]['host']
entryPort = hostsAndInfo[currRoot]['list_port']
hostsAndInfo[idxToStart]['entry_host'] = entryHost
hostsAndInfo[idxToStart]['entry_port'] = entryPort
if entryHost:
entryHost = '-h ' + entryHost
entryPort = '-p ' + str(entryPort)
ifhn = hostsAndInfo[idxToStart]['ifhn']
ncpus = hostsAndInfo[idxToStart]['ncpus']
cons = hostsAndInfo[idxToStart]['cons']
if ifhn:
ifhn = '--ifhn=%s' % (ifhn)
if ncpus:
ncpus = '--ncpus=%s' % (ncpus)
if cons == 'n':
cons = '-n'
mpdArgs = '%s %s %s %s %s ' % (cons,entryHost,entryPort,ifhn,ncpus)
if tmpdir:
mpdArgs += ' --tmpdir=%s' % (tmpdir)
(mpdPID,mpdFD) = launch_one_mpd(idxToStart,currRoot,mpdArgs,hostsAndInfo)
hostsAndInfo[idxToStart]['pid'] = mpdPID
hostsSeen[hostsAndInfo[idxToStart]['host']] = 1
fd2idx[mpdFD] = idxToStart
if idxToStart == 0:
handle_mpd_output(mpdFD,fd2idx,hostsAndInfo)
numStarted += 1
else:
numUnderCurrRoot += 1
fdsToSelect.append(mpdFD)
numStarting += 1
idxToStart += 1
else:
if possRoots:
currRoot = possRoots.pop()
numUnderCurrRoot = 0
selectTime = 0.01
else:
selectTime = 0.1
try:
(readyFDs,unused1,unused2) = select(fdsToSelect,[],[],selectTime)
except error, errmsg:
mpd_print(1,'mpdboot: select failed: errmsg=:%s:' % (errmsg) )
exit(-1)
for fd in readyFDs:
handle_mpd_output(fd,fd2idx,hostsAndInfo)
numStarted += 1
numStarting -= 1
possRoots.append(fd2idx[fd])
fdsToSelect.remove(fd)
fd.close()
### tempManSocket = manSocket
### manSocket = 0
### done += 1
# exit(-1)
if not gdb:
myExitStatus = -1 # used in main
exit(myExitStatus) # really forces jump back into main
elif errmsg.args == 'SIGTSTP':
if manSocket:
msgToSend = {'cmd': 'signal', 'signo': 'SIGTSTP'}
mpd_send_one_msg(manSocket, msgToSend)
signal(SIGTSTP, SIG_DFL) # stop myself
kill(getpid(), SIGTSTP)
signal(SIGTSTP, sig_handler) # restore this handler
elif errmsg.args == 'SIGALRM':
mpd_print(1, 'mpdrun terminating due to timeout %d seconds' % \
(jobTimeout))
if manSocket:
msgToSend = {'cmd': 'signal', 'signo': 'SIGKILL'}
mpd_send_one_msg(manSocket, msgToSend)
manSocket.close()
myExitStatus = -1 # used in main
exit(myExitStatus) # really forces jump back into main
if mergingOutput:
print_ready_merged_lines(1)
if mshipPid:
(donePid, status) = wait() # waitpid(mshipPid,0)
if outXmlFile:
print >> outXmlFile, outXmlDoc.toprettyxml(indent=' ')
outXmlFile.close()
kill(appPid,SIGKILL)
except:
pass
if __name__ == '__main__': # so I can be imported by pydoc
signal(SIGINT,sig_handler)
signal(SIGUSR1,sig_handler)
mpd_set_my_id('mpdgdbdrv')
## mpd_print(1,"RMB:GDBDRV: ARGS=%s" % argv)
if argv[1] == '-attach':
gdb_args = '%s %s' % (argv[2],argv[3]) # userpgm and userpid
else:
if len(argv) > 2:
mpd_print(1, "when using gdb, pass cmd-line args to user pgms via the 'run' cmd")
exit(-1)
gdb_args = argv[1]
gdb_info = Popen4('gdb -q %s' % (gdb_args), 0 )
gdbPid = gdb_info.pid
# print "PID=%d GDBPID=%d" % (getpid(),gdbPid) ; stdout.flush()
gdb_sin = gdb_info.tochild
gdb_sin_fileno = gdb_sin.fileno()
gdb_sout_serr = gdb_info.fromchild
gdb_sout_serr_fileno = gdb_sout_serr.fileno()
write(gdb_sin_fileno,'set prompt (gdb)\\n\n')
gdb_line = gdb_sout_serr.readline()
# check if gdb reports any errors
if findall(r'.*: No such file or directory.',gdb_line) != []:
print gdb_line, ; stdout.flush()
exit(-1)
def mpdboot():
global myHost, fullDirName, topMPDBoot, user
mpd_set_my_id('mpdboot_rank_notset')
fullDirName = path.abspath(path.split(argv[0])[0])
rshCmd = 'ssh'
user = mpd_get_my_username()
mpdCmd = path.join(fullDirName, 'mpd.py')
mpdbootCmd = path.join(fullDirName, 'mpdboot.py')
hostsFilename = 'mpd.hosts'
totalNum = 1 # may get chgd below
debug = 0
verbosity = 0
localConsoleArg = ''
remoteConsoleArg = ''
myConsoleVal = ''
oneMPDPerHost = 1
entryHost = ''
entryPort = ''
topMPDBoot = 1
myHost = gethostname()
myNcpus = 1
myIfhn = ''
try:
shell = path.split(environ['SHELL'])[-1]
except:
shell = 'csh'
argidx = 1 # skip arg 0
while argidx < len(argv):
if argv[argidx] == '-h' or argv[argidx] == '--help':
usage()
elif argv[argidx] == '-zentry': # entry host and port
if ':' not in argv[argidx + 1]:
print 'invalid pair of entry host and entry port for -zentry option'
usage()
(entryHost, entryPort) = argv[argidx + 1].split(':')
try:
ip = gethostbyname_ex(entryHost)[2] # may fail if invalid host
except:
print 'invalid entry host ', entryHost
stdout.flush()
usage()
if not entryPort.isdigit():
print 'invalid (nonumeric) entry port ', entryPort
stdout.flush()
usage()
entryHost = entryHost
entryPort = entryPort
argidx += 2
elif argv[argidx] == '-zrank':
topMPDBoot = 0
myBootRank = int(argv[argidx + 1])
argidx += 2
elif argv[argidx] == '-zhosts':
zhosts = argv[argidx + 1]
zhosts = zhosts.split(',')
hostsAndInfo = []
for zhost in zhosts:
(host, ncpus, ifhn) = zhost.split(':')
hostsAndInfo.append({
'host': host,
'ncpus': ncpus,
'ifhn': ifhn
})
argidx += 2
elif argv[argidx] == '-r': # or --rsh=
rshCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--rsh'):
splitArg = argv[argidx].split('=')
try:
rshCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-u': # or --user=
user = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--user'):
splitArg = argv[argidx].split('=')
try:
user = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-m': # or --mpd=
mpdCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--mpd'):
splitArg = argv[argidx].split('=')
try:
mpdCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-f': # or --file=
hostsFilename = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--file'):
splitArg = argv[argidx].split('=')
try:
hostsFilename = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ncpus'):
splitArg = argv[argidx].split('=')
try:
myNcpus = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ifhn'):
splitArg = argv[argidx].split('=')
myIfhn = splitArg[1]
myHost = splitArg[1]
argidx += 1
elif argv[argidx] == '-n': # or --totalnum=
totalNum = int(argv[argidx + 1])
argidx += 2
elif argv[argidx].startswith('--totalnum'):
splitArg = argv[argidx].split('=')
try:
totalNum = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-d' or argv[argidx] == '--debug':
debug = 1
argidx += 1
elif argv[argidx] == '-s' or argv[argidx] == '--shell':
shell = 'bourne'
argidx += 1
elif argv[argidx] == '-v' or argv[argidx] == '--verbose':
verbosity = 1
argidx += 1
elif argv[argidx] == '-1':
oneMPDPerHost = 0
argidx += 1
elif argv[argidx] == '--loccons':
localConsoleArg = '--loccons'
argidx += 1
elif argv[argidx] == '--remcons':
remoteConsoleArg = '--remcons'
argidx += 1
else:
print 'mpdboot: unrecognized argument:', argv[argidx]
usage()
if topMPDBoot:
lines = []
if totalNum > 1:
try:
f = open(hostsFilename, 'r')
for line in f:
lines.append(line)
except:
print 'unable to open (or read) hostsfile %s' % (hostsFilename)
exit(-1)
hostsAndInfo = [{'host': myHost, 'ncpus': myNcpus, 'ifhn': myIfhn}]
for line in lines:
line = line.strip()
if not line or line[0] == '#':
continue
splitLine = re.split(r'\s+', line)
host = splitLine[0]
ncpus = 1 # default
if ':' in host:
(host, ncpus) = host.split(':', 1)
ncpus = int(ncpus)
ifhn = '' # default
for kv in splitLine[1:]:
(k, v) = kv.split('=', 1)
if k == 'ifhn':
ifhn = v
hostsAndInfo.append({'host': host, 'ncpus': ncpus, 'ifhn': ifhn})
if oneMPDPerHost and totalNum > 1:
oldHosts = hostsAndInfo[:]
hostsAndInfo = []
for x in oldHosts:
keep = 1
for y in hostsAndInfo:
if mpd_same_ips(x['host'], y['host']):
keep = 0
break
if keep:
hostsAndInfo.append(x)
if len(hostsAndInfo) < totalNum: # one is local
print 'totalNum=%d num hosts=%d' % (totalNum, len(hostsAndInfo))
print 'there are not enough hosts on which to start all processes'
exit(-1)
myBootRank = 0
if localConsoleArg:
myConsoleVal = '-n'
else:
if remoteConsoleArg:
myConsoleVal = '-n'
anMPDalreadyHere = 0
for i in range(myBootRank):
if mpd_same_ips(hostsAndInfo[i]['host'],
myHost): # if one before me on this host
myConsoleVal = '-n'
anMPDalreadyHere = 1
break
if not anMPDalreadyHere:
try:
system('%s/mpdallexit.py > /dev/null' %
(fullDirName)) # stop any current mpds
except:
pass
mpd_set_my_id('mpdboot_%s_%d' % (myHost, myBootRank))
if debug:
mpd_print(1, 'starting')
(parent, lchild,
rchild) = mpd_get_ranks_in_binary_tree(myBootRank, totalNum)
if debug:
mpd_print(1, 'p=%d l=%d r=%d' % (parent, lchild, rchild))
if myIfhn:
ifhnVal = '--if %s' % (myIfhn)
elif hostsAndInfo[myBootRank]['ifhn']:
ifhnVal = '--if %s' % (hostsAndInfo[myBootRank]['ifhn'])
else:
ifhnVal = ''
if entryHost:
cmd = '%s %s -h %s -p %s -d -e --ncpus %s %s' % \
(mpdCmd,myConsoleVal,entryHost,entryPort,myNcpus,ifhnVal)
else:
cmd = '%s %s -d -e --ncpus %s %s' % \
(mpdCmd,myConsoleVal,myNcpus,ifhnVal)
if verbosity:
mpd_print(1, 'starting local mpd on %s' % (myHost))
if debug:
mpd_print(1, 'cmd to run local mpd = :%s:' % (cmd))
if not access(mpdCmd, X_OK):
err_exit('cannot access mpd cmd :%s:' % (mpdCmd))
locMPD = Popen4(cmd, 0)
locMPDFD = locMPD.fromchild
locMPDPort = locMPDFD.readline().strip()
if locMPDPort.isdigit():
# can't do this until he's already in his ring
locMPDSocket = mpd_get_inet_socket_and_connect(myHost, int(locMPDPort))
if locMPDSocket:
msgToSend = {
'cmd': 'ping',
'host': 'ping',
'port': 0
} # dummy host & port
mpd_send_one_msg(locMPDSocket, {
'cmd': 'ping',
'host': myHost,
'port': 0
})
msg = mpd_recv_one_msg(locMPDSocket) # RMB: WITH TIMEOUT ??
if not msg or not msg.has_key('cmd') or msg['cmd'] != 'ping_ack':
err_exit(
'%d: unable to ping local mpd; invalid msg from mpd :%s:' %
(myBootRank, msg))
locMPDSocket.close()
else:
err_exit('failed to connect to mpd')
else:
err_exit('%d: invalid port from mpd %s' %
(myBootRank, str(locMPDPort)))
if not entryHost:
entryHost = myHost
entryPort = locMPDPort
if rshCmd == 'ssh':
xOpt = '-x'
else:
xOpt = ''
lfd = 0
rfd = 0
fdsToSelect = []
if debug:
debugArg = '-d'
else:
debugArg = ''
if verbosity:
verboseArg = '-v'
else:
verboseArg = ''
if lchild >= 0:
zhosts = [
"%s:%s:%s" % (h['host'], h['ncpus'], h['ifhn'])
for h in hostsAndInfo
]
if hostsAndInfo[lchild]['ifhn']:
ifhnVal = '--ifhn=%s' % (hostsAndInfo[lchild]['ifhn'])
else:
ifhnVal = ''
cmd = "%s %s %s -n '%s --ncpus=%s %s -r %s -m %s -n %d %s %s %s -zentry %s:%s -zrank %s -zhosts %s </dev/null ' " % \
(rshCmd, xOpt, hostsAndInfo[lchild]['host'], mpdbootCmd,
hostsAndInfo[lchild]['ncpus'],ifhnVal,
rshCmd, mpdCmd, totalNum, debugArg, verboseArg, remoteConsoleArg, entryHost,
entryPort, lchild,
','.join(zhosts) )
if verbosity:
mpd_print(1, 'starting remote mpd on %s' % (hostsAndInfo[lchild]))
if debug:
mpd_print(1, 'cmd to run lchild boot = :%s:' % (cmd))
lchildMPDBoot = Popen4(cmd, 0)
lfd = lchildMPDBoot.fromchild
fdsToSelect.append(lfd)
if rchild >= 0:
zhosts = [
"%s:%s:%s" % (h['host'], h['ncpus'], h['ifhn'])
for h in hostsAndInfo
]
if hostsAndInfo[rchild]['ifhn']:
ifhnVal = '--ifhn=%s' % (hostsAndInfo[rchild]['ifhn'])
else:
ifhnVal = ''
cmd = "%s %s %s -n '%s --ncpus=%s %s -r %s -m %s -n %d %s %s %s -zentry %s:%s -zrank %s -zhosts %s </dev/null ' " % \
(rshCmd, xOpt, hostsAndInfo[rchild]['host'], mpdbootCmd,
hostsAndInfo[rchild]['ncpus'],ifhnVal,
rshCmd, mpdCmd, totalNum, debugArg, verboseArg, remoteConsoleArg, entryHost,
entryPort, rchild,
','.join(zhosts) )
if verbosity:
mpd_print(1, 'starting remote mpd on %s' % (hostsAndInfo[rchild]))
if debug:
mpd_print(1, 'cmd to run rchild boot = :%s:' % (cmd))
rchildMPDBoot = Popen4(cmd, 0)
rfd = rchildMPDBoot.fromchild
fdsToSelect.append(rfd)
lfd_first_line = 1
rfd_first_line = 1
while fdsToSelect:
try:
(readyFDs, unused1, unused2) = select(fdsToSelect, [], [], 0.1)
except error, errmsg:
mpd_raise('mpdboot: select failed: errmsg=:%s:' % (errmsg))
if lfd and lfd in readyFDs:
line = lfd.readline()
if line:
if line.find('RC=MPDBOOT_ERREXIT') >= 0:
err_exit('RC=MPDBOOT_ERREXIT')
else:
if not verbosity and lfd_first_line:
lfd_first_line = 0
mpd_print(
1,
"error trying to start mpd(boot) at %d %s; output:"
% (lchild, hostsAndInfo[lchild]))
print ' ', line,
stdout.flush()
else:
lfd.close()
fdsToSelect.remove(lfd)
if rfd and rfd in readyFDs:
line = rfd.readline()
if line:
if line.find('RC=MPDBOOT_ERREXIT') >= 0:
err_exit('RC=MPDBOOT_ERREXIT')
else:
if not verbosity and rfd_first_line:
rfd_first_line = 0
mpd_print(
1,
"error trying to start mpd(boot) at %d %s; output:"
% (rchild, hostsAndInfo[rchild]))
print ' ', line,
stdout.flush()
else:
rfd.close()
fdsToSelect.remove(rfd)
def mpdlistjobs():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
signal.signal(signal.SIGINT, sig_handler)
mpd_set_my_id(myid='mpdlistjobs')
uname = ''
jobid = ''
sjobid = ''
jobalias = ''
sssPrintFormat = 0
if len(sys.argv) > 1:
aidx = 1
while aidx < len(sys.argv):
if sys.argv[aidx] == '-h' or sys.argv[aidx] == '--help':
usage()
if sys.argv[aidx] == '-u': # or --user=
uname = sys.argv[aidx + 1]
aidx += 2
elif sys.argv[aidx].startswith('--user'):
splitArg = sys.argv[aidx].split('=')
try:
uname = splitArg[1]
except:
print 'mpdlistjobs: invalid argument:', sys.argv[aidx]
usage()
aidx += 1
elif sys.argv[aidx] == '-j': # or --jobid=
jobid = sys.argv[aidx + 1]
aidx += 2
sjobid = jobid.split('@') # jobnum and originating host
elif sys.argv[aidx].startswith('--jobid'):
splitArg = sys.argv[aidx].split('=')
try:
jobid = splitArg[1]
sjobid = jobid.split('@') # jobnum and originating host
except:
print 'mpdlistjobs: invalid argument:', sys.argv[aidx]
usage()
aidx += 1
elif sys.argv[aidx] == '-a': # or --alias=
jobalias = sys.argv[aidx + 1]
aidx += 2
elif sys.argv[aidx].startswith('--alias'):
splitArg = sys.argv[aidx].split('=')
try:
jobalias = splitArg[1]
except:
print 'mpdlistjobs: invalid argument:', sys.argv[aidx]
usage()
aidx += 1
elif sys.argv[aidx] == '--sss':
sssPrintFormat = 1
aidx += 1
else:
print 'unrecognized arg: %s' % sys.argv[aidx]
sys.exit(-1)
parmdb = MPDParmDB(
orderedSources=['cmdline', 'xml', 'env', 'rcfile', 'thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD': 0,
'MPD_SECRETWORD': '',
}
for (k, v) in parmsToOverride.items():
parmdb[('thispgm', k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if (hasattr(os, 'getuid')
and os.getuid() == 0) or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = os.path.abspath(os.path.split(
sys.argv[0])[0]) # normalize
mpdroot = os.path.join(fullDirName, 'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,
secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = {'cmd': 'mpdlistjobs'}
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1, 'no msg recvd from mpd before timeout')
if msg['cmd'] != 'local_mpdid': # get full id of local mpd for filters later
mpd_print(
1,
'did not recv local_mpdid msg from local mpd; instead, recvd: %s' %
msg)
else:
if len(sjobid) == 1:
sjobid.append(msg['id'])
done = 0
while not done:
msg = conSock.recv_dict_msg()
if not msg.has_key('cmd'):
mpd_print(1, 'mpdlistjobs: INVALID msg=:%s:' % (msg))
sys.exit(-1)
if msg['cmd'] == 'mpdlistjobs_info':
smjobid = msg['jobid'].split(
' ') # jobnum, mpdid, and alias (if present)
if len(smjobid) < 3:
smjobid.append('')
print_based_on_uname = 0 # default
print_based_on_jobid = 0 # default
print_based_on_jobalias = 0 # default
if not uname or uname == msg['username']:
print_based_on_uname = 1
if not jobid and not jobalias:
print_based_on_jobid = 1
print_based_on_jobalias = 1
else:
if sjobid and sjobid[0] == smjobid[0] and sjobid[1] == smjobid[
1]:
print_based_on_jobid = 1
if jobalias and jobalias == smjobid[2]:
print_based_on_jobalias = 1
if not smjobid[2]:
smjobid[2] = ' ' # just for printing
if print_based_on_uname and (print_based_on_jobid
or print_based_on_jobalias):
if sssPrintFormat:
print "%s %s %s" % (msg['host'], msg['clipid'], msg['sid'])
else:
print 'jobid = %s@%s' % (smjobid[0], smjobid[1])
print 'jobalias = %s' % (smjobid[2])
print 'username = %s' % (msg['username'])
print 'host = %s' % (msg['host'])
print 'pid = %s' % (msg['clipid'])
print 'sid = %s' % (msg['sid'])
print 'rank = %s' % (msg['rank'])
print 'pgm = %s' % (msg['pgm'])
print
else: # mpdlistjobs_trailer
done = 1
conSock.close()
def mpdboot():
global myHost, fullDirName, rshCmd, user, mpdCmd, debug, verbose
myHost = gethostname()
mpd_set_my_id('mpdboot_%s' % (myHost))
fullDirName = path.abspath(path.split(argv[0])[0])
rshCmd = 'ssh'
user = mpd_get_my_username()
mpdCmd = path.join(fullDirName, 'mpd.py')
hostsFilename = 'mpd.hosts'
totalnumToStart = 1 # may get chgd below
debug = 0
verbose = 0
localConArg = ''
remoteConArg = ''
oneMPDPerHost = 1
myNcpus = 1
myIfhn = ''
chkupIndicator = 0 # 1 -> chk and start ; 2 -> just chk
maxUnderOneRoot = 4
try:
shell = path.split(environ['SHELL'])[-1]
except:
shell = 'csh'
argidx = 1 # skip arg 0
while argidx < len(argv):
if argv[argidx] == '-h' or argv[argidx] == '--help':
usage()
elif argv[argidx] == '-r': # or --rsh=
rshCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--rsh'):
splitArg = argv[argidx].split('=')
try:
rshCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-u': # or --user=
user = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--user'):
splitArg = argv[argidx].split('=')
try:
user = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-m': # or --mpd=
mpdCmd = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--mpd'):
splitArg = argv[argidx].split('=')
try:
mpdCmd = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-f': # or --file=
hostsFilename = argv[argidx + 1]
argidx += 2
elif argv[argidx].startswith('--file'):
splitArg = argv[argidx].split('=')
try:
hostsFilename = splitArg[1]
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ncpus'):
splitArg = argv[argidx].split('=')
try:
myNcpus = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--ifhn'):
splitArg = argv[argidx].split('=')
myIfhn = splitArg[1]
myHost = splitArg[1]
argidx += 1
elif argv[argidx] == '-n': # or --totalnum=
totalnumToStart = int(argv[argidx + 1])
argidx += 2
elif argv[argidx].startswith('--totalnum'):
splitArg = argv[argidx].split('=')
try:
totalnumToStart = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx].startswith('--maxbranch'):
splitArg = argv[argidx].split('=')
try:
maxUnderOneRoot = int(splitArg[1])
except:
print 'mpdboot: invalid argument:', argv[argidx]
usage()
argidx += 1
elif argv[argidx] == '-d' or argv[argidx] == '--debug':
debug = 1
argidx += 1
elif argv[argidx] == '-s' or argv[argidx] == '--shell':
shell = 'bourne'
argidx += 1
elif argv[argidx] == '-v' or argv[argidx] == '--verbose':
verbose = 1
argidx += 1
elif argv[argidx] == '-c' or argv[argidx] == '--chkup':
chkupIndicator = 1
argidx += 1
elif argv[argidx] == '--chkuponly':
chkupIndicator = 2
argidx += 1
elif argv[argidx] == '-1':
oneMPDPerHost = 0
argidx += 1
elif argv[argidx] == '--loccons':
localConArg = '-n'
argidx += 1
elif argv[argidx] == '--remcons':
remoteConArg = '-n'
argidx += 1
else:
print 'mpdboot: unrecognized argument:', argv[argidx]
usage()
if debug:
print 'debug: starting'
lines = []
if totalnumToStart > 1:
try:
f = open(hostsFilename, 'r')
for line in f:
lines.append(line)
except:
print 'unable to open (or read) hostsfile %s' % (hostsFilename)
exit(-1)
hostsAndInfo = [{'host': myHost, 'ncpus': myNcpus, 'ifhn': myIfhn}]
for line in lines:
line = line.strip()
if not line or line[0] == '#':
continue
splitLine = re.split(r'\s+', line)
host = splitLine[0]
ncpus = 1 # default
if ':' in host:
(host, ncpus) = host.split(':', 1)
ncpus = int(ncpus)
ifhn = '' # default
for kv in splitLine[1:]:
(k, v) = kv.split('=', 1)
if k == 'ifhn':
ifhn = v
hostsAndInfo.append({'host': host, 'ncpus': ncpus, 'ifhn': ifhn})
if oneMPDPerHost and totalnumToStart > 1:
oldHosts = hostsAndInfo[:]
hostsAndInfo = []
for x in oldHosts:
keep = 1
for y in hostsAndInfo:
if mpd_same_ips(x['host'], y['host']):
keep = 0
break
if keep:
hostsAndInfo.append(x)
if len(hostsAndInfo) < totalnumToStart: # one is local
print 'totalnum=%d numhosts=%d' % (totalnumToStart, len(hostsAndInfo))
print 'there are not enough hosts on which to start all processes'
exit(-1)
if chkupIndicator:
hostsToCheck = [hai['host'] for hai in hostsAndInfo[1:totalnumToStart]]
(upList, dnList) = chkupdn(hostsToCheck)
if dnList:
print "these hosts are down; exiting"
print dnList
exit(-1)
print "there are %d hosts up (counting local)" % (len(upList) + 1)
if chkupIndicator == 2: # do the chkup and quit
exit(0)
try:
# stop current (if any) mpds; ignore the output
getoutput('%s/mpdallexit.py' % (fullDirName))
if verbose or debug:
print 'running mpdallexit on %s' % (myHost)
except:
pass
if myIfhn:
ifhn = '--ifhn=%s' % (myIfhn)
else:
ifhn = ''
hostsAndInfo[0]['entry_host'] = ''
hostsAndInfo[0]['entry_port'] = ''
mpdArgs = '%s %s --ncpus=%d' % (localConArg, ifhn, myNcpus)
(mpdPID, mpdFD) = launch_one_mpd(0, 0, mpdArgs, hostsAndInfo)
fd2idx = {mpdFD: 0}
handle_mpd_output(mpdFD, fd2idx, hostsAndInfo)
try:
from os import sysconf
maxfds = sysconf('SC_OPEN_MAX')
except:
maxfds = 1024
maxAtOnce = min(128, maxfds -
8) # -8 for stdeout, etc. + a few more for padding
hostsSeen = {myHost: 1}
fdsToSelect = []
numStarted = 1 # local already going
numStarting = 0
numUnderCurrRoot = 0
possRoots = []
currRoot = 0
idxToStart = 1 # local mpd already going
while numStarted < totalnumToStart:
if numStarting < maxAtOnce and idxToStart < totalnumToStart:
if numUnderCurrRoot < maxUnderOneRoot:
entryHost = hostsAndInfo[currRoot]['host']
entryPort = hostsAndInfo[currRoot]['list_port']
hostsAndInfo[idxToStart]['entry_host'] = entryHost
hostsAndInfo[idxToStart]['entry_port'] = entryPort
if hostsSeen.has_key(hostsAndInfo[idxToStart]['host']):
remoteConArg = '-n'
myNcpus = hostsAndInfo[idxToStart]['ncpus']
ifhn = hostsAndInfo[idxToStart]['ifhn']
if ifhn:
ifhn = '--ifhn=%s' % (ifhn)
mpdArgs = '%s -h %s -p %s %s --ncpus=%d' % (
remoteConArg, entryHost, entryPort, ifhn, myNcpus)
(mpdPID, mpdFD) = launch_one_mpd(idxToStart, currRoot, mpdArgs,
hostsAndInfo)
numStarting += 1
numUnderCurrRoot += 1
hostsAndInfo[idxToStart]['pid'] = mpdPID
hostsSeen[hostsAndInfo[idxToStart]['host']] = 1
fd2idx[mpdFD] = idxToStart
fdsToSelect.append(mpdFD)
idxToStart += 1
else:
if possRoots:
currRoot = possRoots.pop()
numUnderCurrRoot = 0
selectTime = 0.01
else:
selectTime = 0.1
try:
(readyFDs, unused1, unused2) = select(fdsToSelect, [], [],
selectTime)
except error, errmsg:
mpd_print(1, 'mpdboot: select failed: errmsg=:%s:' % (errmsg))
exit(-1)
for fd in readyFDs:
handle_mpd_output(fd, fd2idx, hostsAndInfo)
numStarted += 1
numStarting -= 1
possRoots.append(fd2idx[fd])
fdsToSelect.remove(fd)
fd.close()
def mpdsigjob():
import sys # to get access to excepthook in next line
sys.excepthook = mpd_uncaught_except_tb
if len(argv) < 3 or argv[1] == '-h' or argv[1] == '--help':
usage()
signal(SIGINT, sig_handler)
mpd_set_my_id(myid='mpdsigjob')
sigtype = argv[1]
if sigtype.startswith('-'):
sigtype = sigtype[1:]
if sigtype.startswith('SIG'):
sigtype = sigtype[3:]
import signal as tmpsig # just to get valid SIG's
if sigtype.isdigit():
if int(sigtype) > tmpsig.NSIG:
print 'invalid signum: %s' % (sigtype)
exit(-1)
else:
if not tmpsig.__dict__.has_key('SIG' + sigtype):
print 'invalid sig type: %s' % (sigtype)
exit(-1)
jobalias = ''
jobnum = ''
mpdid = ''
single_or_group = 'g'
i = 2
while i < len(argv):
if argv[i] == '-a':
if jobnum: # should not have both alias and jobid
print '** cannot specify both jobalias and jobid'
usage()
jobalias = argv[i+1]
i += 1
jobnum = '0'
elif argv[i] == '-j':
if jobalias: # should not have both alias and jobid
print '** cannot specify both jobalias and jobid'
usage()
jobid = argv[i+1]
i += 1
sjobid = jobid.split('@')
jobnum = sjobid[0]
if len(sjobid) > 1:
mpdid = sjobid[1]
elif argv[i] == '-s':
single_or_group = 's'
elif argv[i] == '-g':
single_or_group = 'g'
else:
print '** unrecognized arg: %s' % (argv[i])
usage()
i += 1
parmdb = MPDParmDB(orderedSources=['cmdline','xml','env','rcfile','thispgm'])
parmsToOverride = {
'MPD_USE_ROOT_MPD' : 0,
'MPD_SECRETWORD' : '',
}
for (k,v) in parmsToOverride.items():
parmdb[('thispgm',k)] = v
parmdb.get_parms_from_env(parmsToOverride)
parmdb.get_parms_from_rcfile(parmsToOverride)
if getuid() == 0 or parmdb['MPD_USE_ROOT_MPD']:
fullDirName = path.abspath(path.split(argv[0])[0]) # normalize
mpdroot = path.join(fullDirName,'mpdroot')
conSock = MPDConClientSock(mpdroot=mpdroot,secretword=parmdb['MPD_SECRETWORD'])
else:
conSock = MPDConClientSock(secretword=parmdb['MPD_SECRETWORD'])
msgToSend = {'cmd' : 'mpdsigjob', 'sigtype': sigtype, 'jobnum' : jobnum,
'mpdid' : mpdid, 'jobalias' : jobalias, 's_or_g' : single_or_group,
'username' : mpd_get_my_username() }
conSock.send_dict_msg(msgToSend)
msg = conSock.recv_dict_msg(timeout=5.0)
if not msg:
mpd_print(1,'no msg recvd from mpd before timeout')
if msg['cmd'] != 'mpdsigjob_ack':
if msg['cmd'] == 'already_have_a_console':
mpd_print(1,'mpd already has a console (e.g. for long ringtest); try later')
else:
mpd_print(1,'unexpected message from mpd: %s' % (msg) )
exit(-1)
if not msg['handled']:
print 'job not found'
exit(-1)
conSock.close()
