def createXML(self, doc, attr, topElement, final):
for k, v in attr.iteritems():
self.log.debug('_createHadoopSiteXml: ' + str(k) + " " + str(v))
lowport, highport = self.__hadoopPortRange
if (v == "fillinport"):
v = "%d" % (ServiceUtil.getUniqRandomPort(
low=lowport, high=highport, log=self.log))
keyvalpair = ''
if isinstance(v, (tuple, list)):
for item in v:
keyvalpair = "%s%s=%s," % (keyvalpair, k, item)
keyvalpair = keyvalpair[:-1]
else:
keyvalpair = k + '=' + v
self.filledInKeyVals.append(keyvalpair)
if (k == "mapred.job.tracker"): # total hack for time's sake
keyvalpair = k + "=" + v
self.filledInKeyVals.append(keyvalpair)
if (v == "fillinhostport"):
port = "%d" % (ServiceUtil.getUniqRandomPort(
low=lowport, high=highport, log=self.log))
self.log.debug('Setting hostname to: %s' % local_fqdn())
v = local_fqdn() + ':' + port
keyvalpair = ''
if isinstance(v, (tuple, list)):
for item in v:
keyvalpair = "%s%s=%s," % (keyvalpair, k, item)
keyvalpair = keyvalpair[:-1]
else:
keyvalpair = k + '=' + v
self.filledInKeyVals.append(keyvalpair)
if (v == "fillindir"):
v = self.__mrSysDir
pass
prop = None
if isinstance(v, (tuple, list)):
for item in v:
prop = self._createXmlElement(doc, k, item,
"No description", final)
topElement.appendChild(prop)
else:
if k == 'fs.default.name':
prop = self._createXmlElement(doc, k, "hdfs://" + v,
"No description", final)
else:
prop = self._createXmlElement(doc, k, v, "No description",
final)
topElement.appendChild(prop)
def createXML(self, doc, attr, topElement, final):
for k,v in attr.iteritems():
self.log.debug('_createHadoopSiteXml: ' + str(k) + " " + str(v))
lowport, highport = self.__hadoopPortRange
if ( v == "fillinport" ):
v = "%d" % (ServiceUtil.getUniqRandomPort(low=lowport, high=highport, log=self.log))
keyvalpair = ''
if isinstance(v, (tuple, list)):
for item in v:
keyvalpair = "%s%s=%s," % (keyvalpair, k, item)
keyvalpair = keyvalpair[:-1]
else:
keyvalpair = k + '=' + v
self.filledInKeyVals.append(keyvalpair)
if(k == "mapred.job.tracker"): # total hack for time's sake
keyvalpair = k + "=" + v
self.filledInKeyVals.append(keyvalpair)
if ( v == "fillinhostport"):
port = "%d" % (ServiceUtil.getUniqRandomPort(low=lowport, high=highport, log=self.log))
self.log.debug('Setting hostname to: %s' % local_fqdn())
v = local_fqdn() + ':' + port
keyvalpair = ''
if isinstance(v, (tuple, list)):
for item in v:
keyvalpair = "%s%s=%s," % (keyvalpair, k, item)
keyvalpair = keyvalpair[:-1]
else:
keyvalpair = k + '=' + v
self.filledInKeyVals.append(keyvalpair)
if ( v == "fillindir"):
v = self.__mrSysDir
pass
prop = None
if isinstance(v, (tuple, list)):
for item in v:
prop = self._createXmlElement(doc, k, item, "No description", final)
topElement.appendChild(prop)
else:
if k == 'fs.default.name':
prop = self._createXmlElement(doc, k, "hdfs://" + v, "No description", final)
else:
prop = self._createXmlElement(doc, k, v, "No description", final)
topElement.appendChild(prop)
def _serve_forever(self):
if len(self.__ports) > 1:
randomPort = Random(os.getpid())
portSequence = range(self.__ports[0], self.__ports[1])
maxTryCount = abs(self.__ports[0] - self.__ports[1])
tryCount = 0
while True:
somePort = randomPort.choice(portSequence)
self.server_address = (self.__host, int(somePort))
if self.__host == '':
self.server_address = (local_fqdn(), self.server_address[1])
try:
reactor.listenTCP(int(somePort), server.Site(
self.__xmlrpc), interface=self.__host)
reactor.run(installSignalHandlers=0)
except:
self.__logger.debug("Failed to bind to: %s:%s." % (
self.__host, somePort))
tryCount = tryCount + 1
if tryCount > maxTryCount:
self.__logger.warn("Failed to bind to: %s:%s" % (
self.__host, self.__ports))
sys.exit(1)
else:
break
else:
try:
self.server_address = (self.__host, int(self.__ports[0]))
if self.__host == '':
self.server_address = (local_fqdn(), self.server_address[1])
reactor.listenTCP(int(self.__ports[0]), server.Site(self.__xmlrpc),
interface=self.__host)
reactor.run(installSignalHandlers=0)
except:
self.__logger.warn("Failed to bind to: %s:%s."% (
self.__host, self.__ports[0]))
sys.exit(1)
def __init__(self, name, config, xrtype='threaded'):
""" Initialization requires a name string and a config object of type
hodlib.Common.setup.options or hodlib.Common.setup.config."""
self.name = name
self.hostname = local_fqdn()
self._cfg = config
self._xrc = None
self.logs = {}
self._baseLogger = None
self._serviceID = os.getenv('PBS_JOBID')
self.__logDir = None
self.__svcrgy = None
self.__stop = False
self.__xrtype = xrtype
self._init_logging()
if name != 'serviceRegistry': self._init_signals()
self._init_xrc_server()
def __init__(self, name, config, xrtype='threaded'):
""" Initialization requires a name string and a config object of type
hodlib.Common.setup.options or hodlib.Common.setup.config."""
self.name = name
self.hostname = local_fqdn()
self._cfg = config
self._xrc = None
self.logs = {}
self._baseLogger = None
self._serviceID = os.getenv('PBS_JOBID')
self.__logDir = None
self.__svcrgy = None
self.__stop = False
self.__xrtype = xrtype
self._init_logging()
if name != 'serviceRegistry': self._init_signals()
self._init_xrc_server()
raise socket.gaierror, errData
except:
tryCount = tryCount + 1
if tryCount > maxTryCount:
bindError = "bind failure for port range %s:%d" % (
self.ports)
raise socket.error, bindError
else:
break
else:
self.server_address = (self.host, int(self.ports[0]))
self.socket.bind(self.server_address)
if self.host == '':
self.server_address = (local_fqdn(), self.server_address[1])
def _serve_forever(self):
"""Replacement for serve_forever loop.
All baseSocketServers run within a master thread; that thread
imitates serve_forever, but checks an event (self.__stopForever)
before processing new connections.
"""
while not self.__stopForever.isSet():
(rlist, wlist, xlist) = select([self.socket], [], [],
1)
if (len(rlist) > 0 and self.socket == rlist[0]):
self.handle_request()
def process_qsub_attributes():
rawAttributes = self.nodePoolDesc.getAttrs()
# 'W:x' is used to specify torque management extentensions ie -W x= ...
resourceManagementExtensions = ''
if 'W:x' in rawAttributes:
resourceManagementExtensions = rawAttributes['W:x']
if qosLevel:
if len(resourceManagementExtensions) > 0:
resourceManagementExtensions += ';'
resourceManagementExtensions += 'QOS:%s' % (qosLevel)
rawAttributes['W:x'] = resourceManagementExtensions
hostname = local_fqdn()
rawAttributes['l:nodes'] = nodeSet._getNumNodes()
if walltime:
rawAttributes['l:walltime'] = walltime
#create a dict of dictionaries for
# various arguments of torque
cmds = {}
for key in rawAttributes:
value = rawAttributes[key]
if key.find(':') == -1:
raise ValueError, 'Syntax error: missing colon after %s in %s=%s' % (
key, key, value)
[option, subOption] = key.split(':', 1)
if not option in cmds:
cmds[option] = {}
cmds[option][subOption] = value
opts = []
#create a string from this
#dictionary of dictionaries createde above
for k in cmds:
csv = []
nv = cmds[k]
for n in nv:
v = nv[n]
if len(n) == 0:
csv.append(v)
else:
csv.append('%s=%s' % (n, v))
opts.append('-%s' % (k))
opts.append(','.join(csv))
for option in cmds:
commandList = []
for subOption in cmds[option]:
value = cmds[option][subOption]
if len(subOption) == 0:
commandList.append(value)
else:
commandList.append("%s=%s" % (subOption, value))
opts.append('-%s' % option)
opts.append(','.join(commandList))
return opts
def process_qsub_attributes():
rawAttributes = self.nodePoolDesc.getAttrs()
# 'W:x' is used to specify torque management extentensions ie -W x= ...
resourceManagementExtensions = ''
if 'W:x' in rawAttributes:
resourceManagementExtensions = rawAttributes['W:x']
if qosLevel:
if len(resourceManagementExtensions) > 0:
resourceManagementExtensions += ';'
resourceManagementExtensions += 'QOS:%s' % (qosLevel)
rawAttributes['W:x'] = resourceManagementExtensions
hostname = local_fqdn()
rawAttributes['l:nodes'] = nodeSet._getNumNodes()
if walltime:
rawAttributes['l:walltime'] = walltime
#create a dict of dictionaries for
# various arguments of torque
cmds = {}
for key in rawAttributes:
value = rawAttributes[key]
if key.find(':') == -1:
raise ValueError, 'Syntax error: missing colon after %s in %s=%s' % (
key, key, value)
[option, subOption] = key.split(':', 1)
if not option in cmds:
cmds[option] = {}
cmds[option][subOption] = value
opts = []
#create a string from this
#dictionary of dictionaries createde above
for k in cmds:
csv = []
nv = cmds[k]
for n in nv:
v = nv[n]
if len(n) == 0:
csv.append(v)
else:
csv.append('%s=%s' % (n, v))
opts.append('-%s' % (k))
opts.append(','.join(csv))
for option in cmds:
commandList = []
for subOption in cmds[option]:
value = cmds[option][subOption]
if len(subOption) == 0:
commandList.append(value)
else:
commandList.append("%s=%s" % (subOption, value))
opts.append('-%s' % option)
opts.append(','.join(commandList))
return opts
