def createXML(self, doc, attr, topElement, final): for k, v in attr.iteritems(): self.log.debug('_createHadoopSiteXml: ' + str(k) + " " + str(v)) lowport, highport = self.__hadoopPortRange if (v == "fillinport"): v = "%d" % (ServiceUtil.getUniqRandomPort( low=lowport, high=highport, log=self.log)) keyvalpair = '' if isinstance(v, (tuple, list)): for item in v: keyvalpair = "%s%s=%s," % (keyvalpair, k, item) keyvalpair = keyvalpair[:-1] else: keyvalpair = k + '=' + v self.filledInKeyVals.append(keyvalpair) if (k == "mapred.job.tracker"): # total hack for time's sake keyvalpair = k + "=" + v self.filledInKeyVals.append(keyvalpair) if (v == "fillinhostport"): port = "%d" % (ServiceUtil.getUniqRandomPort( low=lowport, high=highport, log=self.log)) self.log.debug('Setting hostname to: %s' % local_fqdn()) v = local_fqdn() + ':' + port keyvalpair = '' if isinstance(v, (tuple, list)): for item in v: keyvalpair = "%s%s=%s," % (keyvalpair, k, item) keyvalpair = keyvalpair[:-1] else: keyvalpair = k + '=' + v self.filledInKeyVals.append(keyvalpair) if (v == "fillindir"): v = self.__mrSysDir pass prop = None if isinstance(v, (tuple, list)): for item in v: prop = self._createXmlElement(doc, k, item, "No description", final) topElement.appendChild(prop) else: if k == 'fs.default.name': prop = self._createXmlElement(doc, k, "hdfs://" + v, "No description", final) else: prop = self._createXmlElement(doc, k, v, "No description", final) topElement.appendChild(prop)
def createXML(self, doc, attr, topElement, final): for k,v in attr.iteritems(): self.log.debug('_createHadoopSiteXml: ' + str(k) + " " + str(v)) lowport, highport = self.__hadoopPortRange if ( v == "fillinport" ): v = "%d" % (ServiceUtil.getUniqRandomPort(low=lowport, high=highport, log=self.log)) keyvalpair = '' if isinstance(v, (tuple, list)): for item in v: keyvalpair = "%s%s=%s," % (keyvalpair, k, item) keyvalpair = keyvalpair[:-1] else: keyvalpair = k + '=' + v self.filledInKeyVals.append(keyvalpair) if(k == "mapred.job.tracker"): # total hack for time's sake keyvalpair = k + "=" + v self.filledInKeyVals.append(keyvalpair) if ( v == "fillinhostport"): port = "%d" % (ServiceUtil.getUniqRandomPort(low=lowport, high=highport, log=self.log)) self.log.debug('Setting hostname to: %s' % local_fqdn()) v = local_fqdn() + ':' + port keyvalpair = '' if isinstance(v, (tuple, list)): for item in v: keyvalpair = "%s%s=%s," % (keyvalpair, k, item) keyvalpair = keyvalpair[:-1] else: keyvalpair = k + '=' + v self.filledInKeyVals.append(keyvalpair) if ( v == "fillindir"): v = self.__mrSysDir pass prop = None if isinstance(v, (tuple, list)): for item in v: prop = self._createXmlElement(doc, k, item, "No description", final) topElement.appendChild(prop) else: if k == 'fs.default.name': prop = self._createXmlElement(doc, k, "hdfs://" + v, "No description", final) else: prop = self._createXmlElement(doc, k, v, "No description", final) topElement.appendChild(prop)
def _serve_forever(self): if len(self.__ports) > 1: randomPort = Random(os.getpid()) portSequence = range(self.__ports[0], self.__ports[1]) maxTryCount = abs(self.__ports[0] - self.__ports[1]) tryCount = 0 while True: somePort = randomPort.choice(portSequence) self.server_address = (self.__host, int(somePort)) if self.__host == '': self.server_address = (local_fqdn(), self.server_address[1]) try: reactor.listenTCP(int(somePort), server.Site( self.__xmlrpc), interface=self.__host) reactor.run(installSignalHandlers=0) except: self.__logger.debug("Failed to bind to: %s:%s." % ( self.__host, somePort)) tryCount = tryCount + 1 if tryCount > maxTryCount: self.__logger.warn("Failed to bind to: %s:%s" % ( self.__host, self.__ports)) sys.exit(1) else: break else: try: self.server_address = (self.__host, int(self.__ports[0])) if self.__host == '': self.server_address = (local_fqdn(), self.server_address[1]) reactor.listenTCP(int(self.__ports[0]), server.Site(self.__xmlrpc), interface=self.__host) reactor.run(installSignalHandlers=0) except: self.__logger.warn("Failed to bind to: %s:%s."% ( self.__host, self.__ports[0])) sys.exit(1)
def __init__(self, name, config, xrtype='threaded'): """ Initialization requires a name string and a config object of type hodlib.Common.setup.options or hodlib.Common.setup.config.""" self.name = name self.hostname = local_fqdn() self._cfg = config self._xrc = None self.logs = {} self._baseLogger = None self._serviceID = os.getenv('PBS_JOBID') self.__logDir = None self.__svcrgy = None self.__stop = False self.__xrtype = xrtype self._init_logging() if name != 'serviceRegistry': self._init_signals() self._init_xrc_server()
raise socket.gaierror, errData except: tryCount = tryCount + 1 if tryCount > maxTryCount: bindError = "bind failure for port range %s:%d" % ( self.ports) raise socket.error, bindError else: break else: self.server_address = (self.host, int(self.ports[0])) self.socket.bind(self.server_address) if self.host == '': self.server_address = (local_fqdn(), self.server_address[1]) def _serve_forever(self): """Replacement for serve_forever loop. All baseSocketServers run within a master thread; that thread imitates serve_forever, but checks an event (self.__stopForever) before processing new connections. """ while not self.__stopForever.isSet(): (rlist, wlist, xlist) = select([self.socket], [], [], 1) if (len(rlist) > 0 and self.socket == rlist[0]): self.handle_request()
def process_qsub_attributes(): rawAttributes = self.nodePoolDesc.getAttrs() # 'W:x' is used to specify torque management extentensions ie -W x= ... resourceManagementExtensions = '' if 'W:x' in rawAttributes: resourceManagementExtensions = rawAttributes['W:x'] if qosLevel: if len(resourceManagementExtensions) > 0: resourceManagementExtensions += ';' resourceManagementExtensions += 'QOS:%s' % (qosLevel) rawAttributes['W:x'] = resourceManagementExtensions hostname = local_fqdn() rawAttributes['l:nodes'] = nodeSet._getNumNodes() if walltime: rawAttributes['l:walltime'] = walltime #create a dict of dictionaries for # various arguments of torque cmds = {} for key in rawAttributes: value = rawAttributes[key] if key.find(':') == -1: raise ValueError, 'Syntax error: missing colon after %s in %s=%s' % ( key, key, value) [option, subOption] = key.split(':', 1) if not option in cmds: cmds[option] = {} cmds[option][subOption] = value opts = [] #create a string from this #dictionary of dictionaries createde above for k in cmds: csv = [] nv = cmds[k] for n in nv: v = nv[n] if len(n) == 0: csv.append(v) else: csv.append('%s=%s' % (n, v)) opts.append('-%s' % (k)) opts.append(','.join(csv)) for option in cmds: commandList = [] for subOption in cmds[option]: value = cmds[option][subOption] if len(subOption) == 0: commandList.append(value) else: commandList.append("%s=%s" % (subOption, value)) opts.append('-%s' % option) opts.append(','.join(commandList)) return opts