def _number_of_tasktrackers(self, jt_hostname, timeout, retries=0): url = "http://%s:50030/jobtracker.jsp" % jt_hostname jt_page = url_get(url, timeout, retries) m = self._NUMBER_OF_TASK_TRACKERS.search(jt_page) if m: return int(m.group(1)) return 0
def _authorize_client_ports(self, client_cidrs=[]): if not client_cidrs: self.logger.debug( "No client CIDRs specified, using local address.") client_ip = url_get('http://checkip.amazonaws.com/').strip() client_cidrs = ("%s/32" % client_ip, ) self.logger.debug("Client CIDRs: %s", client_cidrs) namenode = self._get_namenode() jobtracker = self._get_jobtracker() for client_cidr in client_cidrs: # Allow access to port 80 on namenode from client self.cluster.authorize_role(self.NAMENODE, 80, 80, client_cidr) # Allow access to jobtracker UI on master from client # (so we can see when the cluster is ready) self.cluster.authorize_role(self.JOBTRACKER, 50030, 50030, client_cidr) # Allow access to namenode and jobtracker via public address from each other namenode_ip = socket.gethostbyname(namenode.public_dns_name) jobtracker_ip = socket.gethostbyname(jobtracker.public_dns_name) self.cluster.authorize_role(self.NAMENODE, 8020, 8020, "%s/32" % namenode_ip) self.cluster.authorize_role(self.NAMENODE, 8020, 8020, "%s/32" % jobtracker_ip) self.cluster.authorize_role(self.JOBTRACKER, 8021, 8021, "%s/32" % namenode_ip) self.cluster.authorize_role(self.JOBTRACKER, 8021, 8021, "%s/32" % jobtracker_ip)
def _authorize_client_ports(self, client_cidrs=[]): if not client_cidrs: self.logger.debug("No client CIDRs specified, using local address.") client_ip = url_get('http://checkip.amazonaws.com/').strip() client_cidrs = ("%s/32" % client_ip,) self.logger.debug("Client CIDRs: %s", client_cidrs) namenode = self._get_namenode() jobtracker = self._get_jobtracker() for client_cidr in client_cidrs: # Allow access to port 80 on namenode from client self.cluster.authorize_role(self.NAMENODE, 80, 80, client_cidr) # Allow access to jobtracker UI on master from client # (so we can see when the cluster is ready) self.cluster.authorize_role(self.JOBTRACKER, 50030, 50030, client_cidr) # Allow access to namenode and jobtracker via public address from each other namenode_ip = socket.gethostbyname(namenode.public_dns_name) jobtracker_ip = socket.gethostbyname(jobtracker.public_dns_name) self.cluster.authorize_role(self.NAMENODE, 8020, 8020, "%s/32" % namenode_ip) self.cluster.authorize_role(self.NAMENODE, 8020, 8020, "%s/32" % jobtracker_ip) self.cluster.authorize_role(self.JOBTRACKER, 8021, 8021, "%s/32" % namenode_ip) self.cluster.authorize_role(self.JOBTRACKER, 8021, 8021, "%s/32" % jobtracker_ip)