def get_ticket(self, source, target, crypto, key): # prepare metadata md = { 'requestor': source, 'target': target, 'timestamp': time.time(), 'nonce': struct.unpack('Q', os.urandom(8))[0] } metadata = base64.b64encode(jsonutils.dumps(md)) # sign metadata signature = crypto.sign(key, metadata) # HTTP request reply = self._get_ticket({ 'metadata': metadata, 'signature': signature }) # verify reply signature = crypto.sign(key, (reply['metadata'] + reply['ticket'])) if signature != reply['signature']: raise InvalidEncryptedTicket(md['source'], md['destination']) md = jsonutils.loads(base64.b64decode(reply['metadata'])) if ((md['source'] != source or md['destination'] != target or md['expiration'] < time.time())): raise InvalidEncryptedTicket(md['source'], md['destination']) # return ticket data tkt = jsonutils.loads(crypto.decrypt(key, reply['ticket'])) return tkt, md['expiration']
def _exec_ambari_command(self, auth, body, cmd_uri): LOG.debug('PUT URI: {0}'.format(cmd_uri)) result = requests.put(cmd_uri, data=body, auth=auth) if result.status_code == 202: # don't hard code request id LOG.debug( 'PUT response: {0}'.format(result.text)) json_result = json.loads(result.text) href = json_result['href'] + '/tasks?fields=Tasks/status' success = self._wait_for_async_request(href, auth) if success: LOG.info( "Successfully changed state of Hadoop components ") else: LOG.critical('Failed to change state of Hadoop ' 'components') raise RuntimeError('Failed to change state of Hadoop ' 'components') else: LOG.error( 'Command failed. Status: {0}, response: {1}'. format(result.status_code, result.text)) raise RuntimeError('Hadoop/Ambari command failed.')
def _start_components(self, ambari_info, auth, cluster_name, servers): # query for all the host components on one of the hosts in the # INSTALLED state, then get a list of the client services in the list installed_uri = 'http://{0}/api/v1/clusters/{' \ '1}/host_components?HostRoles/state=INSTALLED&' \ 'HostRoles/host_name.in({2})' \ .format(ambari_info.get_address(), cluster_name, self._get_host_list(servers)) result = requests.get(installed_uri, auth=auth) if result.status_code == 200: LOG.debug( 'GET response: {0}'.format(result.text)) json_result = json.loads(result.text) items = json_result['items'] # select non-CLIENT items inclusion_list = list(set([x['HostRoles']['component_name'] for x in items if "CLIENT" not in x['HostRoles']['component_name']])) # query and start all non-client components on the given set of # hosts body = '{"HostRoles": {"state" : "STARTED"}}' start_uri = 'http://{0}/api/v1/clusters/{' \ '1}/host_components?HostRoles/state=INSTALLED&' \ 'HostRoles/host_name.in({2})' \ '&HostRoles/component_name.in({3})'.format( ambari_info.get_address(), cluster_name, self._get_host_list(servers), ",".join(inclusion_list)) self._exec_ambari_command(auth, body, start_uri) else: raise RuntimeError('Unable to determine installed service ' 'components in scaled instances. status' ' code returned = {0}'.format(result.status))
def __init__(self, cluster_template, cluster=None): self.services = [] self.configurations = {} self.node_groups = {} self.str = cluster_template servers = [] if cluster is not None: if hasattr(cluster, "node_groups"): servers = self._get_servers_from_savanna_cluster(cluster) else: servers = cluster.instances host_manifest = self._generate_host_manifest(servers) # TODO(jspeidel): don't hard code ambari server ambari_server = self._get_ambari_host(servers) if ambari_server is not None: cluster_template = cluster_template.replace("%AMBARI_HOST%", ambari_server.fqdn) else: raise RuntimeError("No Ambari server host found") self.str = self._add_manifest_to_config(cluster_template, host_manifest) template_json = json.loads(self.str) self._parse_services(template_json) self._parse_configurations(template_json) self._parse_host_component_mappings(template_json)
def _wait_for_host_registrations(self, num_hosts, ambari_host): LOG.info('Waiting for all Ambari agents to register with server ...') url = 'http://{0}:8080/api/v1/hosts'.format(ambari_host.management_ip) result = None json_result = None #TODO(jspeidel): timeout while result is None or len(json_result['items']) < num_hosts: context.sleep(5) try: result = requests.get(url, auth=(self.ambari_user, self.ambari_password)) json_result = json.loads(result.text) # TODO(jspeidel): just for debug LOG.info('Registered Hosts: {0} of {1}'.format( len(json_result['items']), num_hosts)) for hosts in json_result['items']: LOG.debug('Registered Host: {0}'.format( hosts['Hosts']['host_name'])) except requests.ConnectionError: #TODO(jspeidel): max wait time LOG.info('Waiting to connect to ambari server ...')
def _wait_for_host_registrations(self, num_hosts, ambari_info): LOG.info( 'Waiting for all Ambari agents to register with server ...') url = 'http://{0}/api/v1/hosts'.format(ambari_info.get_address()) result = None json_result = None #TODO(jspeidel): timeout while result is None or len(json_result['items']) < num_hosts: context.sleep(5) try: result = requests.get(url, auth=(ambari_info.user, ambari_info.password)) json_result = json.loads(result.text) # TODO(jspeidel): just for debug LOG.info('Registered Hosts: {0} of {1}'.format( len(json_result['items']), num_hosts)) for hosts in json_result['items']: LOG.debug('Registered Host: {0}'.format( hosts['Hosts']['host_name'])) except requests.ConnectionError: #TODO(jspeidel): max wait time LOG.info('Waiting to connect to ambari server ...')
def _decode_esek(self, key, source, target, timestamp, esek): """This function decrypts the esek buffer passed in and returns a KeyStore to be used to check and decrypt the received message. :param key: The key to use to decrypt the ticket (esek) :param source: The name of the source service :param traget: The name of the target service :param timestamp: The incoming message timestamp :param esek: a base64 encoded encrypted block containing a JSON string """ rkey = None try: s = self._crypto.decrypt(key, esek) j = jsonutils.loads(s) rkey = base64.b64decode(j['key']) expiration = j['timestamp'] + j['ttl'] if j['timestamp'] > timestamp or timestamp > expiration: raise InvalidExpiredTicket(source, target) except Exception: raise InvalidEncryptedTicket(source, target) info = '%s,%s,%s' % (source, target, str(j['timestamp'])) sek = self._hkdf.expand(rkey, info, len(key) * 2) return self._split_key(sek, len(key))
def __init__(self, cluster_template, cluster=None): self.services = [] self.configurations = {} self.node_groups = {} self.str = cluster_template servers = [] if cluster is not None: if hasattr(cluster, 'node_groups'): servers = self._get_servers_from_savanna_cluster(cluster) else: servers = cluster.instances host_manifest = self._generate_host_manifest(servers) #TODO(jspeidel): don't hard code ambari server ambari_server = self._get_ambari_host(servers) if ambari_server is not None: cluster_template = cluster_template.replace( '%AMBARI_HOST%', ambari_server.fqdn) else: raise RuntimeError('No Ambari server host found') self.str = self._add_manifest_to_config(cluster_template, host_manifest) template_json = json.loads(self.str) self._parse_services(template_json) self._parse_configurations(template_json) self._parse_host_component_mappings(template_json)
def __init__(self, config, version='1.3.2'): self._config_template = config self.services = [] self.configurations = {} self.node_groups = {} self.version = version self.user_input_handlers = {} cluster_template = json.loads(config) self._parse_services(cluster_template) self._parse_configurations(cluster_template) self._process_node_groups(template_json=cluster_template)
def _get_reply(self, url, resp): if resp.text: try: body = jsonutils.loads(resp.text) reply = body['reply'] except (KeyError, TypeError, ValueError): msg = "Failed to decode reply: %s" % resp.text raise CommunicationError(url, msg) else: msg = "No reply data was returned." raise CommunicationError(url, msg) return reply
def _unpack_json_msg(self, msg): """Load the JSON data in msg if msg.content_type indicates that it is necessary. Put the loaded data back into msg.content and update msg.content_type appropriately. A Qpid Message containing a dict will have a content_type of 'amqp/map', whereas one containing a string that needs to be converted back from JSON will have a content_type of JSON_CONTENT_TYPE. :param msg: a Qpid Message object :returns: None """ if msg.content_type == JSON_CONTENT_TYPE: msg.content = jsonutils.loads(msg.content) msg.content_type = 'amqp/map'
def create_operational_config(self, cluster, user_inputs, scaled_groups=None): if scaled_groups is None: scaled_groups = {} self._determine_deployed_services(cluster) self._process_node_groups(cluster=cluster) for ng_id in scaled_groups: existing = next(group for group in self.node_groups.values() if group.id == ng_id) existing.count = scaled_groups[ng_id] self.validate_node_groups(cluster) self._finalize_ng_components() self._parse_configurations(json.loads(self._config_template)) self._process_user_inputs(user_inputs) self._replace_config_tokens()
def decode(self, version, metadata, message, signature): """This is the main decoding function. It takes a version, metadata, message and signature strings and returns a tuple with a (decrypted) message and metadata or raises an exception in case of error. :param version: the current envelope version :param metadata: a JSON serialized object with metadata for validation :param message: a JSON serialized (base64 encoded encrypted) message :param signature: a base64 encoded signature """ md = jsonutils.loads(metadata) check_args = ('source', 'destination', 'timestamp', 'nonce', 'esek', 'encryption') for arg in check_args: if arg not in md: raise InvalidMetadata('Missing metadata "%s"' % arg) if md['destination'] != self._name: # TODO(simo) handle group keys by checking target raise UnknownDestinationName(md['destination']) try: skey, ekey = self._decode_esek(self._key, md['source'], md['destination'], md['timestamp'], md['esek']) except InvalidExpiredTicket: raise except Exception: raise InvalidMetadata('Failed to decode ESEK for %s/%s' % (md['source'], md['destination'])) sig = self._crypto.sign(skey, version + metadata + message) if sig != signature: raise InvalidSignature(md['source'], md['destination']) if md['encryption'] is True: msg = self._crypto.decrypt(ekey, message) else: msg = message return (md, msg)
def _wait_for_async_request(self, request_url, auth): started = False while not started: result = requests.get(request_url, auth=auth) LOG.debug( 'async request ' + request_url + ' response:\n' + result.text) json_result = json.loads(result.text) started = True for items in json_result['items']: status = items['Tasks']['status'] if status == 'FAILED' or status == 'ABORTED': return False else: if status != 'COMPLETED': started = False context.sleep(5) return started
def __init__(self, cluster_template, cluster=None): self.services = [] self.configurations = {} self.node_groups = {} self.servers = None self.str = cluster_template if cluster: self.servers = self._get_servers_from_savanna_cluster(cluster) host_manifest = self._generate_host_manifest() cluster_template = self._replace_config_tokens(cluster_template) self.str = self._add_manifest_to_config( cluster_template, host_manifest) template_json = json.loads(self.str) self._parse_services(template_json) self._parse_configurations(template_json) self._parse_host_component_mappings(template_json)
def deserialize_remote_exception(conf, data): failure = jsonutils.loads(str(data)) trace = failure.get('tb', []) message = failure.get('message', "") + "\n" + "\n".join(trace) name = failure.get('class') module = failure.get('module') # NOTE(ameade): We DO NOT want to allow just any module to be imported, in # order to prevent arbitrary code execution. if module not in conf.allowed_rpc_exception_modules: return RemoteError(name, failure.get('message'), trace) try: mod = importutils.import_module(module) klass = getattr(mod, name) if not issubclass(klass, Exception): raise TypeError("Can only deserialize Exceptions") failure = klass(*failure.get('args', []), **failure.get('kwargs', {})) except (AttributeError, TypeError, ImportError): return RemoteError(name, failure.get('message'), trace) ex_type = type(failure) str_override = lambda self: message new_ex_type = type(ex_type.__name__ + _REMOTE_POSTFIX, (ex_type, ), { '__str__': str_override, '__unicode__': str_override }) new_ex_type.__module__ = '%s%s' % (module, _REMOTE_POSTFIX) try: # NOTE(ameade): Dynamically create a new exception type and swap it in # as the new type for the exception. This only works on user defined # Exceptions and not core python exceptions. This is important because # we cannot necessarily change an exception message so we must override # the __str__ method. failure.__class__ = new_ex_type except TypeError: # NOTE(ameade): If a core exception then just add the traceback to the # first exception argument. failure.args = (message, ) + failure.args[1:] return failure
def deserialize_msg(msg): # NOTE(russellb): Hang on to your hats, this road is about to # get a little bumpy. # # Robustness Principle: # "Be strict in what you send, liberal in what you accept." # # At this point we have to do a bit of guessing about what it # is we just received. Here is the set of possibilities: # # 1) We received a dict. This could be 2 things: # # a) Inspect it to see if it looks like a standard message envelope. # If so, great! # # b) If it doesn't look like a standard message envelope, it could either # be a notification, or a message from before we added a message # envelope (referred to as version 1.0). # Just return the message as-is. # # 2) It's any other non-dict type. Just return it and hope for the best. # This case covers return values from rpc.call() from before message # envelopes were used. (messages to call a method were always a dict) if not isinstance(msg, dict): # See #2 above. return msg base_envelope_keys = (_VERSION_KEY, _MESSAGE_KEY) if not all(map(lambda key: key in msg, base_envelope_keys)): # See #1.b above. return msg # At this point we think we have the message envelope # format we were expecting. (#1.a above) if not version_is_compatible(_RPC_ENVELOPE_VERSION, msg[_VERSION_KEY]): raise UnsupportedRpcEnvelopeVersion(version=msg[_VERSION_KEY]) raw_msg = jsonutils.loads(msg[_MESSAGE_KEY]) return raw_msg
def _wait_for_async_request(self, request_id, cluster_name, ambari_host): request_url = "http://{0}:8080/api/v1/clusters/{1}/requests/{" "2}/tasks?fields=Tasks/status".format( ambari_host.management_ip, cluster_name, request_id ) started = False while not started: result = requests.get(request_url, auth=(self.ambari_user, self.ambari_password)) LOG.debug("async request " + request_url + " response:\n" + result.text) json_result = json.loads(result.text) started = True for items in json_result["items"]: status = items["Tasks"]["status"] if status == "FAILED" or status == "ABORTED": return False else: if status != "COMPLETED": started = False context.sleep(5) return started
def deserialize_remote_exception(conf, data): failure = jsonutils.loads(str(data)) trace = failure.get("tb", []) message = failure.get("message", "") + "\n" + "\n".join(trace) name = failure.get("class") module = failure.get("module") # NOTE(ameade): We DO NOT want to allow just any module to be imported, in # order to prevent arbitrary code execution. if module not in conf.allowed_rpc_exception_modules: return RemoteError(name, failure.get("message"), trace) try: mod = importutils.import_module(module) klass = getattr(mod, name) if not issubclass(klass, Exception): raise TypeError("Can only deserialize Exceptions") failure = klass(*failure.get("args", []), **failure.get("kwargs", {})) except (AttributeError, TypeError, ImportError): return RemoteError(name, failure.get("message"), trace) ex_type = type(failure) str_override = lambda self: message new_ex_type = type( ex_type.__name__ + _REMOTE_POSTFIX, (ex_type,), {"__str__": str_override, "__unicode__": str_override} ) new_ex_type.__module__ = "%s%s" % (module, _REMOTE_POSTFIX) try: # NOTE(ameade): Dynamically create a new exception type and swap it in # as the new type for the exception. This only works on user defined # Exceptions and not core python exceptions. This is important because # we cannot necessarily change an exception message so we must override # the __str__ method. failure.__class__ = new_ex_type except TypeError: # NOTE(ameade): If a core exception then just add the traceback to the # first exception argument. failure.args = (message,) + failure.args[1:] return failure
def _wait_for_async_request(self, request_id, cluster_name, ambari_host): request_url = 'http://{0}:8080/api/v1/clusters/{1}/requests/{' \ '2}/tasks?fields=Tasks/status'.format( ambari_host.management_ip, cluster_name, request_id) started = False while not started: result = requests.get(request_url, auth=('admin', 'admin')) LOG.debug( 'async request ' + request_url + ' response:\n' + result.text) json_result = json.loads(result.text) started = True for items in json_result['items']: status = items['Tasks']['status'] if status == 'FAILED' or status == 'ABORTED': return False else: if status != 'COMPLETED': started = False context.sleep(5) return started
def _wait_for_host_registrations(self, num_hosts, ambari_host): LOG.info("Waiting for all Ambari agents to register with server ...") url = "http://{0}:8080/api/v1/hosts".format(ambari_host.management_ip) result = None json_result = None # TODO(jspeidel): timeout while result is None or len(json_result["items"]) < num_hosts: context.sleep(5) try: result = requests.get(url, auth=(self.ambari_user, self.ambari_password)) json_result = json.loads(result.text) # TODO(jspeidel): just for debug LOG.info("Registered Hosts: {0} of {1}".format(len(json_result["items"]), num_hosts)) for hosts in json_result["items"]: LOG.debug("Registered Host: {0}".format(hosts["Hosts"]["host_name"])) except requests.ConnectionError: # TODO(jspeidel): max wait time LOG.info("Waiting to connect to ambari server ...")
def _wait_for_async_request(self, request_id, cluster_name, ambari_host): request_url = 'http://{0}:8080/api/v1/clusters/{1}/requests/{' \ '2}/tasks?fields=Tasks/status'.format( ambari_host.management_ip, cluster_name, request_id) started = False while not started: result = requests.get(request_url, auth=(self.ambari_user, self.ambari_password)) LOG.debug('async request ' + request_url + ' response:\n' + result.text) json_result = json.loads(result.text) started = True for items in json_result['items']: status = items['Tasks']['status'] if status == 'FAILED' or status == 'ABORTED': return False else: if status != 'COMPLETED': started = False context.sleep(5) return started
def _deserialize(data): """Deserialization wrapper.""" LOG.debug(_("Deserializing: %s"), data) return jsonutils.loads(data)
def _from_json(self, datastring): try: return jsonutils.loads(datastring) except ValueError: msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg)
def json2obj(self, data): return json.loads(data, object_hook=self._json_object_hook)
def process_result_value(self, value, dialect): if value is not None: value = jsonutils.loads(value) return value