def check_ambari_service_installed(service_name, ambari_config): curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto']) logger.info('Checking if Ambari service: ' + service_name + ' is installed') cluster_name = ambari_config['cluster_name'] request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name attempts = 0 while attempts < 10: output = curl.make_request('GET', request, '-i') if '200 OK' in output[0]: print('Service Installed Sucessfully') logger.info(service_name + ' was installed successfully') return True else: attempts += 1 raw_input('Could not connect.' + str(10-attempts) + ' remaining. Press any key to continue') logger.info(service_name + ' was not installed successfully') return False
def get_zeppelin_session(username, password): '''Gets the JSESSIONID cookie by POSTing to /api/login This is required to retrieve the JSESSIONID on Zeppelin instances that have logins and user permissions. Used when POSTing notebooks as well. Returned in the form of ``'JSESSIONID=**`` Args: username (str): The username to login with password (str): The password to login with Returns: str: ``''`` If the request was unsuccessful, or the cookie wasn't present. Otherwise returns the cookie as a string. ''' conf = config.read_config('global.conf')['ZEPPELIN'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/api/login' opts = '-i -d \'userName='******'&password='******'\'' res = client.make_request('POST', path, options=opts) lines = res[0].split('\n') for line in lines: if 'Set-Cookie:'.lower() in line.lower( ) and 'JSESSIONID' in line.upper(): cookie = line.split('Set-Cookie: ')[1] print 'COOKIE: ' + cookie return cookie.strip() return ''
def post_notebook(notebook_path, session_cookie=''): '''Add a single notebook to a Zeppelin installation via REST API Must have a file called ``global.conf`` inside of the configuration directory. Inside that configuration file we use protocol://server:port to connect to a Zeppelin instance and use the API Args: notebook_path (str): Full file path to the Zeppelin note session_cookie (str, optional): The cookie used after authentication in order to make authorized API calls. Required on the 2.5 TP Sandbox Returns: bool: True if the upload was successful (received 201 created), or False otherwise. ''' conf = config.read_config('global.conf')['ZEPPELIN'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/api/notebook' logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port)) opts = '-i -b \'' + session_cookie + '\' -H "Content-Type: application/json" -d @' + notebook_path output = client.make_request('POST', path, options=opts) if '201 created' in output[0].lower(): logger.info('Note posted successfully') return True else: logger.info('Note failed to be added. User will need to add manually') return False
def post_template(template_path): '''Add a single template to a NiFi instance via REST API Args: template_path (str): The full path to the NiFi template Returns: bool: True if the upload is successful, False otherwise ''' conf = config.read_config('global.conf')['NIFI'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/nifi-api/controller/templates' logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port)) output = client.make_request('POST', path, options='-i -F template=@' + template_path) if '201 Created' in output[0] or '200 OK' in output[0]: logger.info('Template added to NiFi successfully') return True else: logger.info('Template failed to POST to NiFi') return False
def get_zeppelin_session(username, password): '''Gets the JSESSIONID cookie by POSTing to /api/login This is required to retrieve the JSESSIONID on Zeppelin instances that have logins and user permissions. Used when POSTing notebooks as well. Returned in the form of ``'JSESSIONID=**`` Args: username (str): The username to login with password (str): The password to login with Returns: str: ``''`` If the request was unsuccessful, or the cookie wasn't present. Otherwise returns the cookie as a string. ''' conf = config.read_config('global.conf')['ZEPPELIN'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/api/login' opts = '-i -d \'userName='******'&password='******'\'' res = client.make_request('POST', path, options=opts) lines = res[0].split('\n') for line in lines: if 'Set-Cookie:'.lower() in line.lower() and 'JSESSIONID' in line.upper(): cookie = line.split('Set-Cookie: ')[1] print 'COOKIE: ' + cookie return cookie.strip() return ''
def __init__(self, username='', password='', proto='http', server='127.0.0.1', port=8080, service_wait_time=60, config=''): if config != '': if isinstance(config, dict): if 'username' in config: username = config['username'] if 'password' in config: password = config['password'] if 'proto' in config: proto = config['proto'] if 'server' in config: server = config['server'] if 'port' in config: port = config['port'] if 'service_wait_time' in config: service_wait_time = config['service_wait_time'] else: logger.warn( 'Could not instantiate Ambari client through config object. Config is type: ' + str(type(config))) self.client = CurlClient() if service_wait_time > 0: self.set_service_wait_time(service_wait_time) else: self.set_service_wait_time(60) if not username == '': self.set_username(username) if not password == '': self.set_password(password) if not proto == '': self.set_proto(proto) if not server == '': self.set_server(server) if not len(str(port)) == 0: self.set_port(port) logger.info('Created Ambari client: ' + ':'.join([username, password]) + ' ' + ''.join([proto, '://', server, ':', str(port)]))
def post_template(template_path): conf = config.read_config('service-installer.conf')['NIFI'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/nifi-api/controller/templates' logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port)) output = client.make_request('POST', path, options='-i -F template=@' + template_path ) if '201 Created' in output[0] or '200 OK' in output[0]: logger.info('Template added to NiFi successfully') return True else: logger.info('Template failed to POST to NiFi') return False
def post_notebook(notebook_path): conf = config.read_config('service-installer.conf')['ZEPPELIN'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/api/notebook' logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port)) output = client.make_request('POST', path, options='-i -H "Content-Type: application/json" -d @' + notebook_path ) if '201 created' in output[0].lower(): logger.info('Note posted successfully') return True else: logger.info('Note failed to be added. User will need to add manually') return False
def check_ambari_service_installed(service_name, ambari_config): '''Determine whether an Ambari service is installed. Args: service_name (string): Name of service to look for ambari_config (dict): The entire AMBARI section of ``global.conf`` :: read_config('global.conf')['AMBARI'] Returns: bool: True service exists, False if the service does not exist ''' curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto']) logger.info('Checking if Ambari service: ' + service_name + ' is installed') cluster_name = ambari_config['cluster_name'] request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name attempts = 0 while attempts < 10: output = curl.make_request('GET', request, '-i') if '200 OK' in output[0]: print('Service Installed Sucessfully') logger.info(service_name + ' was installed successfully') return True else: attempts += 1 raw_input('Could not connect. ' + str(10-attempts) + ' remaining. Press any key to continue') logger.info(service_name + ' was not installed successfully') return False
def post_template(template_path): '''Add a single template to a NiFi instance via REST API Args: template_path (str): The full path to the NiFi template Returns: bool: True if the upload is successful, False otherwise ''' conf = config.read_config('global.conf')['NIFI'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/nifi-api/controller/templates' logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port)) output = client.make_request('POST', path, options='-i -F template=@' + template_path ) if '201 Created' in output[0] or '200 OK' in output[0]: logger.info('Template added to NiFi successfully') return True else: logger.info('Template failed to POST to NiFi') return False
def check_ambari_service_installed(service_name, ambari_config): '''Determine whether an Ambari service is installed. Args: service_name (string): Name of service to look for ambari_config (dict): The entire AMBARI section of ``global.conf`` :: read_config('global.conf')['AMBARI'] Returns: bool: True service exists, False if the service does not exist ''' curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto']) logger.info('Checking if Ambari service: ' + service_name + ' is installed') cluster_name = ambari_config['cluster_name'] request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name attempts = 0 while attempts < 10: output = curl.make_request('GET', request, '-i') if '200 OK' in output[0]: print('Service Installed Sucessfully') logger.info(service_name + ' was installed successfully') return True else: attempts += 1 raw_input('Could not connect. ' + str(10 - attempts) + ' remaining. Press any key to continue') logger.info(service_name + ' was not installed successfully') return False
def __init__(self, username='', password='', proto='http', server='127.0.0.1', port=8080, service_wait_time=60, config=''): if config != '': if isinstance(config, dict): if 'username' in config: username = config['username'] if 'password' in config: password = config['password'] if 'proto' in config: proto = config['proto'] if 'server' in config: server = config['server'] if 'port' in config: port = config['port'] if 'service_wait_time' in config: service_wait_time = config['service_wait_time'] else: logger.warn('Could not instantiate Ambari client through config object. Config is type: ' + str(type(config))) self.client = CurlClient() if service_wait_time > 0: self.set_service_wait_time(service_wait_time) else: self.set_service_wait_time(60) if not username == '': self.set_username(username) if not password == '': self.set_password(password) if not proto == '': self.set_proto(proto) if not server == '': self.set_server(server) if not len(str(port)) == 0: self.set_port(port) logger.info('Created Ambari client: ' + ':'.join([username, password]) + ' ' + ''.join([proto, '://', server, ':', str(port)]))
def install_nifi(): '''Install NiFi via Ambari. (And Ali's NiFi service) Automatically installs NiFi with NO user interaction. Simply just run the method while on the same Ambari machine and NiFi will be installed. You'll need to start it manually though. Returns: bool: True if installation is successful. Else, the user specifies whether or not they want to continue setting up the demo without Zeppelin. ``True`` if the user specifed Yes (to continue). ``False`` if they specified No (do not continue). Raises: EnvironmentError: Raised when Ambari is not installed on the current host. Or if ``hdp-select`` cannot be installed''' logger.info('Attempting to install NiFi to the cluster') if not is_ambari_installed(): logger.error('Ambari must be installed to install NiFi as well.') raise EnvironmentError( 'You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing' ) if not is_hdp_select_installed(): installed = install_hdp_select() if not installed: logger.error('hdp-select must be installed to install NiFi') raise EnvironmentError( 'hdp-select could not be installed. Please install it manually and then re-run the setup.' ) conf = config.read_config('global.conf') cmds = json.loads(conf['NIFI']['install-commands']) sh = Shell() logger.info('Getting HDP Version') version = sh.run(cmds[0]) logger.info('HDP Version: ' + version[0]) fixed_copy = cmds[2].replace('$VERSION', str(version[0])).replace('\n', '') fixed_remove = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '') logger.info('NiFi Clean Command: ' + fixed_copy) logger.info('NiFi Copy Command: ' + fixed_remove) remove = sh.run(fixed_remove) copy = sh.run(fixed_copy) amc = conf['AMBARI'] cc = CurlClient(amc['username'], amc['password'], amc['proto'], amc['server'], amc['port']) opts = '-H \'X-Requested-By: ambari\'' path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI' print cc.make_request('POST', path, options=opts) path += '/components/NIFI_MASTER' print cc.make_request('POST', path, options=opts) cfg = { 'cmd': 'bash /var/lib/ambari-server/resources/scripts/configs.sh set', 'server': amc['server'], 'cluster': amc['cluster_name'], 'name': 'nifi-ambari-config', 'config_file': config.get_path('nifi/config/nifi-ambari-config.json') } create_cmd = lambda x: ' '.join([ cfg['cmd'], cfg['server'], cfg['cluster'], x, config.get_path('nifi/config/' + x + '.json') ]) logger.debug(sh.run(create_cmd('nifi-ambari-config'))) logger.debug(sh.run(create_cmd('nifi-bootstrap-env'))) logger.debug(sh.run(create_cmd('nifi-flow-env'))) logger.debug(sh.run(create_cmd('nifi-logback-env'))) logger.debug(sh.run(create_cmd('nifi-properties-env'))) path = '/api/v1/clusters/' + amc['cluster_name'] + '/hosts/' + amc[ 'server'] + '/host_components/NIFI_MASTER' logger.debug(path) cc.make_request('POST', path, options=opts) path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI' opts = '-H \'X-Requested-By: ambari\' -d \'{"RequestInfo": {"context" :"Install Nifi"}, "Body": {"ServiceInfo": {"maintenance_state" : "OFF", "state": "INSTALLED"}}}\'' cc.make_request('PUT', path, options=opts) print( "Please open the Ambari Interface and manually deploy the NiFi Service." ) raw_input("Press enter twice to continue...") raw_input("Press enter once to continue...") # We've copied the necessary files. Once that completes we need to add it to Ambari logger.info('Waiting for user to install service in Ambari to continue') print('Checking to make sure service is installed') ambari = config.read_config('global.conf')['AMBARI'] installed = check_ambari_service_installed('NIFI', ambari) logger.info('NiFi installed successfully') cont = '' if not installed: print( 'Unable to contact Ambari Server. Unsure whether or not NiFi was installed' ) while not (cont == 'y' or cont == 'n'): cont = raw_input('Continue attempt to set up NiFi for demo?(y/n)') if not (cont == 'y' or cont == 'n'): print('Please enter "y" or "n"') else: cont = 'y' if cont == 'n': return False elif cont == 'y': return True
def install_nifi(): '''Install NiFi via Ambari. (And Ali's NiFi service) Automatically installs NiFi with NO user interaction. Simply just run the method while on the same Ambari machine and NiFi will be installed. You'll need to start it manually though. Returns: bool: True if installation is successful. Else, the user specifies whether or not they want to continue setting up the demo without Zeppelin. ``True`` if the user specifed Yes (to continue). ``False`` if they specified No (do not continue). Raises: EnvironmentError: Raised when Ambari is not installed on the current host. Or if ``hdp-select`` cannot be installed''' logger.info('Attempting to install NiFi to the cluster') if not is_ambari_installed(): logger.error('Ambari must be installed to install NiFi as well.') raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing') if not is_hdp_select_installed(): installed = install_hdp_select() if not installed: logger.error('hdp-select must be installed to install NiFi') raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.') conf = config.read_config('global.conf') cmds = json.loads(conf['NIFI']['install-commands']) sh = Shell() logger.info('Getting HDP Version') version = sh.run(cmds[0]) logger.info('HDP Version: ' + version[0]) fixed_copy = cmds[2].replace('$VERSION', str(version[0])).replace('\n', '') fixed_remove = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '') logger.info('NiFi Clean Command: ' + fixed_copy) logger.info('NiFi Copy Command: ' + fixed_remove) remove = sh.run(fixed_remove) copy = sh.run(fixed_copy) amc = conf['AMBARI'] cc = CurlClient(amc['username'], amc['password'], amc['proto'], amc['server'], amc['port']) opts = '-H \'X-Requested-By: ambari\'' path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI' print cc.make_request('POST', path, options=opts) path += '/components/NIFI_MASTER' print cc.make_request('POST', path, options=opts) cfg = { 'cmd': 'bash /var/lib/ambari-server/resources/scripts/configs.sh set', 'server': amc['server'], 'cluster': amc['cluster_name'], 'name': 'nifi-ambari-config', 'config_file': config.get_path('nifi/config/nifi-ambari-config.json') } create_cmd = lambda x: ' '.join([cfg['cmd'], cfg['server'], cfg['cluster'], x, config.get_path('nifi/config/' + x + '.json')]) logger.debug(sh.run(create_cmd('nifi-ambari-config'))) logger.debug(sh.run(create_cmd('nifi-bootstrap-env'))) logger.debug(sh.run(create_cmd('nifi-flow-env'))) logger.debug(sh.run(create_cmd('nifi-logback-env'))) logger.debug(sh.run(create_cmd('nifi-properties-env'))) path = '/api/v1/clusters/' + amc['cluster_name'] + '/hosts/' + amc['server'] + '/host_components/NIFI_MASTER' logger.debug(path) cc.make_request('POST', path, options=opts) path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI' opts = '-H \'X-Requested-By: ambari\' -d \'{"RequestInfo": {"context" :"Install Nifi"}, "Body": {"ServiceInfo": {"maintenance_state" : "OFF", "state": "INSTALLED"}}}\'' cc.make_request('PUT', path, options=opts) print("Please open the Ambari Interface and manually deploy the NiFi Service.") raw_input("Press enter twice to continue...") raw_input("Press enter once to continue...") # We've copied the necessary files. Once that completes we need to add it to Ambari logger.info('Waiting for user to install service in Ambari to continue') print('Checking to make sure service is installed') ambari = config.read_config('global.conf')['AMBARI'] installed = check_ambari_service_installed('NIFI', ambari) logger.info('NiFi installed successfully') cont = '' if not installed: print('Unable to contact Ambari Server. Unsure whether or not NiFi was installed') while not (cont == 'y' or cont == 'n'): cont = raw_input('Continue attempt to set up NiFi for demo?(y/n)') if not (cont == 'y' or cont == 'n'): print('Please enter "y" or "n"') else: cont = 'y' if cont == 'n': return False elif cont == 'y': return True
class Ambari: '''Initalize the Ambari client Args: username (str, optional): username to use for authentication (should have admin access) password (str, optional): password to use for authentication (should have admin access) proto (str, optional): Must be one of 'http' or 'https'. Defines which protocol to use. Defaults to 'http' server (str, optional): The hostname (or IP) of the Ambari server. Defaults to 127.0.0.1. port (int, optional): The port that ambari server is running on. Defaults to 8080 service_wait_time (int, optional): The time (in seconds) we should before we decide a service has failed changing states. config (dict, optional): This is a dictionary object which should contain the any of the keys 'username', 'password', 'proto', 'server', 'port', or 'service_wait_time'. Given the config here you can set any of the client's parameters through this object. However, when using this, the config object will override any of the specific arguments passed. Returns: N/A ''' def __init__(self, username='', password='', proto='http', server='127.0.0.1', port=8080, service_wait_time=60, config=''): if config != '': if isinstance(config, dict): if 'username' in config: username = config['username'] if 'password' in config: password = config['password'] if 'proto' in config: proto = config['proto'] if 'server' in config: server = config['server'] if 'port' in config: port = config['port'] if 'service_wait_time' in config: service_wait_time = config['service_wait_time'] else: logger.warn('Could not instantiate Ambari client through config object. Config is type: ' + str(type(config))) self.client = CurlClient() if service_wait_time > 0: self.set_service_wait_time(service_wait_time) else: self.set_service_wait_time(60) if not username == '': self.set_username(username) if not password == '': self.set_password(password) if not proto == '': self.set_proto(proto) if not server == '': self.set_server(server) if not len(str(port)) == 0: self.set_port(port) logger.info('Created Ambari client: ' + ':'.join([username, password]) + ' ' + ''.join([proto, '://', server, ':', str(port)])) username = password = server = port = proto = '' '''class variable''' @staticmethod def load_output(output): '''Load the output from the curl_client into an object The idea behind this function is to try and keep the same behavior on failed requests across the entire client. .. code-block: json :linenos: // Response with error { "message": "curl (6) Could not connect to host" } // Response without error { "href" : "http://sandbox.hortonworks.com:8080/api/v1/clusters/Sandbox/services/YARN?fields=ServiceInfo", "ServiceInfo" : { "cluster_name" : "Sandbox", "maintenance_state" : "OFF", "service_name" : "YARN", "state" : "INSTALLED" } } Args: output: (str): The output from a curl_client action Returns: dict: a dictionary object created from the JSON response of the Ambari request. If the request was unsuccessful a message attribute will be present containing the error from the curl request. Raises: ValueError: This is raised when an object can't be converted into JSON ''' json_str = '' if not len(output[0]) == 0: json_str = output[0] elif not len(output[1]) == 0: # StdErr output res = {'message': output[1]} return res else: json_str = '{ "message" : "No output was returned." }' res = '' try: res = json.loads(json_str) except ValueError as e: logger.error('Could not convert to JSON: ' + res) raise ValueError(e) return res def service_action(self, cluster_name, service_name, action, queue=False): '''Executes an action on a given service inside of a cluster. Action must be one of START, STOP, or RESTART Args: cluster_name (str): the name of the cluster that the service resides in service_name (str): the name of the service which we are acting on action (str): A string of 'START', 'STOP', or 'RESTART' queue (bool): False when we want the function to wait to continue until the service starts. True when we simply just want to add the function to the Ambari task queue. Defaults to False. **Note**: Does not guarantee the service will complete successfully. Returns: bool: True is the action is completed successfully, False if otherwise. Raises: ValueError: Raised when the action is not one of START/STOP/RESTART ''' logger.info('Attempting Service Action: ' + action + '; Service Name: ' + service_name) if not (action == 'START' or action == 'RESTART' or action == 'STOP'): logger.error('Service actions was not one of START, STOP, or RESTART') raise ValueError('Service action must be one of: START, STOP, or RESTART') logger.info(action + 'ing ' + service_name + ' via Ambari REST API') request_payload = {} request_payload['RequestInfo'] = {} request_payload['RequestInfo']['context'] = action + ' ' + service_name + ' via REST' request_payload['Body'] = {} request_payload['Body']['ServiceInfo'] = {} try: service_info = self.get_service(cluster_name, service_name, query='fields=ServiceInfo') before_state = service_info['ServiceInfo']['state'] logger.debug('Service state before attempting to change: ' + str(before_state)) after_state = '' if action == 'STOP': after_state = 'INSTALLED' elif action == 'START': after_state = 'STARTED' elif action == 'RESTART': r1 = self.service_action(service_name, cluster_name, 'STOP', queue) r2 = self.service_action(service_name, cluster_name, 'START', queue) return r1 and r2 request_payload['Body']['ServiceInfo']['state'] = after_state payload = json.dumps(request_payload) res = self.client.make_request('PUT', '/api/v1/clusters/' + cluster_name + '/services/' + service_name, '-i -d \'' + payload + '\' -H "X-Requested-By:ambari"', 'fields=ServiceInfo') if not ('202 Accepted' in res[0] or '200 OK' in res[0]): logger.error('No 200 Level status when attempting to change service state') return False if queue: return True service_state = '' t = 0 while t < self.service_wait_time: logger.debug('Checking for a change in service state') service_state = self.get_service(cluster_name, service_name, query='fields=ServiceInfo') service_state = service_state['ServiceInfo']['state'] if service_state == after_state: logger.info('Service action completed successfully') return True t += 1 time.sleep(1) except KeyError as e: logger.warn('Unable to find fields in Ambari API response') logger.debug(str(e)) return False def get_clusters(self, query=''): '''Returns a list of clusters from the given Ambari host/port. Equivalent to GET /api/v1/clusters Args: query (string, optional): A formatted query string which is appended to the end of the request URL (advanced). Used for filtering results. Returns: dict: An object which is created after being passed to self.load_output''' logger.info('Making request to /api/v1/clusters/') output = self.client.make_request('GET', '/api/v1/clusters', query) res = self.load_output(output) return res def get_services(self, cluster_name, query=''): '''Get a list of services installed on a cluster. Equivalent to GET /api/v1/clusters/{cluster_name}/services Args: cluster_name (string): The name of the cluster to query query (string, optional): A query to be appended to the url for filtering results Returns: dict: a dictionary object built from the HTTP response and self.load_output''' logger.info('Making request to /api/v1/clusters/' + cluster_name + '/services') output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name + '/services', query) return self.load_output(output) def get_cluster_info(self, cluster_name, query=''): '''Get all of the information about a current cluster. Equivalent to GET /api/v1/clusters/{cluster_name} Args: cluster_name (string): The name of the cluster to query. ''' logger.info('Making request to /api/v1/clusters/' + cluster_name) output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name, query) return self.load_output(output) def get_service(self, cluster_name, service_name, query=''): '''Get all of the information about a single service from the Ambari API. Equivalent to GET /api/v1/clusters/{CLUSTER}/services/{SERVICE} Args: cluster_name (str): The name of the cluster to query service_name (str): The name of the service we want to query. query (str, optional): A query to filter results. Will be appended to the end of the string. i.e ``field1=serviceState&field2=AnotherVal`` Returns: dict: An object converted from the JSON response of the Ambari API. Message will denote otherwise if the request was not successful ''' logger.info('Making request to /api/v1/clusters/' + cluster_name) output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name + '/services/' + service_name, query=query) return self.load_output(output) def set_username(self, user): '''Set the authentication username Args: user (str) Returns: N/A ''' self.username = user self.client.set_username(self.username) def set_password(self, password): '''Set the authentication password Args: password (str) Returns: N/A ''' self.password = password self.client.set_password(self.password) def set_proto(self, proto): '''Set the http protocol to be used Args: proto (str) Returns: N/A ''' self.proto = proto self.client.set_proto(self.proto) def set_server(self, server): '''Set the server/hostname which the client connects to Args: server (str) Returns: N/A ''' self.server = server self.client.set_server(self.server) def set_port(self, port): '''Set the port to be used Args: port (int) Returns: N/A ''' self.port = port self.client.set_port(self.port) def set_service_wait_time(self, wait_time): '''Set the timeout (in seconds) when waiting for a service to change states. i.e to Start/Stop/Restart a service. Args: wait_time (int) Returns: N/A ''' if wait_time > 0: self.service_wait_time = wait_time
class Ambari: '''Initalize the Ambari client Args: username (str, optional): username to use for authentication (should have admin access) password (str, optional): password to use for authentication (should have admin access) proto (str, optional): Must be one of 'http' or 'https'. Defines which protocol to use. Defaults to 'http' server (str, optional): The hostname (or IP) of the Ambari server. Defaults to 127.0.0.1. port (int, optional): The port that ambari server is running on. Defaults to 8080 service_wait_time (int, optional): The time (in seconds) we should before we decide a service has failed changing states. config (dict, optional): This is a dictionary object which should contain the any of the keys 'username', 'password', 'proto', 'server', 'port', or 'service_wait_time'. Given the config here you can set any of the client's parameters through this object. However, when using this, the config object will override any of the specific arguments passed. Returns: N/A ''' def __init__(self, username='', password='', proto='http', server='127.0.0.1', port=8080, service_wait_time=60, config=''): if config != '': if isinstance(config, dict): if 'username' in config: username = config['username'] if 'password' in config: password = config['password'] if 'proto' in config: proto = config['proto'] if 'server' in config: server = config['server'] if 'port' in config: port = config['port'] if 'service_wait_time' in config: service_wait_time = config['service_wait_time'] else: logger.warn( 'Could not instantiate Ambari client through config object. Config is type: ' + str(type(config))) self.client = CurlClient() if service_wait_time > 0: self.set_service_wait_time(service_wait_time) else: self.set_service_wait_time(60) if not username == '': self.set_username(username) if not password == '': self.set_password(password) if not proto == '': self.set_proto(proto) if not server == '': self.set_server(server) if not len(str(port)) == 0: self.set_port(port) logger.info('Created Ambari client: ' + ':'.join([username, password]) + ' ' + ''.join([proto, '://', server, ':', str(port)])) username = password = server = port = proto = '' '''class variable''' @staticmethod def load_output(output): '''Load the output from the curl_client into an object The idea behind this function is to try and keep the same behavior on failed requests across the entire client. .. code-block: json :linenos: // Response with error { "message": "curl (6) Could not connect to host" } // Response without error { "href" : "http://sandbox.hortonworks.com:8080/api/v1/clusters/Sandbox/services/YARN?fields=ServiceInfo", "ServiceInfo" : { "cluster_name" : "Sandbox", "maintenance_state" : "OFF", "service_name" : "YARN", "state" : "INSTALLED" } } Args: output: (str): The output from a curl_client action Returns: dict: a dictionary object created from the JSON response of the Ambari request. If the request was unsuccessful a message attribute will be present containing the error from the curl request. Raises: ValueError: This is raised when an object can't be converted into JSON ''' json_str = '' if not len(output[0]) == 0: json_str = output[0] elif not len(output[1]) == 0: # StdErr output res = {'message': output[1]} return res else: json_str = '{ "message" : "No output was returned." }' res = '' try: res = json.loads(json_str) except ValueError as e: logger.error('Could not convert to JSON: ' + res) raise ValueError(e) return res def service_action(self, cluster_name, service_name, action, queue=False): '''Executes an action on a given service inside of a cluster. Action must be one of START, STOP, or RESTART Args: cluster_name (str): the name of the cluster that the service resides in service_name (str): the name of the service which we are acting on action (str): A string of 'START', 'STOP', or 'RESTART' queue (bool): False when we want the function to wait to continue until the service starts. True when we simply just want to add the function to the Ambari task queue. Defaults to False. **Note**: Does not guarantee the service will complete successfully. Returns: bool: True is the action is completed successfully, False if otherwise. Raises: ValueError: Raised when the action is not one of START/STOP/RESTART ''' logger.info('Attempting Service Action: ' + action + '; Service Name: ' + service_name) if not (action == 'START' or action == 'RESTART' or action == 'STOP'): logger.error( 'Service actions was not one of START, STOP, or RESTART') raise ValueError( 'Service action must be one of: START, STOP, or RESTART') logger.info(action + 'ing ' + service_name + ' via Ambari REST API') request_payload = {} request_payload['RequestInfo'] = {} request_payload['RequestInfo'][ 'context'] = action + ' ' + service_name + ' via REST' request_payload['Body'] = {} request_payload['Body']['ServiceInfo'] = {} try: service_info = self.get_service(cluster_name, service_name, query='fields=ServiceInfo') before_state = service_info['ServiceInfo']['state'] logger.debug('Service state before attempting to change: ' + str(before_state)) after_state = '' if action == 'STOP': after_state = 'INSTALLED' elif action == 'START': after_state = 'STARTED' elif action == 'RESTART': r1 = self.service_action(service_name, cluster_name, 'STOP', queue) r2 = self.service_action(service_name, cluster_name, 'START', queue) return r1 and r2 request_payload['Body']['ServiceInfo']['state'] = after_state payload = json.dumps(request_payload) res = self.client.make_request( 'PUT', '/api/v1/clusters/' + cluster_name + '/services/' + service_name, '-i -d \'' + payload + '\' -H "X-Requested-By:ambari"', 'fields=ServiceInfo') if not ('202 Accepted' in res[0] or '200 OK' in res[0]): logger.error( 'No 200 Level status when attempting to change service state' ) return False if queue: return True service_state = '' t = 0 while t < self.service_wait_time: logger.debug('Checking for a change in service state') service_state = self.get_service(cluster_name, service_name, query='fields=ServiceInfo') service_state = service_state['ServiceInfo']['state'] if service_state == after_state: logger.info('Service action completed successfully') return True t += 1 time.sleep(1) except KeyError as e: logger.warn('Unable to find fields in Ambari API response') logger.debug(str(e)) return False def get_clusters(self, query=''): '''Returns a list of clusters from the given Ambari host/port. Equivalent to GET /api/v1/clusters Args: query (string, optional): A formatted query string which is appended to the end of the request URL (advanced). Used for filtering results. Returns: dict: An object which is created after being passed to self.load_output''' logger.info('Making request to /api/v1/clusters/') output = self.client.make_request('GET', '/api/v1/clusters', query) res = self.load_output(output) return res def get_services(self, cluster_name, query=''): '''Get a list of services installed on a cluster. Equivalent to GET /api/v1/clusters/{cluster_name}/services Args: cluster_name (string): The name of the cluster to query query (string, optional): A query to be appended to the url for filtering results Returns: dict: a dictionary object built from the HTTP response and self.load_output''' logger.info('Making request to /api/v1/clusters/' + cluster_name + '/services') output = self.client.make_request( 'GET', '/api/v1/clusters/' + cluster_name + '/services', query) return self.load_output(output) def get_cluster_info(self, cluster_name, query=''): '''Get all of the information about a current cluster. Equivalent to GET /api/v1/clusters/{cluster_name} Args: cluster_name (string): The name of the cluster to query. ''' logger.info('Making request to /api/v1/clusters/' + cluster_name) output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name, query) return self.load_output(output) def get_service(self, cluster_name, service_name, query=''): '''Get all of the information about a single service from the Ambari API. Equivalent to GET /api/v1/clusters/{CLUSTER}/services/{SERVICE} Args: cluster_name (str): The name of the cluster to query service_name (str): The name of the service we want to query. query (str, optional): A query to filter results. Will be appended to the end of the string. i.e ``field1=serviceState&field2=AnotherVal`` Returns: dict: An object converted from the JSON response of the Ambari API. Message will denote otherwise if the request was not successful ''' logger.info('Making request to /api/v1/clusters/' + cluster_name) output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name + '/services/' + service_name, query=query) return self.load_output(output) def set_username(self, user): '''Set the authentication username Args: user (str) Returns: N/A ''' self.username = user self.client.set_username(self.username) def set_password(self, password): '''Set the authentication password Args: password (str) Returns: N/A ''' self.password = password self.client.set_password(self.password) def set_proto(self, proto): '''Set the http protocol to be used Args: proto (str) Returns: N/A ''' self.proto = proto self.client.set_proto(self.proto) def set_server(self, server): '''Set the server/hostname which the client connects to Args: server (str) Returns: N/A ''' self.server = server self.client.set_server(self.server) def set_port(self, port): '''Set the port to be used Args: port (int) Returns: N/A ''' self.port = port self.client.set_port(self.port) def set_service_wait_time(self, wait_time): '''Set the timeout (in seconds) when waiting for a service to change states. i.e to Start/Stop/Restart a service. Args: wait_time (int) Returns: N/A ''' if wait_time > 0: self.service_wait_time = wait_time