def check_ambari_service_installed(service_name, ambari_config):
curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto'])
logger.info('Checking if Ambari service: ' + service_name + ' is installed')
cluster_name = ambari_config['cluster_name']
request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name
attempts = 0
while attempts < 10:
output = curl.make_request('GET', request, '-i')
if '200 OK' in output[0]:
print('Service Installed Sucessfully')
logger.info(service_name + ' was installed successfully')
return True
else:
attempts += 1
raw_input('Could not connect.' + str(10-attempts) + ' remaining. Press any key to continue')
logger.info(service_name + ' was not installed successfully')
return False
def get_zeppelin_session(username, password):
'''Gets the JSESSIONID cookie by POSTing to /api/login
This is required to retrieve the JSESSIONID on Zeppelin instances that have logins and user permissions. Used when POSTing notebooks as well.
Returned in the form of ``'JSESSIONID=**``
Args:
username (str): The username to login with
password (str): The password to login with
Returns:
str: ``''`` If the request was unsuccessful, or the cookie wasn't present. Otherwise returns the cookie as a string.
'''
conf = config.read_config('global.conf')['ZEPPELIN']
client = CurlClient(proto=conf['protocol'],
server=conf['server'],
port=int(conf['port']))
path = '/api/login'
opts = '-i -d \'userName='******'&password='******'\''
res = client.make_request('POST', path, options=opts)
lines = res[0].split('\n')
for line in lines:
if 'Set-Cookie:'.lower() in line.lower(
) and 'JSESSIONID' in line.upper():
cookie = line.split('Set-Cookie: ')[1]
print 'COOKIE: ' + cookie
return cookie.strip()
return ''
def post_notebook(notebook_path, session_cookie=''):
'''Add a single notebook to a Zeppelin installation via REST API
Must have a file called ``global.conf`` inside of the configuration directory.
Inside that configuration file we use protocol://server:port to connect to a Zeppelin instance and use the API
Args:
notebook_path (str): Full file path to the Zeppelin note
session_cookie (str, optional): The cookie used after authentication in order to make authorized API calls. Required on the 2.5 TP Sandbox
Returns:
bool: True if the upload was successful (received 201 created), or False otherwise.
'''
conf = config.read_config('global.conf')['ZEPPELIN']
client = CurlClient(proto=conf['protocol'],
server=conf['server'],
port=int(conf['port']))
path = '/api/notebook'
logger.info('Attempting to POST notebook at ' + client.proto + '://' +
client.server + ':' + str(client.port))
opts = '-i -b \'' + session_cookie + '\' -H "Content-Type: application/json" -d @' + notebook_path
output = client.make_request('POST', path, options=opts)
if '201 created' in output[0].lower():
logger.info('Note posted successfully')
return True
else:
logger.info('Note failed to be added. User will need to add manually')
return False
def post_template(template_path):
'''Add a single template to a NiFi instance via REST API
Args:
template_path (str): The full path to the NiFi template
Returns:
bool: True if the upload is successful, False otherwise
'''
conf = config.read_config('global.conf')['NIFI']
client = CurlClient(proto=conf['protocol'],
server=conf['server'],
port=int(conf['port']))
path = '/nifi-api/controller/templates'
logger.info('Attempting to POST notebook at ' + client.proto + '://' +
client.server + ':' + str(client.port))
output = client.make_request('POST',
path,
options='-i -F template=@' + template_path)
if '201 Created' in output[0] or '200 OK' in output[0]:
logger.info('Template added to NiFi successfully')
return True
else:
logger.info('Template failed to POST to NiFi')
return False
def post_notebook(notebook_path, session_cookie=''):
'''Add a single notebook to a Zeppelin installation via REST API
Must have a file called ``global.conf`` inside of the configuration directory.
Inside that configuration file we use protocol://server:port to connect to a Zeppelin instance and use the API
Args:
notebook_path (str): Full file path to the Zeppelin note
session_cookie (str, optional): The cookie used after authentication in order to make authorized API calls. Required on the 2.5 TP Sandbox
Returns:
bool: True if the upload was successful (received 201 created), or False otherwise.
'''
conf = config.read_config('global.conf')['ZEPPELIN']
client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port']))
path = '/api/notebook'
logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port))
opts = '-i -b \'' + session_cookie + '\' -H "Content-Type: application/json" -d @' + notebook_path
output = client.make_request('POST', path, options=opts)
if '201 created' in output[0].lower():
logger.info('Note posted successfully')
return True
else:
logger.info('Note failed to be added. User will need to add manually')
return False
def get_zeppelin_session(username, password):
'''Gets the JSESSIONID cookie by POSTing to /api/login
This is required to retrieve the JSESSIONID on Zeppelin instances that have logins and user permissions. Used when POSTing notebooks as well.
Returned in the form of ``'JSESSIONID=**``
Args:
username (str): The username to login with
password (str): The password to login with
Returns:
str: ``''`` If the request was unsuccessful, or the cookie wasn't present. Otherwise returns the cookie as a string.
'''
conf = config.read_config('global.conf')['ZEPPELIN']
client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port']))
path = '/api/login'
opts = '-i -d \'userName='******'&password='******'\''
res = client.make_request('POST', path, options=opts)
lines = res[0].split('\n')
for line in lines:
if 'Set-Cookie:'.lower() in line.lower() and 'JSESSIONID' in line.upper():
cookie = line.split('Set-Cookie: ')[1]
print 'COOKIE: ' + cookie
return cookie.strip()
return ''
def __init__(self,
username='',
password='',
proto='http',
server='127.0.0.1',
port=8080,
service_wait_time=60,
config=''):
if config != '':
if isinstance(config, dict):
if 'username' in config:
username = config['username']
if 'password' in config:
password = config['password']
if 'proto' in config:
proto = config['proto']
if 'server' in config:
server = config['server']
if 'port' in config:
port = config['port']
if 'service_wait_time' in config:
service_wait_time = config['service_wait_time']
else:
logger.warn(
'Could not instantiate Ambari client through config object. Config is type: '
+ str(type(config)))
self.client = CurlClient()
if service_wait_time > 0:
self.set_service_wait_time(service_wait_time)
else:
self.set_service_wait_time(60)
if not username == '':
self.set_username(username)
if not password == '':
self.set_password(password)
if not proto == '':
self.set_proto(proto)
if not server == '':
self.set_server(server)
if not len(str(port)) == 0:
self.set_port(port)
logger.info('Created Ambari client: ' +
':'.join([username, password]) + ' ' +
''.join([proto, '://', server, ':',
str(port)]))
def post_template(template_path):
conf = config.read_config('service-installer.conf')['NIFI']
client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port']))
path = '/nifi-api/controller/templates'
logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port))
output = client.make_request('POST', path, options='-i -F template=@' + template_path )
if '201 Created' in output[0] or '200 OK' in output[0]:
logger.info('Template added to NiFi successfully')
return True
else:
logger.info('Template failed to POST to NiFi')
return False
def post_notebook(notebook_path):
conf = config.read_config('service-installer.conf')['ZEPPELIN']
client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port']))
path = '/api/notebook'
logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port))
output = client.make_request('POST', path, options='-i -H "Content-Type: application/json" -d @' + notebook_path )
if '201 created' in output[0].lower():
logger.info('Note posted successfully')
return True
else:
logger.info('Note failed to be added. User will need to add manually')
return False
def check_ambari_service_installed(service_name, ambari_config):
'''Determine whether an Ambari service is installed.
Args:
service_name (string): Name of service to look for
ambari_config (dict): The entire AMBARI section of ``global.conf``
::
read_config('global.conf')['AMBARI']
Returns:
bool: True service exists, False if the service does not exist
'''
curl = CurlClient(username=ambari_config['username'], password=ambari_config['password'], port=ambari_config['port'], server=ambari_config['server'], proto=ambari_config['proto'])
logger.info('Checking if Ambari service: ' + service_name + ' is installed')
cluster_name = ambari_config['cluster_name']
request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name
attempts = 0
while attempts < 10:
output = curl.make_request('GET', request, '-i')
if '200 OK' in output[0]:
print('Service Installed Sucessfully')
logger.info(service_name + ' was installed successfully')
return True
else:
attempts += 1
raw_input('Could not connect. ' + str(10-attempts) + ' remaining. Press any key to continue')
logger.info(service_name + ' was not installed successfully')
return False
def post_template(template_path):
'''Add a single template to a NiFi instance via REST API
Args:
template_path (str): The full path to the NiFi template
Returns:
bool: True if the upload is successful, False otherwise
'''
conf = config.read_config('global.conf')['NIFI']
client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port']))
path = '/nifi-api/controller/templates'
logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port))
output = client.make_request('POST', path, options='-i -F template=@' + template_path )
if '201 Created' in output[0] or '200 OK' in output[0]:
logger.info('Template added to NiFi successfully')
return True
else:
logger.info('Template failed to POST to NiFi')
return False
def check_ambari_service_installed(service_name, ambari_config):
'''Determine whether an Ambari service is installed.
Args:
service_name (string): Name of service to look for
ambari_config (dict): The entire AMBARI section of ``global.conf``
::
read_config('global.conf')['AMBARI']
Returns:
bool: True service exists, False if the service does not exist
'''
curl = CurlClient(username=ambari_config['username'],
password=ambari_config['password'],
port=ambari_config['port'],
server=ambari_config['server'],
proto=ambari_config['proto'])
logger.info('Checking if Ambari service: ' + service_name +
' is installed')
cluster_name = ambari_config['cluster_name']
request = '/api/v1/clusters/' + cluster_name + '/services/' + service_name
attempts = 0
while attempts < 10:
output = curl.make_request('GET', request, '-i')
if '200 OK' in output[0]:
print('Service Installed Sucessfully')
logger.info(service_name + ' was installed successfully')
return True
else:
attempts += 1
raw_input('Could not connect. ' + str(10 - attempts) +
' remaining. Press any key to continue')
logger.info(service_name + ' was not installed successfully')
return False
def __init__(self, username='', password='', proto='http', server='127.0.0.1', port=8080, service_wait_time=60, config=''):
if config != '':
if isinstance(config, dict):
if 'username' in config:
username = config['username']
if 'password' in config:
password = config['password']
if 'proto' in config:
proto = config['proto']
if 'server' in config:
server = config['server']
if 'port' in config:
port = config['port']
if 'service_wait_time' in config:
service_wait_time = config['service_wait_time']
else:
logger.warn('Could not instantiate Ambari client through config object. Config is type: ' + str(type(config)))
self.client = CurlClient()
if service_wait_time > 0:
self.set_service_wait_time(service_wait_time)
else:
self.set_service_wait_time(60)
if not username == '':
self.set_username(username)
if not password == '':
self.set_password(password)
if not proto == '':
self.set_proto(proto)
if not server == '':
self.set_server(server)
if not len(str(port)) == 0:
self.set_port(port)
logger.info('Created Ambari client: ' + ':'.join([username, password]) + ' ' + ''.join([proto, '://', server, ':', str(port)]))
def install_nifi():
'''Install NiFi via Ambari. (And Ali's NiFi service)
Automatically installs NiFi with NO user interaction. Simply just run the method while on the same Ambari machine and NiFi will be installed. You'll need to start it manually though.
Returns:
bool: True if installation is successful. Else, the user specifies whether or not they want to continue setting up the demo without Zeppelin. ``True`` if the user specifed Yes (to continue). ``False`` if they specified No (do not continue).
Raises:
EnvironmentError: Raised when Ambari is not installed on the current host. Or if ``hdp-select`` cannot be installed'''
logger.info('Attempting to install NiFi to the cluster')
if not is_ambari_installed():
logger.error('Ambari must be installed to install NiFi as well.')
raise EnvironmentError(
'You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing'
)
if not is_hdp_select_installed():
installed = install_hdp_select()
if not installed:
logger.error('hdp-select must be installed to install NiFi')
raise EnvironmentError(
'hdp-select could not be installed. Please install it manually and then re-run the setup.'
)
conf = config.read_config('global.conf')
cmds = json.loads(conf['NIFI']['install-commands'])
sh = Shell()
logger.info('Getting HDP Version')
version = sh.run(cmds[0])
logger.info('HDP Version: ' + version[0])
fixed_copy = cmds[2].replace('$VERSION', str(version[0])).replace('\n', '')
fixed_remove = cmds[1].replace('$VERSION',
str(version[0])).replace('\n', '')
logger.info('NiFi Clean Command: ' + fixed_copy)
logger.info('NiFi Copy Command: ' + fixed_remove)
remove = sh.run(fixed_remove)
copy = sh.run(fixed_copy)
amc = conf['AMBARI']
cc = CurlClient(amc['username'], amc['password'], amc['proto'],
amc['server'], amc['port'])
opts = '-H \'X-Requested-By: ambari\''
path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI'
print cc.make_request('POST', path, options=opts)
path += '/components/NIFI_MASTER'
print cc.make_request('POST', path, options=opts)
cfg = {
'cmd': 'bash /var/lib/ambari-server/resources/scripts/configs.sh set',
'server': amc['server'],
'cluster': amc['cluster_name'],
'name': 'nifi-ambari-config',
'config_file': config.get_path('nifi/config/nifi-ambari-config.json')
}
create_cmd = lambda x: ' '.join([
cfg['cmd'], cfg['server'], cfg['cluster'], x,
config.get_path('nifi/config/' + x + '.json')
])
logger.debug(sh.run(create_cmd('nifi-ambari-config')))
logger.debug(sh.run(create_cmd('nifi-bootstrap-env')))
logger.debug(sh.run(create_cmd('nifi-flow-env')))
logger.debug(sh.run(create_cmd('nifi-logback-env')))
logger.debug(sh.run(create_cmd('nifi-properties-env')))
path = '/api/v1/clusters/' + amc['cluster_name'] + '/hosts/' + amc[
'server'] + '/host_components/NIFI_MASTER'
logger.debug(path)
cc.make_request('POST', path, options=opts)
path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI'
opts = '-H \'X-Requested-By: ambari\' -d \'{"RequestInfo": {"context" :"Install Nifi"}, "Body": {"ServiceInfo": {"maintenance_state" : "OFF", "state": "INSTALLED"}}}\''
cc.make_request('PUT', path, options=opts)
print(
"Please open the Ambari Interface and manually deploy the NiFi Service."
)
raw_input("Press enter twice to continue...")
raw_input("Press enter once to continue...")
# We've copied the necessary files. Once that completes we need to add it to Ambari
logger.info('Waiting for user to install service in Ambari to continue')
print('Checking to make sure service is installed')
ambari = config.read_config('global.conf')['AMBARI']
installed = check_ambari_service_installed('NIFI', ambari)
logger.info('NiFi installed successfully')
cont = ''
if not installed:
print(
'Unable to contact Ambari Server. Unsure whether or not NiFi was installed'
)
while not (cont == 'y' or cont == 'n'):
cont = raw_input('Continue attempt to set up NiFi for demo?(y/n)')
if not (cont == 'y' or cont == 'n'):
print('Please enter "y" or "n"')
else:
cont = 'y'
if cont == 'n':
return False
elif cont == 'y':
return True
def install_nifi():
'''Install NiFi via Ambari. (And Ali's NiFi service)
Automatically installs NiFi with NO user interaction. Simply just run the method while on the same Ambari machine and NiFi will be installed. You'll need to start it manually though.
Returns:
bool: True if installation is successful. Else, the user specifies whether or not they want to continue setting up the demo without Zeppelin. ``True`` if the user specifed Yes (to continue). ``False`` if they specified No (do not continue).
Raises:
EnvironmentError: Raised when Ambari is not installed on the current host. Or if ``hdp-select`` cannot be installed'''
logger.info('Attempting to install NiFi to the cluster')
if not is_ambari_installed():
logger.error('Ambari must be installed to install NiFi as well.')
raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing')
if not is_hdp_select_installed():
installed = install_hdp_select()
if not installed:
logger.error('hdp-select must be installed to install NiFi')
raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.')
conf = config.read_config('global.conf')
cmds = json.loads(conf['NIFI']['install-commands'])
sh = Shell()
logger.info('Getting HDP Version')
version = sh.run(cmds[0])
logger.info('HDP Version: ' + version[0])
fixed_copy = cmds[2].replace('$VERSION', str(version[0])).replace('\n', '')
fixed_remove = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '')
logger.info('NiFi Clean Command: ' + fixed_copy)
logger.info('NiFi Copy Command: ' + fixed_remove)
remove = sh.run(fixed_remove)
copy = sh.run(fixed_copy)
amc = conf['AMBARI']
cc = CurlClient(amc['username'], amc['password'], amc['proto'], amc['server'], amc['port'])
opts = '-H \'X-Requested-By: ambari\''
path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI'
print cc.make_request('POST', path, options=opts)
path += '/components/NIFI_MASTER'
print cc.make_request('POST', path, options=opts)
cfg = {
'cmd': 'bash /var/lib/ambari-server/resources/scripts/configs.sh set',
'server': amc['server'],
'cluster': amc['cluster_name'],
'name': 'nifi-ambari-config',
'config_file': config.get_path('nifi/config/nifi-ambari-config.json')
}
create_cmd = lambda x: ' '.join([cfg['cmd'], cfg['server'], cfg['cluster'], x, config.get_path('nifi/config/' + x + '.json')])
logger.debug(sh.run(create_cmd('nifi-ambari-config')))
logger.debug(sh.run(create_cmd('nifi-bootstrap-env')))
logger.debug(sh.run(create_cmd('nifi-flow-env')))
logger.debug(sh.run(create_cmd('nifi-logback-env')))
logger.debug(sh.run(create_cmd('nifi-properties-env')))
path = '/api/v1/clusters/' + amc['cluster_name'] + '/hosts/' + amc['server'] + '/host_components/NIFI_MASTER'
logger.debug(path)
cc.make_request('POST', path, options=opts)
path = '/api/v1/clusters/' + amc['cluster_name'] + '/services/NIFI'
opts = '-H \'X-Requested-By: ambari\' -d \'{"RequestInfo": {"context" :"Install Nifi"}, "Body": {"ServiceInfo": {"maintenance_state" : "OFF", "state": "INSTALLED"}}}\''
cc.make_request('PUT', path, options=opts)
print("Please open the Ambari Interface and manually deploy the NiFi Service.")
raw_input("Press enter twice to continue...")
raw_input("Press enter once to continue...")
# We've copied the necessary files. Once that completes we need to add it to Ambari
logger.info('Waiting for user to install service in Ambari to continue')
print('Checking to make sure service is installed')
ambari = config.read_config('global.conf')['AMBARI']
installed = check_ambari_service_installed('NIFI', ambari)
logger.info('NiFi installed successfully')
cont = ''
if not installed:
print('Unable to contact Ambari Server. Unsure whether or not NiFi was installed')
while not (cont == 'y' or cont == 'n'):
cont = raw_input('Continue attempt to set up NiFi for demo?(y/n)')
if not (cont == 'y' or cont == 'n'):
print('Please enter "y" or "n"')
else:
cont = 'y'
if cont == 'n':
return False
elif cont == 'y':
return True
class Ambari:
'''Initalize the Ambari client
Args:
username (str, optional): username to use for authentication (should have admin access)
password (str, optional): password to use for authentication (should have admin access)
proto (str, optional): Must be one of 'http' or 'https'. Defines which protocol to use. Defaults to 'http'
server (str, optional): The hostname (or IP) of the Ambari server. Defaults to 127.0.0.1.
port (int, optional): The port that ambari server is running on. Defaults to 8080
service_wait_time (int, optional): The time (in seconds) we should before we decide a service has failed changing states.
config (dict, optional): This is a dictionary object which should contain the any of the keys 'username', 'password', 'proto', 'server', 'port', or 'service_wait_time'. Given the config here you can set any of the client's parameters through this object. However, when using this, the config object will override any of the specific arguments passed.
Returns:
N/A
'''
def __init__(self, username='', password='', proto='http', server='127.0.0.1', port=8080, service_wait_time=60, config=''):
if config != '':
if isinstance(config, dict):
if 'username' in config:
username = config['username']
if 'password' in config:
password = config['password']
if 'proto' in config:
proto = config['proto']
if 'server' in config:
server = config['server']
if 'port' in config:
port = config['port']
if 'service_wait_time' in config:
service_wait_time = config['service_wait_time']
else:
logger.warn('Could not instantiate Ambari client through config object. Config is type: ' + str(type(config)))
self.client = CurlClient()
if service_wait_time > 0:
self.set_service_wait_time(service_wait_time)
else:
self.set_service_wait_time(60)
if not username == '':
self.set_username(username)
if not password == '':
self.set_password(password)
if not proto == '':
self.set_proto(proto)
if not server == '':
self.set_server(server)
if not len(str(port)) == 0:
self.set_port(port)
logger.info('Created Ambari client: ' + ':'.join([username, password]) + ' ' + ''.join([proto, '://', server, ':', str(port)]))
username = password = server = port = proto = ''
'''class variable'''
@staticmethod
def load_output(output):
'''Load the output from the curl_client into an object
The idea behind this function is to try and keep the same behavior on failed requests across the entire client.
.. code-block: json
:linenos:
// Response with error
{
"message": "curl (6) Could not connect to host"
}
// Response without error
{
"href" : "http://sandbox.hortonworks.com:8080/api/v1/clusters/Sandbox/services/YARN?fields=ServiceInfo",
"ServiceInfo" : {
"cluster_name" : "Sandbox",
"maintenance_state" : "OFF",
"service_name" : "YARN",
"state" : "INSTALLED"
}
}
Args:
output: (str): The output from a curl_client action
Returns:
dict: a dictionary object created from the JSON response of the Ambari request. If the request was unsuccessful a message attribute will be present containing the error from the curl request.
Raises:
ValueError: This is raised when an object can't be converted into JSON
'''
json_str = ''
if not len(output[0]) == 0:
json_str = output[0]
elif not len(output[1]) == 0:
# StdErr output
res = {'message': output[1]}
return res
else:
json_str = '{ "message" : "No output was returned." }'
res = ''
try:
res = json.loads(json_str)
except ValueError as e:
logger.error('Could not convert to JSON: ' + res)
raise ValueError(e)
return res
def service_action(self, cluster_name, service_name, action, queue=False):
'''Executes an action on a given service inside of a cluster. Action must be one of START, STOP, or RESTART
Args:
cluster_name (str): the name of the cluster that the service resides in
service_name (str): the name of the service which we are acting on
action (str): A string of 'START', 'STOP', or 'RESTART'
queue (bool): False when we want the function to wait to continue until the service starts. True when we simply just want to add the function to the Ambari task queue. Defaults to False. **Note**: Does not guarantee the service will complete successfully.
Returns:
bool: True is the action is completed successfully, False if otherwise.
Raises:
ValueError: Raised when the action is not one of START/STOP/RESTART
'''
logger.info('Attempting Service Action: ' + action + '; Service Name: ' + service_name)
if not (action == 'START' or action == 'RESTART' or action == 'STOP'):
logger.error('Service actions was not one of START, STOP, or RESTART')
raise ValueError('Service action must be one of: START, STOP, or RESTART')
logger.info(action + 'ing ' + service_name + ' via Ambari REST API')
request_payload = {}
request_payload['RequestInfo'] = {}
request_payload['RequestInfo']['context'] = action + ' ' + service_name + ' via REST'
request_payload['Body'] = {}
request_payload['Body']['ServiceInfo'] = {}
try:
service_info = self.get_service(cluster_name, service_name, query='fields=ServiceInfo')
before_state = service_info['ServiceInfo']['state']
logger.debug('Service state before attempting to change: ' + str(before_state))
after_state = ''
if action == 'STOP':
after_state = 'INSTALLED'
elif action == 'START':
after_state = 'STARTED'
elif action == 'RESTART':
r1 = self.service_action(service_name, cluster_name, 'STOP', queue)
r2 = self.service_action(service_name, cluster_name, 'START', queue)
return r1 and r2
request_payload['Body']['ServiceInfo']['state'] = after_state
payload = json.dumps(request_payload)
res = self.client.make_request('PUT', '/api/v1/clusters/' + cluster_name + '/services/' + service_name, '-i -d \'' + payload + '\' -H "X-Requested-By:ambari"', 'fields=ServiceInfo')
if not ('202 Accepted' in res[0] or '200 OK' in res[0]):
logger.error('No 200 Level status when attempting to change service state')
return False
if queue:
return True
service_state = ''
t = 0
while t < self.service_wait_time:
logger.debug('Checking for a change in service state')
service_state = self.get_service(cluster_name, service_name, query='fields=ServiceInfo')
service_state = service_state['ServiceInfo']['state']
if service_state == after_state:
logger.info('Service action completed successfully')
return True
t += 1
time.sleep(1)
except KeyError as e:
logger.warn('Unable to find fields in Ambari API response')
logger.debug(str(e))
return False
def get_clusters(self, query=''):
'''Returns a list of clusters from the given Ambari host/port. Equivalent to GET /api/v1/clusters
Args:
query (string, optional): A formatted query string which is appended to the end of the request URL (advanced). Used for filtering results.
Returns:
dict: An object which is created after being passed to self.load_output'''
logger.info('Making request to /api/v1/clusters/')
output = self.client.make_request('GET', '/api/v1/clusters', query)
res = self.load_output(output)
return res
def get_services(self, cluster_name, query=''):
'''Get a list of services installed on a cluster. Equivalent to GET /api/v1/clusters/{cluster_name}/services
Args:
cluster_name (string): The name of the cluster to query
query (string, optional): A query to be appended to the url for filtering results
Returns:
dict: a dictionary object built from the HTTP response and self.load_output'''
logger.info('Making request to /api/v1/clusters/' + cluster_name + '/services')
output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name + '/services', query)
return self.load_output(output)
def get_cluster_info(self, cluster_name, query=''):
'''Get all of the information about a current cluster. Equivalent to GET /api/v1/clusters/{cluster_name}
Args:
cluster_name (string): The name of the cluster to query.
'''
logger.info('Making request to /api/v1/clusters/' + cluster_name)
output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name, query)
return self.load_output(output)
def get_service(self, cluster_name, service_name, query=''):
'''Get all of the information about a single service from the Ambari API. Equivalent to GET /api/v1/clusters/{CLUSTER}/services/{SERVICE}
Args:
cluster_name (str): The name of the cluster to query
service_name (str): The name of the service we want to query.
query (str, optional): A query to filter results. Will be appended to the end of the string. i.e ``field1=serviceState&field2=AnotherVal``
Returns:
dict: An object converted from the JSON response of the Ambari API. Message will denote otherwise if the request was not successful
'''
logger.info('Making request to /api/v1/clusters/' + cluster_name)
output = self.client.make_request('GET', '/api/v1/clusters/' + cluster_name + '/services/' + service_name, query=query)
return self.load_output(output)
def set_username(self, user):
'''Set the authentication username
Args:
user (str)
Returns:
N/A
'''
self.username = user
self.client.set_username(self.username)
def set_password(self, password):
'''Set the authentication password
Args:
password (str)
Returns:
N/A
'''
self.password = password
self.client.set_password(self.password)
def set_proto(self, proto):
'''Set the http protocol to be used
Args:
proto (str)
Returns:
N/A
'''
self.proto = proto
self.client.set_proto(self.proto)
def set_server(self, server):
'''Set the server/hostname which the client connects to
Args:
server (str)
Returns:
N/A
'''
self.server = server
self.client.set_server(self.server)
def set_port(self, port):
'''Set the port to be used
Args:
port (int)
Returns:
N/A
'''
self.port = port
self.client.set_port(self.port)
def set_service_wait_time(self, wait_time):
'''Set the timeout (in seconds) when waiting for a service to change states. i.e to Start/Stop/Restart a service.
Args:
wait_time (int)
Returns:
N/A
'''
if wait_time > 0:
self.service_wait_time = wait_time
class Ambari:
'''Initalize the Ambari client
Args:
username (str, optional): username to use for authentication (should have admin access)
password (str, optional): password to use for authentication (should have admin access)
proto (str, optional): Must be one of 'http' or 'https'. Defines which protocol to use. Defaults to 'http'
server (str, optional): The hostname (or IP) of the Ambari server. Defaults to 127.0.0.1.
port (int, optional): The port that ambari server is running on. Defaults to 8080
service_wait_time (int, optional): The time (in seconds) we should before we decide a service has failed changing states.
config (dict, optional): This is a dictionary object which should contain the any of the keys 'username', 'password', 'proto', 'server', 'port', or 'service_wait_time'. Given the config here you can set any of the client's parameters through this object. However, when using this, the config object will override any of the specific arguments passed.
Returns:
N/A
'''
def __init__(self,
username='',
password='',
proto='http',
server='127.0.0.1',
port=8080,
service_wait_time=60,
config=''):
if config != '':
if isinstance(config, dict):
if 'username' in config:
username = config['username']
if 'password' in config:
password = config['password']
if 'proto' in config:
proto = config['proto']
if 'server' in config:
server = config['server']
if 'port' in config:
port = config['port']
if 'service_wait_time' in config:
service_wait_time = config['service_wait_time']
else:
logger.warn(
'Could not instantiate Ambari client through config object. Config is type: '
+ str(type(config)))
self.client = CurlClient()
if service_wait_time > 0:
self.set_service_wait_time(service_wait_time)
else:
self.set_service_wait_time(60)
if not username == '':
self.set_username(username)
if not password == '':
self.set_password(password)
if not proto == '':
self.set_proto(proto)
if not server == '':
self.set_server(server)
if not len(str(port)) == 0:
self.set_port(port)
logger.info('Created Ambari client: ' +
':'.join([username, password]) + ' ' +
''.join([proto, '://', server, ':',
str(port)]))
username = password = server = port = proto = ''
'''class variable'''
@staticmethod
def load_output(output):
'''Load the output from the curl_client into an object
The idea behind this function is to try and keep the same behavior on failed requests across the entire client.
.. code-block: json
:linenos:
// Response with error
{
"message": "curl (6) Could not connect to host"
}
// Response without error
{
"href" : "http://sandbox.hortonworks.com:8080/api/v1/clusters/Sandbox/services/YARN?fields=ServiceInfo",
"ServiceInfo" : {
"cluster_name" : "Sandbox",
"maintenance_state" : "OFF",
"service_name" : "YARN",
"state" : "INSTALLED"
}
}
Args:
output: (str): The output from a curl_client action
Returns:
dict: a dictionary object created from the JSON response of the Ambari request. If the request was unsuccessful a message attribute will be present containing the error from the curl request.
Raises:
ValueError: This is raised when an object can't be converted into JSON
'''
json_str = ''
if not len(output[0]) == 0:
json_str = output[0]
elif not len(output[1]) == 0:
# StdErr output
res = {'message': output[1]}
return res
else:
json_str = '{ "message" : "No output was returned." }'
res = ''
try:
res = json.loads(json_str)
except ValueError as e:
logger.error('Could not convert to JSON: ' + res)
raise ValueError(e)
return res
def service_action(self, cluster_name, service_name, action, queue=False):
'''Executes an action on a given service inside of a cluster. Action must be one of START, STOP, or RESTART
Args:
cluster_name (str): the name of the cluster that the service resides in
service_name (str): the name of the service which we are acting on
action (str): A string of 'START', 'STOP', or 'RESTART'
queue (bool): False when we want the function to wait to continue until the service starts. True when we simply just want to add the function to the Ambari task queue. Defaults to False. **Note**: Does not guarantee the service will complete successfully.
Returns:
bool: True is the action is completed successfully, False if otherwise.
Raises:
ValueError: Raised when the action is not one of START/STOP/RESTART
'''
logger.info('Attempting Service Action: ' + action +
'; Service Name: ' + service_name)
if not (action == 'START' or action == 'RESTART' or action == 'STOP'):
logger.error(
'Service actions was not one of START, STOP, or RESTART')
raise ValueError(
'Service action must be one of: START, STOP, or RESTART')
logger.info(action + 'ing ' + service_name + ' via Ambari REST API')
request_payload = {}
request_payload['RequestInfo'] = {}
request_payload['RequestInfo'][
'context'] = action + ' ' + service_name + ' via REST'
request_payload['Body'] = {}
request_payload['Body']['ServiceInfo'] = {}
try:
service_info = self.get_service(cluster_name,
service_name,
query='fields=ServiceInfo')
before_state = service_info['ServiceInfo']['state']
logger.debug('Service state before attempting to change: ' +
str(before_state))
after_state = ''
if action == 'STOP':
after_state = 'INSTALLED'
elif action == 'START':
after_state = 'STARTED'
elif action == 'RESTART':
r1 = self.service_action(service_name, cluster_name, 'STOP',
queue)
r2 = self.service_action(service_name, cluster_name, 'START',
queue)
return r1 and r2
request_payload['Body']['ServiceInfo']['state'] = after_state
payload = json.dumps(request_payload)
res = self.client.make_request(
'PUT', '/api/v1/clusters/' + cluster_name + '/services/' +
service_name,
'-i -d \'' + payload + '\' -H "X-Requested-By:ambari"',
'fields=ServiceInfo')
if not ('202 Accepted' in res[0] or '200 OK' in res[0]):
logger.error(
'No 200 Level status when attempting to change service state'
)
return False
if queue:
return True
service_state = ''
t = 0
while t < self.service_wait_time:
logger.debug('Checking for a change in service state')
service_state = self.get_service(cluster_name,
service_name,
query='fields=ServiceInfo')
service_state = service_state['ServiceInfo']['state']
if service_state == after_state:
logger.info('Service action completed successfully')
return True
t += 1
time.sleep(1)
except KeyError as e:
logger.warn('Unable to find fields in Ambari API response')
logger.debug(str(e))
return False
def get_clusters(self, query=''):
'''Returns a list of clusters from the given Ambari host/port. Equivalent to GET /api/v1/clusters
Args:
query (string, optional): A formatted query string which is appended to the end of the request URL (advanced). Used for filtering results.
Returns:
dict: An object which is created after being passed to self.load_output'''
logger.info('Making request to /api/v1/clusters/')
output = self.client.make_request('GET', '/api/v1/clusters', query)
res = self.load_output(output)
return res
def get_services(self, cluster_name, query=''):
'''Get a list of services installed on a cluster. Equivalent to GET /api/v1/clusters/{cluster_name}/services
Args:
cluster_name (string): The name of the cluster to query
query (string, optional): A query to be appended to the url for filtering results
Returns:
dict: a dictionary object built from the HTTP response and self.load_output'''
logger.info('Making request to /api/v1/clusters/' + cluster_name +
'/services')
output = self.client.make_request(
'GET', '/api/v1/clusters/' + cluster_name + '/services', query)
return self.load_output(output)
def get_cluster_info(self, cluster_name, query=''):
'''Get all of the information about a current cluster. Equivalent to GET /api/v1/clusters/{cluster_name}
Args:
cluster_name (string): The name of the cluster to query.
'''
logger.info('Making request to /api/v1/clusters/' + cluster_name)
output = self.client.make_request('GET',
'/api/v1/clusters/' + cluster_name,
query)
return self.load_output(output)
def get_service(self, cluster_name, service_name, query=''):
'''Get all of the information about a single service from the Ambari API. Equivalent to GET /api/v1/clusters/{CLUSTER}/services/{SERVICE}
Args:
cluster_name (str): The name of the cluster to query
service_name (str): The name of the service we want to query.
query (str, optional): A query to filter results. Will be appended to the end of the string. i.e ``field1=serviceState&field2=AnotherVal``
Returns:
dict: An object converted from the JSON response of the Ambari API. Message will denote otherwise if the request was not successful
'''
logger.info('Making request to /api/v1/clusters/' + cluster_name)
output = self.client.make_request('GET',
'/api/v1/clusters/' + cluster_name +
'/services/' + service_name,
query=query)
return self.load_output(output)
def set_username(self, user):
'''Set the authentication username
Args:
user (str)
Returns:
N/A
'''
self.username = user
self.client.set_username(self.username)
def set_password(self, password):
'''Set the authentication password
Args:
password (str)
Returns:
N/A
'''
self.password = password
self.client.set_password(self.password)
def set_proto(self, proto):
'''Set the http protocol to be used
Args:
proto (str)
Returns:
N/A
'''
self.proto = proto
self.client.set_proto(self.proto)
def set_server(self, server):
'''Set the server/hostname which the client connects to
Args:
server (str)
Returns:
N/A
'''
self.server = server
self.client.set_server(self.server)
def set_port(self, port):
'''Set the port to be used
Args:
port (int)
Returns:
N/A
'''
self.port = port
self.client.set_port(self.port)
def set_service_wait_time(self, wait_time):
'''Set the timeout (in seconds) when waiting for a service to change states. i.e to Start/Stop/Restart a service.
Args:
wait_time (int)
Returns:
N/A
'''
if wait_time > 0:
self.service_wait_time = wait_time
