def get_kafka_topics():
'''List the kafka topics on the current installation.
Requires that Kafka is installed on the same machine and Ambari is up and running. Will start the service and use the Kafka scripts to list out all of the topics.
Args:
N/A
Returns:
list: [0] will contain the list of all the topics in a string, typically separated by newlines. [1] will contain any errors when retrieving the topics.
'''
conf = config.read_config('global.conf')
am_conf = conf['AMBARI']
amc = Ambari(am_conf['username'], am_conf['password'], am_conf['proto'], am_conf['server'], am_conf['port']);
logger.info('Starting Kafka Broker')
if amc.service_action('Sandbox', 'KAFKA', 'START'):
sh = Shell()
topics_script = conf['DEMO']['kafka_topics_script']
zk = conf['DEMO']['zk_connection']
logger.info('Attempting to create new Kafka Topic')
out = sh.run(topics_script + ' --list --zookeeper ' + zk)
if len(out[1]) == 0:
topics = out[0]
topics = topics.strip().split('\n')
logger.info('Kafka topics output: ' + str(topics))
return topics
return ['', 'Unable to get topics. Could not start Kafka Broker']
def get_kafka_topics():
'''List the kafka topics on the current installation.
Requires that Kafka is installed on the same machine and Ambari is up and running. Will start the service and use the Kafka scripts to list out all of the topics.
Args:
N/A
Returns:
list: [0] will contain the list of all the topics in a string, typically separated by newlines. [1] will contain any errors when retrieving the topics.
'''
conf = config.read_config('global.conf')
am_conf = conf['AMBARI']
amc = Ambari(am_conf['username'], am_conf['password'], am_conf['proto'],
am_conf['server'], am_conf['port'])
logger.info('Starting Kafka Broker')
if amc.service_action('Sandbox', 'KAFKA', 'START'):
sh = Shell()
topics_script = conf['DEMO']['kafka_topics_script']
zk = conf['DEMO']['zk_connection']
logger.info('Attempting to create new Kafka Topic')
out = sh.run(topics_script + ' --list --zookeeper ' + zk)
if len(out[1]) == 0:
topics = out[0]
topics = topics.strip().split('\n')
logger.info('Kafka topics output: ' + str(topics))
return topics
return ['', 'Unable to get topics. Could not start Kafka Broker']
def create_demo_kafka_topic():
'''Creates a kafka topic for the demo if it doesn't already exist.
The caveat here in using this is that Kafka must be installed on the same machine as the demo, and thus the same machine as Ambari as well. The function will try to start the Kafka service through Ambari and then once the service is started is will use the location of the Kafka topics script to create the topic
The name for the topic is specified in ``global.conf``.
Args:
N/A
Returns:
bool: True if the creation is successful. False otherwise.
'''
conf = config.read_config('global.conf')
am_conf = conf['AMBARI']
amc = Ambari(am_conf['username'], am_conf['password'], am_conf['proto'], am_conf['server'], am_conf['port']);
logger.info('Starting Kafka Broker')
if amc.service_action('Sandbox', 'KAFKA', 'START'):
sh = Shell()
topics_script = conf['DEMO']['kafka_topics_script']
zk = conf['DEMO']['zk_connection']
topic_name = conf['DEMO']['kafka_topic_name']
logger.info('Attempting to create new Kafka Topic')
out = sh.run(topics_script + ' --create --zookeeper ' + zk + ' --replication-factor 1 --partitions 1 --topic ' + topic_name)
logger.debug(str(out))
if len(out[1]) == 0:
return True
else:
return False
def create_demo_kafka_topic():
'''Creates a kafka topic for the demo if it doesn't already exist.
The caveat here in using this is that Kafka must be installed on the same machine as the demo, and thus the same machine as Ambari as well. The function will try to start the Kafka service through Ambari and then once the service is started is will use the location of the Kafka topics script to create the topic
The name for the topic is specified in ``global.conf``.
Args:
N/A
Returns:
bool: True if the creation is successful. False otherwise.
'''
conf = config.read_config('global.conf')
am_conf = conf['AMBARI']
amc = Ambari(am_conf['username'], am_conf['password'], am_conf['proto'],
am_conf['server'], am_conf['port'])
logger.info('Starting Kafka Broker')
if amc.service_action('Sandbox', 'KAFKA', 'START'):
sh = Shell()
topics_script = conf['DEMO']['kafka_topics_script']
zk = conf['DEMO']['zk_connection']
topic_name = conf['DEMO']['kafka_topic_name']
logger.info('Attempting to create new Kafka Topic')
out = sh.run(topics_script + ' --create --zookeeper ' + zk +
' --replication-factor 1 --partitions 1 --topic ' +
topic_name)
logger.debug(str(out))
if len(out[1]) == 0:
return True
else:
return False
def test_bad_json_res(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
try:
data = client.get_cluster_info('bad/json/res')
self.fail('Should have thrown an exception: ValueError')
except ValueError as e:
assert ('Extra data:' in str(e))
assert not len(str(e)) == 0
pass
def test_bad_json_res(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
try:
data = client.get_cluster_info('bad/json/res');
self.fail('Should have thrown an exception: ValueError')
except ValueError as e:
assert ('Extra data:' in str(e))
assert not len(str(e)) == 0
pass
def test_service_request_start(self, mock1, mock2, mock3, mock4):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
try:
client.service_action('Sandbox', 'YARN', 'BAD_ACTION')
self.fail('Should have thrown value error on BAD_ACTION')
except ValueError as e:
assert str(e) == 'Service action must be one of: START, STOP, or RESTART'
val = client.service_action('Sandbox', 'YARN', 'START')
assert val == False
def test_bad_config_ambari_service(self, mock1):
conf = {
'b': 'test_username',
'a': 'test_password',
'c': 'https',
'd': 'sandbox_server',
'e': 12309
}
amc = Ambari(config=conf)
amc.service_action('KAFKA', 'Sandbox', 'START')
def test_bad_config_ambari_service(self, mock1):
conf = {
'b': 'test_username',
'a': 'test_password',
'c': 'https',
'd': 'sandbox_server',
'e': 12309
}
amc = Ambari(config=conf)
amc.service_action('KAFKA', 'Sandbox', 'START')
def test_service_request_start(self, mock1, mock2, mock3, mock4):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
try:
client.service_action('Sandbox', 'YARN', 'BAD_ACTION')
self.fail('Should have thrown value error on BAD_ACTION')
except ValueError as e:
assert str(
e) == 'Service action must be one of: START, STOP, or RESTART'
val = client.service_action('Sandbox', 'YARN', 'START')
assert val == False
def on_service_start():
'''This method will run every time the service start
Fill in this method with any necessary commands to set up and start other services for the demo
Note that this method will always be the very last thing to be executed upon starting the demo service
'''
print 'Running on_service_start'
cfg = config.read_config('global.conf')
# Ambari Client
amc = Ambari(config=cfg['AMBARI'])
#Queue Services
amc.service_action('Sandbox', 'KAFKA', 'START', queue=True)
amc.service_action('Sandbox', 'ZEPPELIN', 'START', queue=True)
try:
# Not guaranteed to be installed
amc.service_action('Sandbox', 'NIFI', 'START', queue=True)
except:
log.warn('Failed to start NiFi')
service_installer.add_zeppelin_notebooks()
# Add anything else below that might be necessary for when the demo starts
pass
def test_set_wait_time(self):
client = Ambari(service_wait_time=-5)
assert client.service_wait_time == 60
client.set_service_wait_time(-9)
assert client.service_wait_time == 60
client.set_service_wait_time(50)
assert client.service_wait_time == 50
client.set_service_wait_time(25)
assert client.service_wait_time == 25
def on_service_start():
'''This method will run every time the service start
Fill in this method with any necessary commands to set up and start other services for the demo
Note that this method will always be the very last thing to be executed upon starting the demo service
'''
print 'Running on_service_start'
cfg = config.read_config('global.conf')
# Ambari Client
amc = Ambari(config=cfg['AMBARI'])
#Queue Services
amc.service_action('Sandbox', 'KAFKA', 'START', queue=True)
amc.service_action('Sandbox', 'ZEPPELIN', 'START', queue=True)
try:
# Not guaranteed to be installed
amc.service_action('Sandbox', 'NIFI', 'START', queue=True)
except:
log.warn('Failed to start NiFi')
service_installer.add_zeppelin_notebooks()
# Add anything else below that might be necessary for when the demo starts
pass
def test_service_request_bad_req(self, mock1, mock2, mock3, mock4):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
val = client.service_action('Sandbox', 'YARN', 'STOP')
assert val == False
def test_set_wait_time(self): client = Ambari(service_wait_time=-5) assert client.service_wait_time == 60 client.set_service_wait_time(-9) assert client.service_wait_time == 60 client.set_service_wait_time(50) assert client.service_wait_time == 50 client.set_service_wait_time(25) assert client.service_wait_time == 25
def test_ambari_config_object(self):
conf = {
'username': '******',
'password': '******',
'proto': 'https',
'server': 'sandbox_server',
'port': 12309,
'service_wait_time': 45
}
amc = Ambari(config=conf)
assert amc.username == conf['username'], 'Usernames must match'
assert amc.password == conf['password'], 'Passwords must match'
assert amc.proto == conf['proto'], 'protocols must match'
assert amc.server == conf['server'], 'servers must match'
assert amc.port == conf['port'], 'ports must match'
assert amc.service_wait_time == conf[
'service_wait_time'], 'wait_times must match'
amc = Ambari(config='Config String should throw log warning')
def test_missing_cluster(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('')
assert data['status'] == 404
assert 'resource doesn\'t exist' in data['message']
def test_cluster_info_request(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('demo_cluster')
assert 'demo-server:8080' in data['href']
assert len(data['services']) == 16
def test_get_service(self, mock1):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
assert json.loads(yarn_service_res) == client.get_service('Sandbox', 'YARN')
assert json.loads(missing_service_res) == client.get_service('Sandbox', 'Ay')
def test_service_request_started(self, mock1, mock2, mock3):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
val = client.service_action('Sandbox', 'YARN', 'START')
assert val == True
def test_err_json_res(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('err/json/res');
assert not len(str(data['message'])) == 0
assert data['message'] == bad_err_res
def test_err_str(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('bad/request')
assert not len(data['message']) == 0
assert data['message'] == err_res
def test_empty_str(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('empty/res');
assert not len(data['message']) == 0
assert data['message'] == 'No output was returned.'
def test_get_service(self, mock1):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
assert json.loads(yarn_service_res) == client.get_service(
'Sandbox', 'YARN')
assert json.loads(missing_service_res) == client.get_service(
'Sandbox', 'Ay')
def test_missing_cluster(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('');
assert data['status'] == 404
assert 'resource doesn\'t exist' in data['message']
def test_err_str(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('bad/request');
assert not len(data['message']) == 0
assert data['message'] == err_res
def test_cluster_info_request(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('demo_cluster');
assert 'demo-server:8080' in data['href']
assert len(data['services']) == 16
def test_clusters_request(self, mock): client = Ambari(self.un, self.pw, self.proto, self.server, self.port) data = client.get_clusters(); assert 'demo-server:8080' in data['href'] assert data['items'][0]['Clusters']['cluster_name'] == 'demo_cluster' assert data['items'][0]['Clusters']['version'] == 'HDP-2.4'
def test_err_json_res(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('err/json/res')
assert not len(str(data['message'])) == 0
assert data['message'] == bad_err_res
def test_clusters_request(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_clusters()
assert 'demo-server:8080' in data['href']
assert data['items'][0]['Clusters']['cluster_name'] == 'demo_cluster'
assert data['items'][0]['Clusters']['version'] == 'HDP-2.4'
def test_empty_str(self, mock):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
data = client.get_cluster_info('empty/res')
assert not len(data['message']) == 0
assert data['message'] == 'No output was returned.'
def test_service_request_bad_req(self, mock1, mock2, mock3, mock4):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
val = client.service_action('Sandbox', 'YARN', 'STOP')
assert val == False
def test_service_request_started(self, mock1, mock2, mock3):
client = Ambari(self.un, self.pw, self.proto, self.server, self.port)
val = client.service_action('Sandbox', 'YARN', 'START')
assert val == True