def make_ambari_service_effective(service_name, data_service):
"""Name: make_ambari_service_effective
Performs all operations necessary (ie. restart) to register new service with Ambari
"""
log.info('Proceeding with registration & installation of %s service + components on head node', service_name)
time.sleep(data_service.get_delay_base_unit() * 2)
log.info('Restarting Ambari on head node to register %s service', service_name)
with rm.Environment():
rm.Execute(('ambari-server', 'refresh-stack-hash'))
rm.Service('ambari-server', action='restart')
# We have to wait for it to come back up properly
time.sleep(data_service.get_delay_base_unit() * 3)
def restart_ams_if_necessary(ambari_cluster, data_service):
"""Name: restart_ams_if_necessary
Restarts the Ambari Metrics Service collector service if the service is running on this host. This is required to make any new metrics registered in the whitelist file effective.
"""
#Ambari holds the fqdn of each host
ams_collector_host = ambari_cluster.services('AMBARI_METRICS').components('METRICS_COLLECTOR').host_components.next().host_name
this_host = socket.getaddrinfo(socket.gethostname(), 0, 0, 0, 0, socket.AI_CANONNAME)[0][3]
if data_service.is_ams_collector_host(ams_collector_host, this_host):
try:
log.info("Restarting AMS to make new whitelist metrics effective")
with rm.Environment():
rm.Execute([data_service.get_ams_collector_exe(), '--config', data_service.get_ams_collector_config(), 'restart'], user='******', logoutput=True)
except rm.Fail as ex:
log.warning('Failed to successfully restart AMS metrics collector. Details: %s', ex.message)
def initial_part(service_config, cluster, topology_info, num_edge_nodes,
edge_node_tag, edge_dns_suffix, extra_config,
template_base_uri, data_service, not_detached, logfile,
verbosity):
try:
stack_service_base_path = data_service.get_stack_service_base_dir(
cluster.stack.stack_name, cluster.stack.stack_version)
log.debug('Stack installation dir: %s', stack_service_base_path)
# Get the service tar ball
log.debug('Downloading service package from: %s',
service_config['package'])
req = requests.get(service_config['package'], stream=True)
with tempfile.NamedTemporaryFile(prefix="service-tarball-",
suffix=".tar.gz",
delete=False) as fd:
shutil.copyfileobj(req.raw, fd)
fd.seek(0)
tar = tarfile.open(fileobj=fd)
tar.extractall(stack_service_base_path)
# If we build the tarball on Windows, the wrong permissions are set
for root, dirs, files in os.walk(
os.path.join(stack_service_base_path,
service_config['serviceName'])):
os.chmod(root, 0644)
for dir in dirs:
os.chmod(os.path.join(root, dir), 0644)
for file in files:
os.chmod(os.path.join(root, file), 0644)
# Ambari resource_manager requires an Environment instance to be effective
with rm.Environment():
# We need all the Ambari agents to automatically pickup this new service
ambari_conf_file = data_service.get_ambari_properties_location()
log.debug(
'Modifying ambari properties file: %s. Set agent.auto.cache.update=true',
ambari_conf_file)
rm.ModifyPropertiesFile(
ambari_conf_file,
properties={'agent.auto.cache.update': 'true'})
# If the service has a WebUX (exposed via edge node endpoint), then create the jar file that exposes this capability via an IFrame viewer
# TODO: Add support for richer, separate UX
if 'iframeViewer' in service_config and service_config[
'iframeViewer']:
if edge_dns_suffix and template_base_uri:
try:
edge_dns_name = '{0}-{1}.apps.azurehdinsight.net'.format(
cluster.cluster_name, edge_dns_suffix)
view_jarname = shared_lib.create_iframe_viewer(
service_config['serviceName'],
service_config['displayName'], edge_dns_name,
template_base_uri, data_service)
if view_jarname:
views_dir = os.path.join(
data_service.get_ambari_resources_base_dir(),
'views')
if not os.access(views_dir, os.F_OK):
os.makedirs(views_dir)
dir, fname = os.path.split(view_jarname)
view_fullname = os.path.join(views_dir, fname)
log.debug(
'Moving dynamically constructed viewer jar file from %s to %s',
view_jarname, view_fullname)
shutil.copy(view_jarname, view_fullname)
except:
# We can tolerate failure here, just log the warning & move on
log.warn(
'Failed to create IFrame View for service. The service UX will not be available via Ambari Views. Details: ',
exc_info=True)
else:
log.warn(
'Not installing IFrame view for service as required arguments --edge-dns-suffix and --template-base-uri were not specified'
)
log.info(
'Starting detached installation component. Will wait for cluster installation to complete prior to rebooting Ambari'
)
if not_detached:
detached_part(service_config, cluster, topology_info,
num_edge_nodes, edge_node_tag, extra_config,
data_service, logfile, verbosity)
else:
# Launch the detached script that must wait for all Ambari installations (including edge nodes) to complete
# before proceeding with the complete service installation
detached = Process(target=detached_part,
args=(
service_config,
cluster,
topology_info,
num_edge_nodes,
edge_node_tag,
extra_config,
data_service,
logfile,
verbosity,
))
detached.daemon = True
detached.start()
# Completely detach the child process from this one - an exit handler terminates & waits
# for the child process to close before exiting itself.
current_process()._children.clear()
return True
except:
log.fatal('FATAL: Failure during initial installation part. Details:',
exc_info=True)
return False