def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
logger, chout, cherr = Logger.initialize_logger(__name__)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(command_name):
self.save_component_version_to_structured_out()
def get_live_status(pid_file, flume_conf_directory):
"""
Gets the status information of a flume agent, including source, sink, and
channel counts.
:param pid_file: the PID file of the agent to check
:param flume_conf_directory: the configuration directory (ie /etc/flume/conf)
:return: a dictionary of information about the flume agent
"""
pid_file_part = pid_file.split(os.sep).pop()
res = {}
res['name'] = pid_file_part
if pid_file_part.endswith(".pid"):
res['name'] = pid_file_part[:-4]
res['status'] = 'RUNNING' if is_flume_process_live(pid_file) else 'NOT_RUNNING'
res['sources_count'] = 0
res['sinks_count'] = 0
res['channels_count'] = 0
flume_agent_conf_dir = flume_conf_directory + os.sep + res['name']
flume_agent_meta_file = flume_agent_conf_dir + os.sep + 'ambari-meta.json'
try:
with open(flume_agent_meta_file) as fp:
meta = json.load(fp)
res['sources_count'] = meta['sources_count']
res['sinks_count'] = meta['sinks_count']
res['channels_count'] = meta['channels_count']
except:
Logger.logger.exception(format("Error reading {flume_agent_meta_file}: "))
return res
def delete_storm_local_data(self, env):
"""
Deletes Storm data from local directories. This will create a marker file
with JSON data representing the upgrade stack and request/stage ID. This
will prevent multiple Storm components on the same host from removing
the local directories more than once.
:return:
"""
import params
Logger.info('Clearing Storm data from local directories...')
storm_local_directory = params.local_dir
if storm_local_directory is None:
raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified")
request_id = default("/requestId", None)
stage_id = default("/stageId", None)
stack_version = params.version
stack_name = params.stack_name
json_map = {}
json_map["requestId"] = request_id
json_map["stageId"] = stage_id
json_map["stackVersion"] = stack_version
json_map["stackName"] = stack_name
temp_directory = params.tmp_dir
upgrade_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version))
if os.path.exists(upgrade_file):
try:
with open(upgrade_file) as file_pointer:
existing_json_map = json.load(file_pointer)
if cmp(json_map, existing_json_map) == 0:
Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for request {2} and stage {3}".format(
stack_name, stack_version, request_id, stage_id))
# nothing else to do here for this as it appears to have already been
# removed by another component being upgraded
return
except:
Logger.error("The upgrade file {0} appears to be corrupt; removing...".format(upgrade_file))
File(upgrade_file, action="delete")
else:
# delete the upgrade file since it does not match
File(upgrade_file, action="delete")
# delete from local directory
Directory(storm_local_directory, action="delete", recursive=True)
# recreate storm local directory
Directory(storm_local_directory, mode=0755, owner = params.storm_user,
group = params.user_group, recursive = True)
# the file doesn't exist, so create it
with open(upgrade_file, 'w') as file_pointer:
json.dump(json_map, file_pointer, indent=2)
def loadJson(path):
try:
with open(path, 'r') as f:
return json.load(f)
except Exception as err:
traceback.print_exc()
raise StackAdvisorException("Error loading file at: {0}".format(path))
def __load_definitions(self):
"""
Loads all alert definitions from a file. All clusters are stored in
a single file.
:return:
"""
definitions = []
all_commands = None
alerts_definitions_path = os.path.join(self.cachedir, self.FILENAME)
try:
with open(alerts_definitions_path) as fp:
all_commands = json.load(fp)
except:
logger.warning('[AlertScheduler] {0} not found or invalid. No alerts will be scheduled until registration occurs.'.format(alerts_definitions_path))
return definitions
for command_json in all_commands:
clusterName = '' if not 'clusterName' in command_json else command_json['clusterName']
hostName = '' if not 'hostName' in command_json else command_json['hostName']
for definition in command_json['alertDefinitions']:
alert = self.__json_to_callable(clusterName, hostName, definition)
if alert is None:
continue
alert.set_helpers(self._collector, self._cluster_configuration)
definitions.append(alert)
return definitions
def get_live_status(pid_file, flume_conf_directory):
"""
Gets the status information of a flume agent, including source, sink, and
channel counts.
:param pid_file: the PID file of the agent to check
:param flume_conf_directory: the configuration directory (ie /etc/flume/conf)
:return: a dictionary of information about the flume agent
"""
pid_file_part = pid_file.split(os.sep).pop()
res = {}
res['name'] = pid_file_part
if pid_file_part.endswith(".pid"):
res['name'] = pid_file_part[:-4]
res['status'] = 'RUNNING' if is_flume_process_live(pid_file) else 'NOT_RUNNING'
res['sources_count'] = 0
res['sinks_count'] = 0
res['channels_count'] = 0
flume_agent_conf_dir = flume_conf_directory + os.sep + res['name']
flume_agent_meta_file = flume_agent_conf_dir + os.sep + 'ambari-meta.json'
try:
with open(flume_agent_meta_file) as fp:
meta = json.load(fp)
res['sources_count'] = meta['sources_count']
res['sinks_count'] = meta['sinks_count']
res['channels_count'] = meta['channels_count']
except:
pass
return res
def __init__(self, cluster_config_cache_dir):
"""
Initializes the configuration cache.
:param cluster_config_cache_dir:
:return:
"""
self.cluster_config_cache_dir = cluster_config_cache_dir
# keys are cluster names, values are configurations
self.__configurations = {}
self.__file_lock = threading.RLock()
self.__cache_lock = threading.RLock()
self.__config_json_file = os.path.join(self.cluster_config_cache_dir, self.FILENAME)
# ensure that our cache directory exists
if not os.path.exists(cluster_config_cache_dir):
try:
os.makedirs(cluster_config_cache_dir)
except:
logger.critical("Could not create the cluster configuration cache directory {0}".format(cluster_config_cache_dir))
# if the file exists, then load it
try:
if os.path.isfile(self.__config_json_file):
with open(self.__config_json_file, 'r') as fp:
self.__configurations = json.load(fp)
except Exception, exception:
logger.warning("Unable to load configurations from {0}. This file will be regenerated on registration".format(self.__config_json_file))
def get_policycache_service_name(service_name, repo_name, cache_service_list):
service_name_exist_flag = False
policycache_path = os.path.join('/etc', 'ranger', repo_name, 'policycache')
try:
for cache_service in cache_service_list:
policycache_json_file = format(
'{policycache_path}/{cache_service}_{repo_name}.json')
if os.path.isfile(policycache_json_file
) and os.path.getsize(policycache_json_file) > 0:
with open(policycache_json_file) as json_file:
json_data = json.load(json_file)
if 'serviceName' in json_data and json_data[
'serviceName'] == repo_name:
Logger.info(
"Skipping Ranger API calls, as policy cache file exists for {0}"
.format(service_name))
Logger.warning(
"If service name for {0} is not created on Ranger Admin, then to re-create it delete policy cache file: {1}"
.format(service_name, policycache_json_file))
service_name_exist_flag = True
break
except Exception, err:
Logger.error(
"Error occurred while fetching service name from policy cache file.\nError: {0}"
.format(err))
def read_mapping():
if os.path.isfile(Options.MR_MAPPING_FILE):
if Options.MR_MAPPING is not None:
return Options.MR_MAPPING
else:
Options.MR_MAPPING = json.load(open(Options.MR_MAPPING_FILE))
return Options.MR_MAPPING
else:
raise FatalException(-1, "MAPREDUCE host mapping file, mr_mapping, is not available or badly formatted. Execute "
"action save-mr-mapping. Ensure the file is present in the directory where you are "
"executing this command.")
def load_version(struct_out_file):
"""
Load version from file. Made a separate method for testing
"""
try:
with open(struct_out_file, 'r') as fp:
json_info = json.load(fp)
return json_info['version']
except (IOError, KeyError, TypeError):
return None
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
# parse arguments
if len(sys.argv) < 7:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logging.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def delete_storm_local_data(self, env):
"""
Deletes Storm data from local directories. This will create a marker file
with JSON data representing the upgrade stack and request/stage ID. This
will prevent multiple Storm components on the same host from removing
the local directories more than once.
:return:
"""
import params
Logger.info('Clearing Storm data from local directories...')
storm_local_directory = params.local_dir
if storm_local_directory is None:
raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified")
request_id = default("/requestId", None)
stack_name = params.stack_name
stack_version = params.version
upgrade_direction = params.upgrade_direction
json_map = {}
json_map["requestId"] = request_id
json_map["stackName"] = stack_name
json_map["stackVersion"] = stack_version
json_map["direction"] = upgrade_direction
temp_directory = params.tmp_dir
marker_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version))
Logger.info("Marker file for upgrade/downgrade of Storm, {0}".format(marker_file))
if os.path.exists(marker_file):
Logger.info("The marker file exists.")
try:
with open(marker_file) as file_pointer:
existing_json_map = json.load(file_pointer)
if cmp(json_map, existing_json_map) == 0:
Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for "
"request {2} and direction {3}. Nothing else to do.".format(stack_name, stack_version, request_id, upgrade_direction))
# Nothing else to do here for this as it appears to have already been
# removed by another component being upgraded
return
else:
Logger.info("The marker file differs from the new value. Will proceed to delete Storm local dir, "
"and generate new file. Current marker file: {0}".format(str(existing_json_map)))
except Exception, e:
Logger.error("The marker file {0} appears to be corrupt; removing it. Error: {1}".format(marker_file, str(e)))
File(marker_file, action="delete")
def load_structured_out(self):
Script.structuredOut = {}
if os.path.exists(self.stroutfile):
with open(self.stroutfile, 'r') as fp:
Script.structuredOut = json.load(fp)
# version is only set in a specific way and should not be carried
if "version" in Script.structuredOut:
del Script.structuredOut["version"]
# reset security issues and errors found on previous runs
if "securityIssuesFound" in Script.structuredOut:
del Script.structuredOut["securityIssuesFound"]
if "securityStateErrorInfo" in Script.structuredOut:
del Script.structuredOut["securityStateErrorInfo"]
def load_version(struct_out_file):
"""
Load version from file. Made a separate method for testing
"""
json_version = None
try:
if os.path.exists(struct_out_file):
with open(struct_out_file, 'r') as fp:
json_info = json.load(fp)
json_version = json_info['version']
except:
pass
return json_version
def load_version(struct_out_file):
"""
Load version from file. Made a separate method for testing
"""
json_version = None
try:
if os.path.exists(struct_out_file):
with open(struct_out_file, 'r') as fp:
json_info = json.load(fp)
json_version = json_info['version']
except:
pass
return json_version
def read_result_from_files(self, out_path, err_path, structured_out_path):
out = open(out_path, 'r').read()
error = open(err_path, 'r').read()
try:
with open(structured_out_path, 'r') as fp:
structured_out = json.load(fp)
except Exception:
if os.path.exists(structured_out_path):
errMsg = 'Unable to read structured output from ' + structured_out_path
structured_out = {'msg': errMsg}
logger.warn(structured_out)
else:
structured_out = {}
return out, error, structured_out
def load_structured_out(self):
Script.structuredOut = {}
if os.path.exists(self.stroutfile):
if os.path.getsize(self.stroutfile) > 0:
with open(self.stroutfile, 'r') as fp:
Script.structuredOut = json.load(fp)
# version is only set in a specific way and should not be carried
if "version" in Script.structuredOut:
del Script.structuredOut["version"]
# reset security issues and errors found on previous runs
if "securityIssuesFound" in Script.structuredOut:
del Script.structuredOut["securityIssuesFound"]
if "securityStateErrorInfo" in Script.structuredOut:
del Script.structuredOut["securityStateErrorInfo"]
def read_result_from_files(self, out_path, err_path, structured_out_path):
out = open(out_path, 'r').read()
error = open(err_path, 'r').read()
try:
with open(structured_out_path, 'r') as fp:
structured_out = json.load(fp)
except (TypeError, ValueError):
structured_out = {
"msg":
"Unable to read structured output from " + structured_out_path
}
self.logger.warn(structured_out)
except (OSError, IOError):
structured_out = {}
return out, error, structured_out
def read_result_from_files(self, out_path, err_path, structured_out_path):
out = open(out_path, 'r').read()
error = open(err_path, 'r').read()
try:
with open(structured_out_path, 'r') as fp:
structured_out = json.load(fp)
except Exception:
if os.path.exists(structured_out_path):
errMsg = 'Unable to read structured output from ' + structured_out_path
structured_out = {
'msg' : errMsg
}
logger.warn(structured_out)
else:
structured_out = {}
return out, error, structured_out
def read_file(self, filename):
runDir = self.findRunDir()
fullname = os.path.join(runDir, filename)
if os.path.isfile(fullname):
res = None
conf_file = open(os.path.join(runDir, filename), 'r')
try:
res = json.load(conf_file)
if (0 == len(res)):
res = None
except Exception, e:
logger.error("Error parsing " + filename + ": " + repr(e))
res = None
pass
conf_file.close()
return res
def cached_kinit_executor(kinit_path,
exec_user,
keytab_file,
principal,
hostname,
temp_dir,
expiration_time=5):
"""
Main cached kinit executor - Uses a temporary file on the FS to cache executions. Each command
will have its own file and only one entry (last successful execution) will be stored
"""
key = str(hash("%s|%s" % (principal, keytab_file)))
filename = key + "_tmp.txt"
file_path = temp_dir + os.sep + "kinit_executor_cache"
output = None
# First execution scenario dir file existence check
if not os.path.exists(file_path):
os.makedirs(file_path)
file_path += os.sep + filename
# If the file does not exist create before read
if not os.path.isfile(file_path):
with open(file_path, 'w+') as new_file:
new_file.write("{}")
try:
with open(file_path, 'r') as cache_file:
output = json.load(cache_file)
except:
# In the extraordinary case the temporary file gets corrupted the cache should be reset to avoid error loop
with open(file_path, 'w+') as cache_file:
cache_file.write("{}")
if (not output) or (key not in output) or ("last_successful_execution"
not in output[key]):
new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user,
keytab_file, principal, hostname)
else:
last_run_time = output[key]["last_successful_execution"]
now = datetime.now()
if (now - datetime.strptime(last_run_time, "%Y-%m-%d %H:%M:%S.%f") >
timedelta(minutes=expiration_time)):
new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user,
keytab_file, principal, hostname)
def __load_definitions(self):
"""
Loads all alert definitions from a file. All clusters are stored in
a single file. This wil also populate the cluster-to-hash dictionary.
:return:
"""
definitions = []
alerts_definitions_path = os.path.join(self.cachedir, self.FILENAME)
try:
with open(alerts_definitions_path) as fp:
all_commands = json.load(fp)
except:
logger.warning(
'[AlertScheduler] {0} not found or invalid. No alerts will be scheduled until registration occurs.'
.format(alerts_definitions_path))
return definitions
for command_json in all_commands:
clusterName = '' if not 'clusterName' in command_json else command_json[
'clusterName']
hostName = '' if not 'hostName' in command_json else command_json[
'hostName']
clusterHash = None if not 'hash' in command_json else command_json[
'hash']
# cache the cluster and cluster hash after loading the JSON
if clusterName != '' and clusterHash is not None:
logger.info(
'[AlertScheduler] Caching cluster {0} with alert hash {1}'.
format(clusterName, clusterHash))
self._cluster_hashes[clusterName] = clusterHash
for definition in command_json['alertDefinitions']:
alert = self.__json_to_callable(clusterName, hostName,
definition)
if alert is None:
continue
alert.set_helpers(self._collector, self._cluster_configuration)
definitions.append(alert)
return definitions
def load_structured_out(self):
Script.structuredOut = {}
if os.path.exists(self.stroutfile):
if os.path.getsize(self.stroutfile) > 0:
with open(self.stroutfile, 'r') as fp:
try:
Script.structuredOut = json.load(fp)
except Exception:
errMsg = 'Unable to read structured output from ' + self.stroutfile
Logger.logger.exception(errMsg)
pass
# version is only set in a specific way and should not be carried
if "version" in Script.structuredOut:
del Script.structuredOut["version"]
# reset security issues and errors found on previous runs
if "securityIssuesFound" in Script.structuredOut:
del Script.structuredOut["securityIssuesFound"]
if "securityStateErrorInfo" in Script.structuredOut:
del Script.structuredOut["securityStateErrorInfo"]
def load_structured_out(self):
Script.structuredOut = {}
if os.path.exists(self.stroutfile):
if os.path.getsize(self.stroutfile) > 0:
with open(self.stroutfile, 'r') as fp:
try:
Script.structuredOut = json.load(fp)
except Exception:
errMsg = 'Unable to read structured output from ' + self.stroutfile
Logger.logger.exception(errMsg)
pass
# version is only set in a specific way and should not be carried
if "version" in Script.structuredOut:
del Script.structuredOut["version"]
# reset security issues and errors found on previous runs
if "securityIssuesFound" in Script.structuredOut:
del Script.structuredOut["securityIssuesFound"]
if "securityStateErrorInfo" in Script.structuredOut:
del Script.structuredOut["securityStateErrorInfo"]
def __init__(self, cluster_cache_dir):
"""
Initializes the cache.
:param cluster_cache_dir:
:return:
"""
self.cluster_cache_dir = cluster_cache_dir
self.__current_cache_json_file = os.path.join(self.cluster_cache_dir, self.get_cache_name()+'.json')
self.__current_cache_hash_file = os.path.join(self.cluster_cache_dir, '.'+self.get_cache_name()+'.hash')
self._cache_lock = threading.RLock()
self.__file_lock = ClusterCache.file_locks[self.__current_cache_json_file]
self.hash = None
cache_dict = {}
try:
with self.__file_lock:
if os.path.isfile(self.__current_cache_json_file):
with open(self.__current_cache_json_file, 'r') as fp:
cache_dict = json.load(fp)
if os.path.isfile(self.__current_cache_hash_file):
with open(self.__current_cache_hash_file, 'r') as fp:
self.hash = fp.read()
except (IOError,ValueError):
logger.exception("Cannot load data from {0} and {1}".format(self.__current_cache_json_file, self.__current_cache_hash_file))
self.hash = None
cache_dict = {}
try:
self.rewrite_cache(cache_dict, self.hash)
except:
# Example: hostname change and restart causes old topology loading to fail with exception
logger.exception("Loading saved cache for {0} failed".format(self.__class__.__name__))
self.rewrite_cache({}, None)
def __load_definitions(self):
"""
Loads all alert definitions from a file. All clusters are stored in
a single file. This wil also populate the cluster-to-hash dictionary.
:return:
"""
definitions = []
alerts_definitions_path = os.path.join(self.cachedir, self.FILENAME)
try:
with open(alerts_definitions_path) as fp:
all_commands = json.load(fp)
except:
logger.warning('[AlertScheduler] {0} not found or invalid. No alerts will be scheduled until registration occurs.'.format(alerts_definitions_path))
return definitions
for command_json in all_commands:
clusterName = '' if not 'clusterName' in command_json else command_json['clusterName']
hostName = '' if not 'hostName' in command_json else command_json['hostName']
clusterHash = None if not 'hash' in command_json else command_json['hash']
# cache the cluster and cluster hash after loading the JSON
if clusterName != '' and clusterHash is not None:
logger.info('[AlertScheduler] Caching cluster {0} with alert hash {1}'.format(clusterName, clusterHash))
self._cluster_hashes[clusterName] = clusterHash
for definition in command_json['alertDefinitions']:
alert = self.__json_to_callable(clusterName, hostName, definition)
if alert is None:
continue
alert.set_helpers(self._collector, self._cluster_configuration)
definitions.append(alert)
return definitions
def cached_kinit_executor(kinit_path, exec_user, keytab_file, principal, hostname, temp_dir,
expiration_time=5):
"""
Main cached kinit executor - Uses a temporary file on the FS to cache executions. Each command
will have its own file and only one entry (last successful execution) will be stored
"""
key = str(hash("%s|%s" % (principal, keytab_file)))
filename = key + "_tmp.txt"
file_path = temp_dir + os.sep + "kinit_executor_cache"
output = None
# First execution scenario dir file existence check
if not os.path.exists(file_path):
os.makedirs(file_path)
file_path += os.sep + filename
# If the file does not exist create before read
if not os.path.isfile(file_path):
with open(file_path, 'w+') as new_file:
new_file.write("{}")
try:
with open(file_path, 'r') as cache_file:
output = json.load(cache_file)
except:
# In the extraordinary case the temporary file gets corrupted the cache should be reset to avoid error loop
with open(file_path, 'w+') as cache_file:
cache_file.write("{}")
if (not output) or (key not in output) or ("last_successful_execution" not in output[key]):
new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user, keytab_file, principal, hostname)
else:
last_run_time = output[key]["last_successful_execution"]
now = datetime.now()
if (now - datetime.strptime(last_run_time, "%Y-%m-%d %H:%M:%S.%f") > timedelta(minutes=expiration_time)):
new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user, keytab_file, principal, hostname)
def create_ams_dashboards():
"""
Create dashboards in grafana from the json files
"""
import params
server = Server(protocol = params.ams_grafana_protocol.strip(),
host = params.ams_grafana_host.strip(),
port = params.ams_grafana_port,
user = params.ams_grafana_admin_user,
password = params.ams_grafana_admin_pwd)
dashboard_files = params.get_grafana_dashboard_defs()
version = params.get_ambari_version()
Logger.info("Checking dashboards to update for Ambari version : %s" % version)
# Friendly representation of dashboard
Dashboard = namedtuple('Dashboard', ['uri', 'id', 'title', 'tags'])
existing_dashboards = []
response = perform_grafana_get_call(GRAFANA_SEARCH_BULTIN_DASHBOARDS, server)
if response and response.status == 200:
data = response.read()
try:
dashboards = json.loads(data)
except:
Logger.error("Unable to parse JSON response from grafana request: %s" %
GRAFANA_SEARCH_BULTIN_DASHBOARDS)
Logger.info(data)
return
for dashboard in dashboards:
if dashboard['title'] == 'HBase - Performance':
perform_grafana_delete_call("/api/dashboards/" + dashboard['uri'], server)
else:
existing_dashboards.append(
Dashboard(uri = dashboard['uri'], id = dashboard['id'],
title = dashboard['title'], tags = dashboard['tags'])
)
pass
else:
Logger.error("Failed to execute search query on Grafana dashboards. "
"query = %s\n statuscode = %s\n reason = %s\n data = %s\n" %
(GRAFANA_SEARCH_BULTIN_DASHBOARDS, response.status, response.reason, response.read()))
return
Logger.debug('Dashboard definitions found = %s' % str(dashboard_files))
if dashboard_files:
for dashboard_file in dashboard_files:
try:
with open(dashboard_file, 'r') as file:
dashboard_def = json.load(file)
except Exception, e:
Logger.error('Unable to load dashboard json file %s' % dashboard_file)
Logger.error(str(e))
continue
if dashboard_def:
update_def = True
# Make sure static json does not have id
if "id" in dashboard_def:
dashboard_def['id'] = None
# Set correct tags
if 'tags' in dashboard_def:
dashboard_def['tags'].append('builtin')
dashboard_def['tags'].append(version)
else:
dashboard_def['tags'] = [ 'builtin', version ]
for dashboard in existing_dashboards:
if dashboard.title == dashboard_def['title']:
if version not in dashboard.tags:
# Found existing dashboard with wrong version - update dashboard
update_def = True
else:
update_def = False # Skip update
pass
if update_def:
Logger.info("Updating dashboard definition for %s with tags: %s" %
(dashboard_def['title'], dashboard_def['tags']))
# Discrepancy in grafana export vs import format
dashboard_def_payload = { "dashboard" : dashboard_def, 'overwrite': True }
paylaod = json.dumps(dashboard_def_payload).strip()
(response, data) = perform_grafana_post_call(GRAFANA_DASHBOARDS_URL, paylaod, server)
if response and response.status == 200:
Logger.info("Dashboard created successfully.\n %s" % str(data))
else:
Logger.error("Failed creating dashboard: %s" % dashboard_def['title'])
pass
else:
Logger.info('No update needed for dashboard = %s' % dashboard_def['title'])
pass
pass
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
help="use this option to enable outputting *.out files of the service pre-start")
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
if len(sys.argv) >= 9:
Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
# make sure that script has forced https protocol and ca_certs file passed from agent
ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
from resource_management.libraries.functions import lzo_utils
repo_tags_to_skip = set()
if not lzo_utils.is_gpl_license_accepted():
repo_tags_to_skip.add("GPL")
Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post', env)
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out(self.command_name)
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option(
"-o",
"--out-files-logging",
dest="log_out_files",
action="store_true",
help=
"use this option to enable outputting *.out files of the service pre-start"
)
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script argument for forcing https protocol
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
Logger.logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if self.command_name == "start" and not self.is_hook():
self.pre_start()
method(env)
if self.command_name == "start" and not self.is_hook():
self.post_start()
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def delete_storm_local_data(self, env):
"""
Deletes Storm data from local directories. This will create a marker file
with JSON data representing the upgrade stack and request/stage ID. This
will prevent multiple Storm components on the same host from removing
the local directories more than once.
:return:
"""
import params
Logger.info('Clearing Storm data from local directories...')
storm_local_directory = params.local_dir
if storm_local_directory is None:
raise Fail(
"The storm local directory specified by storm-site/storm.local.dir must be specified"
)
request_id = default("/requestId", None)
stage_id = default("/stageId", None)
stack_version = params.version
stack_name = params.stack_name
json_map = {}
json_map["requestId"] = request_id
json_map["stageId"] = stage_id
json_map["stackVersion"] = stack_version
json_map["stackName"] = stack_name
temp_directory = params.tmp_dir
upgrade_file = os.path.join(
temp_directory, "storm-upgrade-{0}.json".format(stack_version))
if os.path.exists(upgrade_file):
try:
with open(upgrade_file) as file_pointer:
existing_json_map = json.load(file_pointer)
if cmp(json_map, existing_json_map) == 0:
Logger.info(
"The storm upgrade has already removed the local directories for {0}-{1} for request {2} and stage {3}"
.format(stack_name, stack_version, request_id,
stage_id))
# nothing else to do here for this as it appears to have already been
# removed by another component being upgraded
return
except:
Logger.error(
"The upgrade file {0} appears to be corrupt; removing...".
format(upgrade_file))
File(upgrade_file, action="delete")
else:
# delete the upgrade file since it does not match
File(upgrade_file, action="delete")
# delete from local directory
Directory(storm_local_directory, action="delete", recursive=True)
# recreate storm local directory
Directory(storm_local_directory,
mode=0755,
owner=params.storm_user,
group=params.user_group,
recursive=True)
# the file doesn't exist, so create it
with open(upgrade_file, 'w') as file_pointer:
json.dump(json_map, file_pointer, indent=2)
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
stack_version_override=None,
skip_if_rangeradmin_down=True,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user_principal=None,
component_user_keytab=None,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
"/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
if stack_version_override is None:
stack_version = get_stack_version(component_select_name)
else:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
service_name_exist = False
policycache_path = os.path.join('/etc', 'ranger', repo_name,
'policycache')
try:
for cache_service in cache_service_list:
policycache_json_file = format(
'{policycache_path}/{cache_service}_{repo_name}.json')
if os.path.isfile(policycache_json_file) and os.path.getsize(
policycache_json_file) > 0:
with open(policycache_json_file) as json_file:
json_data = json.load(json_file)
if 'serviceName' in json_data and json_data[
'serviceName'] == repo_name:
service_name_exist = True
Logger.info(
"Skipping Ranger API calls, as policy cache file exists for {0}"
.format(service_name))
Logger.warning(
"If service name for {0} is not created on Ranger Admin UI, then to re-create it delete policy cache file: {1}"
.format(service_name, policycache_json_file))
break
except Exception, err:
Logger.error(
"Error occurred while fetching service name from policy cache file.\nError: {0}"
.format(err))
if not service_name_exist:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user,
is_security_enabled, is_stack_supports_ranger_kerberos,
component_user, component_user_principal,
component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_password_property = 'xasecure.audit.destination.db.password'
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(plugin_audit_properties)
if plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
plugin_audit_password_property] = "crypted"
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
# remove plain-text password from xml configs
plugin_password_properties = [
'xasecure.policymgr.clientssl.keystore.password',
'xasecure.policymgr.clientssl.truststore.password'
]
plugin_policymgr_ssl_properties_copy = {}
plugin_policymgr_ssl_properties_copy.update(
plugin_policymgr_ssl_properties)
for prop in plugin_password_properties:
if prop in plugin_policymgr_ssl_properties_copy:
plugin_policymgr_ssl_properties_copy[prop] = "crypted"
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
# creating symblink should be done by rpm package
# setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(
service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user, component_group, java_home,
cred_lib_path_override, cred_setup_prefix_override)
