def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
logger, chout, cherr = Logger.initialize_logger(__name__)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(command_name):
self.save_component_version_to_structured_out()
def executeScript(self, path, classname=None, command=None, config_file=None,
config_dict=None,
# common mocks for all the scripts
config_overrides = None,
hdp_stack_version = None,
checked_call_mocks = itertools.cycle([(0, "OK.")]),
call_mocks = itertools.cycle([(0, "OK.")]),
os_type=('Suse','11','Final'),
kinit_path_local="/usr/bin/kinit",
os_env={'PATH':'/bin'},
target=TARGET_STACKS,
mocks_dict={},
try_install=False,
command_args=[]):
norm_path = os.path.normpath(path)
src_dir = RMFTestCase.get_src_folder()
if target == self.TARGET_STACKS:
stack_version = norm_path.split(os.sep)[0]
base_path = os.path.join(src_dir, PATH_TO_STACKS)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
elif target == self.TARGET_CUSTOM_ACTIONS:
base_path = os.path.join(src_dir, PATH_TO_CUSTOM_ACTIONS)
configs_path = os.path.join(src_dir, PATH_TO_CUSTOM_ACTION_TESTS, "configs")
elif target == self.TARGET_COMMON_SERVICES:
base_path = os.path.join(src_dir, PATH_TO_COMMON_SERVICES)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, hdp_stack_version, "configs")
else:
raise RuntimeError("Wrong target value %s", target)
script_path = os.path.join(base_path, norm_path)
if config_file is not None and config_dict is None:
config_file_path = os.path.join(configs_path, config_file)
try:
with open(config_file_path, "r") as f:
self.config_dict = json.load(f)
except IOError:
raise RuntimeError("Can not read config file: "+ config_file_path)
elif config_dict is not None and config_file is None:
self.config_dict = config_dict
else:
raise RuntimeError("Please specify either config_file_path or config_dict parameter")
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform, 'linux_distribution', return_value=os_type):
script_module = imp.load_source(classname, script_path)
script_class_inst = RMFTestCase._get_attr(script_module, classname)()
method = RMFTestCase._get_attr(script_class_inst, command)
except IOError, err:
raise RuntimeError("Cannot load class %s from %s: %s" % (classname, norm_path, err.message))
def execute(configurations={}, parameters={}, host_name=None):
BASE_URL = "http://{0}:{1}/pxf/".format(host_name, PXF_PORT)
try:
# Get delegation token if security is enabled
if CLUSTER_ENV_SECURITY in configurations and configurations[
CLUSTER_ENV_SECURITY].lower() == "true":
if 'dfs.nameservices' in configurations[HDFS_SITE]:
namenode_address = get_active_namenode(
ConfigDictionary(configurations[HDFS_SITE]),
configurations[CLUSTER_ENV_SECURITY],
configurations[HADOOP_ENV_HDFS_USER])[1]
else:
namenode_address = configurations[HDFS_SITE][
'dfs.namenode.http-address']
token = _get_delegation_token(
namenode_address, configurations[HADOOP_ENV_HDFS_USER],
configurations[HADOOP_ENV_HDFS_USER_KEYTAB],
configurations[HADOOP_ENV_HDFS_PRINCIPAL_NAME], None)
commonPXFHeaders.update({"X-GP-TOKEN": token})
if _get_pxf_protocol_version(BASE_URL).startswith("v"):
return (RESULT_STATE_OK, ['PXF is functional'])
message = "Unable to determine PXF version"
logger.exception(message)
raise Exception(message)
except Exception as e:
message = 'PXF is not functional on host, {0}: {1}'.format(
host_name, e)
logger.exception(message)
return (RESULT_STATE_WARNING, [message])
def __generate_config_files(self):
"""
Generates pxf-env.sh file from jinja template and sets the classpath for HDP
"""
import params
import shutil
hdp_stack = "HDP"
# Create file pxf-env.sh from jinja template
File("{0}/pxf-env.sh".format(params.pxf_conf_dir),
content=Template("pxf-env.j2"))
# Classpath is set for PHD by default. If stack is HDP, set classpath for HDP
if (params.stack_name == hdp_stack):
shutil.copy2(
"{0}/pxf-privatehdp.classpath".format(params.pxf_conf_dir),
"{0}/pxf-private.classpath".format(params.pxf_conf_dir))
if params.security_enabled:
pxf_site_dict = dict(params.config['configurations']['pxf-site'])
pxf_site_dict[
'pxf.service.kerberos.principal'] = "{0}/_HOST@{1}".format(
params.pxf_user, params.realm_name)
pxf_site = ConfigDictionary(pxf_site_dict)
else:
pxf_site = params.config['configurations']['pxf-site']
XmlConfig(
"pxf-site.xml",
conf_dir=params.pxf_conf_dir,
configurations=pxf_site,
configuration_attributes=params.config['configuration_attributes']
['pxf-site'])
def __update_hdfs_client():
"""
Writes hdfs-client.xml on the local filesystem on hawq nodes.
If hdfs ha is enabled, appends related parameters to hdfs-client.xml
"""
import params
hdfs_client_dict = params.hdfs_client.copy()
# security
if params.security_enabled:
hdfs_client_dict["hadoop.security.authentication"] = "kerberos"
else:
hdfs_client_dict.pop("hadoop.security.authentication",
None) # remove the entry
XmlConfig(
"hdfs-client.xml",
conf_dir=hawq_constants.hawq_config_dir,
configurations=ConfigDictionary(hdfs_client_dict),
configuration_attributes=params.config['configuration_attributes']
['hdfs-client'],
owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group,
mode=0644)
def __update_hawq_site():
"""
Sets up hawq-site.xml
"""
import params
hawq_site_modifiable = dict(params.hawq_site)
if params.security_enabled:
hawq_site_modifiable["enable_secure_filesystem"] = "ON"
hawq_site_modifiable[
"krb_server_keyfile"] = hawq_constants.hawq_keytab_file
else:
hawq_site_modifiable.pop("enable_secure_filesystem",
None) # remove the entry
hawq_site_modifiable.pop("krb_server_keyfile",
None) # remove the entry
XmlConfig(
"hawq-site.xml",
conf_dir=hawq_constants.hawq_config_dir,
configurations=ConfigDictionary(hawq_site_modifiable),
configuration_attributes=params.config['configuration_attributes']
['hawq-site'],
owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group,
mode=0644)
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
# set up logging (two separate loggers for stderr and stdout with different loglevels)
logger = logging.getLogger('resource_management')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
chout = logging.StreamHandler(sys.stdout)
chout.setLevel(logging.INFO)
chout.setFormatter(formatter)
cherr = logging.StreamHandler(sys.stderr)
cherr.setLevel(logging.ERROR)
cherr.setFormatter(formatter)
logger.addHandler(cherr)
logger.addHandler(chout)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
command_data_file = sys.argv[2]
basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
try:
with open(command_data_file, "r") as f:
pass
Script.config = ConfigDictionary(json.load(f))
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(basedir) as env:
method(env)
except ClientComponentHasNoStatus or ComponentIsNotRunning:
# Support of component status checks.
# Non-zero exit code is interpreted as an INSTALLED status of a component
sys.exit(1)
except Fail:
logger.exception(
"Error while executing command '{0}':".format(command_name))
sys.exit(1)
def executeScript(self, path, classname=None, command=None, config_file=None,
# common mocks for all the scripts
config_overrides = None,
shell_mock_value = (0, "OK."),
os_type=('Suse','11','Final'),
kinit_path_local="/usr/bin/kinit"
):
norm_path = os.path.normpath(path)
src_dir = RMFTestCase._getSrcFolder()
stack_version = norm_path.split(os.sep)[0]
stacks_path = os.path.join(src_dir, PATH_TO_STACKS)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
script_path = os.path.join(stacks_path, norm_path)
config_file_path = os.path.join(configs_path, config_file)
try:
with open(config_file_path, "r") as f:
self.config_dict = json.load(f)
except IOError:
raise RuntimeError("Can not read config file: "+ config_file_path)
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform, 'linux_distribution', return_value=os_type):
script_module = imp.load_source(classname, script_path)
except IOError:
raise RuntimeError("Cannot load class %s from %s",classname, norm_path)
script_class_inst = RMFTestCase._get_attr(script_module, classname)()
method = RMFTestCase._get_attr(script_class_inst, command)
# Reload params import, otherwise it won't change properties during next import
if 'params' in sys.modules:
del(sys.modules["params"])
# run
with Environment(basedir, test_mode=True) as RMFTestCase.env:
with patch('resource_management.core.shell.checked_call', return_value=shell_mock_value): # we must always mock any shell calls
with patch.object(Script, 'get_config', return_value=self.config_dict): # mocking configurations
with patch.object(Script, 'get_tmp_dir', return_value="/tmp"):
with patch.object(Script, 'install_packages'):
with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local):
with patch.object(platform, 'linux_distribution', return_value=os_type):
method(RMFTestCase.env)
sys.path.remove(scriptsdir)
def __update_hawq_site():
"""
Sets up hawq-site.xml
"""
import params
XmlConfig(
"hawq-site.xml",
conf_dir=hawq_constants.hawq_config_dir,
configurations=ConfigDictionary(params.hawq_site),
configuration_attributes=params.config['configuration_attributes']
['hawq-site'],
owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group,
mode=0644)
def __update_hdfs_client():
"""
Writes hdfs-client.xml on the local filesystem on hawq nodes.
If hdfs ha is enabled, appends related parameters to hdfs-client.xml
"""
import params
hdfs_client_dict = params.hdfs_client.copy()
XmlConfig(
"hdfs-client.xml",
conf_dir=hawq_constants.hawq_config_dir,
configurations=ConfigDictionary(hdfs_client_dict),
configuration_attributes=params.config['configuration_attributes']
['hdfs-client'],
owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group,
mode=0644)
def executeScript(
self,
path,
classname=None,
command=None,
config_file=None,
config_dict=None,
# common mocks for all the scripts
config_overrides=None,
stack_version=None,
checked_call_mocks=itertools.cycle([(0, "OK.")]),
call_mocks=itertools.cycle([(0, "OK.")]),
os_type=('Suse', '11', 'Final'),
kinit_path_local="/usr/bin/kinit",
os_env={'PATH': '/bin'},
target=TARGET_STACKS,
mocks_dict={},
try_install=False,
command_args=[],
log_out_files=False):
norm_path = os.path.normpath(path)
if target == self.TARGET_STACKS:
stack_version = norm_path.split(os.sep)[0]
base_path, configs_path = self._get_test_paths(target, stack_version)
script_path = os.path.join(base_path, norm_path)
if config_file is not None and config_dict is None:
self.config_dict = self.get_config_file(configs_path, config_file)
elif config_dict is not None and config_file is None:
self.config_dict = config_dict
else:
raise RuntimeError(
"Please specify either config_file_path or config_dict parameter"
)
# add the stack tools & features from the stack if the test case's JSON file didn't have them
if "stack_tools" not in self.config_dict["configurations"][
"cluster-env"]:
self.config_dict["configurations"]["cluster-env"][
"stack_tools"] = RMFTestCase.get_stack_tools()
if "stack_features" not in self.config_dict["configurations"][
"cluster-env"]:
self.config_dict["configurations"]["cluster-env"][
"stack_features"] = RMFTestCase.get_stack_features()
if "stack_packages" not in self.config_dict["configurations"][
"cluster-env"]:
self.config_dict["configurations"]["cluster-env"][
"stack_packages"] = RMFTestCase.get_stack_packages()
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform,
'linux_distribution',
return_value=os_type):
script_module = imp.load_source(classname, script_path)
Script.instance = None
script_class_inst = RMFTestCase._get_attr(
script_module, classname)()
script_class_inst.log_out_files = log_out_files
method = RMFTestCase._get_attr(script_class_inst, command)
except IOError, err:
raise RuntimeError("Cannot load class %s from %s: %s" %
(classname, norm_path, err.message))
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option(
"-o",
"--out-files-logging",
dest="log_out_files",
action="store_true",
help=
"use this option to enable outputting *.out files of the service pre-start"
)
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script argument for forcing https protocol
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
Logger.logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if self.command_name == "start" and not self.is_hook():
self.pre_start()
method(env)
if self.command_name == "start" and not self.is_hook():
self.post_start()
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
help="use this option to enable outputting *.out files of the service pre-start")
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
if len(sys.argv) >= 9:
Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
# make sure that script has forced https protocol and ca_certs file passed from agent
ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
from resource_management.libraries.functions import lzo_utils
repo_tags_to_skip = set()
if not lzo_utils.is_gpl_license_accepted():
repo_tags_to_skip.add("GPL")
Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post', env)
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out(self.command_name)
def __update_yarn_client():
"""
Writes yarn-client.xml on the local filesystem on hawq nodes.
If yarn ha is enabled, appends related parameters to yarn-client.xml
"""
import params
yarn_client_dict = params.yarn_client.copy()
if params.yarn_ha_enabled:
# Temporary logic, this logic will be moved in ambari-web to expose these parameters on UI once Yarn HA is enabled
rm_ids = [
rm_id.strip() for rm_id in params.config['configurations']
['yarn-site']['yarn.resourcemanager.ha.rm-ids'].split(',')
]
rm_id1 = rm_ids[0]
rm_id2 = rm_ids[1]
# Identify the hostname for yarn resource managers
rm_host1 = params.config['configurations']['yarn-site'][
'yarn.resourcemanager.hostname.{0}'.format(rm_id1)]
rm_host2 = params.config['configurations']['yarn-site'][
'yarn.resourcemanager.hostname.{0}'.format(rm_id2)]
# Ambari does not update yarn.resourcemanager.address.${rm_id} and yarn.resourcemanager.scheduler.address.${rm_id}
# property as its derived automatically at yarn.
# Hawq uses these properties to use yarn ha. If these properties are defined at Ambari use them, else derive them.
# Use port 8032 to derive hawq.resourcemanager.address.${rm_id}:port value if needed
rm_default_port = 8032
# Use port 8030 to derive hawq.resourcemanager.scheduler.address.${rm_id}:port value if needed
rm_scheduler_default_port = 8030
rm_address_host1 = params.config['configurations']['yarn-site'].get(
'yarn.resourcemanager.address.{0}'.format(rm_id1))
if rm_address_host1 is None:
rm_address_host1 = "{0}:{1}".format(rm_host1, rm_default_port)
rm_address_host2 = params.config['configurations']['yarn-site'].get(
'yarn.resourcemanager.address.{0}'.format(rm_id2))
if rm_address_host2 is None:
rm_address_host2 = "{0}:{1}".format(rm_host2, rm_default_port)
rm_scheduler_address_host1 = params.config['configurations'][
'yarn-site'].get(
'yarn.resourcemanager.scheduler.address.{0}'.format(rm_id1))
if rm_scheduler_address_host1 is None:
rm_scheduler_address_host1 = "{0}:{1}".format(
rm_host1, rm_scheduler_default_port)
rm_scheduler_address_host2 = params.config['configurations'][
'yarn-site'].get(
'yarn.resourcemanager.scheduler.address.{0}'.format(rm_id2))
if rm_scheduler_address_host2 is None:
rm_scheduler_address_host2 = "{0}:{1}".format(
rm_host2, rm_scheduler_default_port)
yarn_client_dict['yarn.resourcemanager.ha'] = "{0},{1}".format(
rm_address_host1, rm_address_host2)
yarn_client_dict[
'yarn.resourcemanager.scheduler.ha'] = "{0},{1}".format(
rm_scheduler_address_host1, rm_scheduler_address_host2)
XmlConfig(
"yarn-client.xml",
conf_dir=hawq_constants.hawq_config_dir,
configurations=ConfigDictionary(yarn_client_dict),
configuration_attributes=params.config['configuration_attributes']
['yarn-client'],
owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group,
mode=0644)