def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
logger, chout, cherr = Logger.initialize_logger()
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
if command_name == "install":
self.set_version()
except ClientComponentHasNoStatus or ComponentIsNotRunning:
# Support of component status checks.
# Non-zero exit code is interpreted as an INSTALLED status of a component
sys.exit(1)
except Fail:
logger.exception("Error while executing command '{0}':".format(command_name))
sys.exit(1)
finally:
if self.should_expose_component_version(command_name):
self.save_component_version_to_structured_out()
def executeScript(
self,
path,
classname=None,
command=None,
config_file=None,
config_dict=None,
# common mocks for all the scripts
config_overrides=None,
hdp_stack_version=None,
checked_call_mocks=itertools.cycle([(0, "OK.")]),
call_mocks=itertools.cycle([(0, "OK.")]),
os_type=('Suse', '11', 'Final'),
kinit_path_local="/usr/bin/kinit",
os_env={'PATH': '/bin'},
target=TARGET_STACKS,
mocks_dict={},
try_install=False):
norm_path = os.path.normpath(path)
src_dir = RMFTestCase.get_src_folder()
if target == self.TARGET_STACKS:
stack_version = norm_path.split(os.sep)[0]
base_path = os.path.join(src_dir, PATH_TO_STACKS)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS,
stack_version, "configs")
elif target == self.TARGET_CUSTOM_ACTIONS:
base_path = os.path.join(src_dir, PATH_TO_CUSTOM_ACTIONS)
configs_path = os.path.join(src_dir, PATH_TO_CUSTOM_ACTION_TESTS,
"configs")
elif target == self.TARGET_COMMON_SERVICES:
base_path = os.path.join(src_dir, PATH_TO_COMMON_SERVICES)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS,
hdp_stack_version, "configs")
else:
raise RuntimeError("Wrong target value %s", target)
script_path = os.path.join(base_path, norm_path)
if config_file is not None and config_dict is None:
config_file_path = os.path.join(configs_path, config_file)
try:
with open(config_file_path, "r") as f:
self.config_dict = json.load(f)
except IOError:
raise RuntimeError("Can not read config file: " +
config_file_path)
elif config_dict is not None and config_file is None:
self.config_dict = config_dict
else:
raise RuntimeError(
"Please specify either config_file_path or config_dict parameter"
)
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform,
'linux_distribution',
return_value=os_type):
script_module = imp.load_source(classname, script_path)
except IOError, err:
raise RuntimeError("Cannot load class %s from %s: %s" %
(classname, norm_path, err.message))
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option(
"-o",
"--out-files-logging",
dest="log_out_files",
action="store_true",
help=
"use this option to enable outputting *.out files of the service pre-start"
)
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script argument for forcing https protocol
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
Logger.logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post',
env)
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def executeScript(
self,
path,
classname=None,
command=None,
config_file=None,
config_dict=None,
# common mocks for all the scripts
config_overrides=None,
stack_version=None,
checked_call_mocks=itertools.cycle([(0, "OK.")]),
call_mocks=itertools.cycle([(0, "OK.")]),
os_type=('Suse', '11', 'Final'),
kinit_path_local="/usr/bin/kinit",
os_env={'PATH': '/bin'},
target=TARGET_STACKS,
mocks_dict={},
try_install=False,
command_args=[],
log_out_files=False,
available_packages_in_repos=[]):
norm_path = os.path.normpath(path)
if target == self.TARGET_STACKS:
stack_version = norm_path.split(os.sep)[0]
base_path, configs_path = self._get_test_paths(target, stack_version)
script_path = os.path.join(base_path, norm_path)
if config_file is not None and config_dict is None:
self.config_dict = self.get_config_file(configs_path, config_file)
elif config_dict is not None and config_file is None:
self.config_dict = config_dict
else:
raise RuntimeError(
"Please specify either config_file or config_dict parameter")
# add the stack tools & features from the stack if the test case's JSON file didn't have them
if "stack_tools" not in self.config_dict["configurations"][
"cluster-env"]:
self.config_dict["configurations"]["cluster-env"][
"stack_tools"] = RMFTestCase.get_stack_tools()
if "stack_features" not in self.config_dict["configurations"][
"cluster-env"]:
self.config_dict["configurations"]["cluster-env"][
"stack_features"] = RMFTestCase.get_stack_features()
if "stack_packages" not in self.config_dict["configurations"][
"cluster-env"]:
self.config_dict["configurations"]["cluster-env"][
"stack_packages"] = RMFTestCase.get_stack_packages()
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform,
'linux_distribution',
return_value=os_type):
script_module = imp.load_source(classname, script_path)
Script.instance = None
script_class_inst = RMFTestCase._get_attr(
script_module, classname)()
script_class_inst.log_out_files = log_out_files
script_class_inst.available_packages_in_repos = available_packages_in_repos
Script.repository_util = RepositoryUtil(
self.config_dict, set())
method = RMFTestCase._get_attr(script_class_inst, command)
except IOError, err:
raise RuntimeError("Cannot load class %s from %s: %s" %
(classname, norm_path, err.message))
def __update_yarn_client():
"""
Writes yarn-client.xml on the local filesystem on hawq nodes.
If yarn ha is enabled, appends related parameters to yarn-client.xml
"""
import params
yarn_client_dict = params.yarn_client.copy()
if params.yarn_ha_enabled:
# Temporary logic, this logic will be moved in ambari-web to expose these parameters on UI once Yarn HA is enabled
rm_ids = [
rm_id.strip() for rm_id in params.config['configurations']
['yarn-site']['yarn.resourcemanager.ha.rm-ids'].split(',')
]
rm_id1 = rm_ids[0]
rm_id2 = rm_ids[1]
# Identify the hostname for yarn resource managers
rm_host1 = params.config['configurations']['yarn-site'][
'yarn.resourcemanager.hostname.{0}'.format(rm_id1)]
rm_host2 = params.config['configurations']['yarn-site'][
'yarn.resourcemanager.hostname.{0}'.format(rm_id2)]
# Ambari does not update yarn.resourcemanager.address.${rm_id} and yarn.resourcemanager.scheduler.address.${rm_id}
# property as its derived automatically at yarn.
# Hawq uses these properties to use yarn ha. If these properties are defined at Ambari use them, else derive them.
# Use port 8032 to derive hawq.resourcemanager.address.${rm_id}:port value if needed
rm_default_port = 8032
# Use port 8030 to derive hawq.resourcemanager.scheduler.address.${rm_id}:port value if needed
rm_scheduler_default_port = 8030
rm_address_host1 = params.config['configurations']['yarn-site'].get(
'yarn.resourcemanager.address.{0}'.format(rm_id1))
if rm_address_host1 is None:
rm_address_host1 = "{0}:{1}".format(rm_host1, rm_default_port)
rm_address_host2 = params.config['configurations']['yarn-site'].get(
'yarn.resourcemanager.address.{0}'.format(rm_id2))
if rm_address_host2 is None:
rm_address_host2 = "{0}:{1}".format(rm_host2, rm_default_port)
rm_scheduler_address_host1 = params.config['configurations'][
'yarn-site'].get(
'yarn.resourcemanager.scheduler.address.{0}'.format(rm_id1))
if rm_scheduler_address_host1 is None:
rm_scheduler_address_host1 = "{0}:{1}".format(
rm_host1, rm_scheduler_default_port)
rm_scheduler_address_host2 = params.config['configurations'][
'yarn-site'].get(
'yarn.resourcemanager.scheduler.address.{0}'.format(rm_id2))
if rm_scheduler_address_host2 is None:
rm_scheduler_address_host2 = "{0}:{1}".format(
rm_host2, rm_scheduler_default_port)
yarn_client_dict['yarn.resourcemanager.ha'] = "{0},{1}".format(
rm_address_host1, rm_address_host2)
yarn_client_dict[
'yarn.resourcemanager.scheduler.ha'] = "{0},{1}".format(
rm_scheduler_address_host1, rm_scheduler_address_host2)
XmlConfig(
"yarn-client.xml",
conf_dir=hawq_constants.hawq_config_dir,
configurations=ConfigDictionary(yarn_client_dict),
configuration_attributes=params.config['configuration_attributes']
['yarn-client'],
owner=hawq_constants.hawq_user,
group=hawq_constants.hawq_group,
mode=0644)
def executeScript(
self,
path,
classname=None,
command=None,
config_file=None,
# common mocks for all the scripts
config_overrides=None,
shell_mock_value=(0, "OK."),
os_type=('Suse', '11', 'Final'),
kinit_path_local="/usr/bin/kinit"):
norm_path = os.path.normpath(path)
src_dir = RMFTestCase._getSrcFolder()
stack_version = norm_path.split(os.sep)[0]
stacks_path = os.path.join(src_dir, PATH_TO_STACKS)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS,
stack_version, "configs")
script_path = os.path.join(stacks_path, norm_path)
config_file_path = os.path.join(configs_path, config_file)
try:
with open(config_file_path, "r") as f:
self.config_dict = json.load(f)
except IOError:
raise RuntimeError("Can not read config file: " + config_file_path)
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform,
'linux_distribution',
return_value=os_type):
script_module = imp.load_source(classname, script_path)
except IOError:
raise RuntimeError("Cannot load class %s from %s", classname,
norm_path)
script_class_inst = RMFTestCase._get_attr(script_module, classname)()
method = RMFTestCase._get_attr(script_class_inst, command)
# Reload params import, otherwise it won't change properties during next import
if 'params' in sys.modules:
del (sys.modules["params"])
# run
with Environment(basedir, test_mode=True) as RMFTestCase.env:
with patch('resource_management.core.shell.checked_call',
return_value=shell_mock_value
): # we must always mock any shell calls
with patch.object(Script,
'get_config',
return_value=self.config_dict
): # mocking configurations
with patch.object(Script, 'install_packages'):
with patch(
'resource_management.libraries.functions.get_kinit_path',
return_value=kinit_path_local):
with patch.object(platform,
'linux_distribution',
return_value=os_type):
method(RMFTestCase.env)
sys.path.remove(scriptsdir)