def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
logger, chout, cherr = Logger.initialize_logger(__name__)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(command_name):
self.save_component_version_to_structured_out()
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
if OSCheck.is_windows_family():
if name[-4:] == ".msi":
#TODO all msis must be located in resource folder of server, change it to repo later
Msi(name, http_source=os.path.join(config['hostLevelParams']['jdk_location']))
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"),
config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
# RepoInstaller.remove_repos(config)
pass
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
if OSCheck.is_windows_family():
if name[-4:] == ".msi":
#TODO all msis must be located in resource folder of server, change it to repo later
Msi(name, http_source=os.path.join(config['hostLevelParams']['jdk_location']))
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"),
config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
pass
def install_windows_msi(msi_url, save_dir, save_file, hadoop_user,
hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("Global\\hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info(
"Some other task currently installing hdp.msi, waiting for 10 min for finish"
)
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
hdp_stack_version = format_hdp_stack_version(stack_version)
hdp_22_specific_props = ''
if hdp_stack_version != "" and compare_versions(
hdp_stack_version, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir)
# install msi
try:
download_file(msi_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=msi_url))
File(os.path.join(msi_save_dir, "properties.txt"),
content=cluster_properties.format(
hdp_log_dir=hdp_log_dir,
hdp_data_dir=hdp_data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi"))
hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log"))
hdp_layout_path = os_utils.quote_path(
os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path,
hdp_log_path=hdp_log_path,
hdp_layout_path=hdp_layout_path,
hadoop_user=hadoop_user,
hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_user, hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(
config['hostLevelParams']
['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(
config['hostLevelParams']['agent_stack_retry_count'])
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if self.check_package_condition(package):
name = self.format_package_name(package['name'])
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("Global\\hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish")
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
stack_version_formatted = format_stack_version(stack_version)
hdp_22_specific_props = ''
if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(data_dir=data_dir)
# MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation
msi_file = ''
for save_file in save_files:
if save_file.lower().endswith(".msi"):
msi_file = save_file
file_url = urlparse.urljoin(url_base, save_file)
try:
download_file(file_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=file_url))
File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir,
data_dir=data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
# install msi
msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file))
log_path = os_utils.quote_path(os.path.join(save_dir, msi_file[:-3] + "log"))
layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path,
hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_user, hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
# parse arguments
if len(sys.argv) < 7:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logging.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count'])
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if Script.check_package_condition(package):
name = self.format_package_name(package['name'])
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
install_windows_msi(config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], ["hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab"],
hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def install_windows_msi(msi_url, save_dir, save_file, hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish")
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
hdp_stack_version = format_hdp_stack_version(stack_version)
hdp_22_specific_props = ''
if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir)
# install msi
download_file(msi_url, os.path.join(msi_save_dir, save_file))
File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(hdp_log_dir=hdp_log_dir,
hdp_data_dir=hdp_data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi"))
hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log"))
hdp_layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path,
hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
self.set_version()
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
if OSCheck.is_windows_family():
if name[-4:] == ".msi":
#TODO all msis must be located in resource folder of server, change it to repo later
Msi(name,
http_source=os.path.join(
config['hostLevelParams']
['jdk_location']))
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
os.path.join(config['hostLevelParams']['jdk_location'],
"hdp.msi"),
config["hostLevelParams"]["agentCacheDir"], "hdp.msi",
hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
pass
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option(
"-o",
"--out-files-logging",
dest="log_out_files",
action="store_true",
help=
"use this option to enable outputting *.out files of the service pre-start"
)
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script argument for forcing https protocol
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
Logger.logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if self.command_name == "start" and not self.is_hook():
self.pre_start()
method(env)
if self.command_name == "start" and not self.is_hook():
self.post_start()
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
help="use this option to enable outputting *.out files of the service pre-start")
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
if len(sys.argv) >= 9:
Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
# make sure that script has forced https protocol and ca_certs file passed from agent
ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
from resource_management.libraries.functions import lzo_utils
repo_tags_to_skip = set()
if not lzo_utils.is_gpl_license_accepted():
repo_tags_to_skip.add("GPL")
Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post', env)
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out(self.command_name)
def install_windows_msi(url_base, save_dir, save_files, hadoop_user,
hadoop_password, stack_version):
global _working_dir
_working_dir = save_dir
save_dir = os.path.abspath(save_dir)
msi_save_dir = save_dir
# system wide lock to prevent simultaneous installations(when first task failed on timeout)
install_lock = SystemWideLock("Global\\hdp_msi_lock")
try:
# try to acquire lock
if not install_lock.lock():
Logger.info(
"Some other task currently installing hdp.msi, waiting for 10 min for finish"
)
if not install_lock.lock(600000):
raise Fail("Timeout on acquiring lock")
if _validate_msi_install():
Logger.info("hdp.msi already installed")
return
stack_version_formatted = format_stack_version(stack_version)
hdp_22_specific_props = ''
if stack_version_formatted != "" and compare_versions(
stack_version_formatted, '2.2') >= 0:
hdp_22_specific_props = hdp_22.format(data_dir=data_dir)
# MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation
msi_file = ''
for save_file in save_files:
if save_file.lower().endswith(".msi"):
msi_file = save_file
file_url = urlparse.urljoin(url_base, save_file)
try:
download_file(file_url, os.path.join(msi_save_dir, save_file))
except:
raise Fail("Failed to download {url}".format(url=file_url))
File(os.path.join(msi_save_dir, "properties.txt"),
content=cluster_properties.format(
log_dir=log_dir,
data_dir=data_dir,
local_host=local_host,
db_flavor=db_flavor,
hdp_22_specific_props=hdp_22_specific_props))
# install msi
msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file))
log_path = os_utils.quote_path(
os.path.join(save_dir, msi_file[:-3] + "log"))
layout_path = os_utils.quote_path(
os.path.join(save_dir, "properties.txt"))
hadoop_password_arg = os_utils.quote_path(hadoop_password)
Execute(
INSTALL_MSI_CMD.format(msi_path=msi_path,
log_path=log_path,
layout_path=layout_path,
hadoop_user=hadoop_user,
hadoop_password_arg=hadoop_password_arg))
reload_windows_env()
# create additional services manually due to hdp.msi limitaitons
_ensure_services_created(hadoop_user, hadoop_password)
_create_symlinks(stack_version)
# finalizing install
_write_marker()
_validate_msi_install()
finally:
install_lock.unlock()