def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ logger, chout, cherr = Logger.initialize_logger(__name__) # parse arguments if len(sys.argv) < 7: logger.error("Script expects at least 6 arguments") print USAGE.format(os.path.basename( sys.argv[0])) # print to stdout sys.exit(1) command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] logging_level_str = logging._levelNames[self.logging_level] chout.setLevel(logging_level_str) logger.setLevel(logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration( k, Script.config) and get_path_from_configuration( v, Script.config): Script.passwords[get_path_from_configuration( k, Script.config)] = get_path_from_configuration( v, Script.config) except IOError: logger.exception( "Can not read json file with command parameters: ") sys.exit(1) # Run class method depending on a command type try: method = self.choose_method_to_execute(command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir method(env) finally: if self.should_expose_component_version(command_name): self.save_component_version_to_structured_out()
def install_packages(self, env, exclude_packages=[]): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list """ config = self.get_config() try: package_list_str = config['hostLevelParams']['package_list'] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if not package['name'] in exclude_packages: name = package['name'] if OSCheck.is_windows_family(): if name[-4:] == ".msi": #TODO all msis must be located in resource folder of server, change it to repo later Msi(name, http_source=os.path.join(config['hostLevelParams']['jdk_location'])) else: Package(name) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"), config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"), str(config['hostLevelParams']['stack_version'])) reload_windows_env() # RepoInstaller.remove_repos(config) pass
def install_packages(self, env, exclude_packages=[]): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list """ config = self.get_config() try: package_list_str = config['hostLevelParams']['package_list'] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if not package['name'] in exclude_packages: name = package['name'] if OSCheck.is_windows_family(): if name[-4:] == ".msi": #TODO all msis must be located in resource folder of server, change it to repo later Msi(name, http_source=os.path.join(config['hostLevelParams']['jdk_location'])) else: Package(name) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"), config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"), str(config['hostLevelParams']['stack_version'])) reload_windows_env() pass
def install_windows_msi(msi_url, save_dir, save_file, hadoop_user, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("Global\\hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info( "Some other task currently installing hdp.msi, waiting for 10 min for finish" ) if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return hdp_stack_version = format_hdp_stack_version(stack_version) hdp_22_specific_props = '' if hdp_stack_version != "" and compare_versions( hdp_stack_version, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir) # install msi try: download_file(msi_url, os.path.join(msi_save_dir, save_file)) except: raise Fail("Failed to download {url}".format(url=msi_url)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format( hdp_log_dir=hdp_log_dir, hdp_data_dir=hdp_data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi")) hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log")) hdp_layout_path = os_utils.quote_path( os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path, hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_user, hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()
def install_packages(self, env): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list exclude_packages - list of regexes (possibly raw strings as well), the packages which match the regex won't be installed. NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ? """ config = self.get_config() if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: Logger.info("Node has all packages pre-installed. Skipping.") return pass try: package_list_str = config['hostLevelParams']['package_list'] agent_stack_retry_on_unavailability = bool( config['hostLevelParams'] ['agent_stack_retry_on_unavailability']) agent_stack_retry_count = int( config['hostLevelParams']['agent_stack_retry_count']) if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if self.check_package_condition(package): name = self.format_package_name(package['name']) # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support # <osFamily>any<osFamily> which would cause installation failure on Windows. if OSCheck.is_windows_family(): if "ambari-metrics" in name: Package(name) else: Package(name, retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented hadoop_user = config["configurations"]["cluster-env"][ "hadoop.user.name"] install_windows_msi( config['hostLevelParams']['jdk_location'], config["hostLevelParams"]["agentCacheDir"], [ "hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab" ], hadoop_user, self.get_password(hadoop_user), str(config['hostLevelParams']['stack_version'])) reload_windows_env()
def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("Global\\hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish") if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return stack_version_formatted = format_stack_version(stack_version) hdp_22_specific_props = '' if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(data_dir=data_dir) # MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation msi_file = '' for save_file in save_files: if save_file.lower().endswith(".msi"): msi_file = save_file file_url = urlparse.urljoin(url_base, save_file) try: download_file(file_url, os.path.join(msi_save_dir, save_file)) except: raise Fail("Failed to download {url}".format(url=file_url)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(log_dir=log_dir, data_dir=data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) # install msi msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file)) log_path = os_utils.quote_path(os.path.join(save_dir, msi_file[:-3] + "log")) layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path, hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_user, hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()
def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ # parse arguments if len(sys.argv) < 7: print "Script expects at least 6 arguments" print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout sys.exit(1) self.command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] logging_level_str = logging._levelNames[self.logging_level] Logger.initialize_logger(__name__, logging_level=logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config): Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config) except IOError: Logging.logger.exception("Can not read json file with command parameters: ") sys.exit(1) # Run class method depending on a command type try: method = self.choose_method_to_execute(self.command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir method(env) finally: if self.should_expose_component_version(self.command_name): self.save_component_version_to_structured_out()
def install_packages(self, env): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list exclude_packages - list of regexes (possibly raw strings as well), the packages which match the regex won't be installed. NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ? """ config = self.get_config() if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: Logger.info("Node has all packages pre-installed. Skipping.") return pass try: package_list_str = config['hostLevelParams']['package_list'] agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if Script.check_package_condition(package): name = self.format_package_name(package['name']) # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support # <osFamily>any<osFamily> which would cause installation failure on Windows. if OSCheck.is_windows_family(): if "ambari-metrics" in name: Package(name) else: Package(name, retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] install_windows_msi(config['hostLevelParams']['jdk_location'], config["hostLevelParams"]["agentCacheDir"], ["hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab"], hadoop_user, self.get_password(hadoop_user), str(config['hostLevelParams']['stack_version'])) reload_windows_env()
def install_windows_msi(msi_url, save_dir, save_file, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info("Some other task currently installing hdp.msi, waiting for 10 min for finish") if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return hdp_stack_version = format_hdp_stack_version(stack_version) hdp_22_specific_props = '' if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(hdp_data_dir=hdp_data_dir) # install msi download_file(msi_url, os.path.join(msi_save_dir, save_file)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format(hdp_log_dir=hdp_log_dir, hdp_data_dir=hdp_data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) hdp_msi_path = os_utils.quote_path(os.path.join(save_dir, "hdp.msi")) hdp_log_path = os_utils.quote_path(os.path.join(save_dir, "hdp.log")) hdp_layout_path = os_utils.quote_path(os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(hdp_msi_path=hdp_msi_path, hdp_log_path=hdp_log_path, hdp_layout_path=hdp_layout_path, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()
def install_packages(self, env, exclude_packages=[]): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list """ config = self.get_config() if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: Logger.info("Node has all packages pre-installed. Skipping.") return pass try: package_list_str = config['hostLevelParams']['package_list'] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if not package['name'] in exclude_packages: name = package['name'] # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support # <osFamily>any<osFamily> which would cause installation failure on Windows. if OSCheck.is_windows_family(): if "ambari-metrics" in name: Package(name) else: Package(name) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented hadoop_user = config["configurations"]["cluster-env"][ "hadoop.user.name"] install_windows_msi( config['hostLevelParams']['jdk_location'], config["hostLevelParams"]["agentCacheDir"], [ "hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab" ], hadoop_user, self.get_password(hadoop_user), str(config['hostLevelParams']['stack_version'])) reload_windows_env() self.set_version()
def install_packages(self, env, exclude_packages=[]): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list """ config = self.get_config() if 'host_sys_prepped' in config['hostLevelParams']: # do not install anything on sys-prepped host if config['hostLevelParams']['host_sys_prepped'] == True: Logger.info("Node has all packages pre-installed. Skipping.") return pass try: package_list_str = config['hostLevelParams']['package_list'] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if not package['name'] in exclude_packages: name = package['name'] if OSCheck.is_windows_family(): if name[-4:] == ".msi": #TODO all msis must be located in resource folder of server, change it to repo later Msi(name, http_source=os.path.join( config['hostLevelParams'] ['jdk_location'])) else: Package(name) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented hadoop_user = config["configurations"]["cluster-env"][ "hadoop.user.name"] install_windows_msi( os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"), config["hostLevelParams"]["agentCacheDir"], "hdp.msi", hadoop_user, self.get_password(hadoop_user), str(config['hostLevelParams']['stack_version'])) reload_windows_env() pass
def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ parser = OptionParser() parser.add_option( "-o", "--out-files-logging", dest="log_out_files", action="store_true", help= "use this option to enable outputting *.out files of the service pre-start" ) (self.options, args) = parser.parse_args() self.log_out_files = self.options.log_out_files # parse arguments if len(args) < 6: print "Script expects at least 6 arguments" print USAGE.format(os.path.basename( sys.argv[0])) # print to stdout sys.exit(1) self.command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] # optional script argument for forcing https protocol if len(sys.argv) >= 8: Script.force_https_protocol = sys.argv[7] logging_level_str = logging._levelNames[self.logging_level] Logger.initialize_logger(__name__, logging_level=logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration( k, Script.config) and get_path_from_configuration( v, Script.config): Script.passwords[get_path_from_configuration( k, Script.config)] = get_path_from_configuration( v, Script.config) except IOError: Logger.logger.exception( "Can not read json file with command parameters: ") sys.exit(1) # Run class method depending on a command type try: method = self.choose_method_to_execute(self.command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir if self.command_name == "start" and not self.is_hook(): self.pre_start() method(env) if self.command_name == "start" and not self.is_hook(): self.post_start() except Fail as ex: ex.pre_raise() raise finally: if self.should_expose_component_version(self.command_name): self.save_component_version_to_structured_out()
def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ parser = OptionParser() parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true", help="use this option to enable outputting *.out files of the service pre-start") (self.options, args) = parser.parse_args() self.log_out_files = self.options.log_out_files # parse arguments if len(args) < 6: print "Script expects at least 6 arguments" print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout sys.exit(1) self.command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] # optional script arguments for forcing https protocol and ca_certs file if len(sys.argv) >= 8: Script.force_https_protocol = sys.argv[7] if len(sys.argv) >= 9: Script.ca_cert_file_path = sys.argv[8] logging_level_str = logging._levelNames[self.logging_level] Logger.initialize_logger(__name__, logging_level=logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() # !!! status commands re-use structured output files; if the status command doesn't update the # the file (because it doesn't have to) then we must ensure that the file is reset to prevent # old, stale structured output from a prior status command from being used if self.command_name == "status": Script.structuredOut = {} self.put_structured_out({}) # make sure that script has forced https protocol and ca_certs file passed from agent ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path()) try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config): Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config) except IOError: Logger.logger.exception("Can not read json file with command parameters: ") sys.exit(1) from resource_management.libraries.functions import lzo_utils repo_tags_to_skip = set() if not lzo_utils.is_gpl_license_accepted(): repo_tags_to_skip.add("GPL") Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip) # Run class method depending on a command type try: method = self.choose_method_to_execute(self.command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir if not self.is_hook(): self.execute_prefix_function(self.command_name, 'pre', env) method(env) if not self.is_hook(): self.execute_prefix_function(self.command_name, 'post', env) except Fail as ex: ex.pre_raise() raise finally: if self.should_expose_component_version(self.command_name): self.save_component_version_to_structured_out(self.command_name)
def install_windows_msi(url_base, save_dir, save_files, hadoop_user, hadoop_password, stack_version): global _working_dir _working_dir = save_dir save_dir = os.path.abspath(save_dir) msi_save_dir = save_dir # system wide lock to prevent simultaneous installations(when first task failed on timeout) install_lock = SystemWideLock("Global\\hdp_msi_lock") try: # try to acquire lock if not install_lock.lock(): Logger.info( "Some other task currently installing hdp.msi, waiting for 10 min for finish" ) if not install_lock.lock(600000): raise Fail("Timeout on acquiring lock") if _validate_msi_install(): Logger.info("hdp.msi already installed") return stack_version_formatted = format_stack_version(stack_version) hdp_22_specific_props = '' if stack_version_formatted != "" and compare_versions( stack_version_formatted, '2.2') >= 0: hdp_22_specific_props = hdp_22.format(data_dir=data_dir) # MSIs cannot be larger than 2GB. HDPWIN 2.3 needed split in order to accommodate this limitation msi_file = '' for save_file in save_files: if save_file.lower().endswith(".msi"): msi_file = save_file file_url = urlparse.urljoin(url_base, save_file) try: download_file(file_url, os.path.join(msi_save_dir, save_file)) except: raise Fail("Failed to download {url}".format(url=file_url)) File(os.path.join(msi_save_dir, "properties.txt"), content=cluster_properties.format( log_dir=log_dir, data_dir=data_dir, local_host=local_host, db_flavor=db_flavor, hdp_22_specific_props=hdp_22_specific_props)) # install msi msi_path = os_utils.quote_path(os.path.join(save_dir, msi_file)) log_path = os_utils.quote_path( os.path.join(save_dir, msi_file[:-3] + "log")) layout_path = os_utils.quote_path( os.path.join(save_dir, "properties.txt")) hadoop_password_arg = os_utils.quote_path(hadoop_password) Execute( INSTALL_MSI_CMD.format(msi_path=msi_path, log_path=log_path, layout_path=layout_path, hadoop_user=hadoop_user, hadoop_password_arg=hadoop_password_arg)) reload_windows_env() # create additional services manually due to hdp.msi limitaitons _ensure_services_created(hadoop_user, hadoop_password) _create_symlinks(stack_version) # finalizing install _write_marker() _validate_msi_install() finally: install_lock.unlock()