def falcon(type, action = None, upgrade_type=None): import params if action == 'config': env = Environment.get_instance() # These 2 parameters are used in ../templates/client.properties.j2 env.config.params["falcon_host"] = params.falcon_host env.config.params["falcon_port"] = params.falcon_port File(os.path.join(params.falcon_conf_dir, 'falcon-env.sh'), content = InlineTemplate(params.falcon_env_sh_template)) PropertiesFile(os.path.join(params.falcon_conf_dir, 'runtime.properties'), properties = params.falcon_runtime_properties) PropertiesFile(os.path.join(params.falcon_conf_dir, 'startup.properties'), properties = params.falcon_startup_properties) PropertiesFile(os.path.join(params.falcon_conf_dir, 'client.properties'), properties = params.falcon_client_properties) if type == 'server': ServiceConfig(params.falcon_win_service_name, action = "change_user", username = params.falcon_user, password = Script.get_password(params.falcon_user)) if action == 'start': Service(params.falcon_win_service_name, action = "start") if action == 'stop': Service(params.falcon_win_service_name, action = "stop")
def generate_configs(self, env): """ Generates config files and stores them as an archive in tmp_dir based on xml_configs_list and env_configs_list from commandParams """ import params env.set_params(params) config = self.get_config() xml_configs_list = config['commandParams']['xml_configs_list'] env_configs_list = config['commandParams']['env_configs_list'] properties_configs_list = config['commandParams'][ 'properties_configs_list'] Directory(self.get_tmp_dir(), create_parents=True) conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) os.chmod(conf_tmp_dir, 0700) output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) try: for file_dict in xml_configs_list: for filename, dict in file_dict.iteritems(): XmlConfig(filename, conf_dir=conf_tmp_dir, mode=0600, **self.generate_configs_get_xml_file_content( filename, dict)) for file_dict in env_configs_list: for filename, dicts in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), mode=0600, content=InlineTemplate( self.generate_configs_get_template_file_content( filename, dicts))) for file_dict in properties_configs_list: for filename, dict in file_dict.iteritems(): PropertiesFile( os.path.join(conf_tmp_dir, filename), mode=0600, properties=self.generate_configs_get_xml_file_dict( filename, dict)) with closing(tarfile.open(output_filename, "w:gz")) as tar: os.chmod(output_filename, 0600) try: tar.add(conf_tmp_dir, arcname=os.path.basename(".")) finally: tar.close() finally: Directory(conf_tmp_dir, action="delete")
def generate_configs(self, env): """ Generates config files and stores them as an archive in tmp_dir based on xml_configs_list and env_configs_list from commandParams """ import params env.set_params(params) config = self.get_config() xml_configs_list = config['commandParams']['xml_configs_list'] env_configs_list = config['commandParams']['env_configs_list'] properties_configs_list = config['commandParams'][ 'properties_configs_list'] Directory(self.get_tmp_dir(), recursive=True) conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir()) output_filename = os.path.join(self.get_tmp_dir(), config['commandParams']['output_file']) try: for file_dict in xml_configs_list: for filename, dict in file_dict.iteritems(): XmlConfig(filename, conf_dir=conf_tmp_dir, **self.generate_configs_get_xml_file_content( filename, dict)) for file_dict in env_configs_list: for filename, dicts in file_dict.iteritems(): File(os.path.join(conf_tmp_dir, filename), content=InlineTemplate( self.generate_configs_get_template_file_content( filename, dicts))) for file_dict in properties_configs_list: for filename, dict in file_dict.iteritems(): PropertiesFile( os.path.join(conf_tmp_dir, filename), properties=self.generate_configs_get_xml_file_dict( filename, dict)) archive_dir(output_filename, conf_tmp_dir) finally: Directory(conf_tmp_dir, action="delete")
def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): """ :param env: Python environment :param type: Spark component type :param upgrade_type: If in a stack upgrade, either UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING :param action: Action to perform, such as generate configs :param config_dir: Optional config directory to write configs to. """ import params # ensure that matching LZO libraries are installed for Spark lzo_utils.install_lzo_if_needed() if config_dir is None: config_dir = params.spark_conf Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group, mode=0775, create_parents = True, cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.spark_user, mode=0775 ) params.HdfsResource(None, action="execute") PropertiesFile(os.path.join(config_dir, "spark-defaults.conf"), properties = params.config['configurations']['spark-defaults'], key_value_delimiter = " ", owner=params.spark_user, group=params.spark_group, mode=0644 ) # create spark-env.sh in etc/conf dir File(os.path.join(config_dir, 'spark-env.sh'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_env_sh), mode=0644, ) #create log4j.properties in etc/conf dir File(os.path.join(config_dir, 'log4j.properties'), owner=params.spark_user, group=params.spark_group, content=params.spark_log4j_properties, mode=0644, ) #create metrics.properties in etc/conf dir File(os.path.join(config_dir, 'metrics.properties'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_metrics_properties), mode=0644 ) Directory(params.spark_logs_dir, owner=params.spark_user, group=params.spark_group, mode=0755, ) if params.is_hive_installed: XmlConfig("hive-site.xml", conf_dir=config_dir, configurations=params.spark_hive_properties, owner=params.spark_user, group=params.spark_group, mode=0644) if params.has_spark_thriftserver: PropertiesFile(params.spark_thrift_server_conf_file, properties = params.config['configurations']['spark-thrift-sparkconf'], owner = params.hive_user, group = params.user_group, key_value_delimiter = " ", mode=0644 ) effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks if effective_version: effective_version = format_stack_version(effective_version) if check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version): File(os.path.join(params.spark_conf, 'java-opts'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_javaopts_properties), mode=0644 ) else: File(os.path.join(params.spark_conf, 'java-opts'), action="delete" ) if params.spark_thrift_fairscheduler_content and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"), owner=params.spark_user, group=params.spark_group, mode=0755, content=InlineTemplate(params.spark_thrift_fairscheduler_content) )
def falcon(type, action = None, upgrade_type=None): import params if action == 'config': Directory(params.falcon_pid_dir, owner = params.falcon_user, create_parents = True, mode = 0755, cd_access = "a", ) Directory(params.falcon_log_dir, owner = params.falcon_user, create_parents = True, mode = 0755, cd_access = "a", ) Directory(params.falcon_webapp_dir, owner = params.falcon_user, create_parents = True) Directory(params.falcon_home, owner = params.falcon_user, create_parents = True) Directory(params.etc_prefix_dir, mode = 0755, create_parents = True) Directory(params.falcon_conf_dir, owner = params.falcon_user, create_parents = True) File(params.falcon_conf_dir + '/falcon-env.sh', content = InlineTemplate(params.falcon_env_sh_template), owner = params.falcon_user, group=params.user_group, ) PropertiesFile(params.falcon_conf_dir + '/client.properties', properties = params.falcon_client_properties, mode = 0644, owner = params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/runtime.properties', properties = params.falcon_runtime_properties, mode = 0644, owner = params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/startup.properties', properties = params.falcon_startup_properties, mode = 0644, owner = params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, owner = params.falcon_user, group = params.user_group, mode = 0775, create_parents = True, cd_access = "a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, owner = params.falcon_user, group = params.user_group, mode = 0775, create_parents = True, cd_access = "a") setup_atlas_falcon() if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, type = "directory", action = "create_on_execute", owner = params.falcon_user, mode = 0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner = params.falcon_user, create_parents = True) # TODO change to proper mode params.HdfsResource(params.falcon_apps_dir, type = "directory", action = "create_on_execute", owner = params.falcon_user, mode = 0777) # In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS. if params.supports_data_mirroring: params.HdfsResource(params.dfs_data_mirroring_dir, type = "directory", action = "create_on_execute", owner = params.falcon_user, group = params.proxyuser_group, recursive_chown = True, recursive_chmod = True, mode = 0770, source = params.local_data_mirroring_dir) if params.supports_falcon_extensions: params.HdfsResource(params.falcon_extensions_dest_dir, type = "directory", action = "create_on_execute", owner = params.falcon_user, group = params.proxyuser_group, recursive_chown = True, recursive_chmod = True, mode = 0755, source = params.falcon_extensions_source_dir) # Create the extensons HiveDR store params.HdfsResource(os.path.join(params.falcon_extensions_dest_dir, "mirroring"), type = "directory", action = "create_on_execute", owner = params.falcon_user, group = params.proxyuser_group, mode = 0770) # At least one HDFS Dir should be created, so execute the change now. params.HdfsResource(None, action = "execute") Directory(params.falcon_local_dir, owner = params.falcon_user, create_parents = True, cd_access = "a") if params.falcon_embeddedmq_enabled == True: Directory( os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")), owner = params.falcon_user, create_parents = True) Directory(params.falcon_embeddedmq_data, owner = params.falcon_user, create_parents = True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' # still points to older binaries; it's safer to just pass in the # hadoop home directory to use environment_dictionary = { "HADOOP_HOME" : params.hadoop_home_dir } if action == 'start': try: Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'), user = params.falcon_user, path = params.hadoop_bin_dir, environment=environment_dictionary) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if action == 'stop': try: Execute(format('{falcon_home}/bin/falcon-stop'), user = params.falcon_user, path = params.hadoop_bin_dir, environment=environment_dictionary) except: show_logs(params.falcon_log_dir, params.falcon_user) raise File(params.server_pid_file, action = 'delete')
def setup_spark(env, type, upgrade_type = None, action = None): import params # ensure that matching LZO libraries are installed for Spark lzo_utils.install_lzo_if_needed() Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group, mode=0775, create_parents = True, cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.spark_user, mode=0775 ) params.HdfsResource(None, action="execute") PropertiesFile(format("{spark_conf}/spark-defaults.conf"), properties = params.config['configurations']['spark-defaults'], key_value_delimiter = " ", owner=params.spark_user, group=params.spark_group, mode=0644 ) # create spark-env.sh in etc/conf dir File(os.path.join(params.spark_conf, 'spark-env.sh'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_env_sh), mode=0644, ) #create log4j.properties in etc/conf dir File(os.path.join(params.spark_conf, 'log4j.properties'), owner=params.spark_user, group=params.spark_group, content=params.spark_log4j_properties, mode=0644, ) #create metrics.properties in etc/conf dir File(os.path.join(params.spark_conf, 'metrics.properties'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_metrics_properties), mode=0644 ) if params.is_hive_installed: XmlConfig("hive-site.xml", conf_dir=params.spark_conf, configurations=params.spark_hive_properties, owner=params.spark_user, group=params.spark_group, mode=0644) if params.has_spark_thriftserver: PropertiesFile(params.spark_thrift_server_conf_file, properties = params.config['configurations']['spark-thrift-sparkconf'], owner = params.hive_user, group = params.user_group, key_value_delimiter = " ", mode=0644 ) effective_version = params.version if upgrade_type is not None else params.stack_version_formatted if effective_version: effective_version = format_stack_version(effective_version) if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(params.spark_conf,"spark-thrift-fairscheduler.xml"), owner=params.spark_user, group=params.spark_group, mode=0755, content=InlineTemplate(params.spark_thrift_fairscheduler_content) )
def falcon(type, action=None, upgrade_type=None): import params if action == 'config': Directory( params.falcon_pid_dir, owner=params.falcon_user, create_parents=True, mode=0755, cd_access="a", ) Directory( params.falcon_log_dir, owner=params.falcon_user, create_parents=True, mode=0755, cd_access="a", ) Directory(params.falcon_webapp_dir, owner=params.falcon_user, create_parents=True) Directory(params.falcon_home, owner=params.falcon_user, create_parents=True) Directory(params.etc_prefix_dir, mode=0755, create_parents=True) Directory(params.falcon_conf_dir, owner=params.falcon_user, create_parents=True) File( params.falcon_conf_dir + '/falcon-env.sh', content=InlineTemplate(params.falcon_env_sh_template), owner=params.falcon_user, group=params.user_group, ) PropertiesFile(params.falcon_conf_dir + '/client.properties', properties=params.falcon_client_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/runtime.properties', properties=params.falcon_runtime_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/startup.properties', properties=params.falcon_startup_properties, mode=0644, owner=params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, owner=params.falcon_user, group=params.user_group, mode=0775, create_parents=True, cd_access="a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, owner=params.falcon_user, group=params.user_group, mode=0775, create_parents=True, cd_access="a") # Generate atlas-application.properties.xml file if params.falcon_atlas_support: # If Atlas is added later than Falcon, this package will be absent. install_atlas_hook_packages( params.atlas_plugin_package, params.atlas_ubuntu_plugin_package, params.host_sys_prepped, params.agent_stack_retry_on_unavailability, params.agent_stack_retry_count) atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) # Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH. # Setup symlinks for older versions. if params.current_version_formatted and check_stack_feature( StackFeature.FALCON_ATLAS_SUPPORT_2_3, params.current_version_formatted): setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib) if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner=params.falcon_user, create_parents=True) # TODO change to proper mode params.HdfsResource(params.falcon_apps_dir, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0777) # In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS. if params.supports_data_mirroring: params.HdfsResource(params.dfs_data_mirroring_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0770, source=params.local_data_mirroring_dir) # Falcon Extensions were supported in HDP 2.5 and higher. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version( params.version) supports_falcon_extensions = effective_version and check_stack_feature( StackFeature.FALCON_EXTENSIONS, effective_version) if supports_falcon_extensions: params.HdfsResource(params.falcon_extensions_dest_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0755, source=params.falcon_extensions_source_dir) # Create the extensons HiveDR store params.HdfsResource(os.path.join( params.falcon_extensions_dest_dir, "mirroring"), type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, mode=0770) # At least one HDFS Dir should be created, so execute the change now. params.HdfsResource(None, action="execute") Directory(params.falcon_local_dir, owner=params.falcon_user, create_parents=True, cd_access="a") if params.falcon_embeddedmq_enabled == True: Directory(os.path.abspath( os.path.join(params.falcon_embeddedmq_data, "..")), owner=params.falcon_user, create_parents=True) Directory(params.falcon_embeddedmq_data, owner=params.falcon_user, create_parents=True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' # still points to older binaries; it's safer to just pass in the # hadoop home directory to use environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir} pid = get_user_call_output.get_user_call_output( format("cat {server_pid_file}"), user=params.falcon_user, is_checked_call=False)[1] process_exists = format("ls {server_pid_file} && ps -p {pid}") if action == 'start': try: Execute( format('{falcon_home}/bin/falcon-config.sh server falcon'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary, not_if=process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if not os.path.exists(params.target_jar_file): try: File(params.target_jar_file, content=DownloadSource(params.bdb_resource_name), mode=0755) except: exc_msg = traceback.format_exc() exception_message = format( "Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}" ) Logger.error(exception_message) if not os.path.isfile(params.target_jar_file): error_message = """ If you are using bdb as the Falcon graph db store, please run ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar> on the ambari server host. Otherwise falcon startup will fail. Otherwise please configure Falcon to use HBase as the backend as described in the Falcon documentation. """ Logger.error(error_message) try: Execute( format( '{falcon_home}/bin/falcon-start -port {falcon_port}'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary, not_if=process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if action == 'stop': try: Execute(format('{falcon_home}/bin/falcon-stop'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary) except: show_logs(params.falcon_log_dir, params.falcon_user) raise File(params.server_pid_file, action='delete')
def falcon(type, action=None): import params if action == 'config': Directory(params.falcon_pid_dir, owner=params.falcon_user, recursive=True) Directory(params.falcon_log_dir, owner=params.falcon_user, recursive=True) Directory(params.falcon_webapp_dir, owner=params.falcon_user, recursive=True) Directory(params.falcon_home, owner=params.falcon_user, recursive=True) Directory(params.etc_prefix_dir, mode=0755, recursive=True) Directory(params.falcon_conf_dir, owner=params.falcon_user, recursive=True) File(params.falcon_conf_dir + '/falcon-env.sh', content=InlineTemplate(params.falcon_env_sh_template), owner=params.falcon_user) File(params.falcon_conf_dir + '/client.properties', content=Template('client.properties.j2'), mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/runtime.properties', properties=params.falcon_runtime_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/startup.properties', properties=params.falcon_startup_properties, mode=0644, owner=params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, owner=params.falcon_user, group=params.user_group, mode=0775, recursive=True, cd_access="a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, owner=params.falcon_user, group=params.user_group, mode=0775, recursive=True, cd_access="a") if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner=params.falcon_user, recursive=True) # TODO change to proper mode params.HdfsResource(params.flacon_apps_dir, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0777) if params.falcon_store_uri[0:4] == "hdfs": params.HdfsResource(params.falcon_store_uri, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0755) elif params.falcon_store_uri[0:4] == "file": Directory(params.falcon_store_uri[7:], owner=params.falcon_user, recursive=True) if params.supports_hive_dr: params.HdfsResource(params.dfs_data_mirroring_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0770, source=params.local_data_mirroring_dir) params.HdfsResource(None, action="execute") Directory(params.falcon_local_dir, owner=params.falcon_user, recursive=True, cd_access="a") if params.falcon_embeddedmq_enabled == True: Directory(os.path.abspath( os.path.join(params.falcon_embeddedmq_data, "..")), owner=params.falcon_user, recursive=True) Directory(params.falcon_embeddedmq_data, owner=params.falcon_user, recursive=True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' # still points to older binaries; it's safer to just pass in the # hadoop home directory to use environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir} if action == 'start': Execute( format('{falcon_home}/bin/falcon-start -port {falcon_port}'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary) if action == 'stop': Execute(format('{falcon_home}/bin/falcon-stop'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary) File(params.server_pid_file, action='delete')