def sqoop(type=None): import params Link(params.sqoop_lib + "/mysql-connector-java.jar", to='/usr/share/java/mysql-connector-java.jar') jdbc_connector() Directory(params.sqoop_conf_dir, owner=params.sqoop_user, group=params.user_group, create_parents=True) configs = {} sqoop_site_config = get_config('sqoop-site') if sqoop_site_config: configs.update(sqoop_site_config) XmlConfig( "sqoop-site.xml", conf_dir=params.sqoop_conf_dir, configurations=configs, configuration_attributes=params.config['configurationAttributes'] ['sqoop-site'], owner=params.sqoop_user, group=params.user_group) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group) setup_atlas_jar_symlinks("sqoop", params.sqoop_lib) File(format("{sqoop_conf_dir}/sqoop-env.sh"), owner=params.sqoop_user, group=params.user_group, content=InlineTemplate(params.sqoop_env_sh_template)) update_config_permissions( ["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"]) pass
def ensure_base_directories(): import params import status_params Directory( [ params.streamline_log_dir, status_params.streamline_pid_dir, params.conf_dir, params.streamline_agent_dir, params.streamline_bootstrap_dir, params.streamline_libs ], mode=0755, cd_access='a', owner=params.streamline_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) if params.enable_atlas_hook: storm_extlib_dir = os.path.join(params.storm_client_home, "extlib") setup_atlas_jar_symlinks("storm", storm_extlib_dir)
def storm(name=None): import params import os Directory( params.log_dir, owner=params.storm_user, group=params.user_group, mode=0777, create_parents=True, cd_access="a", ) Directory( [params.pid_dir, params.local_dir], owner=params.storm_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) Directory( params.conf_dir, group=params.user_group, create_parents=True, cd_access="a", ) File(format("{conf_dir}/config.yaml"), content=Template("config.yaml.j2"), owner=params.storm_user, group=params.user_group) configurations = params.config['configurations']['storm-site'] File(format("{conf_dir}/storm.yaml"), content=yaml_config_template(configurations), owner=params.storm_user, group=params.user_group) File(format("{conf_dir}/storm-env.sh"), owner=params.storm_user, content=InlineTemplate(params.storm_env_sh_template)) # Generate atlas-application.properties.xml file and symlink the hook jars if params.enable_atlas_hook: atlas_hook_filepath = os.path.join(params.conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.STORM, params.storm_atlas_application_properties, atlas_hook_filepath, params.storm_user, params.user_group) storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib") setup_atlas_jar_symlinks("storm", storm_extlib_dir) if params.has_metric_collector: File(format("{conf_dir}/storm-metrics2.properties"), owner=params.storm_user, group=params.user_group, content=Template("storm-metrics2.properties.j2")) # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2 Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"), action="delete") # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete") if check_stack_feature(StackFeature.STORM_METRICS_APACHE_CLASSES, params.version_for_stack_feature_checks): sink_jar = params.metric_collector_sink_jar else: sink_jar = params.metric_collector_legacy_sink_jar Execute(format( "{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar" ), not_if=format( "ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"), only_if=format("ls {sink_jar}")) if params.storm_logs_supported: Directory(params.log4j_dir, owner=params.storm_user, group=params.user_group, mode=0755, create_parents=True) File(format("{log4j_dir}/cluster.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_cluster_log4j_content)) File(format("{log4j_dir}/worker.xml"), owner=params.storm_user, content=InlineTemplate(params.storm_worker_log4j_content)) if params.security_enabled: TemplateConfig(format("{conf_dir}/storm_jaas.conf"), owner=params.storm_user) if params.stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.storm_user) minRuid = configurations[ '_storm.min.ruid'] if configurations.has_key( '_storm.min.ruid') else '' min_user_ruid = int( minRuid) if minRuid.isdigit() else _find_real_user_min_uid() File(format("{conf_dir}/worker-launcher.cfg"), content=Template("worker-launcher.cfg.j2", min_user_ruid=min_user_ruid), owner='root', group=params.user_group)
def falcon(type, action=None, upgrade_type=None): import params if action == 'config': Directory( params.falcon_pid_dir, owner=params.falcon_user, create_parents=True, mode=0755, cd_access="a", ) Directory( params.falcon_log_dir, owner=params.falcon_user, create_parents=True, mode=0755, cd_access="a", ) Directory(params.falcon_webapp_dir, owner=params.falcon_user, create_parents=True) Directory(params.falcon_home, owner=params.falcon_user, create_parents=True) Directory(params.etc_prefix_dir, mode=0755, create_parents=True) Directory(params.falcon_conf_dir, owner=params.falcon_user, create_parents=True) File( params.falcon_conf_dir + '/falcon-env.sh', content=InlineTemplate(params.falcon_env_sh_template), owner=params.falcon_user, group=params.user_group, ) PropertiesFile(params.falcon_conf_dir + '/client.properties', properties=params.falcon_client_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/runtime.properties', properties=params.falcon_runtime_properties, mode=0644, owner=params.falcon_user) PropertiesFile(params.falcon_conf_dir + '/startup.properties', properties=params.falcon_startup_properties, mode=0644, owner=params.falcon_user) if params.falcon_graph_storage_directory: Directory(params.falcon_graph_storage_directory, owner=params.falcon_user, group=params.user_group, mode=0775, create_parents=True, cd_access="a") if params.falcon_graph_serialize_path: Directory(params.falcon_graph_serialize_path, owner=params.falcon_user, group=params.user_group, mode=0775, create_parents=True, cd_access="a") # Generate atlas-application.properties.xml file if params.falcon_atlas_support: # If Atlas is added later than Falcon, this package will be absent. install_atlas_hook_packages( params.atlas_plugin_package, params.atlas_ubuntu_plugin_package, params.host_sys_prepped, params.agent_stack_retry_on_unavailability, params.agent_stack_retry_count) atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group) # Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH. # Setup symlinks for older versions. if params.current_version_formatted and check_stack_feature( StackFeature.FALCON_ATLAS_SUPPORT_2_3, params.current_version_formatted): setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib) if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": params.HdfsResource(params.store_uri, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0755) elif params.store_uri[0:4] == "file": Directory(params.store_uri[7:], owner=params.falcon_user, create_parents=True) # TODO change to proper mode params.HdfsResource(params.falcon_apps_dir, type="directory", action="create_on_execute", owner=params.falcon_user, mode=0777) # In HDP 2.4 and earlier, the data-mirroring directory was copied to HDFS. if params.supports_data_mirroring: params.HdfsResource(params.dfs_data_mirroring_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0770, source=params.local_data_mirroring_dir) # Falcon Extensions were supported in HDP 2.5 and higher. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version( params.version) supports_falcon_extensions = effective_version and check_stack_feature( StackFeature.FALCON_EXTENSIONS, effective_version) if supports_falcon_extensions: params.HdfsResource(params.falcon_extensions_dest_dir, type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, recursive_chown=True, recursive_chmod=True, mode=0755, source=params.falcon_extensions_source_dir) # Create the extensons HiveDR store params.HdfsResource(os.path.join( params.falcon_extensions_dest_dir, "mirroring"), type="directory", action="create_on_execute", owner=params.falcon_user, group=params.proxyuser_group, mode=0770) # At least one HDFS Dir should be created, so execute the change now. params.HdfsResource(None, action="execute") Directory(params.falcon_local_dir, owner=params.falcon_user, create_parents=True, cd_access="a") if params.falcon_embeddedmq_enabled == True: Directory(os.path.abspath( os.path.join(params.falcon_embeddedmq_data, "..")), owner=params.falcon_user, create_parents=True) Directory(params.falcon_embeddedmq_data, owner=params.falcon_user, create_parents=True) # although Falcon's falcon-config.sh will use 'which hadoop' to figure # this out, in an upgraded cluster, it's possible that 'which hadoop' # still points to older binaries; it's safer to just pass in the # hadoop home directory to use environment_dictionary = {"HADOOP_HOME": params.hadoop_home_dir} pid = get_user_call_output.get_user_call_output( format("cat {server_pid_file}"), user=params.falcon_user, is_checked_call=False)[1] process_exists = format("ls {server_pid_file} && ps -p {pid}") if action == 'start': try: Execute( format('{falcon_home}/bin/falcon-config.sh server falcon'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary, not_if=process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if not os.path.exists(params.target_jar_file): try: File(params.target_jar_file, content=DownloadSource(params.bdb_resource_name), mode=0755) except: exc_msg = traceback.format_exc() exception_message = format( "Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}" ) Logger.error(exception_message) if not os.path.isfile(params.target_jar_file): error_message = """ If you are using bdb as the Falcon graph db store, please run ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar> on the ambari server host. Otherwise falcon startup will fail. Otherwise please configure Falcon to use HBase as the backend as described in the Falcon documentation. """ Logger.error(error_message) try: Execute( format( '{falcon_home}/bin/falcon-start -port {falcon_port}'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary, not_if=process_exists, ) except: show_logs(params.falcon_log_dir, params.falcon_user) raise if action == 'stop': try: Execute(format('{falcon_home}/bin/falcon-stop'), user=params.falcon_user, path=params.hadoop_bin_dir, environment=environment_dictionary) except: show_logs(params.falcon_log_dir, params.falcon_user) raise File(params.server_pid_file, action='delete')