def new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user, keytab_file, principal, hostname): """ Entry point of an actual execution - triggered when timeout on the cache expired or on fresh execution """ now = datetime.now() temp_kinit_cache_fd, temp_kinit_cache_filename = mkstemp(dir=temp_dir) command = "%s -c %s -kt %s %s" % \ (kinit_path, temp_kinit_cache_filename, keytab_file, principal.replace("_HOST", hostname)) os.close(temp_kinit_cache_fd) try: # Ensure the proper user owns this file File(temp_kinit_cache_filename, owner=exec_user, mode=0600) # Execute the kinit Execute(command, user=exec_user) with open(file_path, 'w+') as cache_file: result = {key: {"last_successful_execution": str(now)}} json.dump(result, cache_file) finally: File(temp_kinit_cache_filename, action='delete')
def metadata(): import params Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile(format('{metadata_home}/server/webapp/atlas.war'))) PropertiesFile(format('{conf_dir}/application.properties'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=StaticFile('atlas-log4j.xml'))
def setup_livy(env, type, upgrade_type=None, action=None): import params Directory([params.livy2_pid_dir, params.livy2_log_dir], owner=params.livy2_user, group=params.user_group, mode=0775, create_parents=True) if type == 'server' and action == 'config': params.HdfsResource(params.livy2_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0775) params.HdfsResource(None, action="execute") # create livy-env.sh in etc/conf dir File( os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, group=params.livy2_group, content=InlineTemplate(params.livy2_env_sh), mode=0644, ) # create livy.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy.conf"), properties=params.config['configurations']['livy2-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create log4j.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'log4j.properties'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_log4j_properties, mode=0644, ) # create spark-blacklist.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'spark-blacklist.conf'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_spark_blacklist_properties, mode=0644, ) Directory( params.livy2_logs_dir, owner=params.livy2_user, group=params.livy2_group, mode=0755, )
def stop(self, env, upgrade_type=None): import params env.set_params(params) daemon_cmd = format( 'source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}' ) # If the pid dir doesn't exist, this means either # 1. The user just added Atlas service and issued a restart command (stop+start). So stop should be a no-op # since there's nothing to stop. # OR # 2. The user changed the value of the pid dir config and incorrectly issued a restart command. # In which case the stop command cannot do anything since Ambari doesn't know which process to kill. # The start command will spawn another instance. # The user should have issued a stop, changed the config, and then started it. if not os.path.isdir(params.pid_dir): Logger.info( "*******************************************************************" ) Logger.info( "Will skip the stop command since this is the first time stopping/restarting Atlas " "and the pid dir does not exist, %s\n" % params.pid_dir) return try: Execute( daemon_cmd, user=params.metadata_user, ) except: show_logs(params.log_dir, params.metadata_user) raise File(params.pid_file, action="delete")
def setup_config(self, env): import params env.set_params(params) XmlConfig( "hdfs-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'], owner=params.hdfs_user, group=params.user_group) XmlConfig( "core-site.xml", conf_dir=params.hadoop_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configuration_attributes'] ['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) File( format("{params.ambari_libs_dir}/fast-hdfs-resource.jar"), mode=0644, content=StaticFile( "/var/lib/ambari-agent/cache/stack-hooks/before-START/files/fast-hdfs-resource.jar" ))
def update_credential_provider_path(config, config_type, dest_provider_path, file_owner, file_group): """ Copies the JCEKS file for the specified config from the default location to the given location, and sets the ACLs for the specified owner and group. Also updates the config type's configuration hadoop credential store provider with the copied file name. :param config: configurations['configurations'][config_type] :param config_type: Like hive-site, oozie-site, etc. :param dest_provider_path: The full path to the file where the JCEKS provider file is to be copied to. :param file_owner: File owner :param file_group: Group :return: A copy of the config that was modified or the input config itself if nothing was modified. """ # Get the path to the provider <config_type>.jceks if HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME in config: provider_path = config[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] src_provider_path = provider_path[len('jceks://file'):] File(dest_provider_path, owner=file_owner, group=file_group, mode=0640, content=StaticFile(src_provider_path)) # make a copy of the config dictionary since it is read-only config_copy = config.copy() # overwrite the provider path with the path specified config_copy[ HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file{0}'.format( dest_provider_path) return config_copy return config
def configure_airflow(self, env): import params env.set_params(params) File("/etc/systemd/system/airflow_webserver.service", content=Template("webserver.j2", configurations=params), owner='root', group='root', mode=0o0600) File("/etc/systemd/system/airflow_scheduler.service", content=Template("scheduler.j2", configurations=params), owner='root', group='root', mode=0o0600) Execute('sudo systemctl daemon-reload')
def configure_ac(self, env): import params env.set_params(params) print("!!!!!!!!!!!!!!!!!!!!!!!creating file!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") File("/etc/systemd/system/airflow_portforward.service", content=Template("portforward_service.j2", configurations=params), owner='root', group='root', mode=0o0600 ) Execute('sudo systemctl daemon-reload')
def configure_ac(self, env): import params env.set_params(params) File("/etc/systemd/system/jupyter_portforward.service", content=Template("portforward_service_jupyter.j2", configurations=params), owner=params.anaconda_user, group=params.anaconda_group, mode=0o0600) Execute('sudo systemctl daemon-reload')
def stop(self, env, upgrade_type=None): import params env.set_params(params) daemon_cmd = format('source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}') try: Execute(daemon_cmd, user=params.metadata_user, ) except: show_logs(params.log_dir, params.metadata_user) raise File(params.pid_file, action="delete")
def install_jce_policy(): import params jcePolicyInfo = JcePolicyInfo(params.java_home) if jcePolicyInfo.is_unlimited_key_jce_policy(): major, minor = java_version() Logger.info( "The unlimited key JCE policy is required, and appears to have been installed." ) java_security_dir = format("{java_home}/jre/lib/security") Logger.debug("Removing existing JCE policy JAR files: {0}.".format( java_security_dir)) File(format("{java_security_dir}/US_export_policy.jar"), action="delete") File(format("{java_security_dir}/local_policy.jar"), action="delete") src_url = jce_zip_target = '' if minor == 7: # jdk7 src_url = 'http://yum.example.com/hadoop/UnlimitedJCEPolicyJDK7.zip' jce_zip_target = '/tmp/UnlimitedJCEPolicyJDK7.zip' elif minor == 8: # jdk8 src_url = 'http://yum.example.com/hadoop/jce_policy-8.zip' jce_zip_target = '/tmp/jce_policy-8.zip' if src_url and jce_zip_target: Execute('wget ' + src_url + ' -O ' + jce_zip_target) Logger.debug( "Unzipping the unlimited key JCE policy files from {0} into {1}." .format(jce_zip_target, java_security_dir)) extract_cmd = ("unzip", "-o", "-j", "-q", jce_zip_target, "-d", java_security_dir) Execute( extract_cmd, only_if=format( "test -e {java_security_dir} && test -f {jce_zip_target}"), path=['/bin/', '/usr/bin'], sudo=True)
def configure_ac(self, env): import params env.set_params(params) conf = dict() for key in params.config['configurations']['jupyter-env']: conf[key] = params.config['configurations']['jupyter-env'][key] conf['jupyter_password'] = params.hashText(conf['jupyter_password']) print("----------------------------\n", conf, "\n------------------------------------------------") File("{0}jupyter_notebook_config.py".format(params.config_dir), content=Template("jupyter_notebook_config.py.j2", configurations=conf), owner=params.anaconda_user, group=params.anaconda_group, mode=0o0600 )
def metadata(): import params Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, recursive=True ) Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, recursive=True ) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, recursive=True ) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, recursive=True ) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, recursive=True ) metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war') if not os.path.isfile(metadata_war_file): raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file)) Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir)) shutil.copy2(metadata_war_file, params.expanded_war_dir) File(format('{conf_dir}/application.properties'), content=InlineTemplate(params.application_properties_content), mode=0644, owner=params.metadata_user, group=params.user_group ) File(format("{conf_dir}/metadata-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content) ) File(format("{conf_dir}/log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=StaticFile('log4j.xml') )
def setup_livy(env, type, upgrade_type=None, action=None): import params Directory([params.livy2_pid_dir, params.livy2_log_dir], owner=params.livy2_user, group=params.user_group, mode=0775, cd_access='a', create_parents=True) if type == 'server' and action == 'config': params.HdfsResource(params.livy2_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0775) params.HdfsResource(None, action="execute") if params.livy2_recovery_store == 'filesystem': params.HdfsResource(params.livy2_recovery_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0700) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'spark2', Template("input.config-spark2.json.j2", extra_imports=[default])) # create livy-env.sh in etc/conf dir File( os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, group=params.livy2_group, content=InlineTemplate(params.livy2_env_sh), mode=0644, ) # create livy-client.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy-client.conf"), properties=params.config['configurations']['livy2-client-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create livy.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy.conf"), properties=params.config['configurations']['livy2-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create log4j.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'log4j.properties'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_log4j_properties, mode=0644, ) # create spark-blacklist.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'spark-blacklist.conf'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_spark_blacklist_properties, mode=0644, ) Directory( params.livy2_logs_dir, owner=params.livy2_user, group=params.livy2_group, mode=0755, )
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) if type == 'server' and params.search_backend_solr and params.has_logsearch_solr: solr_cloud_util.setup_solr_client(params.config) random_num = random.random() upload_conf_set('basic_configs', random_num) create_collection('vertex_index', 'basic_configs') create_collection('edge_index', 'basic_configs') create_collection('fulltext_index', 'basic_configs') if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user)
def metadata(): import params Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.atlas_hbase_log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.atlas_hbase_data_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents = True ) File(format("{expanded_war_dir}/atlas.war"), content = StaticFile(format('{metadata_home}/server/webapp/atlas.war')) ) PropertiesFile(format('{conf_dir}/{conf_file}'), properties = params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group ) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content) ) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content) ) # hbase-site for embedded hbase used by Atlas XmlConfig( "hbase-site.xml", conf_dir = params.atlas_hbase_conf_dir, configurations = params.config['configurations']['atlas-hbase-site'], configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'], owner = params.metadata_user, group = params.user_group ) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user)