Esempio n. 1
0
    def test_action_create_dir_exist(self, isdir_mock, dirname_mock):
        """
    Tests if 'create' action fails when path is existent directory
    """
        isdir_mock.side_effect = [True, False]
        try:
            with Environment('/') as env:
                File('/existent_directory',
                     action='create',
                     mode=0777,
                     content='file-content')

            self.fail("Must fail when directory with name 'path' exist")
        except Fail as e:
            self.assertEqual(
                'Applying File[\'/existent_directory\'] failed, directory with name /existent_directory exists',
                str(e))
        self.assertFalse(dirname_mock.called)
Esempio n. 2
0
    def test_action_create_replace(self, isdir_mock, exists_mock,
                                   create_file_mock, read_file_mock,
                                   ensure_mock):
        """
    Tests if 'create' action rewrite existent file with new data
    """
        isdir_mock.side_effect = [False, True]
        exists_mock.return_value = True

        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 backup=False,
                 content='new-content')

        read_file_mock.assert_called_with('/directory/file')
        create_file_mock.assert_called_with('/directory/file', 'new-content')
Esempio n. 3
0
    def install(self, env):
        Logger.info('Installing R Server Client...')
        tmp_dir = Script.tmp_dir
        Logger.debug('Using temp dir: {0}'.format(tmp_dir))
        Logger.info(
            "Will download and install the following rpm's from {0}: {1}".
            format(rpm_location, rpms))

        for rpm in rpms:
            Logger.info('Downloading {0}'.format(rpm))
            rpmFile = '{0}/{1}'.format(tmp_dir, rpm)
            File(rpmFile, \
                 content = DownloadSource(rpm_location + rpm), \
                 mode = 0644)
            Logger.info('Installing {0}'.format(rpm))
            Package(rpmFile)

        Logger.info('Installed R Server')
Esempio n. 4
0
    def test_attribute_path(self, isdir_mock):
        """
    Tests 'path' attribute
    """
        isdir_mock.side_effect = [True, False]

        try:
            with Environment('/') as env:
                File('/existent_directory',
                     action='create',
                     mode=0777,
                     content='file-content')
            env.run()
            self.fail("Must fail when directory with name 'path' exist")
        except Fail as e:
            pass

        isdir_mock.assert_called_with('/existent_directory')
Esempio n. 5
0
    def generate_configs(self, env):
        """
    Generates config files and stores them as an archive in tmp_dir
    based on xml_configs_list and env_configs_list from commandParams
    """
        import params
        env.set_params(params)

        config = self.get_config()

        xml_configs_list = config['commandParams']['xml_configs_list']
        env_configs_list = config['commandParams']['env_configs_list']
        properties_configs_list = config['commandParams'][
            'properties_configs_list']

        Directory(self.get_tmp_dir(), recursive=True)

        conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir())
        output_filename = os.path.join(self.get_tmp_dir(),
                                       config['commandParams']['output_file'])

        try:
            for file_dict in xml_configs_list:
                for filename, dict in file_dict.iteritems():
                    XmlConfig(filename,
                              conf_dir=conf_tmp_dir,
                              **self.generate_configs_get_xml_file_content(
                                  filename, dict))
            for file_dict in env_configs_list:
                for filename, dicts in file_dict.iteritems():
                    File(os.path.join(conf_tmp_dir, filename),
                         content=InlineTemplate(
                             self.generate_configs_get_template_file_content(
                                 filename, dicts)))

            for file_dict in properties_configs_list:
                for filename, dict in file_dict.iteritems():
                    PropertiesFile(
                        os.path.join(conf_tmp_dir, filename),
                        properties=self.generate_configs_get_xml_file_dict(
                            filename, dict))
            archive_dir(output_filename, conf_tmp_dir)
        finally:
            Directory(conf_tmp_dir, action="delete")
Esempio n. 6
0
def setup_ranger_plugin(component_select_name, service_name,
                        downloaded_custom_connector, driver_curl_source, 
                        driver_curl_target, java_home,
                        repo_name, plugin_repo_dict, 
                        ranger_env_properties, plugin_properties,
                        policy_user, policymgr_mgr_url,
                        plugin_enabled):
  File(downloaded_custom_connector,
       content = DownloadSource(driver_curl_source)
  )

  Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
          not_if=format("test -f {driver_curl_target}"),
          sudo=True
  )

  hdp_version = get_hdp_version(component_select_name)
  file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
  
  if not os.path.isfile(file_path):
    raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
  
  ModifyPropertiesFile(file_path,
    properties = plugin_properties
  )

  if plugin_enabled:
    cmd = (format('enable-{service_name}-plugin.sh'),)
    
    ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
    ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
                                            ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], 
                                            ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], 
                                            policy_user)
  else:
    cmd = (format('disable-{service_name}-plugin.sh'),)
    
  cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
  
  Execute(cmd, 
        environment=cmd_env, 
        logoutput=True,
        sudo=True,
  )                    
Esempio n. 7
0
    def action_create(self):
        filename = self.resource.filename
        xml_config_provider_config_dir = self.resource.conf_dir
        configuration_attrs = {} if is_empty(
            self.resource.configuration_attributes
        ) else self.resource.configuration_attributes

        # |e - for html-like escaping of <,>,',"
        config_content = InlineTemplate(
            '''  <configuration>
    {% for key, value in configurations_dict|dictsort %}
    <property>
      <name>{{ key|e }}</name>
      <value>{{ resource_management.core.source.InlineTemplate(unicode(value)).get_content() |e }}</value>
      {%- if not configuration_attrs is none -%}
      {%- for attrib_name, attrib_occurances in  configuration_attrs.items() -%}
      {%- for property_name, attrib_value in  attrib_occurances.items() -%}
      {% if property_name == key and attrib_name %}
      <{{attrib_name|e}}>{{attrib_value|e}}</{{attrib_name|e}}>
      {%- endif -%}
      {%- endfor -%}
      {%- endfor -%}
      {%- endif %}
    </property>
    {% endfor %}
  </configuration>''',
            extra_imports=[
                time, resource_management, resource_management.core,
                resource_management.core.source
            ],
            configurations_dict=self.resource.configurations,
            configuration_attrs=configuration_attrs)

        xml_config_dest_file_path = os.path.join(
            xml_config_provider_config_dir, filename)
        Logger.info("Generating config: {0}".format(xml_config_dest_file_path))

        File(xml_config_dest_file_path,
             content=config_content,
             owner=self.resource.owner,
             group=self.resource.group,
             mode=self.resource.mode,
             encoding=self.resource.encoding)
Esempio n. 8
0
    def convert_properties_to_dict(jdk64_home, nifi_registry_props_file,
                                   nifi_registry_bootstrap):
        dict = {}
        if sudo.path_isfile(nifi_registry_props_file):
            encrypt_tool_script = nifi_toolkit_util.get_toolkit_script(
                'encrypt-config.sh')
            File(encrypt_tool_script, mode=0755)

            command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' ' + encrypt_tool_script + ' --nifiRegistry --decrypt -r ' + nifi_registry_props_file + ' -b ' + nifi_registry_bootstrap
            code, out = shell.call(command, quiet=True, logoutput=False)

            lines = out.split('\n')
            for line in lines:
                props = line.rstrip().split('=')
                if len(props) == 2:
                    dict[props[0]] = props[1]
                elif len(props) == 1:
                    dict[props[0]] = ''
        return dict
def setup_configuration_file_for_required_plugins(component_user, component_group, create_core_site_path,
                                                  configurations={}, configuration_attributes={}, file_name='core-site.xml',
                                                  xml_include_file=None, xml_include_file_content=None):
  XmlConfig(file_name,
    conf_dir = create_core_site_path,
    configurations = configurations,
    configuration_attributes = configuration_attributes,
    owner = component_user,
    group = component_group,
    mode = 0644,
    xml_include_file = xml_include_file
  )

  if xml_include_file_content:
    File(xml_include_file,
         owner=component_user,
         group=component_group,
         content=xml_include_file_content
         )
Esempio n. 10
0
    def check_nifi_portal_with_toolkit(urls, jdk64_home, nifi_dir,
                                       nifi_bootstrap, toolkit_tmp_dir,
                                       stack_version_buildnum):

        node_manager_script = nifi_toolkit_util_common.get_toolkit_script(
            'node-manager.sh', toolkit_tmp_dir, stack_version_buildnum)
        File(node_manager_script, mode=0755)
        command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' ' + node_manager_script + ' -d ' + nifi_dir + ' -b ' + nifi_bootstrap + ' -o status -u "' + ','.join(
            urls) + '"'
        code, out = shell.call(command, quiet=True, logoutput=False)

        if code > 0:
            raise Fail(
                "Call to admin-toolkit encountered error: {0}".format(out))
        else:
            if out.find('did not complete due to exception') > -1:
                raise Fail(
                    "Error connecting to one or more nifi nodes: {0}".format(
                        out))
Esempio n. 11
0
def create_topology_script():
    import params

    path = params.net_topology_script_file_path
    parent_dir = os.path.dirname(path)
    # only create the parent directory and set its permission if it does not exist
    if not os.path.exists(parent_dir):
        Directory(parent_dir,
                  create_parents=True,
                  owner=params.hdfs_user,
                  group=params.user_group)

    # installing the topology script to the specified location
    File(
        path,
        content=StaticFile('topology_script.py'),
        mode=0755,
        only_if=format("test -d {net_topology_script_dir}"),
    )
Esempio n. 12
0
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
                                 stack_version, credential_file,
                                 xa_audit_db_password, ssl_truststore_password,
                                 ssl_keystore_password, component_user,
                                 component_group, java_home):

    stack_root = Script.get_stack_root()
    cred_lib_path = format(
        '{stack_root}/{stack_version}/ranger-{service_name}-plugin/install/lib/*'
    )
    cred_setup_prefix = (format(
        '{stack_root}/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'
    ), '-l', cred_lib_path)

    if audit_db_is_enabled:
        cred_setup = cred_setup_prefix + (
            '-f', credential_file, '-k', 'auditDBCred', '-v',
            PasswordString(xa_audit_db_password), '-c', '1')
        Execute(cred_setup,
                environment={'JAVA_HOME': java_home},
                logoutput=True,
                sudo=True)

    cred_setup = cred_setup_prefix + (
        '-f', credential_file, '-k', 'sslKeyStore', '-v',
        PasswordString(ssl_keystore_password), '-c', '1')
    Execute(cred_setup,
            environment={'JAVA_HOME': java_home},
            logoutput=True,
            sudo=True)

    cred_setup = cred_setup_prefix + (
        '-f', credential_file, '-k', 'sslTrustStore', '-v',
        PasswordString(ssl_truststore_password), '-c', '1')
    Execute(cred_setup,
            environment={'JAVA_HOME': java_home},
            logoutput=True,
            sudo=True)

    File(credential_file,
         owner=component_user,
         group=component_group,
         mode=0640)
Esempio n. 13
0
    def test_action_create_non_existent_file(self, isdir_mock, exists_mock,
                                             open_mock, ensure_mock):
        """
    Tests if 'create' action create new non existent file and write proper data
    """
        isdir_mock.side_effect = [False, True]
        exists_mock.return_value = False
        new_file = MagicMock()
        open_mock.return_value = new_file
        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 content='file-content')

        open_mock.assert_called_with('/directory/file', 'wb')
        new_file.__enter__().write.assert_called_with('file-content')
        self.assertEqual(open_mock.call_count, 1)
        ensure_mock.assert_called()
Esempio n. 14
0
    def test_action_create_non_existent_file(self, isdir_mock, exists_mock,
                                             create_file_mock, read_file_mock,
                                             ensure_mock):
        """
    Tests if 'create' action create new non existent file and write proper data
    """
        isdir_mock.side_effect = [False, True]
        exists_mock.return_value = False
        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 content='file-content')

        create_file_mock.assert_called_once('/directory/file',
                                            'file-content',
                                            encoding=None,
                                            on_file_created=ANY)
        ensure_mock.assert_called()
Esempio n. 15
0
def oozie(is_server=False):
    import params

    XmlConfig(
        "oozie-site.xml",
        conf_dir=params.oozie_conf_dir,
        configurations=params.config['configurations']['oozie-site'],
        owner=params.oozie_user,
        mode='f',
        configuration_attributes=params.config['configuration_attributes']
        ['oozie-site'])

    File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
         owner=params.oozie_user,
         content=InlineTemplate(params.oozie_env_cmd_template))

    Directory(
        params.oozie_tmp_dir,
        owner=params.oozie_user,
        recursive=True,
    )
    download_file(
        os.path.join(params.config['hostLevelParams']['jdk_location'],
                     "sqljdbc4.jar"),
        os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar"))
    webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server",
                                        "webapps", "oozie", "WEB-INF", "lib",
                                        "sqljdbc4.jar")
    if os.path.isfile(webapps_sqljdbc_path):
        download_file(
            os.path.join(params.config['hostLevelParams']['jdk_location'],
                         "sqljdbc4.jar"), webapps_sqljdbc_path)
    download_file(
        os.path.join(params.config['hostLevelParams']['jdk_location'],
                     "sqljdbc4.jar"),
        os.path.join(params.oozie_home, "share", "lib", "oozie",
                     "sqljdbc4.jar"))
    download_file(
        os.path.join(params.config['hostLevelParams']['jdk_location'],
                     "sqljdbc4.jar"),
        os.path.join(params.oozie_home, "temp", "WEB-INF", "lib",
                     "sqljdbc4.jar"))
Esempio n. 16
0
    def service_check(self, env):
        import params
        env.set_params(params)

        unique = get_unique_id_and_date()

        File("/tmp/wordCount.jar", content=StaticFile("wordCount.jar"))

        cmd = format(
            "storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}"
        )

        Execute(cmd,
                logoutput=True,
                path=params.storm_bin_dir,
                user=params.storm_user)

        Execute(format("storm kill WordCount{unique}"),
                path=params.storm_bin_dir,
                user=params.storm_user)
Esempio n. 17
0
    def test_action_create_encoding(self, isdir_mock, exists_mock,
                                    create_file_mock, read_file_mock,
                                    get_content_mock, ensure_mock):

        isdir_mock.side_effect = [False, True]
        content_mock = MagicMock()
        old_content_mock = MagicMock()
        get_content_mock.return_value = content_mock
        read_file_mock.return_value = old_content_mock
        exists_mock.return_value = True
        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 content='file-content',
                 encoding="UTF-8")

        read_file_mock.assert_called_with('/directory/file')
        content_mock.encode.assert_called_with('UTF-8')
        old_content_mock.decode.assert_called_with('UTF-8')
Esempio n. 18
0
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password,
                                ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home):

  cred_lib_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install/lib/*')
  cred_setup_prefix = format('python /usr/hdp/{hdp_version}/ranger-{service_name}-plugin/ranger_credential_helper.py -l "{cred_lib_path}"')

  if audit_db_is_enabled:
    cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "auditDBCred" -v {xa_audit_db_password!p} -c 1')
    Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True)

  cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "sslKeyStore" -v {ssl_keystore_password!p} -c 1')
  Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True)

  cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "sslTrustStore" -v {ssl_truststore_password!p} -c 1')
  Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True)

  File(credential_file,
    owner = component_user,
    group = component_group
  )
Esempio n. 19
0
  def action_create(self):
    filename = self.resource.filename
    comment_symbols = self.resource.comment_symbols
    delimiter = self.resource.key_value_delimiter
    properties = self.resource.properties
    unsaved_values = properties.keys()
    new_content_lines = []
    
    if sudo.path_isfile(filename):
      file_content = sudo.read_file(filename, encoding=self.resource.encoding)
      new_content_lines += file_content.split('\n')

      Logger.info(format("Modifying existing properties file: {filename}"))
      
      for line_num in range(len(new_content_lines)):
        line = new_content_lines[line_num]
        
        if line.lstrip() and not line.lstrip()[0] in comment_symbols and delimiter in line:
          in_var_name = line.split(delimiter)[0].strip()
          in_var_value = line.split(delimiter)[1].strip()
          
          if in_var_name in properties:
            value = InlineTemplate(unicode(properties[in_var_name])).get_content().strip()
            new_content_lines[line_num] = u"{0}{1}{2}".format(unicode(in_var_name), delimiter, value)
            unsaved_values.remove(in_var_name)
    else:
      Logger.info(format("Creating new properties file as {filename} doesn't exist"))
       
    for property_name in unsaved_values:
      value = InlineTemplate(unicode(properties[property_name])).get_content().strip()
      line = u"{0}{1}{2}".format(unicode(property_name), delimiter, value)
      new_content_lines.append(line)
          
    with Environment.get_instance_copy() as env:
      File (filename,
            content = u"\n".join(new_content_lines) + "\n",
            owner = self.resource.owner,
            group = self.resource.group,
            mode = self.resource.mode,
            encoding = self.resource.encoding,
      )
Esempio n. 20
0
    def test_action_create_parent_dir_non_exist(self, isdir_mock,
                                                dirname_mock):
        """
    Tests if 'create' action fails when parent directory of path
    doesn't exist
    """
        isdir_mock.side_effect = [False, False]
        dirname_mock.return_value = "/non_existent_directory"
        try:
            with Environment('/') as env:
                File('/non_existent_directory/file',
                     action='create',
                     mode=0777,
                     content='file-content')

            self.fail('Must fail on non existent parent directory')
        except Fail as e:
            self.assertEqual(
                "Applying File['/non_existent_directory/file'] failed, parent directory /non_existent_directory doesn't exist",
                str(e))
        self.assertTrue(dirname_mock.called)
Esempio n. 21
0
def create_topology_mapping():
    import params

    path = params.net_topology_mapping_data_file_path
    parent_dir = os.path.dirname(path)
    # only create the parent directory and set its permission if it does not exist
    if not os.path.exists(parent_dir):
        Directory(parent_dir,
                  create_parents=True,
                  owner=params.hdfs_user,
                  group=params.user_group)

    # placing the mappings file in the same folder where the topology script is located
    File(
        path,
        content=Template("topology_mappings.data.j2"),
        owner=params.hdfs_user,
        group=params.user_group,
        # if there is no hadoop components, don't create the script
        only_if=format("test -d {net_topology_script_dir}"),
    )
Esempio n. 22
0
  def action_create(self):
    filename = self.resource.filename
    dir = self.resource.dir
    if dir == None:
      filepath = filename
    else:
      filepath = os.path.join(dir, filename)

    config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
    {% for key, value in properties_dict|dictsort %}
{{key}}={{value}}{% endfor %}
    ''', extra_imports=[time], properties_dict=self.resource.properties)

    Logger.info(format("Generating properties file: {filepath}"))

    with Environment.get_instance_copy() as env:
      File (format("{filepath}"),
            content = config_content,
            owner = self.resource.owner,
            group = self.resource.group,
            mode = self.resource.mode
      )
Esempio n. 23
0
  def action_create(self):
    filename = self.resource.filename
    dir = self.resource.dir
    if dir == None:
      filepath = filename
    else:
      filepath = os.path.join(dir, filename)

    config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
    {% for key, value in properties_dict|dictsort %}
{{key}}{{key_value_delimiter}}{{ resource_management.core.source.InlineTemplate(str(value)).get_content() }}{% endfor %}
    ''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], properties_dict=self.resource.properties, key_value_delimiter=self.resource.key_value_delimiter)

    Logger.info(format("Generating properties file: {filepath}"))

    File (format("{filepath}"),
          content = config_content,
          owner = self.resource.owner,
          group = self.resource.group,
          mode = self.resource.mode,
          encoding = self.resource.encoding,
    )
Esempio n. 24
0
def oozie_smoke_shell_file(file_name):
    import params

    File(format("{tmp_dir}/{file_name}"),
         content=StaticFile(file_name),
         mode=0755)

    if params.security_enabled:
        sh_cmd = format(
            "{tmp_dir}/{file_name} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}"
        )
    else:
        sh_cmd = format(
            "{tmp_dir}/{file_name} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled}"
        )

    Execute(format("{tmp_dir}/{file_name}"),
            command=sh_cmd,
            path=params.execute_path,
            tries=3,
            try_sleep=5,
            logoutput=True)
Esempio n. 25
0
    def test_action_delete_is_directory(self, isdir_mock, exist_mock,
                                        unlink_mock):
        """
    Tests if 'delete' action fails when path is directory
    """
        isdir_mock.return_value = True

        try:
            with Environment('/') as env:
                File('/directory/file',
                     action='delete',
                     mode=0777,
                     backup=False,
                     content='new-content')

            self.fail("Should fail when deleting directory")
        except Fail:
            pass

        self.assertEqual(isdir_mock.call_count, 1)
        self.assertEqual(exist_mock.call_count, 0)
        self.assertEqual(unlink_mock.call_count, 0)
Esempio n. 26
0
    def test_action_create_encoding(self, isdir_mock, exists_mock, open_mock,
                                    get_content_mock, ensure_mock):

        isdir_mock.side_effect = [False, True]
        exists_mock.return_value = True
        content_mock = MagicMock()
        old_content_mock = MagicMock()
        get_content_mock.return_value = content_mock
        new_file = MagicMock()
        open_mock.return_value = new_file
        enter_file_mock = MagicMock()
        enter_file_mock.read = MagicMock(return_value=old_content_mock)
        new_file.__enter__ = MagicMock(return_value=enter_file_mock)
        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 content='file-content',
                 encoding="UTF-8")

        open_mock.assert_called_with('/directory/file', 'wb')
        content_mock.encode.assert_called_with('UTF-8')
        old_content_mock.decode.assert_called_with('UTF-8')
Esempio n. 27
0
    def test_attribute_replace(self, isdir_mock, exists_mock, open_mock,
                               ensure_mock):
        """
    Tests 'replace' attribute
    """
        isdir_mock.side_effect = [False, True]
        old_file, new_file = MagicMock(), MagicMock()
        open_mock.side_effect = [old_file, new_file]
        old_file.read.return_value = 'old-content'
        exists_mock.return_value = True

        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 backup=False,
                 content='new-content',
                 replace=False)

        old_file.read.assert_called()
        self.assertEqual(new_file.__enter__().write.call_count, 0)
        ensure_mock.assert_called()
        self.assertEqual(open_mock.call_count, 0)
Esempio n. 28
0
    def test_action_create_replace(self, isdir_mock, exists_mock, open_mock,
                                   ensure_mock):
        """
    Tests if 'create' action rewrite existent file with new data
    """
        isdir_mock.side_effect = [False, True]
        old_file, new_file = MagicMock(), MagicMock()
        open_mock.side_effect = [old_file, new_file]
        old_file.read.return_value = 'old-content'
        exists_mock.return_value = True

        with Environment('/') as env:
            File('/directory/file',
                 action='create',
                 mode=0777,
                 backup=False,
                 content='new-content')

        old_file.read.assert_called()
        new_file.__enter__().write.assert_called_with('new-content')
        ensure_mock.assert_called()
        self.assertEqual(open_mock.call_count, 2)
        open_mock.assert_any_call('/directory/file', 'rb')
        open_mock.assert_any_call('/directory/file', 'wb')
Esempio n. 29
0
def setup_ranger_knox(upgrade_type=None):
    import params

    if params.enable_ranger_knox:

        stack_version = None
        if upgrade_type is not None:
            stack_version = params.version

        if params.retryAble:
            Logger.info(
                "Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !"
            )
        else:
            Logger.info(
                "Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
            )

        if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled:
            if params.has_namenode:
                params.HdfsResource("/ranger/audit",
                                    type="directory",
                                    action="create_on_execute",
                                    owner=params.hdfs_user,
                                    group=params.hdfs_user,
                                    mode=0755,
                                    recursive_chmod=True)
                params.HdfsResource("/ranger/audit/knox",
                                    type="directory",
                                    action="create_on_execute",
                                    owner=params.knox_user,
                                    group=params.knox_user,
                                    mode=0700,
                                    recursive_chmod=True)
                params.HdfsResource(None, action="execute")

                if params.namenode_hosts is not None and len(
                        params.namenode_hosts) > 1:
                    Logger.info(
                        'Ranger Knox plugin is enabled in NameNode HA environment along with audit to Hdfs enabled, creating hdfs-site.xml'
                    )
                    XmlConfig("hdfs-site.xml",
                              conf_dir=params.knox_conf_dir,
                              configurations=params.config['configurations']
                              ['hdfs-site'],
                              configuration_attributes=params.
                              config['configurationAttributes']['hdfs-site'],
                              owner=params.knox_user,
                              group=params.knox_group,
                              mode=0644)
                else:
                    File(format('{knox_conf_dir}/hdfs-site.xml'),
                         action="delete")

        if params.xml_configurations_supported:
            api_version = None
            if params.stack_supports_ranger_kerberos:
                api_version = 'v2'
            from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
            setup_ranger_plugin(
                'knox-server',
                'knox',
                params.previous_jdbc_jar,
                params.downloaded_custom_connector,
                params.driver_curl_source,
                params.driver_curl_target,
                params.java_home,
                params.repo_name,
                params.knox_ranger_plugin_repo,
                params.ranger_env,
                params.ranger_plugin_properties,
                params.policy_user,
                params.policymgr_mgr_url,
                params.enable_ranger_knox,
                conf_dict=params.knox_conf_dir,
                component_user=params.knox_user,
                component_group=params.knox_group,
                cache_service_list=['knox'],
                plugin_audit_properties=params.config['configurations']
                ['ranger-knox-audit'],
                plugin_audit_attributes=params.
                config['configurationAttributes']['ranger-knox-audit'],
                plugin_security_properties=params.config['configurations']
                ['ranger-knox-security'],
                plugin_security_attributes=params.
                config['configurationAttributes']['ranger-knox-security'],
                plugin_policymgr_ssl_properties=params.config['configurations']
                ['ranger-knox-policymgr-ssl'],
                plugin_policymgr_ssl_attributes=params.
                config['configurationAttributes']['ranger-knox-policymgr-ssl'],
                component_list=['knox-server'],
                audit_db_is_enabled=params.xa_audit_db_is_enabled,
                credential_file=params.credential_file,
                xa_audit_db_password=params.xa_audit_db_password,
                ssl_truststore_password=params.ssl_truststore_password,
                ssl_keystore_password=params.ssl_keystore_password,
                stack_version_override=stack_version,
                skip_if_rangeradmin_down=not params.retryAble,
                api_version=api_version,
                is_security_enabled=params.security_enabled,
                is_stack_supports_ranger_kerberos=params.
                stack_supports_ranger_kerberos,
                component_user_principal=params.knox_principal_name
                if params.security_enabled else None,
                component_user_keytab=params.knox_keytab_path
                if params.security_enabled else None)
        else:
            from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
            setup_ranger_plugin(
                'knox-server',
                'knox',
                params.previous_jdbc_jar,
                params.downloaded_custom_connector,
                params.driver_curl_source,
                params.driver_curl_target,
                params.java_home,
                params.repo_name,
                params.knox_ranger_plugin_repo,
                params.ranger_env,
                params.ranger_plugin_properties,
                params.policy_user,
                params.policymgr_mgr_url,
                params.enable_ranger_knox,
                conf_dict=params.knox_conf_dir,
                component_user=params.knox_user,
                component_group=params.knox_group,
                cache_service_list=['knox'],
                plugin_audit_properties=params.config['configurations']
                ['ranger-knox-audit'],
                plugin_audit_attributes=params.
                config['configurationAttributes']['ranger-knox-audit'],
                plugin_security_properties=params.config['configurations']
                ['ranger-knox-security'],
                plugin_security_attributes=params.
                config['configurationAttributes']['ranger-knox-security'],
                plugin_policymgr_ssl_properties=params.config['configurations']
                ['ranger-knox-policymgr-ssl'],
                plugin_policymgr_ssl_attributes=params.
                config['configurationAttributes']['ranger-knox-policymgr-ssl'],
                component_list=['knox-server'],
                audit_db_is_enabled=params.xa_audit_db_is_enabled,
                credential_file=params.credential_file,
                xa_audit_db_password=params.xa_audit_db_password,
                ssl_truststore_password=params.ssl_truststore_password,
                ssl_keystore_password=params.ssl_keystore_password,
                stack_version_override=stack_version,
                skip_if_rangeradmin_down=not params.retryAble)
        if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.security_enabled:
            if params.has_namenode:
                Logger.info(
                    "Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
                )
                setup_configuration_file_for_required_plugins(
                    component_user=params.knox_user,
                    component_group=params.knox_group,
                    create_core_site_path=params.knox_conf_dir,
                    configurations=params.config['configurations']
                    ['core-site'],
                    configuration_attributes=params.
                    config['configurationAttributes']['core-site'],
                    file_name='core-site.xml')
            else:
                Logger.info(
                    "Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
                )
                setup_configuration_file_for_required_plugins(
                    component_user=params.knox_user,
                    component_group=params.knox_group,
                    create_core_site_path=params.knox_conf_dir,
                    configurations={
                        'hadoop.security.authentication':
                        'kerberos' if params.security_enabled else 'simple'
                    },
                    configuration_attributes={},
                    file_name='core-site.xml')
        else:
            Logger.info(
                "Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
            )

    else:
        Logger.info('Ranger Knox plugin is not enabled')
Esempio n. 30
0
def hbase(name=None):
    import params

    # ensure that matching LZO libraries are installed for HBase
    lzo_utils.install_lzo_if_needed()

    Directory(params.etc_prefix_dir, mode=0755)

    Directory(params.hbase_conf_dir,
              owner=params.hbase_user,
              group=params.user_group,
              create_parents=True)

    Directory(params.java_io_tmpdir, create_parents=True, mode=0777)

    # If a file location is specified in ioengine parameter,
    # ensure that directory exists. Otherwise create the
    # directory with permissions assigned to hbase:hadoop.
    ioengine_input = params.ioengine_param
    if ioengine_input != None:
        if ioengine_input.startswith("file:/"):
            ioengine_fullpath = ioengine_input[5:]
            ioengine_dir = os.path.dirname(ioengine_fullpath)
            Directory(ioengine_dir,
                      owner=params.hbase_user,
                      group=params.user_group,
                      create_parents=True,
                      mode=0755)

    parent_dir = os.path.dirname(params.tmp_dir)
    # In case if we have several placeholders in path
    while ("${" in parent_dir):
        parent_dir = os.path.dirname(parent_dir)
    if parent_dir != os.path.abspath(os.sep):
        Directory(
            parent_dir,
            create_parents=True,
            cd_access="a",
        )
        Execute(("chmod", "1777", parent_dir), sudo=True)

    XmlConfig("hbase-site.xml",
              conf_dir=params.hbase_conf_dir,
              configurations=params.config['configurations']['hbase-site'],
              configuration_attributes=params.config['configurationAttributes']
              ['hbase-site'],
              owner=params.hbase_user,
              group=params.user_group)

    if check_stack_feature(StackFeature.PHOENIX_CORE_HDFS_SITE_REQUIRED,
                           params.version_for_stack_feature_checks):
        XmlConfig(
            "core-site.xml",
            conf_dir=params.hbase_conf_dir,
            configurations=params.config['configurations']['core-site'],
            configuration_attributes=params.config['configurationAttributes']
            ['core-site'],
            owner=params.hbase_user,
            group=params.user_group)
        if 'hdfs-site' in params.config['configurations']:
            XmlConfig(
                "hdfs-site.xml",
                conf_dir=params.hbase_conf_dir,
                configurations=params.config['configurations']['hdfs-site'],
                configuration_attributes=params.
                config['configurationAttributes']['hdfs-site'],
                owner=params.hbase_user,
                group=params.user_group)
    else:
        File(format("{params.hbase_conf_dir}/hdfs-site.xml"), action="delete")
        File(format("{params.hbase_conf_dir}/core-site.xml"), action="delete")

    if 'hbase-policy' in params.config['configurations']:
        XmlConfig(
            "hbase-policy.xml",
            conf_dir=params.hbase_conf_dir,
            configurations=params.config['configurations']['hbase-policy'],
            configuration_attributes=params.config['configurationAttributes']
            ['hbase-policy'],
            owner=params.hbase_user,
            group=params.user_group)
    # Manually overriding ownership of file installed by hadoop package
    else:
        File(format("{params.hbase_conf_dir}/hbase-policy.xml"),
             owner=params.hbase_user,
             group=params.user_group)

    File(
        format("{hbase_conf_dir}/hbase-env.sh"),
        owner=params.hbase_user,
        content=InlineTemplate(params.hbase_env_sh_template),
        group=params.user_group,
    )

    # On some OS this folder could be not exists, so we will create it before pushing there files
    Directory(params.limits_conf_dir,
              create_parents=True,
              owner='root',
              group='root')

    File(os.path.join(params.limits_conf_dir, 'hbase.conf'),
         owner='root',
         group='root',
         mode=0644,
         content=Template("hbase.conf.j2"))

    hbase_TemplateConfig(
        params.metric_prop_file_name,
        tag='GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS')

    hbase_TemplateConfig('regionservers')

    if params.security_enabled:
        hbase_TemplateConfig(format("hbase_{name}_jaas.conf"))

    if name != "client":
        Directory(
            params.pid_dir,
            owner=params.hbase_user,
            create_parents=True,
            cd_access="a",
            mode=0755,
        )

        Directory(
            params.log_dir,
            owner=params.hbase_user,
            create_parents=True,
            cd_access="a",
            mode=0755,
        )

    if (params.log4j_props != None):
        File(format("{params.hbase_conf_dir}/log4j.properties"),
             mode=0644,
             group=params.user_group,
             owner=params.hbase_user,
             content=InlineTemplate(params.log4j_props))
    elif (os.path.exists(format("{params.hbase_conf_dir}/log4j.properties"))):
        File(format("{params.hbase_conf_dir}/log4j.properties"),
             mode=0644,
             group=params.user_group,
             owner=params.hbase_user)
    if name == "master" and params.default_fs:
        if not params.hbase_hdfs_root_dir_protocol or params.hbase_hdfs_root_dir_protocol == urlparse(
                params.default_fs).scheme:
            params.HdfsResource(params.hbase_hdfs_root_dir,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hbase_user)
        params.HdfsResource(params.hbase_staging_dir,
                            type="directory",
                            action="create_on_execute",
                            owner=params.hbase_user,
                            mode=0711)
        if params.create_hbase_home_directory:
            params.HdfsResource(params.hbase_home_directory,
                                type="directory",
                                action="create_on_execute",
                                owner=params.hbase_user,
                                mode=0755)
        params.HdfsResource(None, action="execute")

    if name in ('master', 'regionserver') and not params.default_fs:
        Directory(
            params.hbase_staging_dir,
            owner=params.hbase_user,
            create_parents=True,
            cd_access="a",
            mode=0711,
        )

    if params.phoenix_enabled:
        Package(params.phoenix_package,
                retry_on_repo_unavailability=params.
                agent_stack_retry_on_unavailability,
                retry_count=params.agent_stack_retry_count)