Exemplo n.º 1
0
  def test_download_source_get_content_cache_existent_md5_unmatch(self, exists_mock, makedirs_mock, open_mock,
                                                                  urlopen_mock):
    """
    Testing DownloadSource.get_content with cache on cached resource with md5 check
    """
    exists_mock.side_effect = [True, True, False]
    fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
    file_mock = MagicMock(name = 'file_mock')
    file_mock.__enter__.return_value = file_mock
    file_mock.read.return_value = 'cached_content'
    open_mock.return_value = file_mock
    web_file_mock = MagicMock()
    web_file_mock.read.return_value = 'web_content'
    urlopen_mock.return_value = web_file_mock

    with Environment("/base") as env:
      download_source = DownloadSource("http://download/source", cache=True, md5sum=fake_md5)
    content = download_source.get_content()

    self.assertEqual('web_content', content)
    self.assertEqual(open_mock.call_count, 2)
    open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
    open_mock.assert_once_called('/var/tmp/downloads/source')
    self.assertEqual(file_mock.read.call_count, 1)
    self.assertEqual(urlopen_mock.call_count, 1)
    urlopen_mock.assert_called_with('http://download/source')
Exemplo n.º 2
0
def download_database_connector_if_needed():
    """
  Downloads the database connector to use when connecting to the metadata storage
  """
    import params
    if params.metadata_storage_type != 'mysql' or not params.jdbc_driver_jar:
        return

    File(params.check_db_connection_jar,
         content=DownloadSource(
             format("{jdk_location}{check_db_connection_jar_name}")))

    target_jar_with_directory = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar

    if not os.path.exists(target_jar_with_directory):
        File(params.downloaded_custom_connector,
             content=DownloadSource(params.connector_curl_source))

        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             target_jar_with_directory),
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(target_jar_with_directory,
             owner=params.druid_user,
             group=params.user_group)
Exemplo n.º 3
0
    def test_download_source_get_content_cache_existent_md5_unmatch(
            self, exists_mock, makedirs_mock, open_mock, urlopen_mock):
        """
    Testing DownloadSource.get_content with cache on cached resource with md5 check
    """
        exists_mock.side_effect = [True, True, False]
        fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
        file_mock = MagicMock(name='file_mock')
        file_mock.__enter__.return_value = file_mock
        file_mock.read.return_value = 'cached_content'
        open_mock.return_value = file_mock
        web_file_mock = MagicMock()
        web_file_mock.read.return_value = 'web_content'
        urlopen_mock.return_value = web_file_mock

        with Environment("/base") as env:
            download_source = DownloadSource("http://download/source",
                                             cache=True,
                                             md5sum=fake_md5)
        content = download_source.get_content()

        self.assertEqual('web_content', content)
        self.assertEqual(open_mock.call_count, 2)
        open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
        open_mock.assert_once_called('/var/tmp/downloads/source')
        self.assertEqual(file_mock.read.call_count, 1)
        self.assertEqual(urlopen_mock.call_count, 1)
        urlopen_mock.assert_called_with('http://download/source')
Exemplo n.º 4
0
    def test_download_source_get_content_cache_new(self, exists_mock,
                                                   makedirs_mock, opener_mock,
                                                   request_mock, open_mock):
        """
    Testing DownloadSource.get_content with cache on non-cached resource
    """
        exists_mock.side_effect = [True, False]
        web_file_mock = MagicMock()
        web_file_mock.read.return_value = 'web_content'
        opener_mock.return_value.open = MagicMock(return_value=web_file_mock)

        file_mock = MagicMock(name='file_mock')
        file_mock.__enter__.return_value = file_mock
        file_mock.read.return_value = 'content'
        open_mock.return_value = file_mock

        with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
            download_source = DownloadSource("http://download/source",
                                             redownload_files=False)
        content = download_source.get_content()

        self.assertEqual('web_content', content)
        self.assertEqual(opener_mock.call_count, 1)
        request_mock.assert_called_with('http://download/source')
        self.assertEqual(web_file_mock.read.call_count, 1)
    def patch_mysql_driver(self):
        import params

        Logger.info("Patching mysql driver")

        Logger.info("Mysql Jar source :" + params.driver_source)
        Logger.info("Mysql Jar target :" + params.mysql_driver_target)

        if params.jdbc_jar_name is None:
            raise Fail("Mysql JDBC driver not installed on ambari-server")

        File(params.mysql_driver_target,
             content=DownloadSource(params.driver_source),
             mode=0644)

        start_script = params.dpprofiler_home + "/bin/profiler-agent"

        Logger.info("Updating script : " + start_script)

        search_string = "declare -r app_classpath=\""
        replace_string = "declare -r app_classpath=\"" + params.mysql_driver_target + ":"

        data = open(start_script).read()
        if replace_string not in data:
            Logger.info("Adding mysql in classpath")
            data = data.replace(search_string, replace_string)
            f = open(start_script, 'w')
            f.write(data)
            f.close()
Exemplo n.º 6
0
def jdbc_connector():
  import params
  from urllib2 import HTTPError
  from resource_management import Fail
  for jar_name in params.sqoop_jdbc_drivers_dict:
    if 'mysql-connector-java.jar' in jar_name:
      continue
    downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
    jdbc_symlink_remote = params.sqoop_jdbc_drivers_dict[jar_name]
    jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
    driver_curl_source = format("{jdk_location}/{jdbc_symlink_remote}")
    environment = {
      "no_proxy": format("{ambari_server_hostname}")
    }
    try:
      File(downloaded_custom_connector,
           content = DownloadSource(driver_curl_source),
           mode = 0644,
      )
    except HTTPError:
      error_string = format("Could not download {driver_curl_source}\n\
                 Please upload jdbc driver to server by run command:\n\
                 ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
                 at {ambari_server_hostname}") 
      raise Fail(error_string)
                 
Exemplo n.º 7
0
  def setup_jce(self):
    import params

    if not params.jdk_name:
      return
    jce_curl_target = None
    if params.jce_policy_zip is not None:
      jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
      Directory(params.artifact_dir,
                create_parents = True,
                )
      File(jce_curl_target,
           content = DownloadSource(format("{jce_location}/{jce_policy_zip}")),
           )
    elif params.security_enabled:
      # Something weird is happening
      raise Fail("Security is enabled, but JCE policy zip is not specified.")

    # The extraction will occur only after the security flag is set
    if params.security_enabled:
      security_dir = format("{java_home}/jre/lib/security")

      File([format("{security_dir}/US_export_policy.jar"), format("{security_dir}/local_policy.jar")],
           action = "delete",
           )

      extract_cmd = ("unzip", "-o", "-j", "-q", jce_curl_target, "-d", security_dir)
      Execute(extract_cmd,
              only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
              path = ['/bin/','/usr/bin'],
              sudo = True
      )
Exemplo n.º 8
0
def download_database_library_if_needed():
    """
  Downloads the library to use when connecting to the Oozie database, if
  necessary. The library will be downloaded to 'params.target' unless
  otherwise specified.
  :param target_directory: the location where the database library will be
  downloaded to.
  :return:
  """
    import params
    # check to see if the JDBC driver name is in the list of ones that need to
    # be downloaded
    if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
       params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
        File(
            params.downloaded_custom_connector,
            content=DownloadSource(params.driver_curl_source),
        )
        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             params.target),
            #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(params.target, owner=params.oozie_user, group=params.user_group)
Exemplo n.º 9
0
def jdbc_connector():
  import params

  if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
    environment = {
      "no_proxy": format("{ambari_server_hostname}")
    }

    # TODO: should be removed after ranger_hive_plugin will not provide jdbc
    Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
            path=["/bin", "/usr/bin/"],
            sudo = True)
    
    File(params.downloaded_custom_connector,
         content = DownloadSource(params.driver_curl_source),
    )

    Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
            #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo = True)

  else:
    #for default hive db (Mysql)
    Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), params.target),
            #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo=True
    )
    
  File(params.target,
       mode = 0644,
  )
Exemplo n.º 10
0
def download_database_library_if_needed(target_directory = None):
  """
  Downloads the library to use when connecting to the Oozie database, if
  necessary. The library will be downloaded to 'params.target' unless
  otherwise specified.
  :param target_directory: the location where the database library will be
  downloaded to.
  :return:
  """
  import params
  jdbc_drivers = ["com.mysql.jdbc.Driver",
    "com.microsoft.sqlserver.jdbc.SQLServerDriver",
    "oracle.jdbc.driver.OracleDriver"]

  # check to see if the JDBC driver name is in the list of ones that need to
  # be downloaded
  if params.jdbc_driver_name not in jdbc_drivers:
    return

  # if the target directory is not specified
  if target_directory is None:
    target_jar_with_directory = params.target
  else:
    # create the full path using the supplied target directory and the JDBC JAR
    target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar

  File(params.downloaded_custom_connector,
    content = DownloadSource(params.driver_curl_source))

  Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory),
    path=["/bin", "/usr/bin/"],
    sudo = True)

  File(target_jar_with_directory, owner = params.oozie_user,
    group = params.user_group)
Exemplo n.º 11
0
def setup_ranger_admin(rolling_upgrade=False):
  import params

  ranger_home = params.ranger_home
  ranger_conf = params.ranger_conf

  if rolling_upgrade:
    ranger_home = format("/usr/hdp/{version}/ranger-admin")
    ranger_conf = format("/usr/hdp/{version}/ranger-admin/conf")

  File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
    content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
  )

  cp = format("{check_db_connection_jar}")
  cp = cp + os.pathsep + format("{driver_curl_target}")
  cp = cp + os.pathsep + format("{ranger_home}/ews/webapp/WEB-INF/lib/*")

  db_connection_check_command = format(
    "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}")

  Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)

  Execute(('ln','-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'), format('{ranger_home}/conf')),
    not_if=format("ls {ranger_home}/conf"),
    only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"),
    sudo=True)

  if rolling_upgrade:
    src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml')
    dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml')
    Execute(('cp', '-f', src_file, dst_file), sudo=True)

    src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml')
    dst_file = format('{ranger_home}/conf/security-applicationContext.xml')

    Execute(('cp', '-f', src_file, dst_file), sudo=True)

  Execute(('chown','-R',format('{unix_user}:{unix_group}'), format('{ranger_home}/')), sudo=True)

  Execute(('ln','-sf', format('{ranger_home}/ews/ranger-admin-services.sh'),'/usr/bin/ranger-admin'),
    not_if=format("ls /usr/bin/ranger-admin"),
    only_if=format("ls {ranger_home}/ews/ranger-admin-services.sh"),
    sudo=True)

  XmlConfig("ranger-admin-site.xml",
    conf_dir=ranger_conf,
    configurations=params.config['configurations']['ranger-admin-site'],
    configuration_attributes=params.config['configuration_attributes']['ranger-admin-site'],
    owner=params.unix_user,
    group=params.unix_group,
    mode=0644)

  Directory(os.path.join(ranger_conf,'ranger_jaas'),
    mode=0700,
    owner=params.unix_user,
    group=params.unix_group,
  )

  do_keystore_setup(rolling_upgrade=rolling_upgrade)
Exemplo n.º 12
0
def jdbc_connector():
    import params
    from urllib2 import HTTPError
    from resource_management import Fail
    for jar_name in params.sqoop_jdbc_drivers_dict:
        if not jar_name or 'mysql' in jar_name:
            continue
        downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
        custom_connector_to_remove = format(
            "{sqoop_lib}/" +
            str(params.sqoop_jdbc_drivers_to_remove[jar_name]))
        jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
        driver_curl_source = format("{jdk_location}/{jar_name}")
        environment = {"no_proxy": format("{ambari_server_hostname}")}
        try:
            if custom_connector_to_remove and os.path.isfile(
                    custom_connector_to_remove):
                File(custom_connector_to_remove, action='delete')

            File(
                downloaded_custom_connector,
                content=DownloadSource(driver_curl_source),
                mode=0644,
            )
        except HTTPError:
            error_string = format("Could not download {driver_curl_source}\n\
                 Please upload jdbc driver to server by run command:\n\
                 ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
                 at {ambari_server_hostname}")
            raise Fail(error_string)
Exemplo n.º 13
0
def copy_jdbc_connector(kms_home):
  import params

  if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
    error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
    Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location)
    Logger.error(error_message)

  if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
    if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
      File(params.previous_jdbc_jar, action='delete')

  driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")

  File(params.downloaded_custom_connector,
    content = DownloadSource(params.driver_curl_source),
    mode = 0644
  )

  Directory(os.path.join(kms_home, 'ews', 'lib'),
    mode=0755
  )

  if params.db_flavor.lower() == 'sqla':
    Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)

    Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
      path=["/bin", "/usr/bin/"],
      sudo=True)

    Directory(params.jdbc_libs_dir,
      cd_access="a",
      create_parents=True)

    Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
      path=["/bin", "/usr/bin/"])

    File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644)
  else:
    Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
      path=["/bin", "/usr/bin/"],
      sudo=True)

    File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)

  ModifyPropertiesFile(format("{kms_home}/install.properties"),
    properties = params.config['configurations']['kms-properties'],
    owner = params.kms_user
  )

  if params.db_flavor.lower() == 'sqla':
    ModifyPropertiesFile(format("{kms_home}/install.properties"),
      properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')},
      owner = params.kms_user,
    )
  else:
    ModifyPropertiesFile(format("{kms_home}/install.properties"),
      properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
      owner = params.kms_user,
    )
Exemplo n.º 14
0
def setup_kms_jce():
    import params

    if params.jce_name is not None:
        Directory(params.jce_source_dir, create_parents=True)

        jce_target = format('{jce_source_dir}/{jce_name}')

        File(
            jce_target,
            content=DownloadSource(format('{jdk_location}/{jce_name}')),
            mode=0644,
        )

        File(
            [
                format("{java_home}/jre/lib/security/local_policy.jar"),
                format("{java_home}/jre/lib/security/US_export_policy.jar")
            ],
            action="delete",
        )

        unzip_cmd = ("unzip", "-o", "-j", "-q", jce_target, "-d",
                     format("{java_home}/jre/lib/security"))

        Execute(
            unzip_cmd,
            only_if=format(
                "test -e {java_home}/jre/lib/security && test -f {jce_target}"
            ),
            path=['/bin/', '/usr/bin'],
            sudo=True)
    else:
        Logger.warning(
            "Required jce policy zip is not available, need to setup manually")
Exemplo n.º 15
0
def jdbc_connector(target, hive_previous_jdbc_jar):
    """
    Shared by Hive Batch, Hive Metastore, and Hive Interactive
    :param target: Target of jdbc jar name, which could be for any of the components above.
    """
    import params

    if not params.jdbc_jar_name:
        return

    if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
        environment = {"no_proxy": format("{ambari_server_hostname}")}

        if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar):
            File(hive_previous_jdbc_jar, action='delete')

        File(params.downloaded_custom_connector,
             content=DownloadSource(params.driver_curl_source))

        Execute(('cp', '-r', params.downloaded_custom_connector, target),
                path=["/bin", "/usr/bin/"],
                sudo=True)

    else:
        # for default hive db (Mysql)
        Execute(
            ('cp', '-r', format('/usr/share/java/{jdbc_jar_name}'), target),
            path=["/bin", "/usr/bin/"],
            sudo=True)
    pass

    File(
        target,
        mode=0644,
    )
Exemplo n.º 16
0
def jdbc_connector(target, hive_previous_jdbc_jar):
  """
  Shared by Hive Batch, Hive Metastore, and Hive Interactive
  :param target: Target of jdbc jar name, which could be for any of the components above.
  """
  import params

  if not params.jdbc_jar_name:
    return

  if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
    environment = {
      "no_proxy": format("{ambari_server_hostname}")
    }

    if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar):
      File(hive_previous_jdbc_jar, action='delete')

    # TODO: should be removed after ranger_hive_plugin will not provide jdbc
    if params.prepackaged_jdbc_name != params.jdbc_jar_name:
      Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
              path=["/bin", "/usr/bin/"],
              sudo = True)
    
    File(params.downloaded_custom_connector,
         content = DownloadSource(params.driver_curl_source))

    # maybe it will be more correcvly to use db type
    if params.sqla_db_used:
      untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir)

      Execute(untar_sqla_type2_driver, sudo = True)

      Execute(format("yes | {sudo} cp {jars_path_in_archive} {hive_lib}"))

      Directory(params.jdbc_libs_dir,
                create_parents = True)

      Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}"))

      Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))

    else:
      Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target),
            #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo = True)

  else:
    #for default hive db (Mysql)
    Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), target),
            #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo=True
    )
  pass

  File(target,
       mode = 0644,
  )
Exemplo n.º 17
0
def copy_jdbc_connector(stack_version=None):
  import params

  if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
    error_message = format("{db_flavor} jdbc driver cannot be downloaded from {jdk_location}\nPlease run 'ambari-server setup --jdbc-db={db_flavor} --jdbc-driver={{path_to_jdbc}}' on ambari-server host.")
    raise Fail(error_message)

  if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
    if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
      File(params.previous_jdbc_jar, action='delete')

  File(params.downloaded_custom_connector,
    content = DownloadSource(params.driver_curl_source),
    mode = 0644
  )

  ranger_home = params.ranger_home
  if stack_version is not None:
    ranger_home = format("{stack_root}/{stack_version}/ranger-admin")

  driver_curl_target = format("{ranger_home}/ews/lib/{jdbc_jar_name}")

  if params.db_flavor.lower() == 'sqla':
    Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)

    Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')),
      path=["/bin", "/usr/bin/"],
      sudo=True)

    File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)

    Directory(params.jdbc_libs_dir,
      cd_access="a",
      create_parents=True)

    Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
            path=["/bin", "/usr/bin/"])
  else:
    Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')),
      path=["/bin", "/usr/bin/"],
      sudo=True)

    File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)

  ModifyPropertiesFile(format("{ranger_home}/install.properties"),
    properties = params.config['configurations']['admin-properties'],
    owner = params.unix_user,
  )

  if params.db_flavor.lower() == 'sqla':
    ModifyPropertiesFile(format("{ranger_home}/install.properties"),
      properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')},
      owner = params.unix_user,
    )
  else:
    ModifyPropertiesFile(format("{ranger_home}/install.properties"),
      properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
       owner = params.unix_user,
    )
Exemplo n.º 18
0
  def test_download_source_get_content_nocache(self, exists_mock, request_mock, opener_mock):
    """
    Testing DownloadSource.get_content without cache
    """
    exists_mock.return_value = True
    web_file_mock = MagicMock()
    web_file_mock.read.return_value = 'web_content'
    opener_mock.return_value.open = MagicMock(return_value=web_file_mock)

    with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
      download_source = DownloadSource("http://download/source", redownload_files=True)
    content = download_source.get_content()

    self.assertEqual('web_content', content)
    self.assertEqual(opener_mock.call_count, 1)
    request_mock.assert_called_with('http://download/source')
    self.assertEqual(web_file_mock.read.call_count, 1)
Exemplo n.º 19
0
  def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, urlopen_mock):
    """
    Testing DownloadSource.get_content with cache on non-cached resource
    """
    exists_mock.side_effect = [True, False]
    web_file_mock = MagicMock()
    web_file_mock.read.return_value = 'web_content'
    urlopen_mock.return_value = web_file_mock

    with Environment("/base") as env:
      download_source = DownloadSource("http://download/source", cache=True)
    content = download_source.get_content()

    self.assertEqual('web_content', content)
    self.assertEqual(urlopen_mock.call_count, 1)
    urlopen_mock.assert_called_with('http://download/source')
    self.assertEqual(web_file_mock.read.call_count, 1)
Exemplo n.º 20
0
def jdbc_connector():
    import params

    if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
        environment = {"no_proxy": format("{ambari_server_hostname}")}

        # TODO: should be removed after ranger_hive_plugin will not provide jdbc
        Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
                path=["/bin", "/usr/bin/"],
                sudo=True)

        File(params.downloaded_custom_connector,
             content=DownloadSource(params.driver_curl_source))

        # maybe it will be more correcvly to use db type
        if params.sqla_db_used:
            untar_sqla_type2_driver = ('tar', '-xvf',
                                       params.downloaded_custom_connector,
                                       '-C', params.tmp_dir)

            Execute(untar_sqla_type2_driver, sudo=True)

            Execute(as_sudo([
                'yes', '|', 'cp', params.jars_path_in_archive, params.hive_lib
            ],
                            auto_escape=False),
                    path=["/bin", "/usr/bin/"])

            Directory(params.jdbc_libs_dir, recursive=True)

            Execute(as_sudo([
                'yes', '|', 'cp', params.libs_path_in_archive,
                params.jdbc_libs_dir
            ],
                            auto_escape=False),
                    path=["/bin", "/usr/bin/"])

        else:
            Execute(
                ('cp', '--remove-destination',
                 params.downloaded_custom_connector, params.target),
                #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
                path=["/bin", "/usr/bin/"],
                sudo=True)

    else:
        #for default hive db (Mysql)
        Execute(
            ('cp', '--remove-destination',
             format('/usr/share/java/{jdbc_jar_name}'), params.target),
            #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo=True)

    File(
        params.target,
        mode=0644,
    )
Exemplo n.º 21
0
def download_mysql_driver():
    import params

    if params.jdbc_jar_name is None:
        raise Fail("Mysql JDBC driver not installed on ambari-server")

    File(params.mysql_driver_target,
         content=DownloadSource(params.driver_source),
         mode=0644)
Exemplo n.º 22
0
def downloadjar(cs_lib_path, jdk_location):
    # Try to download CredentialUtil.jar from ambari-server resources
    credential_util_dir = cs_lib_path.split('*')[0].split(":")[-1] # Remove the trailing '*' and get the last directory if an entire path is passed
    credential_util_path = os.path.join(credential_util_dir, credential_util_jar)
    credential_util_url =  jdk_location + '/' + credential_util_jar
    File(credential_util_path,
         content = DownloadSource(credential_util_url),
         mode = 0755,
         )
Exemplo n.º 23
0
def hive(name=None):
  import params
    
  Directory(params.hive_etc_dir_prefix,
            mode=0755
  )

  # We should change configurations for client as well as for server.
  # The reason is that stale-configs are service-level, not component.
  Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
  for conf_dir in params.hive_conf_dirs_list:
    fill_conf_dir(conf_dir)

  XmlConfig("hive-site.xml",
            conf_dir=params.hive_config_dir,
            configurations=params.hive_site_config,
            configuration_attributes=params.config['configuration_attributes']['hive-site'],
            owner=params.hive_user,
            group=params.user_group,
            mode=0644)

  # Generate atlas-application.properties.xml file
  if has_atlas_in_cluster():
    atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename)
    setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
  
  File(format("{hive_config_dir}/hive-env.sh"),
       owner=params.hive_user,
       group=params.user_group,
       content=InlineTemplate(params.hive_env_sh_template)
  )

  # On some OS this folder could be not exists, so we will create it before pushing there files
  Directory(params.limits_conf_dir,
            create_parents = True,
            owner='root',
            group='root'
            )

  File(os.path.join(params.limits_conf_dir, 'hive.conf'),
       owner='root',
       group='root',
       mode=0644,
       content=Template("hive.conf.j2")
       )

  File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
       content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
       mode = 0644,
  )

  if name != "client":
    setup_non_client()
  if name == 'hiveserver2':
    setup_hiveserver2()
  if name == 'metastore':
    setup_metastore() # schematool work
Exemplo n.º 24
0
    def test_download_source_get_content_cache_existent(self, exists_mock, open_mock):
        """
    Testing DownloadSource.get_content with cache on cached resource
    """
        exists_mock.side_effect = [True, True]

        file_mock = MagicMock(name="file_mock")
        file_mock.__enter__.return_value = file_mock
        file_mock.read.return_value = "cached_content"
        open_mock.return_value = file_mock

        with Environment("/base", tmp_dir="/var/tmp/downloads") as env:
            download_source = DownloadSource("http://download/source", redownload_files=False)
        content = download_source.get_content()

        self.assertEqual("cached_content", content)
        self.assertEqual(open_mock.call_count, 1)
        self.assertEqual(file_mock.read.call_count, 1)
Exemplo n.º 25
0
  def test_download_source_get_content_cache_existent(self, exists_mock, open_mock):
    """
    Testing DownloadSource.get_content with cache on cached resource
    """
    exists_mock.side_effect = [True, True]

    file_mock = MagicMock(name = 'file_mock')
    file_mock.__enter__.return_value = file_mock
    file_mock.read.return_value = 'cached_content'
    open_mock.return_value = file_mock


    with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
      download_source = DownloadSource("http://download/source", redownload_files=False)
    content = download_source.get_content()

    self.assertEqual('cached_content', content)
    self.assertEqual(open_mock.call_count, 1)
    self.assertEqual(file_mock.read.call_count, 1)
Exemplo n.º 26
0
    def test_download_source_get_content_nocache(self, exists_mock,
                                                 makedirs_mock, urlopen_mock):
        """
    Testing DownloadSource.get_content without cache
    """
        exists_mock.return_value = True
        web_file_mock = MagicMock()
        web_file_mock.read.return_value = 'web_content'
        urlopen_mock.return_value = web_file_mock

        with Environment("/base") as env:
            download_source = DownloadSource("http://download/source",
                                             cache=False)
        content = download_source.get_content()

        self.assertEqual('web_content', content)
        self.assertEqual(urlopen_mock.call_count, 1)
        urlopen_mock.assert_called_with('http://download/source')
        self.assertEqual(web_file_mock.read.call_count, 1)
Exemplo n.º 27
0
def setup_ranger_plugin(component_select_name, service_name,
                        downloaded_custom_connector, driver_curl_source,
                        driver_curl_target, java_home,
                        repo_name, plugin_repo_dict, 
                        ranger_env_properties, plugin_properties,
                        policy_user, policymgr_mgr_url,
                        plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs):
  File(downloaded_custom_connector,
      content = DownloadSource(driver_curl_source),
      mode = 0644
  )

  Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
          path=["/bin", "/usr/bin/"],
          sudo=True
  )

  File(driver_curl_target, mode=0644)

  hdp_version = get_hdp_version(component_select_name)
  file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
  
  if not os.path.isfile(file_path):
    raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
  
  ModifyPropertiesFile(file_path,
    properties = plugin_properties
  )

  custom_plugin_properties = dict()
  custom_plugin_properties['CUSTOM_USER'] = component_user
  custom_plugin_properties['CUSTOM_GROUP'] = component_group
  ModifyPropertiesFile(file_path,properties = custom_plugin_properties)

  if plugin_enabled:
    cmd = (format('enable-{service_name}-plugin.sh'),)
    if api_version == 'v2' and api_version is not None:
      ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
    else:
      ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)

    ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
                                            ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], 
                                            ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], 
                                            policy_user)
  else:
    cmd = (format('disable-{service_name}-plugin.sh'),)
    
  cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
  
  Execute(cmd, 
        environment=cmd_env, 
        logoutput=True,
        sudo=True,
  )
Exemplo n.º 28
0
def setup_ranger_db(rolling_upgrade=False):
    import params

    File(params.downloaded_custom_connector,
         content=DownloadSource(params.driver_curl_source))

    Directory(params.java_share_dir, mode=0755)

    if not os.path.isfile(params.driver_curl_target):
        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             params.driver_curl_target),
            path=["/bin", "/usr/bin/"],
            not_if=format("test -f {driver_curl_target}"),
            sudo=True)

    ranger_home = params.ranger_home
    if rolling_upgrade:
        ranger_home = format("/usr/hdp/{version}/ranger-admin")

    if not os.path.isfile(
            os.path.join(params.ranger_home, 'ews', 'lib',
                         params.jdbc_jar_name)):
        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             os.path.join(params.ranger_home, 'ews', 'lib')),
            path=["/bin", "/usr/bin/"],
            sudo=True)

    ModifyPropertiesFile(
        format("{ranger_home}/install.properties"),
        properties=params.config['configurations']['admin-properties'])

    # User wants us to setup the DB user and DB?
    if params.create_db_dbuser:
        Logger.info('Setting up Ranger DB and DB User')
        dba_setup = format('python {ranger_home}/dba_script.py -q')
        Execute(dba_setup,
                environment={
                    'RANGER_ADMIN_HOME': ranger_home,
                    'JAVA_HOME': params.java_home
                },
                logoutput=True)
    else:
        Logger.info(
            'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
        )

    db_setup = format('python {ranger_home}/db_setup.py')
    Execute(db_setup,
            environment={
                'RANGER_ADMIN_HOME': ranger_home,
                'JAVA_HOME': params.java_home
            },
            logoutput=True)
Exemplo n.º 29
0
  def test_download_source_get_content_cache_existent(self, exists_mock, makedirs_mock, open_mock):
    """
    Testing DownloadSource.get_content with cache on cached resource
    """
    exists_mock.side_effect = [True, True, False]

    file_mock = MagicMock(name = 'file_mock')
    file_mock.__enter__.return_value = file_mock
    file_mock.read.return_value = 'cached_content'
    open_mock.return_value = file_mock


    with Environment("/base") as env:
      download_source = DownloadSource("http://download/source", cache=True)
    content = download_source.get_content()

    self.assertEqual('cached_content', content)
    self.assertEqual(open_mock.call_count, 1)
    open_mock.assert_called_with('/var/tmp/downloads/source')
    self.assertEqual(file_mock.read.call_count, 1)
Exemplo n.º 30
0
def download_database_library_if_needed(target_directory = None):
  """
  Downloads the library to use when connecting to the Oozie database, if
  necessary. The library will be downloaded to 'params.target' unless
  otherwise specified.
  :param target_directory: the location where the database library will be
  downloaded to.
  :return:
  """
  import params
  jdbc_drivers = ["com.mysql.jdbc.Driver",
    "com.microsoft.sqlserver.jdbc.SQLServerDriver",
    "oracle.jdbc.driver.OracleDriver","sap.jdbc4.sqlanywhere.IDriver"]

  # check to see if the JDBC driver name is in the list of ones that need to
  # be downloaded
  if params.jdbc_driver_name not in jdbc_drivers or not params.jdbc_driver_jar:
    return

  if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
    File(params.previous_jdbc_jar, action='delete')

  # if the target directory is not specified
  if target_directory is None:
    target_jar_with_directory = params.target
  else:
    # create the full path using the supplied target directory and the JDBC JAR
    target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar

  if not os.path.exists(target_jar_with_directory):
    File(params.downloaded_custom_connector,
      content = DownloadSource(params.driver_curl_source))

    if params.sqla_db_used:
      untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir)

      Execute(untar_sqla_type2_driver, sudo = True)

      Execute(format("yes | {sudo} cp {jars_path_in_archive} {oozie_libext_dir}"))

      Directory(params.jdbc_libs_dir,
                create_parents = True)

      Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}"))

      Execute(format("{sudo} chown -R {oozie_user}:{user_group} {oozie_libext_dir}/*"))

    else:
      Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory),
        path=["/bin", "/usr/bin/"],
        sudo = True)

    File(target_jar_with_directory, owner = params.oozie_user,
      group = params.user_group)
  def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, opener_mock, request_mock, open_mock):
    """
    Testing DownloadSource.get_content with cache on non-cached resource
    """
    exists_mock.side_effect = [True, False]
    web_file_mock = MagicMock()
    web_file_mock.read.return_value = 'web_content'
    opener_mock.return_value.open = MagicMock(return_value=web_file_mock)
    
    file_mock = MagicMock(name = 'file_mock')
    file_mock.__enter__.return_value = file_mock
    file_mock.read.return_value = 'content'
    open_mock.return_value = file_mock

    with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
      download_source = DownloadSource("http://download/source", redownload_files=False)
    content = download_source.get_content()

    self.assertEqual('web_content', content)
    self.assertEqual(opener_mock.call_count, 1)
    request_mock.assert_called_with('http://download/source')
    self.assertEqual(web_file_mock.read.call_count, 1)
Exemplo n.º 32
0
    def test_download_source_get_content_cache_existent_md5_match(
            self, exists_mock, makedirs_mock, open_mock, urlopen_mock):
        """
    Testing DownloadSource.get_content with cache on cached resource with md5 check
    """
        exists_mock.side_effect = [True, True, False]

        file_mock = MagicMock(name='file_mock')
        file_mock.__enter__.return_value = file_mock
        file_mock.read.return_value = 'cached_content'
        open_mock.return_value = file_mock

        with Environment("/base") as env:
            download_source = DownloadSource("http://download/source",
                                             cache=True)
        content = download_source.get_content()

        self.assertEqual('cached_content', content)
        self.assertEqual(open_mock.call_count, 1)
        open_mock.assert_called_with('/var/tmp/downloads/source')
        self.assertEqual(file_mock.read.call_count, 1)
        self.assertEqual(urlopen_mock.call_count, 0)
Exemplo n.º 33
0
    def test_download_source_init_existent_download_directory(
            self, exists_mock, makedirs_mock):
        """
    Testing DownloadSource without cache with existent download directory
    """
        exists_mock.return_value = True

        with Environment("/base") as env:
            static_file = DownloadSource("http://download/source")

        self.assertEqual(makedirs_mock.call_count, 0)
        self.assertEqual(exists_mock.call_count, 1)
        pass
Exemplo n.º 34
0
    def test_download_source_init_nonexistent_download_directory(
            self, exists_mock, makedirs_mock):
        """
    Testing DownloadSource without cache with non-existent download directory
    """
        exists_mock.return_value = False

        with Environment("/base") as env:
            static_file = DownloadSource("http://download/source")

        self.assertEqual(makedirs_mock.call_count, 1)
        makedirs_mock.assert_called_with("/var/tmp/downloads")
        self.assertEqual(exists_mock.call_count, 1)
        pass
Exemplo n.º 35
0
def setup_kms_db():
    import params

    if params.has_ranger_admin:

        File(params.downloaded_custom_connector,
             content=DownloadSource(params.driver_curl_source),
             mode=0644)

        Directory(params.java_share_dir, mode=0755)

        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             params.driver_curl_target),
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(params.driver_curl_target, mode=0644)

        Directory(os.path.join(params.kms_home, 'ews', 'lib'), mode=0755)

        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib',
                          params.jdbc_jar_name),
             mode=0644)

        ModifyPropertiesFile(
            format("/usr/hdp/current/ranger-kms/install.properties"),
            properties=params.config['configurations']['kms-properties'])

        dba_setup = format('python {kms_home}/dba_script.py -q')
        db_setup = format('python {kms_home}/db_setup.py')

        Execute(dba_setup,
                environment={
                    'RANGER_KMS_HOME': params.kms_home,
                    'JAVA_HOME': params.java_home
                },
                logoutput=True)
        Execute(db_setup,
                environment={
                    'RANGER_KMS_HOME': params.kms_home,
                    'JAVA_HOME': params.java_home
                },
                logoutput=True)