def pre_upgrade_restart(self, env, upgrade_type=None):
    """
    Performs the tasks that should be done before an upgrade of oozie. This includes:
      - backing up configurations
      - running hdp-select and conf-select
      - restoring configurations
      - preparing the libext directory
    :param env:
    :return:
    """
    import params
    env.set_params(params)

    # this function should not execute if the version can't be determined or
    # is not at least HDP 2.2.0.0
    if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
      return

    Logger.info("Executing Oozie Server Stack Upgrade pre-restart")

    OozieUpgrade.backup_configuration()

    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
      conf_select.select(params.stack_name, "oozie", params.version)
      hdp_select.select("oozie-server", params.version)

    OozieUpgrade.restore_configuration()
    OozieUpgrade.prepare_libext_directory()
  def configure(self, env, upgrade_type=None):
    import params

    # The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
    if upgrade_type is None:
      restart_type = default("/commandParams/restart_type", "")
      if restart_type.lower() == "rolling_upgrade":
        upgrade_type = UPGRADE_TYPE_ROLLING
      elif restart_type.lower() == "nonrolling_upgrade":
        upgrade_type = UPGRADE_TYPE_NON_ROLLING

    if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
      Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"))
      if compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
        # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
        # oozie, we need to create the symlinks both for server and client.
        # This is required as both need to be pointing to new installed oozie version.

        # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
        hdp_select.select("oozie-client", params.version)
        # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
        hdp_select.select("oozie-server", params.version)

      if compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') >= 0:
        conf_select.select(params.stack_name, "oozie", params.version)

    env.set_params(params)
    oozie(is_server=True)
Beispiel #3
0
    def configure(self, env, upgrade_type=None):
        import params

        # The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary
        if upgrade_type is None:
            restart_type = default("/commandParams/restart_type", "")
            if restart_type.lower() == "rolling_upgrade":
                upgrade_type = UPGRADE_TYPE_ROLLING
            elif restart_type.lower() == "nonrolling_upgrade":
                upgrade_type = UPGRADE_TYPE_NON_ROLLING

        if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None:
            Logger.info(
                format(
                    "Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}"
                ))
            if compare_versions(format_hdp_stack_version(params.version),
                                '2.2.0.0') >= 0:
                # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
                # oozie, we need to create the symlinks both for server and client.
                # This is required as both need to be pointing to new installed oozie version.

                # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
                hdp_select.select("oozie-client", params.version)
                # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
                hdp_select.select("oozie-server", params.version)

            if compare_versions(format_hdp_stack_version(params.version),
                                '2.3.0.0') >= 0:
                conf_select.select(params.stack_name, "oozie", params.version)

        env.set_params(params)
        oozie(is_server=True)
Beispiel #4
0
    def pre_upgrade_restart(self, env, upgrade_type=None):
        """
    Performs the tasks that should be done before an upgrade of oozie. This includes:
      - backing up configurations
      - running hdp-select and conf-select
      - restoring configurations
      - preparing the libext directory
    :param env:
    :return:
    """
        import params
        env.set_params(params)

        # this function should not execute if the version can't be determined or
        # is not at least HDP 2.2.0.0
        if not params.version or compare_versions(
                format_hdp_stack_version(params.version), '2.2.0.0') < 0:
            return

        Logger.info("Executing Oozie Server Stack Upgrade pre-restart")

        OozieUpgrade.backup_configuration()

        if params.version and compare_versions(
                format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
            conf_select.select(params.stack_name, "oozie", params.version)
            hdp_select.select("oozie-server", params.version)

        OozieUpgrade.restore_configuration()
        OozieUpgrade.prepare_libext_directory()
Beispiel #5
0
  def pre_rolling_restart(self, env):
    """
    Performs the tasks surrounding the Oozie startup when a rolling upgrade
    is in progress. This includes backing up the configuration, updating
    the database, preparing the WAR, and installing the sharelib in HDFS.
    :param env:
    :return:
    """
    import params
    env.set_params(params)

    # this function should not execute if the version can't be determined or
    # is not at least HDP 2.2.0.0
    if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
      return

    Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")

    oozie_server_upgrade.backup_configuration()

    conf_select.select(params.stack_name, "oozie", params.version)
    hdp_select.select("oozie-server", params.version)

    oozie_server_upgrade.restore_configuration()
    oozie_server_upgrade.prepare_libext_directory()
    oozie_server_upgrade.upgrade_oozie()
Beispiel #6
0
  def pre_rolling_restart(self, env):
    """
    Performs the tasks surrounding the Oozie startup when a rolling upgrade
    is in progress. This includes backing up the configuration, updating
    the database, preparing the WAR, and installing the sharelib in HDFS.
    :param env:
    :return:
    """
    import params
    env.set_params(params)

    # this function should not execute if the version can't be determined or
    # is not at least HDP 2.2.0.0
    if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
      return

    Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")

    oozie_server_upgrade.backup_configuration()

    Execute(format("hdp-select set oozie-server {version}"))

    oozie_server_upgrade.restore_configuration()
    oozie_server_upgrade.prepare_libext_directory()
    oozie_server_upgrade.upgrade_oozie()
Beispiel #7
0
def prepare_libext_directory():
    """
  Creates /usr/hdp/current/oozie/libext-customer and recursively sets
  777 permissions on it and its parents.
  :return:
  """
    import params

    # some versions of HDP don't need the lzo compression libraries
    target_version_needs_compression_libraries = compare_versions(
        format_hdp_stack_version(params.version), '2.2.1.0') >= 0

    if not os.path.isdir(params.oozie_libext_customer_dir):
        os.makedirs(params.oozie_libext_customer_dir, 0o777)

    # ensure that it's rwx for all
    os.chmod(params.oozie_libext_customer_dir, 0o777)

    # get all hadooplzo* JAR files
    # hdp-select set hadoop-client has not run yet, therefore we cannot use
    # /usr/hdp/current/hadoop-client ; we must use params.version directly
    # however, this only works when upgrading beyond 2.2.0.0; don't do this
    # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
    # This can also be called during a Downgrade.
    # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
    # if lzo is enabled.
    if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE
                               or target_version_needs_compression_libraries):
        hadoop_lzo_pattern = 'hadoop-lzo*.jar'
        hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")

        files = glob.iglob(
            os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
        if not files:
            raise Fail("There are no files at {0} matching {1}".format(
                hadoop_client_new_lib_dir, hadoop_lzo_pattern))

        # copy files into libext
        files_copied = False
        for file in files:
            if os.path.isfile(file):
                Logger.info("Copying {0} to {1}".format(
                    str(file), params.oozie_libext_customer_dir))
                shutil.copy2(file, params.oozie_libext_customer_dir)
                files_copied = True

        if not files_copied:
            raise Fail("There are no files at {0} matching {1}".format(
                hadoop_client_new_lib_dir, hadoop_lzo_pattern))

    # copy ext ZIP to customer dir
    oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
    if not os.path.isfile(oozie_ext_zip_file):
        raise Fail("Unable to copy {0} because it does not exist".format(
            oozie_ext_zip_file))

    Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
                                            params.oozie_libext_customer_dir))
    shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
def prepare_libext_directory():
  """
  Creates /usr/hdp/current/oozie/libext-customer and recursively sets
  777 permissions on it and its parents.
  :return:
  """
  import params

  # some versions of HDP don't need the lzo compression libraries
  target_version_needs_compression_libraries = compare_versions(
    format_hdp_stack_version(params.version), '2.2.1.0') >= 0

  if not os.path.isdir(params.oozie_libext_customer_dir):
    os.makedirs(params.oozie_libext_customer_dir, 0o777)

  # ensure that it's rwx for all
  os.chmod(params.oozie_libext_customer_dir, 0o777)

  # get all hadooplzo* JAR files
  # hdp-select set hadoop-client has not run yet, therefore we cannot use
  # /usr/hdp/current/hadoop-client ; we must use params.version directly
  # however, this only works when upgrading beyond 2.2.0.0; don't do this
  # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
  # This can also be called during a Downgrade.
  # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
  # if lzo is enabled.
  if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
    hadoop_lzo_pattern = 'hadoop-lzo*.jar'
    hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")

    files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
    if not files:
      raise Fail("There are no files at {0} matching {1}".format(
        hadoop_client_new_lib_dir, hadoop_lzo_pattern))

    # copy files into libext
    files_copied = False
    for file in files:
      if os.path.isfile(file):
        Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir))
        shutil.copy2(file, params.oozie_libext_customer_dir)
        files_copied = True

    if not files_copied:
      raise Fail("There are no files at {0} matching {1}".format(
        hadoop_client_new_lib_dir, hadoop_lzo_pattern))

  # copy ext ZIP to customer dir
  oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
  if not os.path.isfile(oozie_ext_zip_file):
    raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))

  Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir))
  shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
Beispiel #9
0
    def configure(self, env, upgrade_type=None):
        import params

        if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \
                params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
            conf_select.select(params.stack_name, "oozie", params.version)
            # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
            # oozie, we need to create the symlinks both for server and client.
            # This is required as both need to be pointing to new installed oozie version.

            # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
            hdp_select.select("oozie-client", params.version)
            # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
            hdp_select.select("oozie-server", params.version)
        env.set_params(params)

        oozie(is_server=True)
Beispiel #10
0
  def prepare_libext_directory():
    """
    Performs the following actions on libext:
      - creates /usr/hdp/current/oozie/libext and recursively
      - set 777 permissions on it and its parents.
      - downloads JDBC driver JAR if needed
      - copies Falcon JAR for the Oozie WAR if needed
    """
    import params

    # some versions of HDP don't need the lzo compression libraries
    target_version_needs_compression_libraries = compare_versions(
      format_hdp_stack_version(params.version), '2.2.1.0') >= 0

    # ensure the directory exists
    Directory(params.oozie_libext_dir, mode = 0777)

    # get all hadooplzo* JAR files
    # hdp-select set hadoop-client has not run yet, therefore we cannot use
    # /usr/hdp/current/hadoop-client ; we must use params.version directly
    # however, this only works when upgrading beyond 2.2.0.0; don't do this
    # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
    # This can also be called during a Downgrade.
    # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
    # if lzo is enabled.
    if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
      hadoop_lzo_pattern = 'hadoop-lzo*.jar'
      hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")

      files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
      if not files:
        raise Fail("There are no files at {0} matching {1}".format(
          hadoop_client_new_lib_dir, hadoop_lzo_pattern))

      # copy files into libext
      files_copied = False
      for file in files:
        if os.path.isfile(file):
          Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
          shutil.copy2(file, params.oozie_libext_dir)
          files_copied = True

      if not files_copied:
        raise Fail("There are no files at {0} matching {1}".format(
          hadoop_client_new_lib_dir, hadoop_lzo_pattern))

    # copy ext ZIP to libext dir
    oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'

    # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
    oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")

    if not os.path.isfile(oozie_ext_zip_file):
      raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))

    Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
    Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
    Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
    File(oozie_ext_zip_target_path,
         mode=0644
    )

    # Redownload jdbc driver to a new current location
    oozie.download_database_library_if_needed()

    # get the upgrade version in the event that it's needed
    upgrade_stack = hdp_select._get_upgrade_stack()
    if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
      raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")

    # something like 2.3.0.0-1234
    stack_version = upgrade_stack[1]

    # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
    # use the versioned falcon directory
    if params.has_falcon_host:
      versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version)
      Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))

      Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
      Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
  def prepare_libext_directory():
    """
    Performs the following actions on libext:
      - creates /usr/hdp/current/oozie/libext and recursively
      - set 777 permissions on it and its parents.
      - downloads JDBC driver JAR if needed
      - copies Falcon JAR for the Oozie WAR if needed
    """
    import params

    # some versions of HDP don't need the lzo compression libraries
    target_version_needs_compression_libraries = compare_versions(
      format_hdp_stack_version(params.version), '2.2.1.0') >= 0

    # ensure the directory exists
    Directory(params.oozie_libext_dir, mode = 0777)

    # get all hadooplzo* JAR files
    # hdp-select set hadoop-client has not run yet, therefore we cannot use
    # /usr/hdp/current/hadoop-client ; we must use params.version directly
    # however, this only works when upgrading beyond 2.2.0.0; don't do this
    # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
    # This can also be called during a Downgrade.
    # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
    # if lzo is enabled.
    if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
      hadoop_lzo_pattern = 'hadoop-lzo*.jar'
      hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")

      files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
      if not files:
        raise Fail("There are no files at {0} matching {1}".format(
          hadoop_client_new_lib_dir, hadoop_lzo_pattern))

      # copy files into libext
      files_copied = False
      for file in files:
        if os.path.isfile(file):
          Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
          shutil.copy2(file, params.oozie_libext_dir)
          files_copied = True

      if not files_copied:
        raise Fail("There are no files at {0} matching {1}".format(
          hadoop_client_new_lib_dir, hadoop_lzo_pattern))

    # copy ext ZIP to libext dir
    oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'

    # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
    oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")

    if not os.path.isfile(oozie_ext_zip_file):
      raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))

    Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
    Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
    Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
    sudo.chmod(oozie_ext_zip_target_path, 0644)

    # Redownload jdbc driver to a new current location
    oozie.download_database_library_if_needed()

    # get the upgrade version in the event that it's needed
    upgrade_stack = hdp_select._get_upgrade_stack()
    if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
      raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")

    # something like 2.3.0.0-1234
    stack_version = upgrade_stack[1]

    # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
    # use the versioned falcon directory
    if params.has_falcon_host:
      versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version)
      Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))

      Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
      Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
Beispiel #12
0
def prepare_libext_directory():
    """
  Creates /usr/hdp/current/oozie/libext-customer and recursively sets
  777 permissions on it and its parents.
  Also, downloads jdbc driver and provides other staff
  """
    import params

    # some versions of HDP don't need the lzo compression libraries
    target_version_needs_compression_libraries = compare_versions(
        format_hdp_stack_version(params.version), '2.2.1.0') >= 0

    if not os.path.isdir(params.oozie_libext_customer_dir):
        os.makedirs(params.oozie_libext_customer_dir, 0o777)

    # ensure that it's rwx for all
    os.chmod(params.oozie_libext_customer_dir, 0o777)

    # get all hadooplzo* JAR files
    # hdp-select set hadoop-client has not run yet, therefore we cannot use
    # /usr/hdp/current/hadoop-client ; we must use params.version directly
    # however, this only works when upgrading beyond 2.2.0.0; don't do this
    # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
    # This can also be called during a Downgrade.
    # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
    # if lzo is enabled.
    if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE
                               or target_version_needs_compression_libraries):
        hadoop_lzo_pattern = 'hadoop-lzo*.jar'
        hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")

        files = glob.iglob(
            os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
        if not files:
            raise Fail("There are no files at {0} matching {1}".format(
                hadoop_client_new_lib_dir, hadoop_lzo_pattern))

        # copy files into libext
        files_copied = False
        for file in files:
            if os.path.isfile(file):
                Logger.info("Copying {0} to {1}".format(
                    str(file), params.oozie_libext_customer_dir))
                shutil.copy2(file, params.oozie_libext_customer_dir)
                files_copied = True

        if not files_copied:
            raise Fail("There are no files at {0} matching {1}".format(
                hadoop_client_new_lib_dir, hadoop_lzo_pattern))

    # copy ext ZIP to customer dir
    oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
    if not os.path.isfile(oozie_ext_zip_file):
        raise Fail("Unable to copy {0} because it does not exist".format(
            oozie_ext_zip_file))

    Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
                                            params.oozie_libext_customer_dir))
    shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)

    # Redownload jdbc driver to a new current location
    if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
                    params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
                    params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
        File(
            params.downloaded_custom_connector,
            content=DownloadSource(params.driver_curl_source),
        )

        Execute(
            ('cp', '--remove-destination', params.downloaded_custom_connector,
             params.target),
            #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
            path=["/bin", "/usr/bin/"],
            sudo=True)

        File(params.target, owner=params.oozie_user, group=params.user_group)