def transfer_product(immutability, destination_host, destination_directory,
                     destination_username, destination_pw,
                     product_filename, cksum_filename):
    '''
    Description:
      Transfers the product and associated checksum to the specified directory
      on the destination host

    Returns:
      cksum_value - The check sum value from the destination
      destination_product_file - The full path on the destination

    Note:
      - It is assumed ssh has been setup for access between the localhost
        and destination system
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    # Create the destination directory on the destination host
    logger.info("Creating destination directory %s on %s"
                % (destination_directory, destination_host))
    cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                    destination_host, 'mkdir', '-p', destination_directory])

    output = ''
    try:
        logger.debug(' '.join(["mkdir cmd:", cmd]))
        output = utilities.execute_cmd(cmd)
    finally:
        if len(output) > 0:
            logger.info(output)

    # Figure out the destination full paths
    destination_cksum_file = os.path.join(destination_directory,
                                          os.path.basename(cksum_filename))
    destination_product_file = os.path.join(destination_directory,
                                            os.path.basename(product_filename))

    # Remove any pre-existing files
    # Grab the first part of the filename, which is not unique
    remote_filename_parts = destination_product_file.split('-')
    remote_filename_parts[-1] = '*'  # Replace the last element of the list
    remote_filename = '-'.join(remote_filename_parts)  # Join with '-'

    # Change the attributes on the files so that we can remove them
    if immutability:
        cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                        destination_host, 'sudo', 'chattr', '-if',
                        remote_filename])
        output = ''
        try:
            logger.debug(' '.join(["chattr remote file cmd:", cmd]))
            output = utilities.execute_cmd(cmd)
        except Exception:
            pass
        finally:
            if len(output) > 0:
                logger.info(output)

    # Remove the files on the remote system
    cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                    destination_host, 'rm', '-f', remote_filename])
    output = ''
    try:
        logger.debug(' '.join(["rm remote file cmd:", cmd]))
        output = utilities.execute_cmd(cmd)
    finally:
        if len(output) > 0:
            logger.info(output)

    # Transfer the checksum file
    transfer.transfer_file('localhost', cksum_filename, destination_host,
                           destination_cksum_file,
                           destination_username=destination_username,
                           destination_pw=destination_pw)

    # Transfer the product file
    transfer.transfer_file('localhost', product_filename, destination_host,
                           destination_product_file,
                           destination_username=destination_username,
                           destination_pw=destination_pw)

    # Change the attributes on the files so that we can't remove them
    if immutability:
        cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                        destination_host, 'sudo', 'chattr', '+i',
                        remote_filename])
        output = ''
        try:
            logger.debug(' '.join(["chattr remote file cmd:", cmd]))
            output = utilities.execute_cmd(cmd)
        finally:
            if len(output) > 0:
                logger.info(output)

    # Get the remote checksum value
    cksum_value = ''
    cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                    destination_host, settings.ESPA_CHECKSUM_TOOL,
                    destination_product_file])
    try:
        logger.debug(' '.join(["ssh cmd:", cmd]))
        cksum_value = utilities.execute_cmd(cmd)
    except Exception:
        if len(cksum_value) > 0:
            logger.error(cksum_value)
        raise

    return (cksum_value, destination_product_file, destination_cksum_file)
def distribute_statistics_remote(immutability, product_id, source_path,
                                 destination_host, destination_path,
                                 destination_username, destination_pw):
    '''
    Description:
      Transfers the statistics to the specified directory on the destination
      host

    Parameters:
        product_id - The unique product ID associated with the files.
        source_path - The full path to where the statistics files to
                      distribute reside.
        destination_host - The hostname/url for where to distribute the files.
        destination_path - The full path on the local system to copy the
                           statistics files into.
        destination_username - The user name to use for FTP
        destination_pw - The password to use for FTP

    Note:
      - It is assumed ssh has been setup for access between the localhost
        and destination system
      - It is assumed a stats directory exists under the current directory
    '''

    logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)

    d_name = 'stats'

    # Save the current directory location
    current_directory = os.getcwd()

    # Attempt X times sleeping between each attempt
    attempt = 0
    sleep_seconds = settings.DEFAULT_SLEEP_SECONDS
    while True:
        # Change to the source directory
        os.chdir(source_path)
        try:
            stats_wildcard = ''.join([product_id, '*'])
            stats_path = os.path.join(destination_path, d_name)
            stats_files = os.path.join(d_name, stats_wildcard)
            remote_stats_wildcard = os.path.join(stats_path, stats_wildcard)

            # Create the statistics directory on the destination host
            logger.info("Creating directory {0} on {1}".
                        format(stats_path, destination_host))
            cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                            destination_host, 'mkdir', '-p', stats_path])

            output = ''
            try:
                logger.debug(' '.join(["mkdir cmd:", cmd]))
                output = utilities.execute_cmd(cmd)
            finally:
                if len(output) > 0:
                    logger.info(output)

            # Change the attributes on the files so that we can remove them
            if immutability:
                cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                                destination_host, 'sudo', 'chattr', '-if',
                                remote_stats_wildcard])
                output = ''
                try:
                    logger.debug(' '.join(["chattr remote stats cmd:", cmd]))
                    output = utilities.execute_cmd(cmd)
                except Exception:
                    pass
                finally:
                    if len(output) > 0:
                        logger.info(output)

            # Remove any pre-existing statistics
            cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                            destination_host, 'rm', '-f',
                            remote_stats_wildcard])
            output = ''
            try:
                logger.debug(' '.join(["rm remote stats cmd:", cmd]))
                output = utilities.execute_cmd(cmd)
            finally:
                if len(output) > 0:
                    logger.info(output)

            # Transfer the stats statistics
            transfer.transfer_file('localhost', stats_files, destination_host,
                                   stats_path,
                                   destination_username=destination_username,
                                   destination_pw=destination_pw)

            logger.info("Verifying statistics transfers")
            # NOTE - Re-purposing the stats_files variable
            stats_files = glob.glob(stats_files)
            for file_name in stats_files:
                local_cksum_value = 'a b'
                remote_cksum_value = 'b c'

                # Generate a local checksum value
                cmd = ' '.join([settings.ESPA_CHECKSUM_TOOL, file_name])
                try:
                    logger.debug(' '.join(["checksum cmd:", cmd]))
                    local_cksum_value = utilities.execute_cmd(cmd)
                except Exception:
                    if len(local_cksum_value) > 0:
                        logger.error(local_cksum_value)
                    raise

                # Generate a remote checksum value
                remote_file = os.path.join(destination_path, file_name)
                cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                                destination_host, settings.ESPA_CHECKSUM_TOOL,
                                remote_file])
                try:
                    remote_cksum_value = utilities.execute_cmd(cmd)
                except Exception:
                    if len(remote_cksum_value) > 0:
                        logger.error(remote_cksum_value)
                    raise

                # Checksum validation
                if (local_cksum_value.split()[0] !=
                        remote_cksum_value.split()[0]):
                    raise ESPAException("Failed checksum validation between"
                                        " %s and %s:%s" % (file_name,
                                                           destination_host,
                                                           remote_file))

            # Change the attributes on the files so that we can't remove them
            if immutability:
                cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no',
                                destination_host, 'sudo', 'chattr', '+i',
                                remote_stats_wildcard])
                output = ''
                try:
                    logger.debug(' '.join(["chattr remote stats cmd:", cmd]))
                    output = utilities.execute_cmd(cmd)
                finally:
                    if len(output) > 0:
                        logger.info(output)

        except Exception:
            logger.exception("An exception occurred processing %s"
                             % product_id)
            if attempt < settings.MAX_DELIVERY_ATTEMPTS:
                sleep(sleep_seconds)  # sleep before trying again
                attempt += 1
                continue
            else:
                raise

        finally:
            # Change back to the previous directory
            os.chdir(current_directory)

        break
Example #3
0
 def transfer(self):
     transfer_file(self.path)