def transfer_product(immutability, destination_host, destination_directory, destination_username, destination_pw, product_filename, cksum_filename): ''' Description: Transfers the product and associated checksum to the specified directory on the destination host Returns: cksum_value - The check sum value from the destination destination_product_file - The full path on the destination Note: - It is assumed ssh has been setup for access between the localhost and destination system ''' logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER) # Create the destination directory on the destination host logger.info("Creating destination directory %s on %s" % (destination_directory, destination_host)) cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'mkdir', '-p', destination_directory]) output = '' try: logger.debug(' '.join(["mkdir cmd:", cmd])) output = utilities.execute_cmd(cmd) finally: if len(output) > 0: logger.info(output) # Figure out the destination full paths destination_cksum_file = os.path.join(destination_directory, os.path.basename(cksum_filename)) destination_product_file = os.path.join(destination_directory, os.path.basename(product_filename)) # Remove any pre-existing files # Grab the first part of the filename, which is not unique remote_filename_parts = destination_product_file.split('-') remote_filename_parts[-1] = '*' # Replace the last element of the list remote_filename = '-'.join(remote_filename_parts) # Join with '-' # Change the attributes on the files so that we can remove them if immutability: cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'sudo', 'chattr', '-if', remote_filename]) output = '' try: logger.debug(' '.join(["chattr remote file cmd:", cmd])) output = utilities.execute_cmd(cmd) except Exception: pass finally: if len(output) > 0: logger.info(output) # Remove the files on the remote system cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'rm', '-f', remote_filename]) output = '' try: logger.debug(' '.join(["rm remote file cmd:", cmd])) output = utilities.execute_cmd(cmd) finally: if len(output) > 0: logger.info(output) # Transfer the checksum file transfer.transfer_file('localhost', cksum_filename, destination_host, destination_cksum_file, destination_username=destination_username, destination_pw=destination_pw) # Transfer the product file transfer.transfer_file('localhost', product_filename, destination_host, destination_product_file, destination_username=destination_username, destination_pw=destination_pw) # Change the attributes on the files so that we can't remove them if immutability: cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'sudo', 'chattr', '+i', remote_filename]) output = '' try: logger.debug(' '.join(["chattr remote file cmd:", cmd])) output = utilities.execute_cmd(cmd) finally: if len(output) > 0: logger.info(output) # Get the remote checksum value cksum_value = '' cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, settings.ESPA_CHECKSUM_TOOL, destination_product_file]) try: logger.debug(' '.join(["ssh cmd:", cmd])) cksum_value = utilities.execute_cmd(cmd) except Exception: if len(cksum_value) > 0: logger.error(cksum_value) raise return (cksum_value, destination_product_file, destination_cksum_file)
def distribute_statistics_remote(immutability, product_id, source_path, destination_host, destination_path, destination_username, destination_pw): ''' Description: Transfers the statistics to the specified directory on the destination host Parameters: product_id - The unique product ID associated with the files. source_path - The full path to where the statistics files to distribute reside. destination_host - The hostname/url for where to distribute the files. destination_path - The full path on the local system to copy the statistics files into. destination_username - The user name to use for FTP destination_pw - The password to use for FTP Note: - It is assumed ssh has been setup for access between the localhost and destination system - It is assumed a stats directory exists under the current directory ''' logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER) d_name = 'stats' # Save the current directory location current_directory = os.getcwd() # Attempt X times sleeping between each attempt attempt = 0 sleep_seconds = settings.DEFAULT_SLEEP_SECONDS while True: # Change to the source directory os.chdir(source_path) try: stats_wildcard = ''.join([product_id, '*']) stats_path = os.path.join(destination_path, d_name) stats_files = os.path.join(d_name, stats_wildcard) remote_stats_wildcard = os.path.join(stats_path, stats_wildcard) # Create the statistics directory on the destination host logger.info("Creating directory {0} on {1}". format(stats_path, destination_host)) cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'mkdir', '-p', stats_path]) output = '' try: logger.debug(' '.join(["mkdir cmd:", cmd])) output = utilities.execute_cmd(cmd) finally: if len(output) > 0: logger.info(output) # Change the attributes on the files so that we can remove them if immutability: cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'sudo', 'chattr', '-if', remote_stats_wildcard]) output = '' try: logger.debug(' '.join(["chattr remote stats cmd:", cmd])) output = utilities.execute_cmd(cmd) except Exception: pass finally: if len(output) > 0: logger.info(output) # Remove any pre-existing statistics cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'rm', '-f', remote_stats_wildcard]) output = '' try: logger.debug(' '.join(["rm remote stats cmd:", cmd])) output = utilities.execute_cmd(cmd) finally: if len(output) > 0: logger.info(output) # Transfer the stats statistics transfer.transfer_file('localhost', stats_files, destination_host, stats_path, destination_username=destination_username, destination_pw=destination_pw) logger.info("Verifying statistics transfers") # NOTE - Re-purposing the stats_files variable stats_files = glob.glob(stats_files) for file_name in stats_files: local_cksum_value = 'a b' remote_cksum_value = 'b c' # Generate a local checksum value cmd = ' '.join([settings.ESPA_CHECKSUM_TOOL, file_name]) try: logger.debug(' '.join(["checksum cmd:", cmd])) local_cksum_value = utilities.execute_cmd(cmd) except Exception: if len(local_cksum_value) > 0: logger.error(local_cksum_value) raise # Generate a remote checksum value remote_file = os.path.join(destination_path, file_name) cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, settings.ESPA_CHECKSUM_TOOL, remote_file]) try: remote_cksum_value = utilities.execute_cmd(cmd) except Exception: if len(remote_cksum_value) > 0: logger.error(remote_cksum_value) raise # Checksum validation if (local_cksum_value.split()[0] != remote_cksum_value.split()[0]): raise ESPAException("Failed checksum validation between" " %s and %s:%s" % (file_name, destination_host, remote_file)) # Change the attributes on the files so that we can't remove them if immutability: cmd = ' '.join(['ssh', '-q', '-o', 'StrictHostKeyChecking=no', destination_host, 'sudo', 'chattr', '+i', remote_stats_wildcard]) output = '' try: logger.debug(' '.join(["chattr remote stats cmd:", cmd])) output = utilities.execute_cmd(cmd) finally: if len(output) > 0: logger.info(output) except Exception: logger.exception("An exception occurred processing %s" % product_id) if attempt < settings.MAX_DELIVERY_ATTEMPTS: sleep(sleep_seconds) # sleep before trying again attempt += 1 continue else: raise finally: # Change back to the previous directory os.chdir(current_directory) break
def transfer(self): transfer_file(self.path)