def distribute_statistics(immutability, source_path, packaging_path, parms):
'''
Description:
Determines if the distribution method is set to local or remote and
calls the correct distribution method.
Returns:
product_file - The full path to the product either on the local system
or the remote destination.
cksum_value - The check sum value of the product.
Parameters:
source_path - The full path to of directory containing the data to
package and distribute.
package_dir - The full path on the local system for where the packaged
product should be placed under.
parms - All the user and system defined parameters.
'''
env = Environment()
distribution_method = env.get_distribution_method()
product_id = parms['product_id']
order_id = parms['orderid']
# The file paths to the distributed product and checksum files
product_file = 'ERROR'
cksum_file = 'ERROR'
if distribution_method == DISTRIBUTION_METHOD_LOCAL:
# Use the local cache path
cache_path = os.path.join(settings.ESPA_LOCAL_CACHE_DIRECTORY,
order_id)
# Adjust the packaging_path to use the cache
package_path = os.path.join(packaging_path, cache_path)
distribute_statistics_local(immutability, product_id, source_path,
package_path)
else: # remote
env = Environment()
# Determine the remote hostname to use
destination_host = utilities.get_cache_hostname(env
.get_cache_host_list())
# Use the remote cache path
cache_path = os.path.join(settings.ESPA_REMOTE_CACHE_DIRECTORY,
order_id)
options = parms['options']
dest_user = options['destination_username']
dest_pw = options['destination_pw']
distribute_statistics_remote(immutability, product_id, source_path,
destination_host, cache_path,
dest_user, dest_pw)
return (product_file, cksum_file)
def stage_remote_statistics_data(stage_dir, work_dir, order_id):
'''
Description:
Stages the statistics using scp from a remote location.
'''
cache_host = utilities.get_cache_hostname()
cache_dir = os.path.join(settings.ESPA_REMOTE_CACHE_DIRECTORY, order_id)
cache_dir = os.path.join(cache_dir, 'stats')
# Transfer the directory using scp
try:
transfer.scp_transfer_directory(cache_host, cache_dir,
'localhost', stage_dir)
except Exception as e:
raise ee.ESPAException(ee.ErrorCodes.staging_data, str(e)), \
None, sys.exc_info()[2]
# Move the staged data to the work directory
try:
stats_files = glob.glob(os.path.join(stage_dir, 'stats/*'))
transfer.move_files_to_directory(stats_files, work_dir)
except Exception as e:
raise ee.ESPAException(ee.ErrorCodes.unpacking, str(e)), \
None, sys.exc_info()[2]
def stage_remote_statistics_data(stage_dir, work_dir, order_id):
'''
Description:
Stages the statistics using scp from a remote location.
'''
cache_host = utilities.get_cache_hostname()
cache_dir = os.path.join(settings.ESPA_REMOTE_CACHE_DIRECTORY, order_id)
cache_dir = os.path.join(cache_dir, 'stats')
# Transfer the directory using scp
transfer.scp_transfer_directory(cache_host, cache_dir,
'localhost', stage_dir)
# Move the staged data to the work directory
stats_files = glob.glob(os.path.join(stage_dir, 'stats/*'))
transfer.move_files_to_directory(stats_files, work_dir)
def stage_remote_statistics_data(stage_dir, work_dir, order_id):
'''
Description:
Stages the statistics using scp from a remote location.
'''
env = Environment()
cache_host = utilities.get_cache_hostname(env.get_cache_host_list())
cache_dir = os.path.join(settings.ESPA_REMOTE_CACHE_DIRECTORY, order_id)
cache_dir = os.path.join(cache_dir, 'stats')
# Transfer the directory using scp
transfer.scp_transfer_directory(cache_host, cache_dir,
'localhost', stage_dir)
# Move the staged data to the work directory
stats_files = glob.glob(os.path.join(stage_dir, 'stats/*'))
transfer.move_files_to_directory(stats_files, work_dir)
def distribute_product_remote(immutability, product_name, source_path,
packaging_path, cache_path, parms):
logger = EspaLogging.get_logger(settings.PROCESSING_LOGGER)
opts = parms['options']
env = Environment()
# Determine the remote hostname to use
destination_host = utilities.get_cache_hostname(env.get_cache_host_list())
# Deliver the product files
# Attempt X times sleeping between each attempt
sleep_seconds = settings.DEFAULT_SLEEP_SECONDS
max_number_of_attempts = settings.MAX_DISTRIBUTION_ATTEMPTS
max_package_attempts = settings.MAX_PACKAGING_ATTEMPTS
max_delivery_attempts = settings.MAX_DELIVERY_ATTEMPTS
attempt = 0
product_file = 'ERROR'
cksum_file = 'ERROR'
while True:
try:
# Package the product files
# Attempt X times sleeping between each sub_attempt
sub_attempt = 0
while True:
try:
(product_full_path, cksum_full_path,
local_cksum_value) = package_product(immutability,
source_path,
packaging_path,
product_name)
except Exception:
logger.exception("An exception occurred processing %s"
% product_name)
if sub_attempt < max_package_attempts:
sleep(sleep_seconds) # sleep before trying again
sub_attempt += 1
continue
else:
raise
break
# Distribute the product
# Attempt X times sleeping between each sub_attempt
sub_attempt = 0
while True:
try:
(remote_cksum_value, product_file, cksum_file) = \
transfer_product(immutability, destination_host,
cache_path,
opts['destination_username'],
opts['destination_pw'],
product_full_path, cksum_full_path)
except Exception:
logger.exception("An exception occurred processing %s"
% product_name)
if sub_attempt < max_delivery_attempts:
sleep(sleep_seconds) # sleep before trying again
sub_attempt += 1
continue
else:
raise
break
# Checksum validation
if local_cksum_value.split()[0] != remote_cksum_value.split()[0]:
raise ESPAException("Failed checksum validation between"
" %s and %s:%s"
% (product_full_path,
destination_host,
product_file))
# Always log where we placed the files
logger.info("Delivered product to %s at location %s"
" and cksum location %s" % (destination_host,
product_file, cksum_file))
except Exception:
if attempt < max_number_of_attempts:
sleep(sleep_seconds) # sleep before trying again
attempt += 1
# adjust for next set
sleep_seconds = int(sleep_seconds * 1.5)
continue
else:
raise
break
return (product_file, cksum_file)
def distribute_statistics(immutability, source_path, packaging_path, parms,
user, group):
"""
Determines if the distribution method is set to local or remote
and calls the correct distribution method.
Args:
immutability (bool): If True, change the file attributes for immutability.
source_path (str): The full path to directory containing the data to
package and distribute.
packaging_path (str): The full path on the local system for where the packaged
product should be placed under.
parms (dict): All the user and system defined parameters.
user (str): The user to take ownership.
group (str): The group to take ownership.
Returns:
product_file (str): The full path to the product either on the local system
or the remote destination.
cksum_value (str): The check sum value of the product.
"""
env = Environment()
distribution_method = env.get_distribution_method()
product_id = parms['product_id']
order_id = parms['orderid']
# The file paths to the distributed product and checksum files
product_file = 'ERROR'
cksum_file = 'ERROR'
if distribution_method == DISTRIBUTION_METHOD_LOCAL:
# Use the local cache path
cache_path = os.path.join(settings.ESPA_LOCAL_CACHE_DIRECTORY,
order_id)
# Adjust the packaging_path to use the cache
package_path = os.path.join(packaging_path, cache_path)
distribute_statistics_local(immutability, product_id, source_path,
package_path, user, group)
else: # remote
env = Environment()
# Determine the remote hostname to use
destination_host = utilities.get_cache_hostname(
env.get_cache_host_list())
# Use the remote cache path
cache_path = os.path.join(settings.ESPA_REMOTE_CACHE_DIRECTORY,
order_id)
options = parms['options']
dest_user = options['destination_username']
dest_pw = options['destination_pw']
distribute_statistics_remote(immutability, product_id, source_path,
destination_host, cache_path, dest_user,
dest_pw)
return (product_file, cksum_file)