def _handle_copy_down(file_name, date_time):
"""
Copy the backup file from the bucket.
"""
remote_bucket_path = _make_remote_bucket_path(file_name + '.bz2',
date_time)
bucket_objects = list_bucket_objects(remote_bucket_path)
copy_from_bucket(bucket_objects, '.')
uncompress(file_name + '.bz2')
def get_stands(stands_file_name, local_path_string=DATA_HOME):
"""
Gets the stands descriptions required to support data processing.
The data source name is fixed.
Parameters
----------
local_path_string is defaulted to the known local airport data location.
"""
destination_path = '/'.join([local_path_string, AIRPORTS_STANDS]) \
if local_path_string == DATA_HOME else local_path_string
objects = list_bucket_objects(AIRPORTS_STANDS)
filtered_objects = [obj for obj in objects if stands_file_name in obj.name]
log.debug("Getting stands file: " + str(filtered_objects) + " to: " +
destination_path)
copy_from_bucket(filtered_objects, destination_path)
return destination_path
def get_apds(apds_filename, local_path_string=UPLOAD_DIR):
"""
Gets the apds descriptions required to support data processing.
Parameters
----------
apds_filename The file name
"""
log.debug("Getting apds file: %s", apds_filename)
destination_path = '/'.join([local_path_string, APDS]) \
if local_path_string == UPLOAD_DIR else local_path_string
_, tail = path.split(UPLOAD_DIR)
source_path = tail + '/' + APDS
objects = list_bucket_objects(source_path)
filtered_objects = [obj for obj in objects if apds_filename in obj.name]
log.debug("Getting apds file: " + str(filtered_objects) + " to: " +
destination_path)
copy_from_bucket(filtered_objects, destination_path)
return destination_path
def get_user_airspaces(user_airspaces_file_name, local_path_string=DATA_HOME):
"""
Gets the user airspaces descriptions required to support data processing.
The data source location and file name is fixed.
Parameters
----------
local_path_string is defaulted to the known local user airspaces data location.
"""
destination_path = '/'.join([local_path_string, AIRSPACES]) \
if local_path_string == DATA_HOME else local_path_string
objects = list_bucket_objects(AIRSPACES)
filtered_objects = [
obj for obj in objects if user_airspaces_file_name in obj.name
]
log.debug("Getting user airspaces file: " + str(filtered_objects) +
" to: " + destination_path)
copy_from_bucket(filtered_objects, destination_path)
return destination_path
def get_unprocessed(data_type, data_date, local_path_string=UPLOAD_DIR):
"""
Gets data from the bucket.
The destination is assuemd to be local.
If no date is given all the dates for the specified data type are copied.
Parameters
----------
data_type describes the type of unprocessed data
data_date the date we are interested in
local_path_string is defaulted to the local upload dir
"""
log.debug("Getting data item from bucket with type: " + str(data_type))
valid, error_message = validate_data_type_and_date(data_type, data_date)
if valid:
_, tail = path.split(UPLOAD_DIR)
source_path = tail + '/' + data_type
destination_path = '/'.join([local_path_string, data_type]) \
if local_path_string == UPLOAD_DIR else local_path_string
log.debug("Source path: " + source_path + " destnation path: " +
destination_path)
objects = list_bucket_objects(source_path)
names = [obj.name for obj in objects]
log.debug("Found names: " + str(names))
# Only include the date specified
if data_date is not None:
names = filter_date(names, data_date, data_type)
log.debug("Getting unprocessed data with names: " + str(names))
# Remove any directory strings
filtered_objects = [
obj for obj in objects
if names.__contains__(obj.name) and not obj.name.endswith('/')
]
log.debug("Getting unprocessed filtered data: " +
str(filtered_objects))
copy_from_bucket(filtered_objects, destination_path)
else:
return error_message
def get_processed(data_type, filenames, local_directory='.'):
"""
Get processed data items from the bucket of the given type and name to a
local path. These are always compressed in the bucket we decompress them
on read. If decompression fails for any reason we fail the copy and the
function returns false otherwise true.
Parameters
----------
data_type the type of data to get. This is prescribed for each data type.
filenames a list of file names
local_directory is the path to copy the file to and must exist.
"""
log.info('Getting data items: %s from bucket: %s to: %s', str(filenames),
data_type, local_directory)
valid, error_messag = validate_data_type_and_date(data_type)
existing_path = path_exists(local_directory)
if valid and existing_path:
log.debug("Getting files %s", str(filenames))
paths = [data_type + "/" + filename + ".bz2" for filename in filenames]
log.debug("paths : %s", str(paths))
# Note here objects_list is a list of lists
objects_list = [list_bucket_objects(path) for path in paths]
[
copy_from_bucket(objects, local_directory)
for objects in objects_list
]
log.debug("Uncompressing files %s", str(filenames))
uncompress_result = [
uncompress(local_directory + "/" + filename + ".bz2")
for filename in filenames
]
success, uncompressed_file_paths = map(list, zip(*uncompress_result))
if success:
# delete the compressed processed files
compressed_file_paths = [
filename + ".bz2" for filename in filenames
]
[os.remove(file_path) for file_path in compressed_file_paths]
return success
else:
log.error(
"Either the data type was invalid or the path does not exist")
return False