def perform_processing(transformer_instance: transformer_class.Transformer, args: argparse.Namespace, metadata: dict) -> dict:
"""Makes the calls to perform the processing
Arguments:
transformer_instance: instance of transformer class
args: the command line arguments
metadata: the loaded metadata
Return:
Returns a dict containing the result of processing
"""
result = {}
# Get the various types of parameters from the transformer instance
if hasattr(transformer_instance, 'get_transformer_params'):
transformer_params = transformer_instance.get_transformer_params(args, metadata)
if not isinstance(transformer_params, dict):
return __internal__.handle_error(-101,
"Invalid return from getting transformer parameters from transformer class instance")
params_result = __internal__.check_params_result_error(transformer_params)
if params_result:
return params_result
else:
logging.info("Transformer class instance does not have get_transformer_params method")
transformer_params = {}
# First check if the transformer thinks everything is in place
if hasattr(transformer, 'check_continue'):
result = __internal__.handle_check_continue(transformer_instance, transformer_params)
if 'code' in result and result['code'] < 0 and 'error' not in result:
result['error'] = "Unknown error returned from check_continue call"
else:
logging.info("Transformer module doesn't have a function named 'check_continue'")
# Retrieve additional files if indicated by return code from the check
if 'error' not in result and 'code' in result and result['code'] == 0:
result = __internal__.handle_retrieve_files(transformer_instance, args, metadata)
# Next make the call to perform the processing
if 'error' not in result:
if hasattr(transformer, 'perform_process'):
result = transformer.perform_process(transformer_instance, **transformer_params)
else:
logging.debug("Transformer module is missing function named 'perform_process'")
return __internal__.handle_error(-102, "Transformer perform_process interface is not available " +
"for processing data")
return result
def handle_retrieve_files(transformer_instance: transformer_class.Transformer, args: argparse.Namespace, metadata: dict) ->\
Optional[dict]:
"""Handles calling the transformer class to retrieve files
Arguments:
transformer_instance: the current transformer environment
args: the command line arguments
metadata: the loaded metadata
Return:
A dict containing error information if a problem occurs and None if no problems are found.
Note:
A side effect of this function is a information message logged if the transformer class instance does not have
a 'retrieve_files' function declared.
"""
if hasattr(transformer_instance, 'retrieve_files'):
transformer_retrieve = transformer_instance.retrieve_files(args, metadata)
retrieve_results = __internal__.check_retrieve_results_error(transformer_retrieve)
if retrieve_results:
return retrieve_results
else:
logging.info("Transformer class doesn't have function named 'retrieve_files'")
return None
"""Tests for 'entrypoint.py'
"""
#Import entrypoint.py and embedded modules
import argparse
import entrypoint
import transformer
from transformer_class import Transformer
#Set up initial testing values
TEST_TRANSFORMER = Transformer()
PARSE = argparse.ArgumentParser(description="test")
TEST_INTERNAL = entrypoint.__internal__()
# pylint: disable=assignment-from-no-return
def test_handle_error():
"""Test for handle error
"""
#Some testing arguments
test_code = 117
test_message = "Test message"
#Initial test using "ideal" args
ideal_example = TEST_INTERNAL.handle_error(test_code, test_message)
#Should return dict type
assert isinstance(ideal_example, dict)
#A secondary test
test_code = None
test_message = False
def perform_process(transformer: transformer_class.Transformer, check_md: dict,
transformer_md: dict, full_md: dict) -> dict:
"""Performs the processing of the data
Arguments:
transformer: instance of transformer class
Return:
Returns a dictionary with the results of processing
"""
# Find the source file to process
source_file = None
for one_file in check_md['list_files']():
if one_file.endswith('.bin'):
source_file = one_file
break
# Initialize local variables
logging.debug("Working with source file: %s", source_file)
out_filename = os.path.splitext(os.path.basename(source_file))[0] + '.tif'
out_file = os.path.join(check_md['working_folder'], out_filename)
bin_type = 'left' if source_file.endswith(
'_left.bin') else 'right' if source_file.endswith(
'_right.bin') else None
if not bin_type:
msg = "Bin file must be a left or right file: '%s'" % source_file
logging.error(msg)
logging.error(" Returning an error")
return {'code': -1000, 'error': msg}
logging.debug("Source image is type: %s", bin_type)
# Process the file
try:
bin_shape = terraref.stereo_rgb.get_image_shape(
check_md['context_md'], bin_type)
gps_bounds_bin = tr_geojson_to_tuples(
check_md['context_md']['spatial_metadata'][bin_type]
['bounding_box'])
except KeyError:
msg = "Spatial metadata is not properly identified. Unable to continue"
logging.error(msg)
return {'code': -1001, 'error': msg}
logging.debug("Image bounds are: %s", str(gps_bounds_bin))
# Perform actual processing
if transformer.args.save_intermediate:
intermediate_filename = os.path.join(check_md['working_folder'],
"intermediate.tif")
logging.info("Generating intermediate image file: %s",
intermediate_filename)
else:
intermediate_filename = None
new_image = terraref.stereo_rgb.process_raw(bin_shape, source_file,
intermediate_filename)
tr_create_geotiff(new_image,
gps_bounds_bin,
out_file,
None,
False,
transformer.generate_transformer_md(),
check_md['context_md'],
compress=True)
return {
'code': 0,
'file': [{
'path': out_file,
'key': configuration.TRANSFORMER_TYPE
}]
}
def perform_process(transformer: transformer_class.Transformer, check_md: dict,
transformer_md: list, full_md: list) -> dict:
"""Performs the processing of the data
Arguments:
transformer: instance of transformer class
Return:
Returns a dictionary with the results of processing
"""
# pylint: disable=unused-argument
file_md = []
start_timestamp = datetime.datetime.utcnow()
file_list = __internal__.get_file_list(check_md['list_files']())
files_count = len(file_list)
# Find the metadata we're interested in for calibration parameters
terra_md = __internal__.find_terra_md(full_md)
if not terra_md:
raise RuntimeError("Unable to find TERRA REF specific metadata")
transformer_md = transformer.generate_transformer_md()
def generate_file_md(file_path: str) -> dict:
"""Returns file metadata for a file
Arguments:
file_path: the file to generate metadata for
Return:
Returns the metadata
"""
return {
'path': file_path,
'key': configuration.TRANSFORMER_SENSOR,
'metadata': {
'data': transformer_md
}
}
# Generate a list of approved file name endings
file_endings = ["{0:0>4}.bin".format(i) for i in range(0, 102)]
files_processed = 0
try:
img_width, img_height = __internal__.get_image_dimensions(terra_md)
gps_bounds = geojson_to_tuples(
terra_md['spatial_metadata']['ps2Top']['bounding_box'])
logging.debug("Image width and height: %s %s", str(img_width),
str(img_height))
logging.debug("Image geo bounds: %s", str(gps_bounds))
png_frames = {}
for one_file in file_list:
if one_file[-8:] in file_endings:
files_processed += 1
logging.debug("Processing file: '%s'", one_file)
try:
pixels = np.fromfile(one_file, np.dtype('uint8')).reshape(
[int(img_height), int(img_width)])
except ValueError:
logging.info(
"Ignoring ValueError exception while loading file '%s'",
one_file)
continue
png_filename = os.path.join(
check_md['working_folder'],
os.path.basename(one_file.replace('.bin', '.png')))
logging.info("Creating: '%s'", png_filename)
create_image(pixels, png_filename)
file_md.append(generate_file_md(png_filename))
png_frames[int(one_file[-8:-4])] = png_filename
tif_filename = os.path.join(
check_md['working_folder'],
os.path.basename(one_file.replace('.bin', '.tif')))
logging.info("Creating: '%s'", tif_filename)
create_geotiff(pixels, gps_bounds, tif_filename, None, False,
transformer_md, terra_md)
file_md.append(generate_file_md(tif_filename))
else:
logging.info("Skipping non-sensor file '%s'", one_file)
if files_processed > 0:
logging.info("Generating aggregates")
hist_path = os.path.join(check_md['working_folder'],
'combined_hist.png')
false_color_path = os.path.join(check_md['working_folder'],
'combined_pseudocolored.png')
__internal__.analyze(png_frames, hist_path, false_color_path)
file_md.append(generate_file_md(hist_path))
file_md.append(generate_file_md(false_color_path))
else:
logging.warning("No files were processed")
result = {
'code': 0,
'file': file_md,
configuration.TRANSFORMER_NAME: {
'version': configuration.TRANSFORMER_VERSION,
'utc_timestamp': datetime.datetime.utcnow().isoformat(),
'processing_time':
str(datetime.datetime.now() - start_timestamp),
'num_files_received': str(files_count),
'files_processed': str(files_processed)
}
}
except Exception as ex:
msg = 'Exception caught converting PSII files'
logging.exception(msg)
result = {'code': -1000, 'error': msg + ': ' + str(ex)}
return result
def perform_process(transformer: transformer_class.Transformer, check_md: dict, transformer_md: list, full_md: list) -> dict:
"""Performs the processing of the data
Arguments:
transformer: instance of transformer class
check_md: request specific metadata
transformer_md: metadata associated with previous runs of the transformer
full_md: the full set of metadata available to the transformer
Return:
Returns a dictionary with the results of processing
"""
# pylint: disable=unused-argument
start_timestamp = datetime.datetime.now()
all_files = check_md['list_files']()
total_file_count = len(all_files)
files_to_process = __internal__.get_files_to_process(all_files)
file_md = []
num_image_files = 0
num_processed_files = 0
for one_file in files_to_process:
logging.debug("Processing file: '%s'", one_file)
num_image_files += 1
if not os.path.exists(one_file):
logging.error("Unable to access file: '%s'. Continuing processing", one_file)
continue
try:
quality_value = __internal__.get_image_quality(one_file)
image_bounds = transformer.get_image_file_geobounds(one_file)
quality_image_bounds = (image_bounds[2], image_bounds[3], image_bounds[0], image_bounds[1])
mac_file_name = os.path.join(check_md['working_folder'], os.path.splitext(os.path.basename(one_file))[0] + '_mac.tif')
logging.info("MAC score %s for file '%s'", str(quality_value), one_file)
logging.debug("Creating quality image: bounds %s name: '%s'", str(quality_image_bounds), mac_file_name)
create_geotiff(np.array([[quality_value, quality_value], [quality_value, quality_value]]), quality_image_bounds,
mac_file_name, None, True, transformer.generate_transformer_md(), full_md[0], compress=True)
num_processed_files += 1
file_md.append(
{
'path': mac_file_name,
'key': 'tif',
'metadata': {
'replace': True,
'data': {
'MAC score': str(quality_value),
'utc_timestamp': datetime.datetime.utcnow().isoformat(),
'source_file': one_file
}
}
}
)
except Exception as ex:
logging.warning("Ignoring exception caught processing image file '%s'", one_file)
logging.debug("Exception: %s", str(ex))
logging.exception('broken')
return {'code': 0,
'files': file_md,
configuration.TRANSFORMER_NAME: {
'version': configuration.TRANSFORMER_VERSION,
'utc_timestamp': datetime.datetime.utcnow().isoformat(),
'processing_time': str(datetime.datetime.now() - start_timestamp),
'num_files_received': str(total_file_count),
'num_image_files': str(num_image_files),
'num_processed_files': str(num_processed_files)
}
}
def perform_process(transformer: transformer_class.Transformer, check_md: dict,
transformer_md: list, full_md: list) -> dict:
"""Performs the processing of the data
Arguments:
transformer: instance of transformer class'
check_md: the metadata for this request
transformer_md: the metadata associated with this transformer
full_md: the full set of original metadata
Return:
Returns a dictionary with the results of processing
"""
# pylint: disable=unused-argument
result = {}
file_md = []
# Loop through the files
try:
for one_file in check_md['list_files']():
# Check file by type
ext = os.path.splitext(one_file)[1].lower()
if ext not in ('.tiff', '.tif'):
continue
if not os.path.exists(one_file):
logging.warning("Unable to access file '%s'", one_file)
continue
mask_source = one_file
# Get the image's EPSG code
epsg = transformer.get_image_file_epsg(mask_source)
if epsg is None:
logging.debug("Skipping image that is not georeferenced: '%s'",
mask_source)
continue
# Check that it's geo referenced and transform it if it'sin the wrong coordinate system
if epsg != transformer.default_epsg:
logging.info(
"Reprojecting image from EPSG %s to default EPSG %s",
str(epsg), str(transformer.default_epsg))
_, tmp_name = tempfile.mkstemp(dir=check_md['working_folder'])
src = gdal.Open(mask_source)
gdal.Warp(tmp_name,
src,
dstSRS='EPSG:' + str(transformer.default_epsg))
mask_source = tmp_name
# Get the bounds of the image to see if we can process it.
bounds = transformer.get_image_file_geobounds(mask_source)
if bounds is None:
logging.warning(
"Unable to get bounds of georeferenced image: '%s'",
os.path.basename(one_file))
if mask_source != one_file:
os.remove(mask_source)
continue
# Get the mask name using the original name as reference
rgb_mask_tif = os.path.join(
check_md['working_folder'],
__internal__.get_maskfilename(one_file))
# Create the mask file
logging.debug("Creating mask file '%s'", rgb_mask_tif)
mask_ratio, mask_rgb = gen_cc_enhanced(mask_source)
# Bands must be reordered to avoid swapping R and B
mask_rgb = cv2.cvtColor(mask_rgb, cv2.COLOR_BGR2RGB)
transformer_info = transformer.generate_transformer_md()
tr_create_geotiff(mask_rgb, bounds, rgb_mask_tif, None, False,
transformer_info, check_md['context_md'])
tr_compress_geotiff(rgb_mask_tif)
# Remove any temporary file
if mask_source != one_file:
os.remove(mask_source)
transformer_md = {
'name': transformer_info['name'],
'version': transformer_info['version'],
'ratio': mask_ratio
}
new_file_md = {
'path': rgb_mask_tif,
'key': configuration.TRANSFORMER_SENSOR,
'metadata': {
'data': transformer_md
}
}
file_md.append(new_file_md)
result['code'] = 0
result['file'] = file_md
except Exception as ex:
result['code'] = -1001
result['error'] = "Exception caught masking files: %s" % str(ex)
return result
def perform_process(transformer: transformer_class.Transformer, check_md: dict,
transformer_md: dict, full_md: list) -> dict:
"""Performs the processing of the data
Arguments:
transformer: instance of transformer class
Return:
Returns a dictionary with the results of processing
"""
# pylint: disable=unused-argument
result = {}
file_md = []
file_list = check_md['list_files']()
# Extract necessary parameters from metadata
for one_metadata in full_md:
if 'sensor_variable_metadata' in one_metadata:
scan_distance = float(one_metadata['sensor_variable_metadata']
['scan_distance_mm']) / MM_PER_METER
scan_direction = int(
one_metadata['sensor_variable_metadata']['scan_direction'])
point_cloud_origin = one_metadata['sensor_variable_metadata'][
'point_cloud_origin_m']['east']
break
try:
ply_files = []
transformer_info = transformer.generate_transformer_md()
for one_file in file_list:
if one_file.endswith(".ply"):
ply_files.append(
os.path.join(check_md['working_folder'], one_file))
if ply_files:
out_file = ply_files[0].replace(".ply", ".las")
generate_las_from_ply(ply_files, out_file, scan_distance,
scan_direction, point_cloud_origin, True)
file_md.append({
'path': out_file,
'key': configuration.TRANSFORMER_SENSOR,
'metadata': {
'data': {
'name': transformer_info['name'],
'version': transformer_info['version'],
'source': ','.join(file_list),
'utc_timestamp': datetime.datetime.now().isoformat()
}
}
})
result['code'] = 0
result['file'] = file_md
result[configuration.TRANSFORMER_NAME] = {
**transformer_info, 'utc_timestamp':
datetime.datetime.now().isoformat()
}
except Exception as ex:
result['code'] = -1
result['error'] = "Exception caught converting PLY files: %s" % str(ex)
return result