def on_success(task):
"""Capture any Task Success here."""
TASK_LOGGER.info(
task=task.get_task_family(),
params=task.to_str_params(),
level1=getattr(task, "level1", ""),
status="success",
)
def on_failure(task, exception):
"""Capture any Task Failure here."""
TASK_LOGGER.exception(task=task.get_task_family(),
params=task.to_str_params(),
level1=getattr(task, 'level1', ''),
stack_info=True,
status='failure',
exception=exception.__str__(),
traceback=traceback.format_exc().splitlines())
def on_failure(task, exception):
"""Capture any Task Failure here."""
TASK_LOGGER.exception(
event="task-failure",
task=task.get_task_family(),
params=task.to_str_params(),
level1=getattr(task, "level1", ""),
granule=getattr(task, "granule", ""),
stack_info=True,
status="failure",
exception=exception.__str__(),
traceback=traceback.format_exc().splitlines(),
)
def run(self):
temp_yaml = pjoin(self.workdir, "gverify",
str(self.output_yaml).format(granule=self.granule))
res = {}
# Read gverify arguments from yaml
with self.input()["runtime_args"].open("r") as _md:
gverify_args = yaml.load(_md)
try:
if ("error_msg" not in gverify_args or gverify_args["error_msg"]
== ""): # Gverify successfully ran
rh, tr, df = parse_gverify(self.input()["results"].path)
res = calculate_gqa(
df,
tr,
gverify_args["ref_resolution"],
self.standard_deviations,
self.iterations,
self.correlation_coefficient,
)
# Add color residual values to the results
res["colors"] = {
_clean_name(i):
_rounded(rh[rh.Color == i].Residual.values[0])
for i in rh.Color.values
}
else:
_LOG.debug("Writing NaNs for residuals; gverify failed to run")
res = {
"final_qa_count": 0,
"residual": _populate_nan_residuals(),
"error_message": gverify_args["error_msg"],
}
except (StopIteration, FileNotFoundError) as _: # noqa: F841
TASK_LOGGER.error(
"Gverify results file contains no tabulated data; {}".format(
self.input()["results"].path))
_LOG.debug("Defaulting to NaN for the residual values.")
res = {
"final_qa_count": 0,
"residual": _populate_nan_residuals(),
"error_message": "No GCP's were found",
}
finally:
metadata = get_gqa_metadata(gverify_args["executable"])
metadata["ref_source_path"] = gverify_args["ref_source_path"]
metadata["ref_source"] = (
_gls_version(metadata["ref_source_path"])
if metadata["ref_source_path"] else ""
) # if ref_source_path is non-empty calculate version
metadata["ref_date"] = gverify_args["ref_date"]
metadata["granule"] = gverify_args["granule"]
_write_gqa_yaml(temp_yaml, {**metadata, **res})
self.output().makedirs()
# copy temp to output final location
shutil.copy(temp_yaml, self.output().path)
if int(self.cleanup):
_cleanup_workspace(pjoin(self.workdir, "gverify"))
def run(self):
# Subdirectory in the task workdir
workdir = pjoin(self.workdir, "gverify")
if not exists(workdir):
os.makedirs(workdir)
# Get acquisition metadata, limit it to executing granule
container = acquisitions(
self.level1, self.acq_parser_hint).get_granule(self.granule,
container=True)
acq_info = acquisition_info(container, self.granule)
# Initialise output variables for error case
error_msg = ""
ref_date = ""
ref_source_path = ""
reference_resolution = ""
try:
# retrieve a set of matching landsat scenes
# lookup is based on polygon for Sentinel-2
landsat_scenes = acq_info.intersecting_landsat_scenes(
self.landsat_scenes_shapefile)
def fixed_extra_parameters():
points_txt = pjoin(workdir, "points.txt")
collect_gcp(self.root_fix_qa_location, landsat_scenes,
points_txt)
return ["-t", "FIXED_LOCATION", "-t_file", points_txt]
if acq_info.is_land_tile(self.ocean_tile_list):
location = acq_info.land_band()
# for sentinel-2 land tiles we prefer grid points
# rather than GCPs
if acq_info.preferred_gverify_method == "grid":
extra = ["-g", self.grid_size]
else:
extra = fixed_extra_parameters()
else:
# for sea tiles we always pick GCPs
location = acq_info.ocean_band()
extra = fixed_extra_parameters()
# Extract the source band from the results archive
with h5py.File(self.input()[0].path, "r") as h5:
band_id = h5[location].attrs["band_id"]
source_band = pjoin(workdir,
"source-BAND-{}.tif".format(band_id))
source_image = h5[location][:]
source_image[source_image == -999] = 0
write_img(
source_image,
source_band,
geobox=GriddedGeoBox.from_dataset(h5[location]),
nodata=0,
options={
"compression": "deflate",
"zlevel": 1
},
)
# returns a reference image from one of ls5/7/8
# the gqa band id will differ depending on if the source image is 5/7/8
reference_imagery = get_reference_imagery(
landsat_scenes,
acq_info.timestamp,
band_id,
acq_info.tag,
[self.reference_directory, self.backup_reference_directory],
)
ref_date = get_reference_date(
basename(reference_imagery[0].filename), band_id, acq_info.tag)
ref_source_path = reference_imagery[0].filename
# reference resolution is required for the gqa calculation
reference_resolution = [
abs(x) for x in most_common(reference_imagery).resolution
]
vrt_file = pjoin(workdir, "reference.vrt")
build_vrt(reference_imagery, vrt_file, workdir)
self._run_gverify(
vrt_file,
source_band,
outdir=workdir,
extra=extra,
resampling=acq_info.preferred_resampling_method,
)
except (ValueError, FileNotFoundError, CommandError) as ve:
error_msg = str(ve)
TASK_LOGGER.error(
task=self.get_task_family(),
params=self.to_str_params(),
level1=self.level1,
exception="gverify was not executed because:\n {}".format(
error_msg),
)
finally:
# Write out runtime data to be processed by the gqa task
run_args = {
"executable": self.executable,
"ref_resolution": reference_resolution,
"ref_date": (ref_date.isoformat() if ref_date else ""),
"ref_source_path": str(ref_source_path),
"granule": str(self.granule),
"error_msg": str(error_msg),
}
with self.output()["runtime_args"].open("w") as fd:
write_yaml(run_args, fd)
# if gverify failed to product the .res file writ out a blank one
if not exists(self.output()["results"].path):
with self.output()["results"].open("w") as fd:
pass