def recombine_geographic_chunks(chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
logger.info("RECOMBINE_GEO")
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
task = SpectralIndicesTask.objects.get(pk=task_id)
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0], autoclose=True))
combined_data = combine_geographic_chunks(chunk_data)
path = os.path.join(task.get_temp_path(), "recombined_geo_{}.nc".format(time_chunk_id))
combined_data.to_netcdf(path)
logger.info("Done combining geographic chunks for time: " + str(time_chunk_id))
return path, metadata, {'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id}
def recombine_geographic_chunks(self, chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
logger.info("recombine_geographic_chunks() begin!")
task = CloudCoverageTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
if len(total_chunks) == 0:
return None
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0]))
combined_data = combine_geographic_chunks(chunk_data)
path = os.path.join(task.get_temp_path(), "recombined_geo_{}.nc".format(time_chunk_id))
export_xarray_to_netcdf(combined_data, path)
logger.info("Done combining geographic chunks for time: " + str(time_chunk_id))
return path, metadata, {'geo_chunk_id': geo_chunk_id, 'time_chunk_id': time_chunk_id}
def recombine_geographic_chunks(self, chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
num_scn_per_chk: The number of scenes per chunk. Used to determine task progress.
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
task = CustomMosaicToolTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
if len(total_chunks) == 0:
return None
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0]))
combined_data = combine_geographic_chunks(chunk_data)
# if we're animating, combine it all and save to disk.
if task.animated_product.animation_id != "none":
base_index = (task.get_chunk_size()['time'] if task.get_chunk_size()
['time'] is not None else 1) * time_chunk_id
for index in range((task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1)):
animated_data = []
for chunk in total_chunks:
geo_chunk_index = chunk[2]['geo_chunk_id']
# if we're animating, combine it all and save to disk.
path = os.path.join(
task.get_temp_path(),
"animation_{}_{}.nc".format(str(geo_chunk_index),
str(base_index + index)))
if os.path.exists(path):
animated_data.append(xr.open_dataset(path))
path = os.path.join(task.get_temp_path(),
"animation_{}.nc".format(base_index + index))
if len(animated_data) > 0:
combine_geographic_chunks(animated_data).to_netcdf(path)
path = os.path.join(task.get_temp_path(),
"recombined_geo_{}.nc".format(time_chunk_id))
combined_data.to_netcdf(path)
logger.info("Done combining geographic chunks for time: " +
str(time_chunk_id))
return path, metadata, {
'geo_chunk_id': geo_chunk_id,
'time_chunk_id': time_chunk_id
}
def recombine_geographic_chunks(chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
logger.info("RECOMBINE_GEO")
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
task = CoastalChangeTask.objects.get(pk=task_id)
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0], autoclose=True))
combined_data = combine_geographic_chunks(chunk_data)
if task.animated_product.animation_id != "none":
path = os.path.join(task.get_temp_path(),
"animation_{}.png".format(time_chunk_id))
animated_data = mask_mosaic_with_coastlines(
combined_data
) if task.animated_product.animation_id == "coastline_change" else mask_mosaic_with_coastal_change(
combined_data)
write_png_from_xr(path,
animated_data,
bands=['red', 'green', 'blue'],
scale=task.satellite.get_scale(),
no_data=task.satellite.no_data_value)
path = os.path.join(task.get_temp_path(),
"recombined_geo_{}.nc".format(time_chunk_id))
combined_data.to_netcdf(path)
logger.info("Done combining geographic chunks for time: " +
str(time_chunk_id))
return path, metadata, {
'geo_chunk_id': geo_chunk_id,
'time_chunk_id': time_chunk_id
}
def recombine_geographic_chunks(self, chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
if len(total_chunks) == 0: return None
task = SpectralAnomalyTask.objects.get(pk=task_id)
if check_cancel_task(self, task): return
metadata = {}
composite_chunk_data = []
out_of_range_chunk_data = []
no_data_chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[3])
composite_chunk_data.append(xr.open_dataset(chunk[0]))
out_of_range_chunk_data.append(xr.open_dataset(chunk[1]))
no_data_chunk_data.append(xr.open_dataset(chunk[2]))
combined_composite_data = combine_geographic_chunks(composite_chunk_data)
combined_out_of_range_data = combine_geographic_chunks(
out_of_range_chunk_data)
combined_no_data = combine_geographic_chunks(no_data_chunk_data)
composite_path = os.path.join(task.get_temp_path(), "full_composite.nc")
export_xarray_to_netcdf(combined_composite_data, composite_path)
composite_out_of_range_path = os.path.join(
task.get_temp_path(), "full_composite_out_of_range.nc")
export_xarray_to_netcdf(combined_out_of_range_data,
composite_out_of_range_path)
no_data_path = os.path.join(task.get_temp_path(),
"full_composite_no_data.nc")
export_xarray_to_netcdf(combined_no_data, no_data_path)
return composite_path, composite_out_of_range_path, no_data_path, metadata
def recombine_geographic_chunks(chunks, task_id=None):
"""Recombine processed data over the geographic indices
For each geographic chunk process spawned by the main task, open the resulting dataset
and combine it into a single dataset. Combine metadata as well, writing to disk.
Args:
chunks: list of the return from the processing_task function - path, metadata, and {chunk ids}
Returns:
path to the output product, metadata dict, and a dict containing the geo/time ids
"""
logger.info("RECOMBINE_GEO")
total_chunks = [chunks] if not isinstance(chunks, list) else chunks
total_chunks = [chunk for chunk in total_chunks if chunk is not None]
geo_chunk_id = total_chunks[0][2]['geo_chunk_id']
time_chunk_id = total_chunks[0][2]['time_chunk_id']
metadata = {}
task = AppNameTask.objects.get(pk=task_id)
chunk_data = []
for index, chunk in enumerate(total_chunks):
metadata = task.combine_metadata(metadata, chunk[1])
chunk_data.append(xr.open_dataset(chunk[0], autoclose=True))
combined_data = combine_geographic_chunks(chunk_data)
# if we're animating, combine it all and save to disk.
# TODO: If there is no animation, delete this block. Otherwise, recombine all the geo chunks for each time chunk
# and save the result to disk.
if task.animated_product.animation_id != "none":
base_index = (task.get_chunk_size()['time'] if task.get_chunk_size()
['time'] is not None else 1) * time_chunk_id
for index in range((task.get_chunk_size()['time'] if
task.get_chunk_size()['time'] is not None else 1)):
animated_data = []
for chunk in total_chunks:
geo_chunk_index = chunk[2]['geo_chunk_id']
# if we're animating, combine it all and save to disk.
path = os.path.join(
task.get_temp_path(),
"animation_{}_{}.nc".format(str(geo_chunk_index),
str(base_index + index)))
if os.path.exists(path):
animated_data.append(xr.open_dataset(path, autoclose=True))
path = os.path.join(task.get_temp_path(),
"animation_{}.nc".format(base_index + index))
if len(animated_data) > 0:
combine_geographic_chunks(animated_data).to_netcdf(path)
path = os.path.join(task.get_temp_path(),
"recombined_geo_{}.nc".format(time_chunk_id))
combined_data.to_netcdf(path)
logger.info("Done combining geographic chunks for time: " +
str(time_chunk_id))
return path, metadata, {
'geo_chunk_id': geo_chunk_id,
'time_chunk_id': time_chunk_id
}
