def test_latlon_to_cell():
# TODO
# Expected outputs:
#
# (120, -19) -> (120, -20)
# (120.1, -19.1) -> (120, -20)
# (120.9, -19.9) -> (120, -20)
# (120, -20) -> (120, -21)
# Should return
latlon_to_cell(120, -20)
def go(self):
cell_x, cell_y = latlon_to_cell(self.latitude, self.longitude)
_log.info("cell is %d %d", cell_x, cell_y)
ndv = get_dataset_type_ndv(self.dataset_type)
with self.get_output_file(self.dataset_type, self.overwrite) as csv_file:
csv_writer = csv.writer(csv_file, delimiter=self.delimiter)
# Output a HEADER
csv_writer.writerow(["SATELLITE", "ACQUISITION DATE"] + self.bands)
for tile in self.get_tiles(x=cell_x, y=cell_y):
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime)
continue
dataset = tile.datasets[self.dataset_type]
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None
wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None
data = retrieve_pixel_value(dataset, pqa, self.mask_pqa_mask, wofs, self.mask_wofs_mask, self.latitude, self.longitude, ndv=ndv)
if has_data(dataset.bands, data, no_data_value=ndv) or self.output_no_data:
csv_writer.writerow([dataset.satellite.name, format_date_time(tile.end_datetime)] +
decode_data(self.dataset_type, dataset, self.bands, data))
def go(self):
cell_x, cell_y = latlon_to_cell(self.latitude, self.longitude)
# TODO - PQ is UNIT16 and WOFS is BYTE (others are INT16) and so -999 NDV doesn't work
ndv = NDV
if self.dataset_type == DatasetType.PQ25:
ndv = UINT16_MAX
elif self.dataset_type == DatasetType.WATER:
ndv = BYTE_MAX
elif self.dataset_type in [DatasetType.NDVI, DatasetType.EVI, DatasetType.NBR, DatasetType.TCI]:
ndv = NAN
with self.get_output_file(self.dataset_type, self.overwrite) as csv_file:
csv_writer = csv.writer(csv_file, delimiter=self.delimiter)
# Output a HEADER
csv_writer.writerow(["SATELLITE", "ACQUISITION DATE"] + self.bands)
for tile in self.get_tiles(x=cell_x, y=cell_y):
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime)
continue
dataset = tile.datasets[self.dataset_type]
pqa = (
(self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets)
and tile.datasets[DatasetType.PQ25]
or None
)
wofs = (
(self.mask_wofs_apply and DatasetType.WATER in tile.datasets)
and tile.datasets[DatasetType.WATER]
or None
)
data = retrieve_pixel_value(
dataset, pqa, self.mask_pqa_mask, wofs, self.mask_wofs_mask, self.latitude, self.longitude, ndv=ndv
)
if has_data(dataset.bands, data, no_data_value=ndv) or self.output_no_data:
csv_writer.writerow(
[dataset.satellite.name, format_date_time(tile.end_datetime)]
+ decode_data(self.dataset_type, dataset, self.bands, data)
)
def run(self):
self.parse_arguments()
config = Config()
_log.debug(config.to_str())
cell_x, cell_y = latlon_to_cell(self.latitude, self.longitude)
# TODO once WOFS is in the cube
if self.dataset_type in dataset_type_database:
# TODO - PQ is UNIT16 (others are INT16) and so -999 NDV doesn't work
ndv = self.dataset_type == DatasetType.PQ25 and UINT16_MAX or NDV
headered = False
with self.get_output_file(self.dataset_type, self.overwrite) as csv_file:
csv_writer = csv.writer(csv_file, delimiter=self.delimiter)
for tile in list_tiles(x=[cell_x], y=[cell_y], acq_min=self.acq_min, acq_max=self.acq_max,
satellites=[satellite for satellite in self.satellites],
dataset_types=[self.dataset_type],
database=config.get_db_database(),
user=config.get_db_username(),
password=config.get_db_password(),
host=config.get_db_host(), port=config.get_db_port()):
# Output a HEADER
if not headered:
header_fields = ["SATELLITE", "ACQUISITION DATE"] + [b.name for b in tile.datasets[self.dataset_type].bands]
csv_writer.writerow(header_fields)
headered = True
pqa = None
# Apply PQA if specified
if self.apply_pqa_filter:
pqa = tile.datasets[DatasetType.PQ25]
data = retrieve_pixel_value(tile.datasets[self.dataset_type], pqa, self.pqa_mask, self.latitude, self.longitude, ndv=ndv)
_log.debug("data is [%s]", data)
if has_data(tile.datasets[self.dataset_type], data, no_data_value=ndv) or self.output_no_data:
csv_writer.writerow([tile.datasets[self.dataset_type].satellite.value, str(tile.end_datetime)] + decode_data(tile.datasets[self.dataset_type], data))
elif self.dataset_type == DatasetType.WATER:
base = "/g/data/u46/wofs/water_f7q/extents/{x:03d}_{y:04d}/LS*_WATER_{x:03d}_{y:04d}_*.tif".format(x=cell_x, y=cell_y)
headered = False
with self.get_output_file(self.dataset_type, self.overwrite) as csv_file:
csv_writer = csv.writer(csv_file, delimiter=self.delimiter)
for f in glob.glob(base):
_log.debug(" *** Found WOFS file [%s]", f)
satellite, dataset_type, x, y, acq_dt = extract_fields_from_filename(os.path.basename(f))
if acq_dt.date() < self.acq_min or acq_dt.date() > self.acq_max:
continue
dataset = DatasetTile.from_path(f)
_log.debug("Found dataset [%s]", dataset)
# Output a HEADER
if not headered:
header_fields = ["SATELLITE", "ACQUISITION DATE"] + [b.name for b in dataset.bands]
csv_writer.writerow(header_fields)
headered = True
data = retrieve_pixel_value(dataset, None, None, self.latitude, self.longitude)
_log.debug("data is [%s]", data)
# TODO
if True or self.output_no_data:
csv_writer.writerow([satellite.value, str(acq_dt), decode_wofs_water_value(data[Wofs25Bands.WATER][0][0]), str(data[Wofs25Bands.WATER][0][0])])