def test_fetch_one_channel():
lon_zenith = -45.0
t0 = satdata.calc_nearest_zenith_time_at_loc(lon_zenith)
t = t0 - datetime.timedelta(days=3)
dt_max = datetime.timedelta(minutes=20)
cli = satdata.Goes16AWS()
keys = cli.query(time=t, region="F", debug=True, dt_max=dt_max, channel=1)
fn = cli.download(keys[0])[0]
assert Path(fn).exists()
def run(self):
cli = satdata.Goes16AWS(offline=False)
filenames = cli.query(
time=self.time,
region="F",
debug=self.debug,
channel=self.channel,
dt_max=self.dt_max,
)
Path(self.output().fn).parent.mkdir(exist_ok=True)
self.output().write(filenames)
def test_parse_L2_key():
key = "noaa-goes16/ABI-L2-TPWC/2020/037/00/OR_ABI-L2-TPWC-M6_G16_s20200370001071_e20200370003443_c20200370005459.nc"
c = satdata.Goes16AWS()
parsed_data = c.parse_key(key)
data = dict(
level="L2",
product="TPW",
region="C",
sensor_mode=6,
)
for k, v in data.items():
assert parsed_data[k] == v
def test_parse_L1_key():
key = "noaa-goes16/ABI-L1b-RadC/2020/037/00/OR_ABI-L1b-RadC-M6C02_G16_s20200370051071_e20200370053443_c20200370053496.nc"
c = satdata.Goes16AWS()
parsed_data = c.parse_key(key)
data = dict(
level="L1b",
product="Rad",
channel=2,
region="C",
sensor_mode=6,
)
for k, v in data.items():
assert parsed_data[k] == v
def test_fetch_one_channel_multi_day():
N_HOURS = 2
lon_zenith = -45.0
dt_max = datetime.timedelta(hours=N_HOURS)
t0 = satdata.calc_nearest_zenith_time_at_loc(lon_zenith)
t = t0 - datetime.timedelta(days=8)
cli = satdata.Goes16AWS()
keys = cli.query(time=t, dt_max=dt_max, region="F", debug=True, channel=1)
# imagery should be available at least every 15 mins in the F region
print("\n".join(keys))
print(len(keys), (1 + 2 * N_HOURS) * 60 / 15)
assert len(keys) > (1 + 2 * N_HOURS) * 60 / 15
def test_query_products(product, region):
lon_zenith = -45.0
dt_max = datetime.timedelta(hours=2)
t0 = satdata.calc_nearest_zenith_time_at_loc(lon_zenith)
t = t0 - datetime.timedelta(days=8)
cli = satdata.Goes16AWS()
def query_func():
return cli.query(time=t, dt_max=dt_max, region=region, product=product)
if product in cli.PRODUCT_REGIONS and region not in cli.PRODUCT_REGIONS[
product]:
with pytest.raises(NotImplementedError):
query_func()
else:
keys = query_func()
assert len(keys) > 0
def run(self):
files_per_channel = {}
for channel, queries in self.input().items():
files_per_channel[channel] = []
for qr in queries:
files_per_channel[channel] += qr.read()
num_files_per_channel = {}
for channel, files in files_per_channel.items():
num_files_per_channel[channel] = len(files)
if len(set(num_files_per_channel.values())) != 1:
# the source data queries have resulted in a different number of
# files being returned for the channels selected, probably because
# the channels are not recorded at the same time and so one fell
# outside the window
def get_time(fn):
attrs = satdata.Goes16AWS.parse_key(fn, parse_times=True)
return attrs["start_time"]
def time_diff(i0, i1, fpc):
# fpc: new sliced files_per_channel dictionary where each list
# now has the same length
channels = list(files_per_channel.keys())
c1 = channels[i0]
c2 = channels[i1]
dt = get_time(fpc[c1][0]) - get_time(fpc[c2][0])
return abs(dt.total_seconds())
def timediff_all(fpc):
return sum([
time_diff(i0, i1, fpc)
for (i0, i1) in [(0, 1), (0, 2), (1, 2)]
])
N_max = max(num_files_per_channel.values())
fpc1 = {
c: len(fns) == N_max and fns[1:] or fns
for (c, fns) in files_per_channel.items()
}
fpc2 = {
c: len(fns) == N_max and fns[:-1] or fns
for (c, fns) in files_per_channel.items()
}
if timediff_all(fpc1) < timediff_all(fpc2):
files_per_channel = fpc1
else:
files_per_channel = fpc2
# now double-check that we've got the right number of files
num_files_per_channel = {}
for channel, files in files_per_channel.items():
num_files_per_channel[channel] = len(files)
assert len(set(num_files_per_channel.values())) == 1
local_storage_dir = Path(self.data_path).expanduser() / SOURCE_DIR
cli = satdata.Goes16AWS(offline=self.offline_cli,
local_storage_dir=local_storage_dir)
# download all the files using the cli (flatting list first...)
all_files = [fn for fns in files_per_channel.values() for fn in fns]
print("Downloading channel files...")
cli.download(all_files)
scenes_files = [list(a) for a in zip(*files_per_channel.values())]
indexed_scenes_files = {
self._make_scene_id(scene_files): scene_files
for scene_files in scenes_files
}
Path(self.output().fn).parent.mkdir(exist_ok=True)
self.output().write(indexed_scenes_files)