def test_properties():
reg_a = ipx.Query(
"ATL06",
[-64, 66, -55, 72],
["2019-02-22", "2019-02-28"],
start_time="03:30:00",
end_time="21:30:00",
version="2",
)
obs_list = [
reg_a.dataset,
reg_a.dates,
reg_a.start_time,
reg_a.end_time,
reg_a.dataset_version,
reg_a.spatial_extent,
]
exp_list = [
"ATL06",
["2019-02-22", "2019-02-28"],
"03:30:00",
"21:30:00",
"002",
["bounding box", [-64, 66, -55, 72]],
]
for obs, expected in zip(obs_list, exp_list):
assert obs == expected
def test_icepyx_boundingbox_query():
reg_a = ipx.Query(
"ATL06",
[-64, 66, -55, 72],
["2019-02-22", "2019-02-28"],
start_time="03:30:00",
end_time="21:30:00",
version="5",
)
obs_tuple = (
reg_a.product,
reg_a.dates,
reg_a.start_time,
reg_a.end_time,
reg_a.product_version,
reg_a.spatial_extent,
)
exp_tuple = (
"ATL06",
["2019-02-22", "2019-02-28"],
"03:30:00",
"21:30:00",
"005",
("bounding box", [-64.0, 66.0, -55.0, 72.0]),
)
assert obs_tuple == exp_tuple
def test_CMRparams():
reg_a = ipx.Query("ATL06", [-64, 66, -55, 72],
["2019-02-22", "2019-02-28"])
reg_a.build_CMR_params()
obs_keys = reg_a.CMRparams.keys()
exp_keys_all = ["short_name", "version", "temporal"]
exp_keys_any = ["bounding_box", "polygon"]
assert all(keys in obs_keys for keys in exp_keys_all)
assert any(key in obs_keys for key in exp_keys_any)
def query(self, parameters):
dataset = parameters['dataset']
date_range = [parameters['start'], parameters['end']]
bounding_box = [
round(float(coord), 4) for coord in parameters['bbox'].split(',')
]
query = ipx.Query(dataset, bounding_box, date_range)
query._email = self.credentials['email']
query._session = self._session.session
return query
def test_correct_granule_list_returned():
reg_a = ipx.Query("ATL06", [-55, 68, -48, 71],
["2019-02-20", "2019-02-28"],
version="2")
reg_a.avail_granules()
obs_grans = [gran["producer_granule_id"] for gran in reg_a.granules.avail]
exp_grans = [
"ATL06_20190221121851_08410203_002_01.h5",
"ATL06_20190222010344_08490205_002_01.h5",
"ATL06_20190225121032_09020203_002_01.h5",
"ATL06_20190226005526_09100205_002_01.h5",
]
assert set(obs_grans) == set(exp_grans)
def test_correct_granule_list_returned():
reg_a = ipx.Query(
"ATL06",
[-55, 68, -48, 71],
["2019-02-20", "2019-02-28"],
version="3",
)
obs_grans, = reg_a.avail_granules(ids=True)
exp_grans = [
"ATL06_20190221121851_08410203_003_01.h5",
"ATL06_20190222010344_08490205_003_01.h5",
"ATL06_20190225121032_09020203_003_01.h5",
"ATL06_20190226005526_09100205_003_01.h5",
]
assert set(obs_grans) == set(exp_grans)
def __init__(
self,
query_obj=None,
product=None,
spatial_extent=None,
date_range=None,
cycles=None,
tracks=None,
):
if query_obj:
pass
else:
query_obj = ipx.Query(
dataset=product,
spatial_extent=spatial_extent,
date_range=date_range,
cycles=cycles,
tracks=tracks,
)
self.product = is2ref._validate_OA_product(query_obj.dataset)
if query_obj.extent_type == "bounding_box":
self.bbox = query_obj._spat_extent
else:
mrc_bound = query_obj._spat_extent.minimum_rotated_rectangle
# generate bounding box
lonmin = min(mrc_bound.exterior.coords.xy[0])
lonmax = max(mrc_bound.exterior.coords.xy[0])
latmin = min(mrc_bound.exterior.coords.xy[1])
latmax = max(mrc_bound.exterior.coords.xy[1])
self.bbox = [lonmin, latmin, lonmax, latmax]
self.date_range = ([
query_obj._start.strftime("%Y-%m-%d"),
query_obj._end.strftime("%Y-%m-%d")
] if hasattr(query_obj, "_start") else None)
self.cycles = query_obj._cycles if hasattr(query_obj,
"_cycles") else None
self.tracks = query_obj._tracks if hasattr(query_obj,
"_tracks") else None
def download(bbox, timeframe, beam, earthdata_uid, email):
import os
import icepyx as ipx
from icepyx import core
import readers as rd
short_name = ATL03 '''fill in with name of whichever DataSet this is a member function of'''
region = ipx.Query(short_name, bbox, timeframe)
region.avail_granules()
print(region.avail_granules())
if ('y' or 'yes' == input("Here's what we've got! Continue? (y/n)")):
region.earthdata_login(earthdata_uid, email)
path = './download/' + short_name
region.download_granules(path)
files = region.avail_granules(ids=True)
print("Downloaded! Have a great day!")
else:
print("Nothing was downloaded")
exit()
for file in files:
f = h5py.File(path+file, 'r')
df = rd.getATL03(f,beam)
#trim to bounding box
df_cut=df[bbox]
#convert time to UTC
epoch=f['/ancillary_data/atlas_sdp_gps_epoch'][0]
df_cut['time']=Time(epoch+df_cut['dt'],format='gps').utc.datetime
#caculate along track distance
df_cut['AT_dist']=df_cut.x-df_cut.x.values[0]
#save a dictionary entry in the DataSet
self{file} = df_cut
def query_icesat2_filelist(self) -> tuple:
"""
Query list of ICESat-2 files for each bounding box
Returns
-------
filelist_tuple : tuple
A tuple of non-empty lists of bounding boxes and corresponding ICESat-2 file lists
"""
# a list of 5*5 bounding boxes
bbox_list = self.grid_bbox()
is2_bbox_list = []
is2_file_list = []
for bbox_i in bbox_list:
try:
region = ipx.Query(
self.product,
bbox_i,
self.date_range,
cycles=self.cycles,
tracks=self.tracks,
)
icesat2_files = region.avail_granules(ids=True)[0]
except (AttributeError, AssertionError):
continue
if not icesat2_files:
continue
else:
all_cycles = list(set(region.avail_granules(cycles=True)[0]))
icesat2_files_latest_cycle = files_in_latest_n_cycles(
icesat2_files, [int(c) for c in all_cycles]
)
is2_bbox_list.append(bbox_i)
is2_file_list.append(icesat2_files_latest_cycle)
is2_file_tuple = zip(is2_bbox_list, is2_file_list)
return is2_file_tuple
def test_reqconfig_params():
reg_a = ipx.Query("ATL06", [-64, 66, -55, 72],
["2019-02-22", "2019-02-28"])
# test for search params
reg_a.build_reqconfig_params("search")
obs_keys = reg_a.reqparams.keys()
exp_keys_all = ["page_size", "page_num"]
assert all(keys in obs_keys for keys in exp_keys_all)
# test for download params
reg_a.reqparams = None
reg_a.build_reqconfig_params("download")
reg_a.reqparams.update({"token": "", "email": ""})
obs_keys = reg_a.reqparams.keys()
exp_keys_all = [
"page_size",
"page_num",
"request_mode",
"token",
"email",
"include_meta",
]
assert all(keys in obs_keys for keys in exp_keys_all)
def test_no_granules_in_search_results():
ermsg = "Your search returned no results; try different search parameters"
with pytest.raises(AssertionError, match=ermsg):
ipx.Query("ATL06", [-55, 68, -48, 71], ["2019-02-20", "2019-02-20"],
version="2").avail_granules()
def read_atl06(spatial_extent,
date_range,
time_start="00:00:00",
time_end="23:59:59",
path="./data/new_ATL06",
user='******',
email='*****@*****.**'):
region_a = ipx.Query("ATL06",
spatial_extent,
date_range,
start_time=time_start,
end_time=time_end)
#region_a = ipd.Icesat2Data("ATL06", spatial_extent, date_range, start_time = time_start, end_time = time_end)
# The last update of icepyx returns a dictonary with the total number of granules and also a list of one element that contains
# a list with the name of the available granules. This is the reason why we have [0] at the end of the next line.
# - check if this is always the case. For now, I will add an assert.
avail_granules = region_a.avail_granules(ids=True)[0]
print("Available Granules:", avail_granules)
assert region_a.avail_granules()['Number of available granules'] == len(
avail_granules), "The number of avail granules does not match"
if len(avail_granules) == 0:
print("No granules for this specification")
return None
### If I already downloaded these files, I don't need to do it again
# I should check first that such directory exists!
if all([
is_file_in_directory(avail_granules[i], path)
for i in range(len(avail_granules))
]):
print("You have already downloaded all the requested files")
else:
region_a.earthdata_login(user, email)
# This lines were commented after the last update of icepyx. See Issue https://github.com/icesat2py/icepyx/issues/145
#region_a.order_vars.append(var_list=['latitude','longitude','h_li','atl06_quality_summary','delta_time',
# 'signal_selection_source', 'snr', 'snr_significance','h_robust_sprd','dh_fit_dx','dh_fit_dy','bsnow_conf',
# 'cloud_flg_asr','cloud_flg_atm','msw_flag','bsnow_h','bsnow_od','layer_flag','bckgrd',
# 'n_fit_photons','end_geoseg','segment_id','w_surface_window_final', 'sc_orient'])
#region_a.subsetparams(Coverage=region_a.order_vars.wanted)
region_a.order_granules()
region_a.download_granules(path)
### The files I need to read should coincide with the requested granueles
requested_files = []
for fname_granules in avail_granules:
file_is_in_folder = False
for fname_folder in list(Path(path).glob('*.h5')):
if fname_granules in str(fname_folder):
requested_files.append(fname_folder)
file_is_in_folder = True
break
if not file_is_in_folder:
print(fname_granules, "not found")
if len(avail_granules) != len(requested_files):
print("You are missing some files. There are a total of",
len(avail_granules), "available granules but you are accessing",
len(requested_files), "h5 files")
dataf = atl06_2_df(requested_files)
#dataf = atl06_2_df(Path(path).glob('*.h5'))
# reset index
dataf = dataf.reset_index()
del dataf['index']
return (dataf)
def reg(scope="module"):
return ipx.Query(
"ATL06", [-55, 68, -48, 71], ["2019-02-22", "2019-02-28"], version="004"
)
"len": 40.0,
"res": 20.0,
"maxi": 1
}
# get command line parameters
parse_command_line(sys.argv, icfg)
parse_command_line(sys.argv, scfg)
parse_command_line(sys.argv, parms)
# bypass service discovery if url is localhost
if scfg["url"] == '127.0.0.1':
scfg["url"] = ['127.0.0.1']
# create icepx region
iregion = icepyx.Query(icfg["short_name"], icfg["spatial_extent"], icfg["date_range"], cycles=icfg["cycles"], tracks=icfg["tracks"])
# visualize icepx region
# iregion.visualize_spatial_extent()
# display summary information
iregion.product_summary_info()
# print("Available Granules:", iregion.avail_granules())
# print("Available Granule IDs:", iregion.avail_granules(ids=True))
# initialize sliderule api
icesat2.init(scfg["url"], verbose=True)
# generate sliderule atl06 elevations
# parms["poly"] = icesat2.toregion(icfg["spatial_extent"])[0]
atl06_sr = ipxapi.atl06p(iregion, parms, scfg["asset"])
def atl03(area, date_range, path, user, email):
"""
Retrieve atl_03 granules corresponding to selected spatio-temporal area
"""
region_a = ipx.Query('ATL03', spatial_extent=area, date_range=date_range)
avail_granules = region_a.avail_granules(ids=True)[0]
print("Available Granules:", avail_granules)
if all([
is_file_in_directory(avail_granules[i], path)
for i in range(len(avail_granules))
]):
print("You have already downloaded all the requested files")
else:
region_a.earthdata_login(user, email)
#region_a.order_vars.append(var_list=['lat_ph', "lon_ph", "h_ph"])
#region_a.subsetparams(Coverage=region_a.order_vars.wanted)
region_a.order_granules()
region_a.download_granules(path)
dataframes = pd.DataFrame(
columns=["h_ph", "lon_ph", "lat_ph", "ground_track"])
requested_files = []
for fname_granules in avail_granules:
file_is_in_folder = False
for fname_folder in list(Path(path).glob('*.h5')):
if fname_granules in str(fname_folder):
requested_files.append(fname_folder)
file_is_in_folder = True
break
if not file_is_in_folder:
print(fname_granules, "not found")
if len(avail_granules) != len(requested_files):
print("You are missing some files. There are a total of",
len(avail_granules), "available granules but you are accessing",
len(requested_files), "h5 files")
for file in requested_files:
with h5py.File(file, 'r') as fi:
for my_gt in filter(fi.keys(), ["gt"]):
lat_ph = fi[my_gt]['heights']["lat_ph"][:]
lon_ph = fi[my_gt]['heights']["lon_ph"][:]
h_ph = fi[my_gt]['heights']["h_ph"][:]
df = pd.DataFrame.from_dict({
"h_ph": h_ph,
"lon_ph": lon_ph,
"lat_ph": lat_ph,
"ground_track": [my_gt] * len(h_ph)
})
dataframes = dataframes.append(df, ignore_index=True)
return (dataframes)