def ingest(filename):
with GdalFileHelper(filename) as gfh:
#modis hdf file consists of subdatasets
band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=0)
nc_vars = gfh.get_nc_vars(["Ht"])
heights = nc_vars["Ht"]
for band_num in settings["bands"]["numbers"]:
ras = GDALRaster(filename, settings["srid"]["in"])
ras.set_band_num(band_num["number"])
variable_name = band_num["name"]
ras.nodata_value = -9999
level = int(numpy.asscalar(heights[band_num["number"]]))
start_time = band_info["start_time"]
end_time = start_time + timedelta(seconds=settings["time_interval_seconds"])
with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
i = 0
mask = sqa.one(Mask, filterr={"name": "alabama"})
dg = upsert_datagranule(sqa, config, provider_name=settings["provider"]["name"],
variable_name=variable_name, format_name=settings["format"], level=level,
extent=ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=start_time,
end_time=end_time, grib_file_name=filename, overwrite_existing_datagranule=True,
name_suffix="ext_%d" % i)
if dg:
block_size = 50, 50
#sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None
insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"],
srid_out=settings["srid"]["out"], block_size=block_size, mask=mask)
def ingest(filenames):
for filename in filenames:
for band in settings["data"]["bands"]:
ras = GDALRaster(filename, settings["srid_in"])
ras.set_band_num(band["number"])
ras.nodata_value = -9999
variable_name = band["name"]
level = 0
start_time = datetime(year=2010, month=1, day=1)
end_time = datetime(year=2020, month=1, day=1)
with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
i = 0
mask = sqa.one(Mask, filterr={"name": "alabama"})
dg = upsert_datagranule(sqa, config, provider_name=settings["provider_name"],
variable_name=variable_name, format_name=settings["format"], level=level,
extent=ras.extent_ewkt(), srid=settings["srid_in"], start_time=start_time,
end_time=end_time, grib_file_name=filename, overwrite_existing_datagranule=True,
name_suffix="ext_%d" % i)
if dg:
block_size = 50, 50
insert_raster(sqa, config=config, datagranule=dg, ras=ras, settings=settings,
block_size=block_size, mask=mask)
def ingest(filenames):
for filename in filenames:
for variable_name in settings["sds"]:
with GdalFileHelper(filename) as gfh:
#modis hdf file consists of subdatasets
dsn = gfh.get_sds(settings["sds"][variable_name])
band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=0)
ras = GDALRaster(dsn, settings["srid"]["in"])
with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
mask = sqa.one(Mask, filterr={"name": "india"})
#mask = sqa.one(Mask, filterr={"name": "bareilly"})
dg = upsert_datagranule(sqa, config, provider_name=settings["provider"]["name"],
variable_name=variable_name, format_name=settings["format"], level=band_info["level"],
extent=ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=band_info["start_time"],
end_time=band_info["end_time"], grib_file_name=filename, overwrite_existing_datagranule=True)
if dg:
block_size = 50, 50
insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"],
srid_out=settings["srid"]["out"], block_size=block_size, mask=mask)
def ingest(filename):
with GdalFileHelper(filename) as gfh:
#modis hdf file consists of subdatasets
band_info = gfh.get_band_info(meta=settings["bands"]["meta"],
band_num=0)
nc_vars = gfh.get_nc_vars(["Ht"])
heights = nc_vars["Ht"]
for band_num in settings["bands"]["numbers"]:
ras = GDALRaster(filename, settings["srid"]["in"])
ras.set_band_num(band_num["number"])
variable_name = band_num["name"]
ras.nodata_value = -9999
level = int(numpy.asscalar(heights[band_num["number"]]))
start_time = band_info["start_time"]
end_time = start_time + timedelta(
seconds=settings["time_interval_seconds"])
with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
i = 0
mask = sqa.one(Mask, filterr={"name": "alabama"})
dg = upsert_datagranule(sqa,
config,
provider_name=settings["provider"]["name"],
variable_name=variable_name,
format_name=settings["format"],
level=level,
extent=ras.extent_ewkt(),
srid=settings["srid"]["in"],
start_time=start_time,
end_time=end_time,
grib_file_name=filename,
overwrite_existing_datagranule=True,
name_suffix="ext_%d" % i)
if dg:
block_size = 50, 50
#sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None
insert_raster(sqa,
config=config,
datagranule=dg,
ras=ras,
srid_in=settings["srid"]["in"],
srid_out=settings["srid"]["out"],
block_size=block_size,
mask=mask)
def ingest(filenames):
for filename in filenames:
for band in settings["data"]["bands"]:
ras = GDALRaster(filename, settings["srid_in"])
ras.set_band_num(band["number"])
ras.nodata_value = -9999
variable_name = band["name"]
level = 0
start_time = datetime(year=2010, month=1, day=1)
end_time = datetime(year=2020, month=1, day=1)
with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
i = 0
mask = sqa.one(Mask, filterr={"name": "alabama"})
dg = upsert_datagranule(
sqa,
config,
provider_name=settings["provider_name"],
variable_name=variable_name,
format_name=settings["format"],
level=level,
extent=ras.extent_ewkt(),
srid=settings["srid_in"],
start_time=start_time,
end_time=end_time,
grib_file_name=filename,
overwrite_existing_datagranule=True,
name_suffix="ext_%d" % i)
if dg:
block_size = 50, 50
insert_raster(sqa,
config=config,
datagranule=dg,
ras=ras,
settings=settings,
block_size=block_size,
mask=mask)
def ingest_gfs_files():
filenames = get_ingest_files("GFS", "*.f000")
for filename in filenames:
for variable_name in settings["bands"]["variables"]:
with GdalFileHelper(filename) as gfh:
band_num = gfh.find_band_num(filterr=settings["bands"]["variables"][variable_name])
band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=band_num)
if band_num > 0:
ras = GDALRaster(filename, settings["srid"]["in"])
ras.set_band_num(band_num)
data = ras.get_data(0, 0, ras.size, ras.size)
gdal_dtype = numpy2gdal(data.dtype.type)
size_x = ras.size[0] / 2
size_y = ras.size[1] - 1
data1 = data[0:size_y, 0:size_x]
array_ras1 = ArrayRaster(
data_array=data1,
size=(size_x, size_y),
ul=(0, 90),
scale=ras.scale,
skew=(0, 0),
srid=4326,
gdal_datatype=gdal_dtype,
nodata_value=-999,
)
data2 = data[0:size_y, size_x : ras.size[0]]
array_ras2 = ArrayRaster(
data_array=data2,
size=(size_x, size_y),
ul=(-180, 90),
scale=ras.scale,
skew=(0, 0),
srid=4326,
gdal_datatype=gdal_dtype,
nodata_value=-999,
)
with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
india_mask = sqa.one(Mask, filterr={"name": "india"})
i = 0
for arr_ras in [array_ras1, array_ras2]:
if not band_info["end_time"]:
band_info["end_time"] = band_info["start_time"] + timedelta(
seconds=settings["time_interval_seconds"]
)
dg = upsert_datagranule(
sqa,
config,
provider_name=settings["provider"]["name"],
variable_name=variable_name,
format_name=settings["format"],
level=band_info["level"],
extent=arr_ras.extent_ewkt(),
srid=settings["srid"]["in"],
start_time=band_info["start_time"],
end_time=band_info["end_time"],
grib_file_name=filename,
overwrite_existing_datagranule=True,
name_suffix="ext_%d" % i,
)
i += 1
if dg:
if settings["format"] == "RASTER":
block_size = 30, 30
else:
block_size = ras.size
insert_raster(
sqa,
config=config,
datagranule=dg,
ras=arr_ras,
srid_in=settings["srid"]["in"],
srid_out=settings["srid"]["out"],
block_size=block_size,
mask=india_mask,
)