def process_file(rap_file):
provider_name = "RAP"
srid = RAP_Spatial_Reference.epsg
logger.info("Ingesting file %s" % rap_file)
for variable in variables:
logger.info("Processing variable %s" % variable)
band_num = find_band_num(rap_file, filterr=variables[variable])
if band_num is None:
#raise Exception("Could not find band for %s" % variable)
logger.error("Could not find band for %s" % variable)
else:
vars = ["GRIB_REF_TIME", "GRIB_VALID_TIME"]
datevalues = get_band_metadata(rap_file, band_num, vars)
startdate_utc_str = (datevalues["GRIB_REF_TIME"].split())[0]
enddate_utc_str = (datevalues["GRIB_VALID_TIME"].split())[0]
start_date = datetime.utcfromtimestamp(float(startdate_utc_str))
#end_date = datetime.fromtimestamp(float(enddate_utc_str))
end_date = start_date + timedelta(hours=1)
block_size = (10, 10)
ras = GDALRaster(rap_file, srid)
ras.set_band_num(band_num)
if variable == "RAP_REFL":
ras.nodata_range = [-999, -9]
level = int((variables[variable]["GRIB_SHORT_NAME"].split("-"))[0])
granule_name = "%s_%s %s_%d" % (provider_name, variable, start_date.strftime("%Y%m%d %H:%M"), level)
table_name = "%s_%s_%s_%d" % (provider_name, variable, start_date.strftime("%Y%m%d%H%M"), level)
bbox = proj_helper.get_bbox(srid)
base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable, granule_name=granule_name,
table_name=granule_name, srid=srid, level=level, block_size=block_size, dynamic=False,
start_time=start_date, end_time=end_date, subset_bbox=bbox, overwrite=True, threshold=None)
def ingest_gtopo_file(fmt):
df = config.datafiles["GTOPO30_ELEV"]
if isinstance(df["wildcard"], list):
files = []
for wc in df["wildcard"]:
files += base_ingestor.get_ingest_files(df["folder"],
df["wildcard"])
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
gtopo_file = files[0]
ras = GDALRaster(gtopo_file, srid)
ras.nodata_value = -9999
bbox = proj_helper.get_bbox(srid)
if fmt == 'ras':
granule_name = "GTOPO30Elev_ras"
level = 0
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
pgdb_helper.submit("""
drop if exists index rastertile_geom_gist_idx;
create index rastertile_geom_gist_idx on rastertile using gist(st_convexhull(rast));
""")
if fmt == 'vec':
granule_name = "GTOPO30Elev_vec"
level = 1
base_ingestor.ingest_vector(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
def ingest_gtopo_file():
provider_name = "GTOPO30"
variable_name = "ELEV"
df = config.datafiles["GTOPO30_ELEV"]
if isinstance(df["wildcard"], list):
files = []
for wc in df["wildcard"]:
files += base_ingestor.get_ingest_files(df["folder"],
df["wildcard"])
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
gtopo_file = files[0]
srid = 4326
band_num = 1
dtime = datetime(year=1979, month=1, day=1, hour=0, minute=0, second=0)
for block_size in block_sizes:
level = block_size[0]
granule_name = "GTOPO30Elev_%d" % level
ras = GDALRaster(gtopo_file, srid)
ras.nodata_value = -9999
bbox = proj_helper.get_bbox(srid)
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
def process_mrms_file(mrms_file):
provider_name = "MRMS"
variable_name = "REFL"
ext_parts = os.path.splitext(mrms_file)
ext = ext_parts[1]
remove_after_process = False
if ext == ".gz":
nc_file_name = ext_parts[0]
nc_file_copy = os.path.join("./", os.path.basename(nc_file_name))
if os.path.exists(nc_file_copy):
mrms_file = nc_file_copy
else:
with open(nc_file_copy, 'wb') as nc_file:
gz_file = gzip.open(mrms_file, 'rb')
gz_bytes = gz_file.read()
nc_file.write(gz_bytes)
gz_file.close()
mrms_file = nc_file_copy
remove_after_process = True
vars = nc_get_1d_vars_as_list(mrms_file, ["Ht", "time"])
heights = vars["Ht"]
times = vars["time"]
srid = 4326
#dtime = datetime(year=2014, month=8, day=18, hour=19, minute=0, second=0)
dtime = datetime.utcfromtimestamp(times[0])
bbox = proj_helper.get_bbox(srid)
start_time = dtime
end_time = dtime + timedelta(minutes=2)
for block_size in block_sizes:
level = block_size[0] #put various tiles in various levels
granule_name = "%s_%s %s_%d" % (provider_name, variable_name,
dtime.strftime("%Y%m%d %H:%M"), level)
table_name = "%s_%s_%s_%d" % (provider_name, variable_name,
dtime.strftime("%Y%m%d%H%M"), level)
bottom_up_data = True
ras = GDALRaster(mrms_file, srid, bottom_up_data)
l = 14
ras.set_band_num(l + 1)
#explicitly override the noddata_value since netcdf file is not correct
ras.nodata_value = -999
ras.nodata_range = (-999, 0)
#ras.reclassifier_callback = cb
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
subset_bbox=bbox,
start_time=start_time,
end_time=end_time,
overwrite=True,
threshold=34)
if remove_after_process:
os.remove(mrms_file)
#end_date = datetime.strptime(datevalues["RANGEENDINGDATE"], "%Y-%m-%d")
end_date = datetime(year=2019, month=1, day=1)
srid = SRID_MODIS
level = 0
block_size = (50, 50)
granule_name = "%s_%s %s_%d" % (provider_name, variable_name,
start_date.strftime("%Y%m%d"), level)
table_name = "%s_%s_%s_%d" % (provider_name, variable_name,
start_date.strftime("%Y%m%d"), level)
ras = GDALRaster(sds, srid)
#ras.nodata_range = [0.5, 256] #only 0 (water)
bbox = proj_helper.get_bbox(srid)
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
subset_bbox=bbox,
start_time=start_date,
end_time=end_date,
overwrite=True)
#also ingest same data as land water mask
files += base_ingestor.get_ingest_files(df["folder"],
df["wildcard"])
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
for gtopo_file in files:
granule_name = "GTOPO30Elev"
srid = 4326
band_num = 1
block_size = (50, 50)
dtime = datetime(year=1979, month=1, day=1, hour=0, minute=0, second=0)
level = 0
ras = GDALRaster(gtopo_file, srid)
ras.nodata_value = -9999
bbox = proj_helper.get_bbox(srid)
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
from datetime import datetime, timedelta
from gdal import gdalconst
from ci.models.spatial_reference import RAP_Spatial_Reference
from ci.models.array_raster import ArrayRaster
from ci.db.pgdbhelper import PGDbHelper
from ci.ingest import proj_helper, config, logger, base_ingestor
__author__ = 'ujjwal'
nodata = 0
scale = 13000, -13000
bbox = proj_helper.get_bbox(RAP_Spatial_Reference.epsg)
ul = (bbox[0] - scale[0] * 0.5, bbox[2] + scale[0] * 0.5)
size = int((bbox[1] - bbox[0]) / scale[0]), int((bbox[3] - bbox[2]) / scale[1])
def save_raster(lats, lons, t_start, t_end):
x, y = proj_helper.latlon2xy1(lats, lons, RAP_Spatial_Reference.proj4)
data = [1 for i in range(0, len(x))]
array_raster = ArrayRaster(ds_name="",
data_array=None,
size=size,
ul=ul,
scale=scale,
skew=(0, 0),
srid=RAP_Spatial_Reference.epsg,
gdal_datatype=gdalconst.GDT_Int16,
nodata_value=999)
array_raster.set_data_with_xy(x=x, y=y, data=data, stat="count")
def process_file(ahps_file):
logger.info("Processing %s" % ahps_file)
#ahps_file = get_ingest_file_path(r'AHPS_Precip_1day/nws_precip_conus_20140722.nc')
vars = nc_get_1d_vars_as_list(
ahps_file, ["timeofdata", "lat", "lon", "true_lat", "true_lon"])
time_chars = vars["timeofdata"]
lat = vars["lat"]
lon = vars["lon"]
true_lat = vars["true_lat"]
true_lon = -1 * vars["true_lon"]
#bottom-left, bottom-right, top-right and top-left
bottom_left = lat[0], -1 * lon[0]
bottom_right = lat[1], -1 * lon[1]
top_right = lat[2], -1 * lon[2]
top_left = lat[3], -1 * lon[3]
bottom_left_xy = proj_helper.latlon2xy(bottom_left[0], bottom_left[1],
SRID_HRAP)
bottom_right_xy = proj_helper.latlon2xy(bottom_right[0], bottom_right[1],
SRID_HRAP)
top_left_xy = proj_helper.latlon2xy(top_left[0], top_left[1], SRID_HRAP)
top_right_xy = proj_helper.latlon2xy(top_right[0], top_right[1], SRID_HRAP)
time_str = "".join([ch for ch in time_chars])
dtime = datetime.strptime(time_str, "%Y%m%d%HZ")
logger.info("write to postgis - %s" % ahps_file)
block_size = (50, 50)
level = 0
ras = GDALRaster(ahps_file, SRID_HRAP)
ras.set_band_num(1)
ras.nodata_value = -1
ras.nodata_range = (-1, 1)
scale_x1 = (top_right_xy[0] - top_left_xy[0]) / ras.size[0]
scale_x2 = (bottom_right_xy[0] - bottom_left_xy[0]) / ras.size[0]
scale_y1 = (bottom_right_xy[1] - top_right_xy[1]) / ras.size[1]
scale_y2 = (bottom_left_xy[1] - top_left_xy[1]) / ras.size[1]
scale_x = scale_x1
scale_y = scale_y1
skew_y = 0
skew_x = 0
ul_x = top_left_xy[0]
ul_y = top_left_xy[1]
#explicitly set project params since netcdf file does not have it
ras.scale = (scale_x, scale_y)
ras.ul = (ul_x, ul_y)
ras.skew = (skew_x, skew_y)
ras.geo_bounds = [
ras.ul[0], ras.ul[0] + ras.size[0] * ras.scale[0], ras.ul[1],
ras.ul[1] + ras.size[1] * ras.scale[1]
]
granule_name = "%s_%s %s_%d" % (provider_name, variable_name,
dtime.strftime("%Y%m%d %H:%M"), level)
table_name = "%s_%s_%s_%d" % (provider_name, variable_name,
dtime.strftime("%Y%m%d%H%M"), level)
bbox = proj_helper.get_bbox(SRID_HRAP)
#bbox = None
start_time = dtime
end_time = dtime + timedelta(days=1)
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=SRID_HRAP,
level=level,
block_size=block_size,
dynamic=False,
subset_bbox=bbox,
start_time=start_time,
end_time=end_time,
overwrite=True)
time_fname_map = []
for f in files:
dt = parse_datetime_from_filename(f)
time_fname_map.append({
"dt": dt,
"file": f
})
lat_file = base_ingestor.get_ingest_file_path(df["folder"] + r'/SATCAST_Original_Latitude_6480_x_2200.dat')
lon_file = base_ingestor.get_ingest_file_path(df["folder"] + r'/SATCAST_Original_Longitude_6480_x_2200.dat')
lats = bin_reader(fname=lat_file, typechar='f', recycle=False, chunk_size=CHUNK_SIZE)
lons = bin_reader(fname=lon_file, typechar='f', recycle=False, chunk_size=CHUNK_SIZE)
_bbox = proj_helper.get_bbox(srid=4326)
all_lats = []
all_lons = []
indexes = []
num_chunk = 0
while True:
try:
lon_chunk = lons.next()
lat_chunk = lats.next()
for c in range(0, len(lon_chunk), 1):
ln = lon_chunk[c]
lt = lat_chunk[c]
if (lt > _bbox[3]) and (lt < _bbox[2]) and (ln > _bbox[0]) and (ln < _bbox[1]):
