def ingest_gtopo_file(fmt):
df = config.datafiles["GTOPO30_ELEV"]
if isinstance(df["wildcard"], list):
files = []
for wc in df["wildcard"]:
files += base_ingestor.get_ingest_files(df["folder"],
df["wildcard"])
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
gtopo_file = files[0]
ras = GDALRaster(gtopo_file, srid)
ras.nodata_value = -9999
bbox = proj_helper.get_bbox(srid)
if fmt == 'ras':
granule_name = "GTOPO30Elev_ras"
level = 0
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
pgdb_helper.submit("""
drop if exists index rastertile_geom_gist_idx;
create index rastertile_geom_gist_idx on rastertile using gist(st_convexhull(rast));
""")
if fmt == 'vec':
granule_name = "GTOPO30Elev_vec"
level = 1
base_ingestor.ingest_vector(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
def ingest_mrms_files():
df = config.datafiles["MRMS_MREFL"]
if isinstance(df["wildcard"], list):
files = []
for wc in df["wildcard"]:
files += base_ingestor.get_ingest_files(df["folder"], wc)
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
for f in files:
process_mrms_file(f)
break
def ingest_gtopo_file():
provider_name = "GTOPO30"
variable_name = "ELEV"
df = config.datafiles["GTOPO30_ELEV"]
if isinstance(df["wildcard"], list):
files = []
for wc in df["wildcard"]:
files += base_ingestor.get_ingest_files(df["folder"],
df["wildcard"])
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
gtopo_file = files[0]
srid = 4326
band_num = 1
dtime = datetime(year=1979, month=1, day=1, hour=0, minute=0, second=0)
for block_size in block_sizes:
level = block_size[0]
granule_name = "GTOPO30Elev_%d" % level
ras = GDALRaster(gtopo_file, srid)
ras.nodata_value = -9999
bbox = proj_helper.get_bbox(srid)
base_ingestor.ingest(ras=ras,
provider_name=provider_name,
variable_name=variable_name,
granule_name=granule_name,
table_name=granule_name,
srid=srid,
level=level,
block_size=block_size,
dynamic=False,
start_time=dtime,
end_time=datetime.max,
subset_bbox=bbox,
overwrite=True)
dynamic=False,
start_time=start_date,
end_time=end_date,
subset_bbox=bbox,
overwrite=True)
if __name__ == "__main__":
from multiprocessing import Pool
df = config.datafiles["LIS"]
if isinstance(df["wildcard"], list):
files = []
for wc in df["wildcard"]:
files += base_ingestor.get_ingest_files(df["folder"], wc)
else:
files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"])
fix_files = True
if fix_files:
# remove 8 trailing bytes to fix this error
# ERROR: Couldn't find 'GRIB' or 'TDLP'
# There were 8 trailing bytes in the file.
fixed_files = []
for file in files:
ofilename = "fixed_%s" % os.path.basename(file)
ofile = os.path.join(os.path.dirname(file), ofilename)
size_bytes = os.path.getsize(file)
with open(file, "rb") as ifh: