def ingest_gtopo_file(fmt): df = config.datafiles["GTOPO30_ELEV"] if isinstance(df["wildcard"], list): files = [] for wc in df["wildcard"]: files += base_ingestor.get_ingest_files(df["folder"], df["wildcard"]) else: files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"]) gtopo_file = files[0] ras = GDALRaster(gtopo_file, srid) ras.nodata_value = -9999 bbox = proj_helper.get_bbox(srid) if fmt == 'ras': granule_name = "GTOPO30Elev_ras" level = 0 base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=srid, level=level, block_size=block_size, dynamic=False, start_time=dtime, end_time=datetime.max, subset_bbox=bbox, overwrite=True) pgdb_helper.submit(""" drop if exists index rastertile_geom_gist_idx; create index rastertile_geom_gist_idx on rastertile using gist(st_convexhull(rast)); """) if fmt == 'vec': granule_name = "GTOPO30Elev_vec" level = 1 base_ingestor.ingest_vector(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=srid, level=level, block_size=block_size, start_time=dtime, end_time=datetime.max, subset_bbox=bbox, overwrite=True)
def ingest_gtopo_file(): provider_name = "GTOPO30" variable_name = "ELEV" df = config.datafiles["GTOPO30_ELEV"] if isinstance(df["wildcard"], list): files = [] for wc in df["wildcard"]: files += base_ingestor.get_ingest_files(df["folder"], df["wildcard"]) else: files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"]) gtopo_file = files[0] srid = 4326 band_num = 1 dtime = datetime(year=1979, month=1, day=1, hour=0, minute=0, second=0) for block_size in block_sizes: level = block_size[0] granule_name = "GTOPO30Elev_%d" % level ras = GDALRaster(gtopo_file, srid) ras.nodata_value = -9999 bbox = proj_helper.get_bbox(srid) base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=srid, level=level, block_size=block_size, dynamic=False, start_time=dtime, end_time=datetime.max, subset_bbox=bbox, overwrite=True)
def process_mrms_file(mrms_file): provider_name = "MRMS" variable_name = "REFL" ext_parts = os.path.splitext(mrms_file) ext = ext_parts[1] remove_after_process = False if ext == ".gz": nc_file_name = ext_parts[0] nc_file_copy = os.path.join("./", os.path.basename(nc_file_name)) if os.path.exists(nc_file_copy): mrms_file = nc_file_copy else: with open(nc_file_copy, 'wb') as nc_file: gz_file = gzip.open(mrms_file, 'rb') gz_bytes = gz_file.read() nc_file.write(gz_bytes) gz_file.close() mrms_file = nc_file_copy remove_after_process = True vars = nc_get_1d_vars_as_list(mrms_file, ["Ht", "time"]) heights = vars["Ht"] times = vars["time"] srid = 4326 #dtime = datetime(year=2014, month=8, day=18, hour=19, minute=0, second=0) dtime = datetime.utcfromtimestamp(times[0]) bbox = proj_helper.get_bbox(srid) start_time = dtime end_time = dtime + timedelta(minutes=2) for block_size in block_sizes: level = block_size[0] #put various tiles in various levels granule_name = "%s_%s %s_%d" % (provider_name, variable_name, dtime.strftime("%Y%m%d %H:%M"), level) table_name = "%s_%s_%s_%d" % (provider_name, variable_name, dtime.strftime("%Y%m%d%H%M"), level) bottom_up_data = True ras = GDALRaster(mrms_file, srid, bottom_up_data) l = 14 ras.set_band_num(l + 1) #explicitly override the noddata_value since netcdf file is not correct ras.nodata_value = -999 ras.nodata_range = (-999, 0) #ras.reclassifier_callback = cb base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=srid, level=level, block_size=block_size, dynamic=False, subset_bbox=bbox, start_time=start_time, end_time=end_time, overwrite=True, threshold=34) if remove_after_process: os.remove(mrms_file)
for wc in df["wildcard"]: files += base_ingestor.get_ingest_files(df["folder"], df["wildcard"]) else: files = base_ingestor.get_ingest_files(df["folder"], df["wildcard"]) for gtopo_file in files: granule_name = "GTOPO30Elev" srid = 4326 band_num = 1 block_size = (50, 50) dtime = datetime(year=1979, month=1, day=1, hour=0, minute=0, second=0) level = 0 ras = GDALRaster(gtopo_file, srid) ras.nodata_value = -9999 bbox = proj_helper.get_bbox(srid) base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=srid, level=level, block_size=block_size, dynamic=False, start_time=dtime, end_time=datetime.max, subset_bbox=bbox, overwrite=True)
def process_file(ahps_file): logger.info("Processing %s" % ahps_file) #ahps_file = get_ingest_file_path(r'AHPS_Precip_1day/nws_precip_conus_20140722.nc') vars = nc_get_1d_vars_as_list( ahps_file, ["timeofdata", "lat", "lon", "true_lat", "true_lon"]) time_chars = vars["timeofdata"] lat = vars["lat"] lon = vars["lon"] true_lat = vars["true_lat"] true_lon = -1 * vars["true_lon"] #bottom-left, bottom-right, top-right and top-left bottom_left = lat[0], -1 * lon[0] bottom_right = lat[1], -1 * lon[1] top_right = lat[2], -1 * lon[2] top_left = lat[3], -1 * lon[3] bottom_left_xy = proj_helper.latlon2xy(bottom_left[0], bottom_left[1], SRID_HRAP) bottom_right_xy = proj_helper.latlon2xy(bottom_right[0], bottom_right[1], SRID_HRAP) top_left_xy = proj_helper.latlon2xy(top_left[0], top_left[1], SRID_HRAP) top_right_xy = proj_helper.latlon2xy(top_right[0], top_right[1], SRID_HRAP) time_str = "".join([ch for ch in time_chars]) dtime = datetime.strptime(time_str, "%Y%m%d%HZ") logger.info("write to postgis - %s" % ahps_file) block_size = (50, 50) level = 0 ras = GDALRaster(ahps_file, SRID_HRAP) ras.set_band_num(1) ras.nodata_value = -1 ras.nodata_range = (-1, 1) scale_x1 = (top_right_xy[0] - top_left_xy[0]) / ras.size[0] scale_x2 = (bottom_right_xy[0] - bottom_left_xy[0]) / ras.size[0] scale_y1 = (bottom_right_xy[1] - top_right_xy[1]) / ras.size[1] scale_y2 = (bottom_left_xy[1] - top_left_xy[1]) / ras.size[1] scale_x = scale_x1 scale_y = scale_y1 skew_y = 0 skew_x = 0 ul_x = top_left_xy[0] ul_y = top_left_xy[1] #explicitly set project params since netcdf file does not have it ras.scale = (scale_x, scale_y) ras.ul = (ul_x, ul_y) ras.skew = (skew_x, skew_y) ras.geo_bounds = [ ras.ul[0], ras.ul[0] + ras.size[0] * ras.scale[0], ras.ul[1], ras.ul[1] + ras.size[1] * ras.scale[1] ] granule_name = "%s_%s %s_%d" % (provider_name, variable_name, dtime.strftime("%Y%m%d %H:%M"), level) table_name = "%s_%s_%s_%d" % (provider_name, variable_name, dtime.strftime("%Y%m%d%H%M"), level) bbox = proj_helper.get_bbox(SRID_HRAP) #bbox = None start_time = dtime end_time = dtime + timedelta(days=1) base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=SRID_HRAP, level=level, block_size=block_size, dynamic=False, subset_bbox=bbox, start_time=start_time, end_time=end_time, overwrite=True)
start_date = datetime(year=2014, month=6, day=1) end_date = datetime(year=2014, month=9, day=1) srid = SRID_MODIS level = 0 block_size = (500, 500) variable_name = "LCT" bbox = proj_helper.get_bbox(srid) granule_name = "%s_%s %s_%d" % (provider_name, variable_name, start_date.strftime("%Y%m%d"), level) table_name = "%s_%s_%s_%d" % (provider_name, variable_name, start_date.strftime("%Y%m%d"), level) ras = GDALRaster(tif_file, srid) ras.nodata_value = 0 base_ingestor.ingest(ras=ras, provider_name=provider_name, variable_name=variable_name, granule_name=granule_name, table_name=granule_name, srid=srid, level=level, block_size=block_size, dynamic=False, subset_bbox=bbox, start_time=start_date, end_time=end_date, overwrite=True) variable_name = "WATERBODY"