mask = sqa.one(Mask, filterr={"name": "alabama"}) dg = upsert_datagranule(sqa, config, provider_name=settings["provider"]["name"], variable_name=variable_name, format_name=settings["format"], level=level, extent=ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=start_time, end_time=end_time, grib_file_name=filename, overwrite_existing_datagranule=True, name_suffix="ext_%d" % i) if dg: block_size = 50, 50 #sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"], srid_out=settings["srid"]["out"], block_size=block_size, mask=mask) if __name__ == "__main__": if get_env() == 'production': mrms_files1 = get_ingest_files(r"mrms/MREFL", "20140722-*.netcdf.gz") mrms_files2 = get_ingest_files(r"mrms/MREFL", "20140723-*.netcdf.gz") mrms_files3 = get_ingest_files(r"mrms/MREFL", "20140724-*.netcdf.gz") mrms_files = mrms_files1 + mrms_files2 + mrms_files3 else: mrms_files = get_ingest_files(r"mrms", "201407*.netcdf.gz") parallel = config.parallel if parallel: n_proc = config.nprocs pool_size = min(n_proc, len(mrms_files)) p = Pool(pool_size) p.map(ingest, mrms_files) else: for file in mrms_files: ingest(file)
def ingest_gfs_files(): filenames = get_ingest_files("GFS", "*.f000") for filename in filenames: for variable_name in settings["bands"]["variables"]: with GdalFileHelper(filename) as gfh: band_num = gfh.find_band_num(filterr=settings["bands"]["variables"][variable_name]) band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=band_num) if band_num > 0: ras = GDALRaster(filename, settings["srid"]["in"]) ras.set_band_num(band_num) data = ras.get_data(0, 0, ras.size, ras.size) gdal_dtype = numpy2gdal(data.dtype.type) size_x = ras.size[0] / 2 size_y = ras.size[1] - 1 data1 = data[0:size_y, 0:size_x] array_ras1 = ArrayRaster( data_array=data1, size=(size_x, size_y), ul=(0, 90), scale=ras.scale, skew=(0, 0), srid=4326, gdal_datatype=gdal_dtype, nodata_value=-999, ) data2 = data[0:size_y, size_x : ras.size[0]] array_ras2 = ArrayRaster( data_array=data2, size=(size_x, size_y), ul=(-180, 90), scale=ras.scale, skew=(0, 0), srid=4326, gdal_datatype=gdal_dtype, nodata_value=-999, ) with SqaAccess(conn_str=config.sqa_connection_string()) as sqa: india_mask = sqa.one(Mask, filterr={"name": "india"}) i = 0 for arr_ras in [array_ras1, array_ras2]: if not band_info["end_time"]: band_info["end_time"] = band_info["start_time"] + timedelta( seconds=settings["time_interval_seconds"] ) dg = upsert_datagranule( sqa, config, provider_name=settings["provider"]["name"], variable_name=variable_name, format_name=settings["format"], level=band_info["level"], extent=arr_ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=band_info["start_time"], end_time=band_info["end_time"], grib_file_name=filename, overwrite_existing_datagranule=True, name_suffix="ext_%d" % i, ) i += 1 if dg: if settings["format"] == "RASTER": block_size = 30, 30 else: block_size = ras.size insert_raster( sqa, config=config, datagranule=dg, ras=arr_ras, srid_in=settings["srid"]["in"], srid_out=settings["srid"]["out"], block_size=block_size, mask=india_mask, )
if dg: block_size = 50, 50 #sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"], srid_out=settings["srid"]["out"], block_size=block_size, mask=mask) if __name__ == "__main__": if get_env() == 'production': mrms_files1 = get_ingest_files(r"mrms/MREFL", "20140722-*.netcdf.gz") mrms_files2 = get_ingest_files(r"mrms/MREFL", "20140723-*.netcdf.gz") mrms_files3 = get_ingest_files(r"mrms/MREFL", "20140724-*.netcdf.gz") mrms_files = mrms_files1 + mrms_files2 + mrms_files3 else: mrms_files = get_ingest_files(r"mrms", "201407*.netcdf.gz") parallel = config.parallel if parallel: n_proc = config.nprocs pool_size = min(n_proc, len(mrms_files)) p = Pool(pool_size) p.map(ingest, mrms_files) else: for file in mrms_files: ingest(file)