Example #1
0
File: mrms.py Project: ujjwaln/namo
            mask = sqa.one(Mask, filterr={"name": "alabama"})
            dg = upsert_datagranule(sqa, config, provider_name=settings["provider"]["name"],
                variable_name=variable_name, format_name=settings["format"], level=level,
                extent=ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=start_time,
                end_time=end_time, grib_file_name=filename, overwrite_existing_datagranule=True,
                name_suffix="ext_%d" % i)

            if dg:
                block_size = 50, 50
                #sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None
                insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"],
                              srid_out=settings["srid"]["out"], block_size=block_size, mask=mask)

if __name__ == "__main__":
    if get_env() == 'production':
        mrms_files1 = get_ingest_files(r"mrms/MREFL", "20140722-*.netcdf.gz")
        mrms_files2 = get_ingest_files(r"mrms/MREFL", "20140723-*.netcdf.gz")
        mrms_files3 = get_ingest_files(r"mrms/MREFL", "20140724-*.netcdf.gz")
        mrms_files = mrms_files1 + mrms_files2 + mrms_files3
    else:
        mrms_files = get_ingest_files(r"mrms", "201407*.netcdf.gz")

    parallel = config.parallel
    if parallel:
        n_proc = config.nprocs
        pool_size = min(n_proc, len(mrms_files))
        p = Pool(pool_size)
        p.map(ingest, mrms_files)
    else:
        for file in mrms_files:
            ingest(file)
Example #2
0
File: gfs.py Project: ujjwaln/namo
def ingest_gfs_files():
    filenames = get_ingest_files("GFS", "*.f000")
    for filename in filenames:
        for variable_name in settings["bands"]["variables"]:
            with GdalFileHelper(filename) as gfh:
                band_num = gfh.find_band_num(filterr=settings["bands"]["variables"][variable_name])
                band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=band_num)

            if band_num > 0:
                ras = GDALRaster(filename, settings["srid"]["in"])
                ras.set_band_num(band_num)

                data = ras.get_data(0, 0, ras.size, ras.size)
                gdal_dtype = numpy2gdal(data.dtype.type)

                size_x = ras.size[0] / 2
                size_y = ras.size[1] - 1

                data1 = data[0:size_y, 0:size_x]
                array_ras1 = ArrayRaster(
                    data_array=data1,
                    size=(size_x, size_y),
                    ul=(0, 90),
                    scale=ras.scale,
                    skew=(0, 0),
                    srid=4326,
                    gdal_datatype=gdal_dtype,
                    nodata_value=-999,
                )

                data2 = data[0:size_y, size_x : ras.size[0]]
                array_ras2 = ArrayRaster(
                    data_array=data2,
                    size=(size_x, size_y),
                    ul=(-180, 90),
                    scale=ras.scale,
                    skew=(0, 0),
                    srid=4326,
                    gdal_datatype=gdal_dtype,
                    nodata_value=-999,
                )

                with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:

                    india_mask = sqa.one(Mask, filterr={"name": "india"})
                    i = 0
                    for arr_ras in [array_ras1, array_ras2]:
                        if not band_info["end_time"]:
                            band_info["end_time"] = band_info["start_time"] + timedelta(
                                seconds=settings["time_interval_seconds"]
                            )

                        dg = upsert_datagranule(
                            sqa,
                            config,
                            provider_name=settings["provider"]["name"],
                            variable_name=variable_name,
                            format_name=settings["format"],
                            level=band_info["level"],
                            extent=arr_ras.extent_ewkt(),
                            srid=settings["srid"]["in"],
                            start_time=band_info["start_time"],
                            end_time=band_info["end_time"],
                            grib_file_name=filename,
                            overwrite_existing_datagranule=True,
                            name_suffix="ext_%d" % i,
                        )
                        i += 1
                        if dg:
                            if settings["format"] == "RASTER":
                                block_size = 30, 30
                            else:
                                block_size = ras.size

                            insert_raster(
                                sqa,
                                config=config,
                                datagranule=dg,
                                ras=arr_ras,
                                srid_in=settings["srid"]["in"],
                                srid_out=settings["srid"]["out"],
                                block_size=block_size,
                                mask=india_mask,
                            )
Example #3
0
File: mrms.py Project: ujjwaln/namo
            if dg:
                block_size = 50, 50
                #sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None
                insert_raster(sqa,
                              config=config,
                              datagranule=dg,
                              ras=ras,
                              srid_in=settings["srid"]["in"],
                              srid_out=settings["srid"]["out"],
                              block_size=block_size,
                              mask=mask)


if __name__ == "__main__":
    if get_env() == 'production':
        mrms_files1 = get_ingest_files(r"mrms/MREFL", "20140722-*.netcdf.gz")
        mrms_files2 = get_ingest_files(r"mrms/MREFL", "20140723-*.netcdf.gz")
        mrms_files3 = get_ingest_files(r"mrms/MREFL", "20140724-*.netcdf.gz")
        mrms_files = mrms_files1 + mrms_files2 + mrms_files3
    else:
        mrms_files = get_ingest_files(r"mrms", "201407*.netcdf.gz")

    parallel = config.parallel
    if parallel:
        n_proc = config.nprocs
        pool_size = min(n_proc, len(mrms_files))
        p = Pool(pool_size)
        p.map(ingest, mrms_files)
    else:
        for file in mrms_files:
            ingest(file)