Esempio n. 1
0
File: mrms.py Progetto: ujjwaln/namo
def ingest(filename):

    with GdalFileHelper(filename) as gfh:
        #modis hdf file consists of subdatasets
        band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=0)
        nc_vars = gfh.get_nc_vars(["Ht"])
        heights = nc_vars["Ht"]

    for band_num in settings["bands"]["numbers"]:
        ras = GDALRaster(filename, settings["srid"]["in"])
        ras.set_band_num(band_num["number"])
        variable_name = band_num["name"]
        ras.nodata_value = -9999
        level = int(numpy.asscalar(heights[band_num["number"]]))
        start_time = band_info["start_time"]
        end_time = start_time + timedelta(seconds=settings["time_interval_seconds"])

        with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
            i = 0
            mask = sqa.one(Mask, filterr={"name": "alabama"})
            dg = upsert_datagranule(sqa, config, provider_name=settings["provider"]["name"],
                variable_name=variable_name, format_name=settings["format"], level=level,
                extent=ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=start_time,
                end_time=end_time, grib_file_name=filename, overwrite_existing_datagranule=True,
                name_suffix="ext_%d" % i)

            if dg:
                block_size = 50, 50
                #sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None
                insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"],
                              srid_out=settings["srid"]["out"], block_size=block_size, mask=mask)
Esempio n. 2
0
File: mrms.py Progetto: ujjwaln/namo
def ingest(filename):

    with GdalFileHelper(filename) as gfh:
        #modis hdf file consists of subdatasets
        band_info = gfh.get_band_info(meta=settings["bands"]["meta"],
                                      band_num=0)
        nc_vars = gfh.get_nc_vars(["Ht"])
        heights = nc_vars["Ht"]

    for band_num in settings["bands"]["numbers"]:
        ras = GDALRaster(filename, settings["srid"]["in"])
        ras.set_band_num(band_num["number"])
        variable_name = band_num["name"]
        ras.nodata_value = -9999
        level = int(numpy.asscalar(heights[band_num["number"]]))
        start_time = band_info["start_time"]
        end_time = start_time + timedelta(
            seconds=settings["time_interval_seconds"])

        with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
            i = 0
            mask = sqa.one(Mask, filterr={"name": "alabama"})
            dg = upsert_datagranule(sqa,
                                    config,
                                    provider_name=settings["provider"]["name"],
                                    variable_name=variable_name,
                                    format_name=settings["format"],
                                    level=level,
                                    extent=ras.extent_ewkt(),
                                    srid=settings["srid"]["in"],
                                    start_time=start_time,
                                    end_time=end_time,
                                    grib_file_name=filename,
                                    overwrite_existing_datagranule=True,
                                    name_suffix="ext_%d" % i)

            if dg:
                block_size = 50, 50
                #sqa, config, datagranule, ras, srid_in, srid_out, block_size, mask=None
                insert_raster(sqa,
                              config=config,
                              datagranule=dg,
                              ras=ras,
                              srid_in=settings["srid"]["in"],
                              srid_out=settings["srid"]["out"],
                              block_size=block_size,
                              mask=mask)
Esempio n. 3
0
def ingest(filenames):
    for filename in filenames:
        for band in settings["data"]["bands"]:
            ras = GDALRaster(filename, settings["srid_in"])
            ras.set_band_num(band["number"])
            ras.nodata_value = -9999
            variable_name = band["name"]
            level = 0
            start_time = datetime(year=2010, month=1, day=1)
            end_time = datetime(year=2020, month=1, day=1)
            with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
                i = 0
                mask = sqa.one(Mask, filterr={"name": "alabama"})
                dg = upsert_datagranule(sqa, config, provider_name=settings["provider_name"],
                    variable_name=variable_name, format_name=settings["format"], level=level,
                    extent=ras.extent_ewkt(), srid=settings["srid_in"], start_time=start_time,
                    end_time=end_time, grib_file_name=filename, overwrite_existing_datagranule=True,
                    name_suffix="ext_%d" % i)

                if dg:
                    block_size = 50, 50
                    insert_raster(sqa, config=config, datagranule=dg, ras=ras, settings=settings,
                          block_size=block_size, mask=mask)
Esempio n. 4
0
def ingest(filenames):
    for filename in filenames:
        for variable_name in settings["sds"]:
            with GdalFileHelper(filename) as gfh:

                #modis hdf file consists of subdatasets
                dsn = gfh.get_sds(settings["sds"][variable_name])
                band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=0)

                ras = GDALRaster(dsn, settings["srid"]["in"])
                with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
                    mask = sqa.one(Mask, filterr={"name": "india"})
                    #mask = sqa.one(Mask, filterr={"name": "bareilly"})

                    dg = upsert_datagranule(sqa, config, provider_name=settings["provider"]["name"],
                        variable_name=variable_name, format_name=settings["format"], level=band_info["level"],
                        extent=ras.extent_ewkt(), srid=settings["srid"]["in"], start_time=band_info["start_time"],
                        end_time=band_info["end_time"], grib_file_name=filename, overwrite_existing_datagranule=True)

                    if dg:
                        block_size = 50, 50
                        insert_raster(sqa, config=config, datagranule=dg, ras=ras, srid_in=settings["srid"]["in"],
                                      srid_out=settings["srid"]["out"], block_size=block_size, mask=mask)
Esempio n. 5
0
def ingest(filenames):
    for filename in filenames:
        for band in settings["data"]["bands"]:
            ras = GDALRaster(filename, settings["srid_in"])
            ras.set_band_num(band["number"])
            ras.nodata_value = -9999
            variable_name = band["name"]
            level = 0
            start_time = datetime(year=2010, month=1, day=1)
            end_time = datetime(year=2020, month=1, day=1)
            with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:
                i = 0
                mask = sqa.one(Mask, filterr={"name": "alabama"})
                dg = upsert_datagranule(
                    sqa,
                    config,
                    provider_name=settings["provider_name"],
                    variable_name=variable_name,
                    format_name=settings["format"],
                    level=level,
                    extent=ras.extent_ewkt(),
                    srid=settings["srid_in"],
                    start_time=start_time,
                    end_time=end_time,
                    grib_file_name=filename,
                    overwrite_existing_datagranule=True,
                    name_suffix="ext_%d" % i)

                if dg:
                    block_size = 50, 50
                    insert_raster(sqa,
                                  config=config,
                                  datagranule=dg,
                                  ras=ras,
                                  settings=settings,
                                  block_size=block_size,
                                  mask=mask)
Esempio n. 6
0
File: gfs.py Progetto: ujjwaln/namo
def ingest_gfs_files():
    filenames = get_ingest_files("GFS", "*.f000")
    for filename in filenames:
        for variable_name in settings["bands"]["variables"]:
            with GdalFileHelper(filename) as gfh:
                band_num = gfh.find_band_num(filterr=settings["bands"]["variables"][variable_name])
                band_info = gfh.get_band_info(meta=settings["bands"]["meta"], band_num=band_num)

            if band_num > 0:
                ras = GDALRaster(filename, settings["srid"]["in"])
                ras.set_band_num(band_num)

                data = ras.get_data(0, 0, ras.size, ras.size)
                gdal_dtype = numpy2gdal(data.dtype.type)

                size_x = ras.size[0] / 2
                size_y = ras.size[1] - 1

                data1 = data[0:size_y, 0:size_x]
                array_ras1 = ArrayRaster(
                    data_array=data1,
                    size=(size_x, size_y),
                    ul=(0, 90),
                    scale=ras.scale,
                    skew=(0, 0),
                    srid=4326,
                    gdal_datatype=gdal_dtype,
                    nodata_value=-999,
                )

                data2 = data[0:size_y, size_x : ras.size[0]]
                array_ras2 = ArrayRaster(
                    data_array=data2,
                    size=(size_x, size_y),
                    ul=(-180, 90),
                    scale=ras.scale,
                    skew=(0, 0),
                    srid=4326,
                    gdal_datatype=gdal_dtype,
                    nodata_value=-999,
                )

                with SqaAccess(conn_str=config.sqa_connection_string()) as sqa:

                    india_mask = sqa.one(Mask, filterr={"name": "india"})
                    i = 0
                    for arr_ras in [array_ras1, array_ras2]:
                        if not band_info["end_time"]:
                            band_info["end_time"] = band_info["start_time"] + timedelta(
                                seconds=settings["time_interval_seconds"]
                            )

                        dg = upsert_datagranule(
                            sqa,
                            config,
                            provider_name=settings["provider"]["name"],
                            variable_name=variable_name,
                            format_name=settings["format"],
                            level=band_info["level"],
                            extent=arr_ras.extent_ewkt(),
                            srid=settings["srid"]["in"],
                            start_time=band_info["start_time"],
                            end_time=band_info["end_time"],
                            grib_file_name=filename,
                            overwrite_existing_datagranule=True,
                            name_suffix="ext_%d" % i,
                        )
                        i += 1
                        if dg:
                            if settings["format"] == "RASTER":
                                block_size = 30, 30
                            else:
                                block_size = ras.size

                            insert_raster(
                                sqa,
                                config=config,
                                datagranule=dg,
                                ras=arr_ras,
                                srid_in=settings["srid"]["in"],
                                srid_out=settings["srid"]["out"],
                                block_size=block_size,
                                mask=india_mask,
                            )