def create_custom_grid(ds, filename, var_name, dl, custom_grid):
    cgrid = xr.open_dataset(custom_grid)
    clon = cgrid['lon']
    clat = cgrid['lat']
    cgrid_def = geometry.GridDefinition(lons=clon, lats=clat)

    ogrid_def = geometry.GridDefinition(lons=ds['lon'], lats=ds['lat'])

    # Convert data to custom grid
    obs_container = image.ImageContainerNearest(ds[var_name].data[0, :, :],
                                                ogrid_def,
                                                radius_of_influence=20000000)
    obs_modelgrid = obs_container.resample(cgrid_def)
    data_out = obs_modelgrid.image_data

    obs_container = image.ImageContainerNearest(ds['error_std'].data[0, :, :],
                                                ogrid_def,
                                                radius_of_influence=20000000)
    obs_modelgrid = obs_container.resample(cgrid_def)
    err_out = obs_modelgrid.image_data

    ds = xr.Dataset(
        {
            var_name: (('time', 'x', 'y'), np.expand_dims(data_out, axis=0)),
            "error_std": (('time', 'x', 'y'), np.expand_dims(err_out, axis=0)),
            "lon": (('x', 'y'), clon),
            "lat": (('x', 'y'), clat)
        },
        coords={
            'time': dl[0:1],
        },
    )
    ds.to_netcdf(filename)
Beispiel #2
0
def reproject2(data,
               lon0,
               lat0,
               lon1,
               lat1,
               fill=-9999,
               nearest=True,
               radius_of_influence=100):
    from pyresample import image, geometry

    source_def = geometry.SwathDefinition(lons=lon0, lats=lat0)
    target_def = geometry.SwathDefinition(lons=lon1, lats=lat1)

    if nearest:
        source_con = image.ImageContainerNearest(
            data, source_def, radius_of_influence=radius_of_influence)
        target_con = source_con.resample(target_def)
        target_con.fill_value = fill
        result = target_con.image_data
    else:
        from pyresample import kd_tree
        result = kd_tree.resample_gauss(
            source_def,
            data,
            target_def,
            radius_of_influence=radius_of_influence,
            neighbours=1,
            sigmas=0,
            fill_value=fill)

    return result
def resample_it(lat,
                lon,
                data,
                file_str,
                channel,
                output_path="/home/off/HSAF_SRC/offline/H35/input"):

    date_ = datetime.datetime.strptime(file_str.split("_")[4], "%Y%m%d%H%M%S")
    date_str = date_.strftime("%Y%m%d")
    time_str = date_.strftime("%H%M")
    from pyresample import kd_tree, image, geometry
    _p_type = 'M01'

    area_def = geometry.AreaDefinition('a', 'b', 'c', {'proj': 'longlat'},
                                       35999, 8999, [-180, 0, 180, 90])
    swath_def = geometry.SwathDefinition(lons=lon, lats=lat)
    swath_con = image.ImageContainerNearest(data,
                                            swath_def,
                                            radius_of_influence=5000)
    area_con = swath_con.resample(area_def)
    # area_con = kd_tree.resample_nearest(swath_def, data.ravel(), area_def, radius_of_influence = 5000, nprocs = 10)
    im_data = area_con.image_data
    with h5py.File(
            os.path.join(
                output_path, "eps_" + _p_type + "_" + date_str + "_" +
                time_str + "_" + time_str + '__' + str(channel) + '.hdf'),
            'w') as h5file:
        h5file.create_dataset('/' + str(channel),
                              shape=im_data.shape,
                              dtype=im_data.dtype,
                              data=im_data)
Beispiel #4
0
 def test_ref_proj(self):
     import h5py
     from pyresample import image, geometry
     rc_file = r"D:\WorkSpace\20200429\project\data\LUT\ref_065_clear_001.h5"
     rc_f = h5py.File(rc_file, 'r')
     ds = rc_f['ref_065_clear']
     ref_065_clear_gll = np.ma.masked_values(ds[...], ds.attrs['fill_value'])
     ref_065_clear_gll = ref_065_clear_gll * ds.attrs['scale_factor']
     ref_lon = rc_f['lon'][...]
     ref_lat = rc_f['lat'][...]
     obj_lat = self.fy4_nav.get_latitude()
     obj_lon = self.fy4_nav.get_longitude()
     obj_swath_def = geometry.SwathDefinition(
         lons=obj_lon,
         lats=obj_lat)
     ref_lon, ref_lat = np.meshgrid(ref_lon, ref_lat)
     ref_swath_def = geometry.SwathDefinition(
         lons=ref_lon,
         lats=ref_lat)
     ref_swath_con = image.ImageContainerNearest(
         ref_065_clear_gll,
         ref_swath_def,
         radius_of_influence=20000,
         fill_value=65535)
     area_con = ref_swath_con.resample(obj_swath_def)
     ref_065_clear_c = area_con.image_data
     plt.imshow(ref_065_clear_c)
     plt.show()
Beispiel #5
0
def goes_2_roi(loaded_goes,
               target_extent,
               target_rows,
               target_cols,
               cartopy_target_proj,
               data_key='Rad',
               radius_of_influence=50000):
    """Function that goes from loaded GOES data to data resampled in a
    projection for an extent"""

    dat = loaded_goes.metpy.parse_cf(data_key)
    geos_crs = dat.metpy.cartopy_crs
    cartopy_source_extent = geos_crs.x_limits + geos_crs.y_limits
    pyresample_source_extent = (cartopy_source_extent[0],
                                cartopy_source_extent[2],
                                cartopy_source_extent[1],
                                cartopy_source_extent[3])
    rad = dat.data
    source_area = geometry.AreaDefinition('GOES-1X', 'Full Disk', 'GOES-1X',
                                          geos_crs.proj4_params, rad.shape[1],
                                          rad.shape[0],
                                          pyresample_source_extent)
    area_target_def = geometry.AreaDefinition('areaTest', 'Target Region',
                                              'areaTest',
                                              cartopy_target_proj.proj4_params,
                                              target_rows, target_cols,
                                              target_extent)
    geos_con_nn = image.ImageContainerNearest(
        rad, source_area, radius_of_influence=radius_of_influence)

    # Here we are using pyresample for the remapping
    area_proj_con_nn = geos_con_nn.resample(area_target_def)
    return area_proj_con_nn.image_data
Beispiel #6
0
def resample(proj_def, data_def, data, fillValue, resampleRad):
    """Resample the data into the projected area using nearest neighbor.

        Input:
            proj_def: pyresample projection (grid) definition
            data_def: pyresample satellite data geometry definition 
                (swath for polar or geostationary grid)
            data: numpy data array
            fillValue: fill value for data grid
            resampleRad: radius in meters to look for nearest neighbors from
                grid pixel in swath/GOES
        Output:
            resampled_data: numpy data array resampled to the proj_def grid
    """
    log.info("Resampling the data to target projection grid")

    data_con = image.ImageContainerNearest(data,
                                           data_def,
                                           radius_of_influence=resampleRad,
                                           fill_value=fillValue,
                                           nprocs=2)

    image_con = data_con.resample(proj_def)
    resampled_data = image_con.image_data

    return resampled_data
Beispiel #7
0
def resample(layer, tl, br, samples=256):
    """
    Returns a grid, which is resampled.
    """

    data_grid = geometry.GridDefinition(lats=layer.lats, lons=layer.lons)
    
    # Form the coordinates for resampling
    rlons = np.linspace(tl[0], br[0], 256)
    rlats = np.linspace(tl[1], br[1], 256)

    resample_grid = geometry.GridDefinition(
        lats=np.tile(rlats, (rlons.size, 1)).T, 
        lons=np.tile(rlons, (rlats.size, 1))
        )

    # Build a resampler.
    resampler = image.ImageContainerNearest(
        layer.values, 
        data_grid, 
        radius_of_influence=6500, 
        reduce_data=True
        )

    # Form the appropriate grid.
    grid = np.flipud(resampler.resample(resample_grid).image_data)
    grid[grid == 0] = np.nan

    return grid
Beispiel #8
0
def reproject(dataset, inAreaDefinition, outAreaDefinition):

    msg_con_nn = image.ImageContainerNearest(dataset,
                                             inAreaDefinition,
                                             radius_of_influence=2000)
    area_con_nn = msg_con_nn.resample(outAreaDefinition)
    return (area_con_nn)
Beispiel #9
0
def interp_to_obs(var, df, lat, lon, radius=12000.):
    """Short summary.

    Parameters
    ----------
    var : type
        Description of parameter `var`.
    df : type
        Description of parameter `df`.
    lat : type
        Description of parameter `lat`.
    lon : type
        Description of parameter `lon`.
    radius : type
        Description of parameter `radius` (the default is 12000.).

    Returns
    -------
    type
        Description of returned object.

    """
    from numpy import NaN, vstack
    from pyresample import geometry, image
    from pandas import to_timedelta, DataFrame
    # define CMAQ pyresample grid (source)
    grid1 = geometry.GridDefinition(lons=lon, lats=lat)
    # get unique sites from df
    dfn = df.drop_duplicates(subset=['Latitude', 'Longitude'])
    # define site grid (target)
    lats = dfn.Latitude.values
    lons = dfn.Longitude.values
    grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
    # Create image container
    i = image.ImageContainerNearest(var.transpose('y', 'x', 'time').values,
                                    grid1,
                                    radius_of_influence=radius,
                                    fill_value=NaN)
    # resample
    ii = i.resample(grid2).image_data.squeeze()
    # recombine data
    e = DataFrame(ii, index=dfn.SCS, columns=var.time.values)
    w = e.stack().reset_index().rename(columns={
        'level_1': 'datetime',
        0: 'model'
    })
    w = w.merge(dfn.drop(['datetime', 'datetime_local', 'Obs'], axis=1),
                on='SCS',
                how='left')
    w = w.merge(df[['datetime', 'SCS', 'Obs']],
                on=['SCS', 'datetime'],
                how='left')
    # calculate datetime local

    w['datetime_local'] = w.datetime + to_timedelta(w.utcoffset, 'H')

    return w
Beispiel #10
0
def map_slice(lon_slice, lat_slice, data_slice):
    """
    given a slice, return data mapped to a laea projection and the crs dicitonary
    for the projection
    
    Parameters
    ----------
    
    lon_slice,lat_slice,data_slice: 2d arrays (float)
       3 2-d arrays of the same shape giving longitudes, latitudes and data values
       for each pixel
       
    Returns
    -------
    
    mapped_data:  2d array (float)
       2-d array with same number of rows and columns as data_slice, projected
       to new coordinate system
    
    dst_crs: dict
       dictionary with keys needed by pyprojto create the destination projection
    
    """
    llcrnrlat = lat_slice[-1, 0]
    llcrnrlon = lon_slice[-1, 0]
    urcrnrlat = lat_slice[0, -1]
    urcrnrlon = lon_slice[0, -1]
    src_crs = dict(units='m', proj='eqc', datum='WGS84')
    src_proj = pyproj.Proj(src_crs)
    llcrnrx, llcrnry = src_proj(llcrnrlon, llcrnrlat)
    urcrnrx, urcrnry = src_proj(urcrnrlon, urcrnrlat)
    src_extent = [llcrnrx, llcrnry, urcrnrx, urcrnry]
    src_height, src_width = data_slice.shape

    from_def = geometry.AreaDefinition('src', 'src image', 'area_src', src_crs,
                                       src_width, src_height, src_extent)
    lat_0 = (lat_slice[0, 0] + lat_slice[-1, 0]) / 2.
    lon_0 = (lon_slice[0, 0] + lon_slice[0, -1]) / 2.
    dst_crs = {
        'datum': 'WGS84',
        'lat_0': lat_0,
        'lon_0': lon_0,
        'proj': 'laea'
    }
    dst_proj = pyproj.Proj(dst_crs)
    llcrnrx, llcrnry = dst_proj(llcrnrlon, llcrnrlat)
    urcrnrx, urcrnry = dst_proj(urcrnrlon, urcrnrlat)
    dst_extent = [llcrnrx, llcrnry, urcrnrx, urcrnry]
    to_def = geometry.AreaDefinition('big', 'big image', 'area_big', dst_crs,
                                     src_width, src_height, dst_extent)
    from_nn = image.ImageContainerNearest(par_slice,
                                          from_def,
                                          radius_of_influence=50000)
    to_nn = from_nn.resample(to_def)
    mapped_data = to_nn.image_data
    return mapped_data, dst_crs
Beispiel #11
0
 def test_nearest_resize(self):
     data = numpy.fromfunction(lambda y, x: y * x * 10 ** -6, (3712, 3712))
     msg_con = image.ImageContainerNearest(
         data, self.msg_area, 50000, segments=1)
     area_con = msg_con.resample(self.msg_area_resize)
     res = area_con.image_data
     cross_sum = res.sum()
     expected = 2212023.0175830
     self.assertAlmostEqual(cross_sum, expected,
                            msg='ImageContainer resampling nearest neighbour failed')
Beispiel #12
0
 def test_nearest_neighbour(self):
     data = numpy.fromfunction(lambda y, x: y * x * 10**-6, (3712, 3712))
     msg_con = image.ImageContainerNearest(data,
                                           self.msg_area,
                                           50000,
                                           segments=1)
     area_con = msg_con.resample(self.area_def)
     res = area_con.image_data
     cross_sum = res.sum()
     expected = 399936.70287099993
     self.assertAlmostEqual(cross_sum, expected)
Beispiel #13
0
    def export_l3_mask(self, griddef, nc_filepath=None):
        """ Create a gridded mask product in pysiral compliant filenaming.
        The argument griddef is needs to be a pysiral.grid.GridDefinition
        instance """

        # Get the area definition for the grid
        if not isinstance(griddef, GridDefinition):
            msg = "griddef needs to be of type pysiral.grid.GridDefinition"
            self.error.add_error("value-error", msg)

        # Resample the mask
        if self.cfg.pyresample_method == "ImageContainerNearest":
            resample = image.ImageContainerNearest(self.source_mask,
                                                   self.source_area_def,
                                                   **self.cfg.pyresample_keyw)
            resample_result = resample.resample(griddef.pyresample_area_def)
            target_mask = resample_result.image_data

        elif self.cfg.pyresample_method == "resample_gauss":

            result, stddev, count = kd_tree.resample_gauss(
                self.source_area_def,
                self.source_mask,
                griddef.pyresample_area_def,
                with_uncert=True,
                **self.cfg.pyresample_keyw)
            target_mask = result

        else:
            msg = "Unrecognized opt pyresample_method: %s need to be %s" % (
                str(self.cfg.pyresample_method),
                "(ImageContainerNearest, resample_gauss)")
            self.error.add_error("invalid-pr-method", msg)
            self.error.add_error()

        # pyresample may use masked arrays -> set nan's to missing data
        try:
            target_mask[np.where(target_mask.mask)] = np.nan
        except AttributeError:
            pass

        if "post_processing" in self.cfg:
            pp_method = getattr(self, self.cfg.post_processing)
            target_mask = pp_method(target_mask, griddef)

        # Write the mask to a netCDF file
        # (the filename will be automatically generated if not specifically
        # passed to this method
        if nc_filepath is None:
            nc_filename = "%s_%s.nc" % (self.mask_name, griddef.grid_id)
            nc_filepath = os.path.join(self.mask_dir, nc_filename)
        self.log.info("Export mask file: %s" % nc_filepath)
        self._write_netcdf(nc_filepath, griddef, target_mask)
Beispiel #14
0
 def test_nearest_swath(self):
     data = numpy.fromfunction(lambda y, x: y * x, (50, 10))
     lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10))
     lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10))
     swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
     swath_con = image.ImageContainerNearest(
         data, swath_def, 50000, segments=1)
     area_con = swath_con.resample(self.area_def)
     res = area_con.image_data
     cross_sum = res.sum()
     expected = 15874591.0
     self.assertEqual(cross_sum, expected,
                      msg='ImageContainer swath resampling nearest failed')
Beispiel #15
0
    def resample_sourcegrid_to_targetgrid(self):
        """ Use pyresample for nearest neighbour resampling for gridded source
        data """

        # Resample the image
        resample_container = image.ImageContainerNearest(
            self.source_thickness,
            self.source_areadef,
            radius_of_influence=25000,
            fill_value=None)
        resample_result = resample_container.resample(TARGET_AREA_DEF)

        # pyresample uses masked arrays -> set nan's to missing data
        data_ease2 = resample_result.image_data
        data_ease2[np.where(data_ease2.mask)] = np.nan
        self.thickness = data_ease2
Beispiel #16
0
 def test_nearest_neighbour_multi(self):
     data1 = numpy.fromfunction(lambda y, x: y * x * 10 ** -6, (3712, 3712))
     data2 = numpy.fromfunction(
         lambda y, x: y * x * 10 ** -6, (3712, 3712)) * 2
     data = numpy.dstack((data1, data2))
     msg_con = image.ImageContainerNearest(
         data, self.msg_area, 50000, segments=1)
     area_con = msg_con.resample(self.area_def)
     res = area_con.image_data
     cross_sum1 = res[:, :, 0].sum()
     expected1 = 399936.783062
     self.assertAlmostEqual(cross_sum1, expected1,
                            msg='ImageContainer resampling nearest neighbour multi failed')
     cross_sum2 = res[:, :, 1].sum()
     expected2 = 399936.783062 * 2
     self.assertAlmostEqual(cross_sum2, expected2,
                            msg='ImageContainer resampling nearest neighbour multi failed')
Beispiel #17
0
def pre_processGOES(nc_folder, latbox, lonbox, path2save, date, bands):
    full_direc = os.listdir(nc_folder)  # creating a list of files in the given folder
    nc_files = [ii for ii in full_direc if ii.endswith('.nc')]  # creating a list with only NetCDF files in the given folder
    for band in range(0, len(nc_files)):  # iterating over all the bands
        g16_data_file = nc_files[band]  # selecting the NetCDF file according to the band
        g16nc = Dataset(os.path.join(nc_folder, g16_data_file) , 'r')  # opening the NetCDF file to read
        print('preprocessing band ', bands[band], g16_data_file)
        cmiValues = g16nc.variables['CMI'][:]  # selecting the CMI variable
        latValues, lonValues, newArea, oldArea = reproj(nc_folder, band)  #calling the reprojection function and assigning the variables
        fileName = 'band'+str(band)+'/'+'goes_'+'{0:02d}'.format(date.hour)+'.nc'  # creating the file name
        dataset = Dataset(os.path.join(path2save, fileName), 'w', format='NETCDF4', set_fill_off=True)  # creating the new netCDF file

        # adding dimensions
        dataset.createDimension('time', None)
        dataset.createDimension('lat', len(latValues))
        dataset.createDimension('lon', len(lonValues))

        # adding variables
        time = dataset.createVariable('time', 'f8', ('time',), zlib=True, shuffle=True, fill_value=0.)
        lon = dataset.createVariable('lon', 'f8', ('lon',), zlib=True, shuffle=True, least_significant_digit=4)
        lat = dataset.createVariable('lat', 'f8', ('lat',), zlib=True, shuffle=True, least_significant_digit=4)
        CMI = dataset.createVariable('CMI', 'f8', ('time', 'lat', 'lon'), fill_value=-1, zlib=True, shuffle=True, least_significant_digit=6)

        # adding variables atributtes
        lat.long_name = 'Latitude'
        lat.units = 'degrees_north'
        lat.standard_name = 'latitude'

        lon.long_name = 'Longitude'
        lon.units = 'degrees_east'
        lon.standard_name = 'longitude'

        time.long_name = 'Time'
        time.units = 'hours since ' + str(date.year) + '-' + str(date.month) + '-' + str(date.day) + ' 00:00:00'
        time.standard_name = 'time'
        time.axis = 'T'

        # assigning the values to the new variables
        CC = open_dataset(nc_files[0], mask_and_scale=True, decode_times=False)
        lat[:] = latValues[:]
        lon[:] = lonValues[:]
        time[:] = [date.hour]
        #  reprojecting the CMI variable to the new domain
        CMI[0, :, :] = image.ImageContainerNearest(np.round(cmiValues, decimals=6), oldArea, radius_of_influence=10000, nprocs=8).resample(newArea).image_data
        dataset.close()  # closing the new NetCDF
 def _remap_a_score_on_an_area(self, plot_area_name='npole', vmin=0.0, vmax=1.0, 
                               score='Kuipers'):
     from pyresample import image, geometry
     area_def = utils.parse_area_file(
         'reshaped_files_plotting/region_config_test.cfg',  
         plot_area_name)[0]
     data = getattr(self, score)
     data = data.copy()
     if np.ma.is_masked(data):
         data[np.logical_and(np.equal(data.mask,False),data>vmax)]=vmax
         data[np.logical_and(np.equal(data.mask,False),data<vmin)]=vmin #do not wan't low ex hitrates set to nodata!
     else:
         data[data>vmax]=vmax
         data[data<vmin]=vmin   
     #lons = np.ma.masked_array(self.lons, mask=data.mask)
     #lats = np.ma.masked_array(self.lats, mask=data.mask)
     lons = self.lons
     lats = self.lats
     swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
     swath_con = image.ImageContainerNearest(
         data, swath_def, 
         radius_of_influence=self.radius_km*1000*2.5,
         epsilon=1.0)
     area_con = swath_con.resample(area_def)
     result = area_con.image_data
     #pr.plot.show_quicklook(area_def, result,
     #                      vmin=vmin, vmax=vmax, label=score)
     if "FAR" in score:
         matplotlib.rcParams['image.cmap']= "BrBG"
     elif "diff" in score:  
         matplotlib.rcParams['image.cmap']= "BrBG"
     elif "mae" in score:
         matplotlib.rcParams['image.cmap']= "Reds"
     else:
         matplotlib.rcParams['image.cmap']= "BrBG"
     plot_label =score    
     if "mae" in score:
         plot_label =""
     pr.plot.save_quicklook(self.PLOT_DIR_SCORE + self.PLOT_FILENAME_START+
                            plot_area_name +'.png',
                            area_def, result, 
                            vmin=vmin, vmax=vmax, label=plot_label)
Beispiel #19
0
def sample_latlon(layer, lat, lon):
    """
    Returns a float which is a value grid, which is resampled.
    """

    data_grid = geometry.GridDefinition(lats=layer.lats, lons=layer.lons)
    
    resampler = image.ImageContainerNearest(
        layer.values, 
        data_grid, 
        radius_of_influence=6500, 
        reduce_data=False
        )

    resample_grid = geometry.GridDefinition(
        lats=np.ones((1, 1)) * lat, 
        lons=np.ones((1, 1)) * lon)

    # Form the appropriate grid.
    grid = resampler.resample(resample_grid).image_data
    return float(grid[0][0])
Beispiel #20
0
def grid_interpolation(src_grid,
                       tar_grid,
                       radius_of_influence=500000,
                       fill_value=None):
    src_lat = src_grid['lat']
    src_lon = src_grid['lon']
    src_data = src_grid['data']

    tar_lat = tar_grid['lat']
    tar_lon = tar_grid['lon']

    src_lon = src_lon % 360.0
    tar_lon = tar_lon % 360.0

    src_grid_def = geometry.GridDefinition(lons=src_lon, lats=src_lat)
    tar_grid_def = geometry.GridDefinition(lons=tar_lon, lats=tar_lat)
    src_img_container = image.ImageContainerNearest(
        src_data,
        src_grid_def,
        radius_of_influence=radius_of_influence,
        fill_value=fill_value)
    tar_data = src_img_container.resample(tar_grid_def)

    return tar_data.image_data
Beispiel #21
0
                         area_extent = area_extent)
    return ioa1

ioa1 = IOAD1()
ioa_lon, ioa_lat = ioa1.get_lonlats()
ioa_lon = ioa_lon[0,:]
ioa_lat = ioa_lat[:,0]


# Initialize vector coordinates for Dataset
arr = np.empty([ioa_lat.shape[0], ioa_lon.shape[0], len(files)])
tt = np.empty([len(files)], dtype = 'datetime64[s]')
for ii in range(len(files)):
    xc = xr.open_dataset(files[ii]).load()
    goesproj = goesAD(xc)
    arr[:,:,ii] = image.ImageContainerNearest(np.round(xc.CMI.data, decimals=6), goesproj, radius_of_influence=10000, nprocs=8).resample(ioa1).image_data
    tt[ii] = xc.t.data


# NEW xarray Dataset
nwnc = xr.Dataset(coords={'lon': ('lon' , ioa_lon), 'lat': ('lat', ioa_lat), 'time': tt})
nwnc['CMI'] = (['lat', 'lon', 'time'], arr)
# Remove variables
del arr, tt, xc


# Basic Attributes for the new NETCDF
nwnc.time.attrs['axis'] = "time"
nwnc.CMI.attrs['units'] = "K"
nwnc.CMI.attrs['axis'] = "lat lon time"
nwnc.CMI.attrs['sensor_band_bit_depth'] = 12
        grib3 = pygrib.open(file3)
        apcp3 = grib3.read(1)[0]
        pcp3 = apcp3.values.data
        pcp3[apcp3.values.mask] = 0.
    except OSError:
        print('File not found: '+file3)
        print('Setting nan field')
        pcp3 = np.full_like(obs_lat, np.nan)


    # sum for 3-hour total
    tot_pcp = pcp1 + pcp2 + pcp3

    # regrid 3-hour total to forecast grid
    print('Regridding 3-hr total precipitation')
    resample_quick = image.ImageContainerNearest(tot_pcp, obs_grid, radius_of_influence=12000)
    resample_obs = resample_quick.resample(fcst_grid_12km)
    new_tot_pcp = resample_obs.image_data

    # write the 3-hour total to file
    print('Writing to file: '+outname_03h)
    total_precipitation[k, :, :] = new_tot_pcp

    # write the time for the 3-hour precipitation
    time_03h[k,] = date2num(date1, time_unit)


    pcp_obj = [pcp1, pcp2, pcp3]
    time_obj = [time1, time2, time3]

    # regrib and write the hourly totals to file
Beispiel #23
0
    def test_Ref063Day(self):
        lut_file_name = 'Ref_063_Day.nc'
        lut_file_path = os.path.join(data_root_dir, 'LUT', lut_file_name)
        # 063 ref
        import h5py
        from pyresample import image, geometry
        rc_file = os.path.join(data_root_dir, 'LUT', "ref_065_clear_001.h5")
        rc_f = h5py.File(rc_file, 'r')
        ds = rc_f['ref_065_clear']
        ref_065_clear_gll = np.ma.masked_values(ds[...], ds.attrs['fill_value'])
        ref_065_clear_gll = ref_065_clear_gll * ds.attrs['scale_factor']
        ref_lon = rc_f['lon'][...]
        ref_lat = rc_f['lat'][...]
        obj_lat = self.fy4_nav.get_latitude()
        obj_lon = self.fy4_nav.get_longitude()
        obj_swath_def = geometry.SwathDefinition(
            lons=obj_lon,
            lats=obj_lat)
        ref_lon, ref_lat = np.meshgrid(ref_lon, ref_lat)
        ref_swath_def = geometry.SwathDefinition(
            lons=ref_lon,
            lats=ref_lat)
        ref_swath_con = image.ImageContainerNearest(
            ref_065_clear_gll,
            ref_swath_def,
            radius_of_influence=20000,
            fill_value=65535)
        area_con = ref_swath_con.resample(obj_swath_def)
        ref_065_clear_c = area_con.image_data * 100
        # obs mask
        ref_065 = self.fy4_l1.get_band_by_channel('ref_065') * 100
        bt_1080 = self.fy4_l1.get_band_by_channel('bt_1080')
        # day mask (covnert to bool)
        sun_zen = self.fy4_geo.get_sun_zenith()
        sat_zen = self.fy4_geo.get_satellite_zenith()
        pix_lat = self.fy4_nav.get_latitude()
        pix_lon = self.fy4_nav.get_longitude()
        sun_glint = self.fy4_geo.get_sun_glint()
        sat_lat = 0
        sat_lon = 104.7
        # scatter angle
        scat_ang = infer_scat_angle_short(pix_lat, pix_lon, sat_lat, sat_lon, sun_zen, sat_zen)
        # air mass
        air_mass = infer_airmass(sat_zen, sun_zen)
        # dem
        dem = self.fy4_nav.get_dem()
        sft = self.fy4_nav.prepare_surface_type_to_cspp()
        # snow mask
        snow_mask = self.fy4_nav.get_snow_mask()
        snow_mask = snow_mask == 3
        # space mask
        space_mask = self.fy4_nav.get_space_mask(b=True)

        ref063day = Ref063Day(lut_file_path=lut_file_path)
        x = ref063day.prepare_feature(ref_065, ref_065_clear_c)
        valid_mask = ref063day.prepare_valid_mask(ref_065, ref_065_clear_c, dem, sft, sun_glint, sun_zen, scat_ang,
                                                  air_mass, snow_mask, space_mask, bt_1080)
        ratio, prob = ref063day.infer(x, sft, valid_mask, space_mask, prob=True)
        fig, ax = plt.subplots(1, 3, figsize=(10, 10))
        ax[0].imshow(valid_mask, 'plasma')
        ax[0].set_title(ref063day.short_name + ' valid mask \n')
        pos = ax[1].imshow(ratio, 'plasma')
        ax[1].set_title(ref063day.short_name + ' Ratio \n')
        fig.colorbar(pos, ax=ax[1])
        pos = ax[2].imshow(prob, 'plasma', vmin=0, vmax=1)
        ax[2].set_title(ref063day.short_name + ' Prob \n')
        fig.colorbar(pos, ax=ax[2])
        plt.show()
Beispiel #24
0
    HH = ABI_time.strftime('%H')
    MM = ABI_time.strftime('%M')  #Minute

    #Loading in the GLM data
    GLM_file = 'GLM5-' + YYYY + mm + dd + '-' + HH + MM + '.nc'
    GLM_data = nc.Dataset(GLM_loc + GLM_file, 'r')
    GLM_x = np.load('/localdata/coordinates/ABI/x2km.npy')
    GLM_y = np.load('/localdata/coordinates/ABI/y2km.npy')
    GLM_var = GLM_data.variables[v[GLMp][1]][:, :]
    GLM_var = np.ma.filled(GLM_var, fill_value=0)

    if (ABIp == 'CMIP13') | (ABIp == 'ACTP'):
        ABI_new = ABI_var
    else:
        swath_con = image.ImageContainerNearest(ABI_var,
                                                swath_def10,
                                                radius_of_influence=10000)
        swath_resampled = swath_con.resample(swath_def2)
        ABI_new = swath_resampled.image_data

    #Creating the composites
    ABI_new = np.expand_dims(ABI_new, axis=0)
    GLM_var = np.expand_dims(GLM_var, axis=0)

    if GLM_composite_a.shape[0] < halfway:
        ABI_composite_a = np.append(ABI_composite_a,
                                    ABI_new[:, ::10, ::10],
                                    axis=0)
        GLM_composite_a = np.append(GLM_composite_a,
                                    GLM_var[:, ::10, ::10],
                                    axis=0)
def write_results(date, enkf_c_dir, ens_out_dir, Nens, save_dir, obs_list):

    smnd = str(date.month) if date.month > 9 else '0' + str(date.month)
    sday = str(date.day) if date.day > 9 else '0' + str(date.day)
    file = save_dir + 'Assim_summary_' + str(date.year) + smnd + sday + '.nc'

    # Generate the netcdf, shoudl contain aice, vice, before and after in addition,
    # mem1 aicen before and after and sst and vice
    # Can add more later on

    # Read in temp file and use this as template for the dimensions
    print(enkf_c_dir + 'ensemble_6565/mem001_temp.nc')
    tt = xr.open_dataset(enkf_c_dir + 'ensemble_6565/mem001_temp.nc')
    temp = tt['temp']

    print(file)
    ds = nc.Dataset(file, 'w', format='NETCDF4')

    time = ds.createDimension('time', None)
    times = ds.createVariable('time', 'f4', ('time', ))
    times[:] = nc.date2num(date,
                           units='days since 1990-01-01',
                           calendar='gregorian')
    times.units = 'days since 1990-01-01'
    times.calendar = 'gregorian'

    de = ds.createDimension('de', Nens)  # Ens
    di = ds.createDimension('di', 5)  # Ice categories
    dz = ds.createDimension('dz', temp.shape[1])  # Depth levels
    dx = ds.createDimension('dx', temp.shape[2])
    dy = ds.createDimension('dy', temp.shape[3])

    des = ds.createVariable('de', 'f4', ('de', ))
    dis = ds.createVariable('di', 'f4', ('di', ))
    dzs = ds.createVariable('dz', 'f4', ('dz', ))
    dxs = ds.createVariable('dx', 'f4', ('dx', ))
    dys = ds.createVariable('dy', 'f4', ('dy', ))

    #print(Nens)
    #print(np.arange(0, Nens, 1.0))
    des[:] = np.arange(0, Nens, 1.0)
    dis[:] = np.arange(0, 5, 1.0)
    dzs[:] = np.arange(0, temp.shape[1], 1.0)
    dxs[:] = np.arange(0, temp.shape[2], 1.0)
    dys[:] = np.arange(0, temp.shape[3], 1.0)

    tt.close()

    aicen_mem1_before = ds.createVariable('aicen1_inn', 'f4', (
        'time',
        'di',
        'dx',
        'dy',
    ))
    aicen_mem1_after = ds.createVariable('aicen1_out', 'f4', (
        'time',
        'di',
        'dx',
        'dy',
    ))

    vicen_mem1_before = ds.createVariable('vicen1_inn', 'f4', (
        'time',
        'di',
        'dx',
        'dy',
    ))
    vicen_mem1_after = ds.createVariable('vicen1_out', 'f4', (
        'time',
        'di',
        'dx',
        'dy',
    ))

    aice_before = ds.createVariable('aice_inn', 'f4', (
        'time',
        'de',
        'dx',
        'dy',
    ))
    aice_after = ds.createVariable('aice_out', 'f4', (
        'time',
        'de',
        'dx',
        'dy',
    ))

    vice_before = ds.createVariable('vice_inn', 'f4', (
        'time',
        'de',
        'dx',
        'dy',
    ))
    vice_after = ds.createVariable('vice_out', 'f4', (
        'time',
        'de',
        'dx',
        'dy',
    ))

    temp_mem1_before = ds.createVariable('temp1_inn', 'f4', (
        'time',
        'dz',
        'dx',
        'dy',
    ))
    temp_mem1_after = ds.createVariable('temp1_out', 'f4', (
        'time',
        'dz',
        'dx',
        'dy',
    ))

    sst_before = ds.createVariable('sst_inn', 'f4', (
        'time',
        'de',
        'dx',
        'dy',
    ))
    sst_after = ds.createVariable('sst_out', 'f4', (
        'time',
        'de',
        'dx',
        'dy',
    ))

    file_ens = open(enkf_c_dir + 'files_in_ensemble', 'r')
    Lines = file_ens.readlines()
    file_count = 0

    for ll in Lines:
        file_count += 1
        sens = str(file_count) if file_count > 9 else '0' + str(file_count)
        file_out_ice = ens_out_dir + 'iced.' + str(
            date.year) + smnd + sday + '_' + ll[0:-1] + '.nc'
        file_out_ocn = ens_out_dir + 'ocean.' + str(
            date.year) + smnd + sday + '_' + ll[0:-1] + '.nc'

        ############ Write the inn first ###################
        # Write aice_inn to res
        file_inn = enkf_c_dir + 'ensemble_6565/mem0' + sens + '_aice.nc'
        handle = xr.open_dataset(file_inn)
        aice_before[0, int(ll[0:-1]) - 1, :, :] = handle['aice'][0, :, :]
        handle.close()

        # Write vice_inn to res
        file_inn = enkf_c_dir + 'ensemble_6565/mem0' + sens + '_vice.nc'
        handle = xr.open_dataset(file_inn)
        vice_before[0, int(ll[0:-1]) - 1, :, :] = handle['vice'][0, :, :]
        handle.close()

        # Write sst_inn to res
        file_inn = enkf_c_dir + 'ensemble_6565/mem0' + sens + '_sst.nc'
        handle = xr.open_dataset(file_inn)
        sst_before[0, int(ll[0:-1]) - 1, :, :] = handle['sst'][0, :, :]
        handle.close()

        # Write the full member 1 states
        if file_count == 1:
            file_inn = enkf_c_dir + 'ensemble_6565/mem0' + sens + '_aicen.nc'
            handle = xr.open_dataset(file_inn)
            aicen_mem1_before[0, :, :, :] = handle['aicen'][0, :, :, :]
            nx_size = handle['aicen'].shape[2]
            handle.close()

            file_inn = enkf_c_dir + 'ensemble_6565/mem0' + sens + '_vicen.nc'
            handle = xr.open_dataset(file_inn)
            vicen_mem1_before[0, :, :, :] = handle['vicen'][0, :, :, :]
            handle.close()

            file_inn = enkf_c_dir + 'ensemble_6565/mem0' + sens + '_temp.nc'
            handle = xr.open_dataset(file_inn)
            temp_mem1_before[0, :, :, :] = handle['temp'][0, :, :, :]
            handle.close()
        ###################################################################

        #####################   Write the out results #####################
        handle = xr.open_dataset(file_out_ocn)
        sst_after[0, int(ll[0:-1]) - 1, :, :] = handle['temp'][0, 41, :, :]
        if file_count == 1:
            temp_mem1_after[0, :, :, :] = handle['temp'][0, :, :, :]
        handle.close()

        handle = xr.open_dataset(file_out_ice)
        nx_size2 = handle['aicen'].shape[1]
        ice_halo_cells = True if nx_size2 > nx_size else False

        if ice_halo_cells:
            aice_after[0,
                       int(ll[0:-1]) - 1, :, :] = np.sum(handle['aicen'][:,
                                                                         1:-1,
                                                                         1:-1],
                                                         axis=0)
            vice_after[0,
                       int(ll[0:-1]) - 1, :, :] = np.sum(handle['vicen'][:,
                                                                         1:-1,
                                                                         1:-1],
                                                         axis=0)
            if file_count == 1:
                aicen_mem1_after[0, :, :, :] = handle['aicen'][:, 1:-1, 1:-1]
                vicen_mem1_after[0, :, :, :] = handle['vicen'][:, 1:-1, 1:-1]
        else:
            aice_after[0, int(ll[0:-1]) - 1, :, :] = np.sum(
                handle['aicen'][:, :, :], axis=0)
            vice_after[0, int(ll[0:-1]) - 1, :, :] = np.sum(
                handle['vicen'][:, :, :], axis=0)
            if file_count == 1:
                aicen_mem1_after[0, :, :, :] = handle['aicen'][:, :, :]
                vicen_mem1_after[0, :, :, :] = handle['vicen'][:, :, :]

    ####################################################################

    ##################### Also write the observations for easy reference? #####
    # Might convert it first so it might be easier to compare? ################
    # With several observations potentially a list of string could be used here,
    #OSISAF
    file_osisaf = enkf_c_dir + 'obs/OSISAF/this_day.nc'
    file_amsr = enkf_c_dir + 'obs/AMSR/this_day.nc'
    grid_file = enkf_c_dir + 'conf/new_grid_ice.nc'

    Nens = ds.createDimension('Nens', None)
    Obs1 = ds.createVariable('Obs1', 'f4', (
        'time',
        'dx',
        'dy',
        'Nens',
    ))

    handle2 = xr.open_dataset(grid_file)
    lon_mod = handle2['lon']
    lat_mod = handle2['lat']
    mod_grid_def = geometry.GridDefinition(lons=lon_mod, lats=lat_mod)

    i = -1
    for obs in obs_list:
        fileobs = enkf_c_dir + '/obs/' + obs + '/this_day.nc'
        if os.path.exists(fileobs):
            i += 1
            if obs == 'AMSR' or obs == 'SSMIS':
                varname = 'ice_conc'
            elif obs == 'SMOS' or obs == 'CRYO':
                varname = 'sit'
            elif obs == 'MUR':
                varname = 'sst'
            handle = xr.open_dataset(fileobs)
            ice_conc = handle[varname]
            lon_obs = handle['lon']
            lat_obs = handle['lat']
            obs_grid_def = geometry.GridDefinition(lons=lon_obs, lats=lat_obs)

            # Fix future warning!
            obs_container = image.ImageContainerNearest(
                ice_conc[0, :, :].values,
                obs_grid_def,
                radius_of_influence=20000)
            obs_modelgrid = obs_container.resample(mod_grid_def)
            res = obs_modelgrid.image_data

            Obs1[0, :, :, i] = res[:]

    ds.close()
Beispiel #26
0
    encodedic['time'] = {'units':'seconds since 2000-01-01 12:00:00', 'calendar':'gregorian'}

    # Organizing data
    for file in range(nfiles):
        frame = xr.open_dataset(file_b[file])
        arrayshape = frame.data.data.shape
        data = frame.data.data.reshape(arrayshape[1], arrayshape[2])
        tt = np.array([str2time(frame)])
        slat = frame.lat.data
        slon = frame.lon.data
        data = gvar2bt(data, nband)
        outframe = makeframe(area1, tt)

        swath = geometry.SwathDefinition(lons=slon, lats=slat)
        containr = image.ImageContainerNearest(data,
                                               swath,
                                               radius_of_influence=5000,
                                               nprocs=6).resample(area1).image_data


        outframe[var0] = (['lat', 'lon', 'time'], containr.reshape(953, 1347, 1))
        outframe[var0].attrs['axis'] = "lat lon time"
        outframe[var0].attrs['resample_method'] = "Nearest_Neighbour"
        
        filer = outframe.time.data
        for tt in filer:
            odate = np.datetime_as_string(tt, unit='s').tolist().split('T')
            singlefile = outframe.sel(time=str(tt))
            singlefile.to_netcdf(path=join(spath,oname.format(odate[0], odate[-1], nband)),
                                 format='netCDF4',
                                 encoding=encodedic,
                                 unlimited_dims=['time'])
Beispiel #27
0
                               src_width, src_height,
                               src_extent)

lat_0=(lat_slice[0,0]+lat_slice[-1,0])/2.
lon_0=(lon_slice[0,0]+lon_slice[0,-1])/2.
dst_crs={'datum': 'WGS84','lat_0': lat_0,'lon_0': lon_0,'proj': 'laea'}
dst_proj=pyproj.Proj(dst_crs)
llcrnrx,llcrnry=dst_proj(llcrnrlon,llcrnrlat)
urcrnrx,urcrnry=dst_proj(urcrnrlon,urcrnrlat)
dst_extent=[llcrnrx,llcrnry,urcrnrx,urcrnry]
to_def = geometry.AreaDefinition('big', 'big image','area_big',
                               dst_crs,
                               src_width,src_height,
                               dst_extent)

from_nn = image.ImageContainerNearest(par_slice,from_def, radius_of_influence=50000)
to_nn = from_nn.resample(to_def)
result_data_nn = to_nn.image_data
dst_extent


# ### plot the reprojected image as a raw bitmap

# In[11]:

fig,ax=plt.subplots(1,1,figsize=(12,12))
ax.imshow(result_data_nn,cmap=cmap,norm=the_norm,origin='upper');


# ### Put the coastline on the map
Beispiel #28
0
def resample_data(lats, lons, data):

    #Grid definition information of existing data
    grid_def = geometry.GridDefinition(lons=lons, lats=lats)

    #Wanted projection
    area_id = 'laps_scan'
    description = 'LAPS Scandinavian domain'
    proj_id = 'stere'
    proj = 'epsg:3995'
    lon_0 = 20.0
    lat_0 = 90.0

    #x_size = 1000
    #y_size = 1000

    #Corner points to be converted to wanted projection
    lon1 = -2.448425
    lat1 = 68.79139
    lon2 = 29.38635
    lat2 = 54.67893

    #Calculate coordinate points in projection
    p = Proj(init=proj)
    x1, y1 = p(lon1, lat1)
    x2, y2 = p(lon2, lat2)

    print x1, y1, x2, y2

    print abs(x1 - x2) / abs(y1 - y2)
    print abs(y1 - y2) / abs(x1 - x2)

    x_size = 1000
    y_size = abs(x1 - x2) / abs(y1 - y2) * 1000

    area_extent = (x1, y1, x2, y2)
    proj_dict = {
        'a': '6371228.0',
        'units': 'm',
        'lon_0': lon_0,
        'proj': proj_id,
        'lat_0': lat_0
    }
    area_def = geometry.AreaDefinition(area_id, description, proj_id,
                                       proj_dict, x_size, y_size, area_extent)

    print area_def

    #Finland domain
    #lon1=16.52893
    #lat1=70.34990
    #lon2=31.85138
    #lat2=58.76623

    #Resampling data
    laps_con_quick = image.ImageContainerQuick(data, grid_def)
    area_con_quick = laps_con_quick.resample(area_def)
    result_data_quick = area_con_quick.image_data
    laps_con_nn = image.ImageContainerNearest(data,
                                              grid_def,
                                              radius_of_influence=50000)
    area_con_nn = laps_con_nn.resample(area_def)
    result_data_nn = area_con_nn.image_data

    print result_data_nn
Beispiel #29
0
    '/home/NCMRWFTEMP/vsprasad/EXP_HY2B/validation/diagStat/prodAOH/prodAOHdiagstat2020040200-scat.nc'
)
llon = x.longitude.values
llat = x.latitude.values
ygrid = x.latitude.values.shape[0]
xgrid = x.longitude.values.shape[0]
s = xr.open_dataset(
    '/home/NCMRWFTEMP/vsprasad/EXP_HY2B/validation/crr/S_NWC_CRR_MSG1_global-VISIR_20200517T070000Z.nc'
)
data = s.crr.data
lats = s.lat.values
lons = s.lon.values
###############
swath_def = geometry.SwathDefinition(lons=lons, lats=lats)
swath_con = image.ImageContainerNearest(data,
                                        swath_def,
                                        radius_of_influence=5000)
area_con = swath_con.resample(area_def)
result = area_con.image_data
variable = 'crr'
######## Create Netcdf
mydata = Dataset('sate.nc', 'w', format='NETCDF3_CLASSIC')
mydata.description = 'CF netcdf regridding '
mydata.history = 'M. Sateesh; NCMRWF; Noida.'

# dimensions
mydata.createDimension("time", 1)
mydata.createDimension('longitude', xgrid)
mydata.createDimension('latitude', ygrid)
lat = mydata.createVariable('latitude', 'f8', ('latitude', ))
lat.long_name = "latitude"
Beispiel #30
0
def wrapper(date):
    print date
    test1 = datetime.datetime.now()
    global_data, time_slot = pinkdust.load_channels(date)
    print 'Loading data to a scene object:', datetime.datetime.now() - test1

    # Reproject it to North Africa - note that this requires a custom
    # projection in areas.def config file
    test2 = datetime.datetime.now()
    #projected_data = global_data.project("NorthAfrica")
    msg_area = utils.load_area('/soge-home/projects/seviri_dust/areas.def',
                               'met09globeFull')
    target_area = utils.load_area('/soge-home/projects/seviri_dust/areas.def',
                                  'NorthAfrica')

    #print global_data.data
    data = global_data[8.7].data
    msg_con_nn = image.ImageContainerNearest(data,
                                             msg_area,
                                             radius_of_influence=50000)
    area_con_nn = msg_con_nn.resample(target_area)
    result_data_nn = area_con_nn.image_data
    lons, lats = target_area.get_lonlats()
    print np.min(lons)
    print np.max(lons)
    print np.min(lats)
    print np.max(lats)
    """

    print 'Constraining the scene to North Africa:', datetime.datetime.now() \
                                                     - test2

    test3 = datetime.datetime.now()
    netcdf4.save(projected_data,
                 '/soge-home/projects/seviri_dust/raw_seviri_data/bt_nc/'
                 + date.strftime(
                     "%B%Y") + '/North_Africa_SEVIRI_BTs_' + str(
                     date.strftime("%Y%m%d%H%M")) + '.nc',
                 compression=True,
                 dtype=np.int16,
                 band_axis=2) #area_aggregation=True
    # #time_dimension=True
    print 'Saving to netCDF:', datetime.datetime.now() - test3
    test4 = datetime.datetime.now()

    # Pull a netCDF dataset object out so the projection coordinates can
    # be obtained
    ncdata = Dataset(
        '/soge-home/projects/seviri_dust/raw_seviri_data/bt_nc/'
                 + date.strftime(
                     "%B%Y") + '/North_Africa_SEVIRI_BTs_' + str(
                     date.strftime("%Y%m%d%H%M")) + '.nc')
    print 'Loading from netCDF:', datetime.datetime.now() - test4


    # Reproject the original geos projection coordinates
    test5 = datetime.datetime.now()
    lons, lats = pinkdust.reproject_to_latlon(ncdata)
    print 'Reprojecting to lat/lon:', datetime.datetime.now() - test5



    # Regrid the irregular to regular lat lon
    test7 = datetime.datetime.now()
    data_array = np.zeros((lons.shape[0], lats.shape[1], 3))
    data_array_ordered = np.zeros((lons.shape[0], lats.shape[1], 3))

    data_labels = np.array(
        [ncdata.variables['band0'][:][0], ncdata.variables['band1'][:][0],
         ncdata.variables['band2'][:][0]])

    data_array[:, :, 0] = ncdata.variables['Image0'][:]
    data_array[:, :, 1] = ncdata.variables['Image1'][:]
    data_array[:, :, 2] = ncdata.variables['Image2'][:]

    IR_indices = np.array([0, 1, 2])

    data_array_ordered[:, :, 0] = data_array[:, :,
                                  IR_indices[data_labels == 'IR_087'][0]]
    data_array_ordered[:, :, 1] = data_array[:, :,
                                  IR_indices[data_labels == 'IR_108'][0]]
    data_array_ordered[:, :, 2] = data_array[:, :,
                                  IR_indices[data_labels == 'IR_120'][0]]

    data_regridded = pinkdust.regrid_data_to_regular(lons, lats,
                                                 data_array_ordered)
    print 'Regridding to a regular lat/lon:', datetime.datetime.now() - test7



    # Pull out cloud mask data to be regridded and added to the nc file
    # glob is used for a wildcard
    #clouddata = pygrib.open(glob.glob(
    #    '/soge-home/projects/seviri_dust/raw_seviri_data/cloudmask_grib/MSG*-'
    #    'SEVI-MSGCLMK-0100-0100-' + date.strftime(
    #        "%Y%m%d%H%M%S") + '*')[0])
    #grb = clouddata.select()[0]
    #cloudmaskarray = grb.values[:, ::-1]
    #cloudlats, cloudlons = grb.latlons()

    #cloudmaskarray[cloudmaskarray >= 3] = np.nan
    #cloudlats[cloudlats > 90] = np.nan
    #cloudlons[cloudlons > 180] = np.nan

    # Generate a regular lat/lon grid for the cloud mask
    regular_lons = np.linspace(np.min(lons), np.max(lons), lons.shape[1])
    regular_lats = np.linspace(np.min(lats), np.max(lats), lats.shape[0])

    # Regrid the cloud mask to the above regular grid (note a new
    # function was defined as it was needed for a previous version of
    # the code...)
    #cloudmask_regridded = pinkdust.regrid_data(cloudlons, cloudlats,
    #                                         regular_lons,
    #                                  regular_lats, cloudmaskarray)
    time = num2date(ncdata.variables['time'][:],
                    ncdata.variables['time'].units)

    """

    f = tables.open_file(
        '/soge-home/projects/seviri_dust/raw_seviri_data'
        '/intermediary_files/BT_087_' + date.strftime('%Y%m%d%H%M%S.hdf'), 'w')
    atom = tables.Atom.from_dtype(data_regridded[:, :, 0].dtype)
    filters = tables.Filters(complib='blosc', complevel=5)
    ds = f.create_carray(f.root,
                         'data',
                         atom,
                         data_regridded[:, :, 0].shape,
                         filters=filters)
    ds[:] = data_regridded[:, :, 0]
    f.close()

    f = tables.open_file(
        '/soge-home/projects/seviri_dust/raw_seviri_data'
        '/intermediary_files/BT_108_' + date.strftime('%Y%m%d%H%M%S.hdf'), 'w')
    atom = tables.Atom.from_dtype(data_regridded[:, :, 1].dtype)
    filters = tables.Filters(complib='blosc', complevel=5)
    ds = f.create_carray(f.root,
                         'data',
                         atom,
                         data_regridded[:, :, 0].shape,
                         filters=filters)
    ds[:] = data_regridded[:, :, 1]
    f.close()

    f = tables.open_file(
        '/soge-home/projects/seviri_dust/raw_seviri_data'
        '/intermediary_files/BT_120_' + date.strftime('%Y%m%d%H%M%S.hdf'), 'w')
    atom = tables.Atom.from_dtype(data_regridded[:, :, 2].dtype)
    filters = tables.Filters(complib='blosc', complevel=5)
    ds = f.create_carray(f.root,
                         'data',
                         atom,
                         data_regridded[:, :, 0].shape,
                         filters=filters)
    ds[:] = data_regridded[:, :, 2]
    f.close()

    #f = tables.open_file('/soge-home/projects/seviri_dust/raw_seviri_data'
    #                     '/intermediary_files/cloudmask_
    # '+date.strftime(
    #    '%Y%m%d%H%M%S.hdf'), 'w')
    #atom = tables.Atom.from_dtype(cloudmask_regridded.dtype)
    #filters = tables.Filters(complib='blosc', complevel=5)
    #ds = f.create_carray(f.root, 'data', atom,
    #                     data_regridded[:, :, 0].shape,
    #                     filters=filters)
    #ds[:] = cloudmask_regridded
    f.close()

    ncdata.close()