def get_odyssey_filename(time_slot, area, outDir, outFile):
    print("    get_odyssey_filename radar for ", area)
    outputDir = format_name(outDir, time_slot, area=area, rgb="radar")
    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)
    filename = format_name('ODY_RATE-' + area + '_%y%m%d%H%M.png',
                           time_slot,
                           area=area)
    return outputDir + filename
def get_TRT_filename(time_slot, area, outDir, outFile):
    print("    get_TRT_filename TRT for ", area)
    outputDir = format_name(outDir, time_slot, area=area, rgb="TRT")
    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)
    filename = format_name('RAD_TRT-' + area + '_%y%m%d%H%M.png',
                           time_slot,
                           area=area)
    return outputDir + filename
def get_cosmo_filename(rgb, time_slot, area, outDir, outFile):
    print("    get_cosmo_filename radar for ", area)

    outputDir = format_name(outDir, time_slot, area=area, rgb=rgb)
    filename = format_name('COSMO_' + rgb + '-' + area + '_%y%m%d%H%M.png',
                           time_slot,
                           area=area)
    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)

    return outputDir + filename
def get_comp_filename(comp_str, sat_nr, time_slot, area, outDir, outFile):
    print("    get_comp_filename ", comp_str)
    outputDir = format_name(outDir,
                            time_slot,
                            area=area,
                            rgb=comp_str.replace("_", "-"),
                            sat_nr=sat_nr)
    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)
    filename = format_name('MSG_' + comp_str.replace("_", "-") + '-' + area +
                           '_%y%m%d%H%M.png',
                           time_slot,
                           area=area,
                           sat_nr=sat_nr)
    return outputDir + '/' + filename
def get_THX_filename(time_slot, area, outDir, outFile):
    print("    get_THX_filename THX for ", area)
    from ConfigParser import ConfigParser
    from mpop import CONFIG_PATH
    conf = ConfigParser()
    conf.read(join(CONFIG_PATH, "swisslightning.cfg"))
    dx = int(conf.get("thx-level2", "dx"))
    dt = int(conf.get("thx-level2", "dt"))
    dt_str = ("%04d" % dt) + "min"
    dx_str = ("%03d" % dx) + "km"
    outputDir = format_name(outDir, time_slot, area=area, rgb="THX")
    rgb = 'dens'
    filename = format_name('THX_' + rgb + '-' + area + '_%y%m%d%H%M_' +
                           dt_str + '_' + dx_str + '.png',
                           time_slot,
                           area=area,
                           rgb="THX")
    return outputDir + filename
def get_OT_filename(rgb, time_slot, area, outDir, outFile):
    print("    get_OT_filename (overshooting top) for ", area)
    outputDir = format_name(outDir,
                            time_slot,
                            area=area,
                            rgb=rgb,
                            sat=sat,
                            sat_nr=sat_nr)
    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)
    filename = format_name(outFile,
                           time_slot,
                           area=area,
                           rgb=rgb,
                           sat=sat,
                           sat_nr=sat_nr)
    return outputDir + filename
def get_sat_filename(rgb, sat, sat_nr, time_slot, area, outDir, outFile):
    print("    get_sat_filename for ", rgb, area)
    #outputDir = "/data/cinesat/out/"
    outputDir = format_name(outDir,
                            time_slot,
                            area=area,
                            rgb=rgb,
                            sat=sat,
                            sat_nr=sat_nr)
    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)
    filename = format_name(outFile,
                           time_slot,
                           area=area,
                           rgb=rgb,
                           sat=sat,
                           sat_nr=sat_nr)
    return outputDir + filename
def get_radar_filename(rgb, time_slot, area, outDir, outFile):
    print("    get_radar_filename radar for ", area)

    if rgb == 'radar':
        # backward comparbility (old convention for file names of radar products, not that good)
        outputDir = format_name(outDir, time_slot, area=area, rgb="radar")
        filename = format_name('RAD_RZC-' + area + '_%y%m%d%H%M.png',
                               time_slot,
                               area=area)
    else:
        # new better file naming
        outputDir = format_name(outDir, time_slot, area=area, rgb=rgb)
        filename = format_name('RAD_' + rgb + '-' + area + '_%y%m%d%H%M.png',
                               time_slot,
                               area=area)

    if not exists(outputDir):
        print('... create output directory: ' + outputDir)
        from os import makedirs
        makedirs(outputDir)

    return outputDir + filename
예제 #9
0
def scatter_rad_rcz(in_msg):

    # get date of the last SEVIRI observation
    if in_msg.datetime is None:
        in_msg.get_last_SEVIRI_date()

    yearS = str(in_msg.datetime.year)
    #yearS = yearS[2:]
    monthS = "%02d" % in_msg.datetime.month
    dayS = "%02d" % in_msg.datetime.day
    hourS = "%02d" % in_msg.datetime.hour
    minS = "%02d" % in_msg.datetime.minute

    dateS = yearS + '-' + monthS + '-' + dayS
    timeS = hourS + '-' + minS

    if in_msg.sat_nr is None:
        in_msg.sat_nr = choose_msg(in_msg.datetime, in_msg.RSS)

    # check if PyResample is loaded
    try:
        # Work around for on demand import of pyresample. pyresample depends
        # on scipy.spatial which memory leaks on multiple imports
        IS_PYRESAMPLE_LOADED = False
        from pyresample import geometry
        from mpop.projector import get_area_def
        IS_PYRESAMPLE_LOADED = True
    except ImportError:
        LOGGER.warning(
            "pyresample missing. Can only work in satellite projection")

    if in_msg.datetime.year > 2012:
        if in_msg.sat_nr == 8:
            area_loaded = get_area_def("EuropeCanary35")
        elif in_msg.sat_nr == 9:  # rapid scan service satellite
            area_loaded = get_area_def("EuropeCanary95")
        elif in_msg.sat_nr == 10:  # default satellite
            area_loaded = get_area_def(
                "met09globeFull"
            )  # full disk service, like EUMETSATs NWC-SAF products
        elif in_msg.sat_nr == 0:  # fake satellite for reprojected ccs4 data in netCDF
            area_loaded = get_area_def("ccs4")  #
            #area_loaded = get_area_def("EuropeCanary")
            #area_loaded = get_area_def("alps")  # new projection of SAM
        else:
            print("*** Error, unknown satellite number ", in_msg.sat_nr)
            area_loaded = get_area_def("hsaf")  #
    else:
        if in_msg.sat_nr == 8:
            area_loaded = get_area_def("EuropeCanary95")
        elif in_msg.sat_nr == 9:  # default satellite
            area_loaded = get_area_def("EuropeCanary")

    # define contour write for coasts, borders, rivers
    cw = ContourWriterAGG(in_msg.mapDir)

    if type(in_msg.sat_nr) is int:
        sat_nr_str = str(in_msg.sat_nr).zfill(2)
    elif type(in_msg.sat_nr) is str:
        sat_nr_str = in_msg.sat_nr
    else:
        print("*** Waring, unknown type of sat_nr", type(in_msg.sat_nr))
        sat_nr_str = in_msg.sat_nr

    if in_msg.verbose:
        print('*** Create plots for ')
        print('    Satellite/Sensor: ' + in_msg.sat + '  ' + sat_nr_str)
        print('    Date/Time:        ' + dateS + ' ' + hourS + ':' + minS +
              'UTC')
        print('    RGBs:            ', in_msg.RGBs)
        print('    Area:            ', in_msg.areas)

    # check if input data is complete
    if in_msg.verbose:
        print("*** check input data")
    RGBs = check_input(in_msg, in_msg.sat + sat_nr_str, in_msg.datetime)
    if len(RGBs) != len(in_msg.RGBs):
        print("*** Warning, input not complete.")
        print("*** Warning, process only: ", RGBs)

    # define time and data object
    global_data = GeostationaryFactory.create_scene(in_msg.sat, sat_nr_str,
                                                    "seviri", in_msg.datetime)
    # print "type(global_data) ", type(global_data)   # <class 'mpop.scene.SatelliteInstrumentScene'>
    # print "dir(global_data)", dir(global_data)  [..., '__init__', ... 'area', 'area_def', 'area_id', 'channel_list', 'channels',
    #      'channels_to_load', 'check_channels', 'fullname', 'get_area', 'image', 'info', 'instrument_name', 'lat', 'load', 'loaded_channels',
    #      'lon', 'number', 'orbit', 'project', 'remove_attribute', 'satname', 'save', 'set_area', 'time_slot', 'unload', 'variant']

    global_data_radar = GeostationaryFactory.create_scene(
        "swissradar", "", "radar", in_msg.datetime)
    global_data_radar.load(['precip'])

    if len(RGBs) == 0:
        return RGBs

    if in_msg.verbose:
        print(
            "*** load satellite channels for " + in_msg.sat + sat_nr_str + " ",
            global_data.fullname)

    # initialize processed RGBs
    RGBs_done = []

    # load all channels / information
    for rgb in RGBs:
        if in_msg.verbose:
            print("    load prerequisites for: ", rgb)

        if rgb in products.MSG or rgb in products.MSG_color:
            for channel in products.MSG:
                if rgb.find(
                        channel
                ) != -1:  # if a channel name (IR_108) is in the rgb name (IR_108c)
                    if in_msg.verbose:
                        print("    load prerequisites by name: ", channel)
                    if in_msg.reader_level is None:
                        global_data.load(
                            [channel], area_extent=area_loaded.area_extent
                        )  # try all reader levels  load the corresponding data
                    else:
                        global_data.load([channel],
                                         area_extent=area_loaded.area_extent,
                                         reader_level=in_msg.reader_level
                                         )  # load the corresponding data

        if rgb in products.RGBs_buildin or rgb in products.RGBs_user:
            obj_image = get_image(global_data,
                                  rgb)  # find corresponding RGB image object
            if in_msg.verbose:
                print("    load prerequisites by function: ",
                      obj_image.prerequisites)
            global_data.load(
                obj_image.prerequisites,
                area_extent=area_loaded.area_extent)  # load prerequisites

        if rgb in products.CMa or rgb in products.CT or rgb in products.CTTH or rgb in products.SPhR:
            if rgb in products.CMa:
                pge = "CloudMask"
            elif rgb in products.CT:
                pge = "CloudType"
            elif rgb in products.CTTH:
                pge = "CTTH"
            elif rgb in products.SPhR:
                pge = "SPhR"
            else:
                print("*** Error in scatter_rad_rcz (" +
                      inspect.getfile(inspect.currentframe()) + ")")
                print("    unknown NWC-SAF PGE ", rgb)
                quit()
            if in_msg.verbose:
                print("    load NWC-SAF product: " + pge)
            global_data.load(
                [pge],
                calibrate=in_msg.nwcsaf_calibrate,
                reader_level="seviri-level3"
            )  # False, area_extent=area_loaded.area_extent (difficulties to find correct h5 input file)
            #print global_data.loaded_channels()
            #loaded_channels = [chn.name for chn in global_data.loaded_channels()]
            #if pge not in loaded_channels:
            #   return []
            if area_loaded != global_data[pge].area:
                print("*** Warning: NWC-SAF input file on a differnt grid (" +
                      global_data[pge].area.name +
                      ") than suggested input area (" + area_loaded.name + ")")
                print("    use " + global_data[pge].area.name +
                      " as standard grid")
                area_loaded = global_data[pge].area
            convert_NWCSAF_to_radiance_format(global_data, area_loaded, rgb,
                                              IS_PYRESAMPLE_LOADED)

        if rgb in products.HSAF:
            if in_msg.verbose:
                print("    load hsaf product by name: ", rgb)
            global_data.load(
                [rgb]
            )  # , area_extent=area_loaded.area_extent load the corresponding data

        if in_msg.HRV_enhancement:
            # load also the HRV channel (there is a check inside in the load function, if the channel is already loaded)
            if in_msg.verbose:
                print(
                    "    load additionally the HRV channel for HR enhancement")
            global_data.load(["HRV"], area_extent=area_loaded.area_extent)

    # loaded_channels = [chn.name for chn in global_data.loaded_channels()]
    # print loaded_channels

    # check if all prerequisites are loaded
    #rgb_complete = []
    #for rgb in RGBs:
    #   all_loaded = True
    #   if rgb in products.RGBs_buildin or rgb in products.RGB_user:
    #      obj_image = get_image(global_data, rgb)
    #      for pre in obj_image.prerequisites:
    #         if pre not in loaded_channels:
    #            all_loaded = False
    #   elif rgb in products.MSG_color:
    #      if rgb.replace("c","") not in loaded_channels:
    #         all_loaded = False
    #   else:
    #      if rgb not in loaded_channels:
    #         all_loaded = False
    #   if all_loaded:
    #      rgb_complete.append(rgb)
    #print "rgb_complete", rgb_complete

    # preprojecting the data to another area
    # --------------------------------------
    for area in in_msg.areas:
        print("")
        obj_area = get_area_def(area)
        if obj_area == area_loaded:
            if in_msg.verbose:
                print("*** Use data for the area loaded: ", area)
            #obj_area = area_loaded
            data = global_data
            resolution = 'l'
        else:
            if in_msg.verbose:
                print("*** Reproject data to area: ", area,
                      "(org projection: ", area_loaded.name, ")")
            obj_area = get_area_def(area)
            # PROJECT data to new area
            data = global_data.project(area)
            resolution = 'i'

        if in_msg.mapResolution is None:
            if area.find("EuropeCanary") != -1:
                resolution = 'l'
            if area.find("ccs4") != -1:
                resolution = 'i'
            if area.find("ticino") != -1:
                resolution = 'h'
        else:
            resolution = in_msg.mapResolution

        # define area
        proj4_string = obj_area.proj4_string
        # e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
        area_extent = obj_area.area_extent
        # e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
        area_tuple = (proj4_string, area_extent)

        # save reprojected data
        if area in in_msg.save_reprojected_data:  # and area != area_loaded
            _sat_nr = int(data.number) - 7 if int(data.number) - 7 > 0 else 0
            nc_dir = (
                global_data.time_slot.strftime(in_msg.reprojected_data_dir) % {
                    "area": area,
                    "msg": "MSG" + str(_sat_nr)
                })
            nc_file = (global_data.time_slot.strftime(
                in_msg.reprojected_data_filename) % {
                    "area": area,
                    "msg": "MSG" + str(_sat_nr)
                })
            ncOutputFile = nc_dir + nc_file
            # check if output directory exists, if not create it
            path = dirname(ncOutputFile)
            if not exists(path):
                if in_msg.verbose:
                    print('... create output directory: ' + path)
                makedirs(path)
            if in_msg.verbose:
                print("... save reprojected data: ncview " + ncOutputFile +
                      " &")
            #data.save(ncOutputFile, to_format="netcdf4", compression=False)
            data.save(ncOutputFile, band_axis=0, concatenate_bands=False)

        # mask for the cloud depths tests (masked data)
        #if area == 'ccs4':
        if area == False:
            print('... apply convective mask')
            mask_depth = data.image.mask_clouddepth()
            #print type(mask_depth.max)
            #print dir(mask_depth.max)
            index = where(
                mask_depth <
                5)  # less than 5 (of 6) tests successfull -> mask out
            for rgb in RGBs:
                if rgb in products.MSG_color:
                    rgb2 = rgb.replace("c", "")
                    data[rgb2].data.mask[index] = True
                    fill_value = data[rgb2].data.fill_value
                    #data["IR_108"].data[index] = fill_value

        #print "data[IR_108].data.min/max ", data["IR_108"].data.min(), data["IR_108"].data.max()
        #if rgb == "IR_108c":
        #   print type(data["IR_108"].data)
        #   print dir(data["IR_108"].data)
        #print data["IR_108"].data.mask

        # save average values
        if in_msg.save_statistics:
            mean_array = zeros(len(RGBs))
            #statisticFile = '/data/COALITION2/database/meteosat/ccs4/'+yearS+'/'+monthS+'/'+dayS+'/MSG_'+area+'_'+yearS[2:]+monthS+dayS+'.txt'
            statisticFile = './' + yearS + '-' + monthS + '-' + dayS + '/MSG_' + area + '_' + yearS[
                2:] + monthS + dayS + '.txt'
            if in_msg.verbose:
                print("*** write statistics (average values) to " +
                      statisticFile)
            f1 = open(statisticFile, 'a')  # mode append
            i_rgb = 0
            for rgb in RGBs:
                if rgb in products.MSG_color:
                    mean_array[i_rgb] = data[rgb.replace("c", "")].data.mean()
                    i_rgb = i_rgb + 1

            # create string to write
            str2write = dateS + ' ' + hourS + ' : ' + minS + ' UTC  '
            for mm in mean_array:
                str2write = str2write + ' ' + "%7.2f" % mm
            str2write = str2write + "\n"
            f1.write(str2write)
            f1.close()

        print("y.shape ", global_data_radar['precip'].data.shape)
        from numpy import copy
        y = copy(global_data_radar['precip'].data)
        y = y.ravel()
        print("y.shape ", y.shape)

        if 1 == 0:
            if 'X' in locals():
                del X
            from numpy import column_stack, append, concatenate
            for rgb in RGBs:
                # poor mans parallax correction
                if rgb in products.MSG_color:
                    rgb2 = rgb.replace("c", "")
                else:
                    rgb2 = rgb
                x1 = data[rgb2].data.ravel()
                if 'X' not in locals():
                    X = x1
                    X = [X]
                else:
                    concatenate((X, [x1]), axis=0)
                print("X.shape ", X.shape)
            X = append(X, [[1] * len(x1)], axis=1)

            print("y.shape ", y.shape)
            #theta = np.linalg.lstsq(X,y)[0]
            return

            ind_gt_1 = y > 1
            x = x[ind_gt_1]
            y = y[ind_gt_1]
            ind_lt_200 = y < 200
            x = x[ind_lt_200]
            y = y[ind_lt_200]

            #ind_gt_0 = x>0
            #x = x[ind_gt_0]
            #y = y[ind_gt_0]

            #X = np.column_stack(x+[[1]*len(x[0])])
            #beta_hat = np.linalg.lstsq(X,y)[0]
            #print beta_hat
            #X_hat= np.dot(X,theta)
            #y_hat = X_hat.reshape((640, 710))

        # creating plots/images
        if in_msg.make_plots:

            ind_cloudy = data['CTH'].data > 0
            ind_clear = data['CTH'].data <= 0
            ind_cloudy = ind_cloudy.ravel()

            for rgb in RGBs:

                if rgb in products.MSG_color:
                    rgb2 = rgb.replace("c", "")
                else:
                    rgb2 = rgb
                if rgb == 'ir108':
                    rgb2 = 'IR_108'

                # poor mans parallax correction
                if 1 == 0:
                    print("... poor mans parallax correction")
                    data[rgb2].data[25:640, :] = data[rgb2].data[0:615, :]
                    #data[rgb2].data[15:640,:] = data[rgb2].data[0:625,:]
                    data[rgb2].data[:, 0:700] = data[rgb2].data[:, 10:710]

                # create output filename
                outputDir = format_name(in_msg.outputDir,
                                        data.time_slot,
                                        area=area,
                                        rgb=rgb,
                                        sat_nr=data.number)
                outputFile = outputDir + format_name(in_msg.outputFile,
                                                     data.time_slot,
                                                     area=area,
                                                     rgb=rgb,
                                                     sat_nr=data.number)

                PIL_image = create_PIL_image(
                    rgb, data, in_msg
                )  # !!! in_msg.colorbar[rgb] is initialized inside (give attention to rgbs) !!!

                if 1 == 1:
                    y = copy(global_data_radar['precip'].data)
                    ind_gt_300 = y > 300  # replace no rain marker with 0mm/h
                    y[ind_gt_300] = 0

                    y = y.ravel()
                    print("y.shape ", y.shape)

                    x = copy(data[rgb2].data)
                    x = x.ravel()

                    ## get rid of clear sky
                    x = x[ind_cloudy]
                    y = y[ind_cloudy]
                    #ind_gt_01 = x>0.1
                    #x = x[ind_gt_01]
                    #y = y[ind_gt_01]

                    # get rid of no rain limits for rainfall
                    ind_gt_01 = y > 0.1
                    x = x[ind_gt_01]
                    y = y[ind_gt_01]
                    ind_lt_300 = y < 300
                    x = x[ind_lt_300]
                    y = y[ind_lt_300]

                    plt.figure()
                    plt.title('Scatterplot precipitation - radiance')
                    plt.xlabel(rgb)
                    plt.ylabel('precipitation in mm/h')
                    plt.scatter(x, y)  #, s=area, c=colors, alpha=0.5
                    outputDir = format_name(in_msg.outputDir,
                                            data.time_slot,
                                            area=area,
                                            rgb=rgb,
                                            sat_nr=data.number)
                    outputFileScatter = outputDir + format_name(
                        'scatterplot_%(area)s_%Y%m%d%H%M_%(rgb)s_precip_pc.png',
                        data.time_slot,
                        area=area,
                        rgb=rgb,
                        sat_nr=data.number)
                    #plt.show()
                    from numpy import arange
                    x_line = arange(x.min(), x.max(), 1)
                    print("*** display " + outputFileScatter + " &")
                    from numpy import ones, linalg, array
                    print(x.min(), x.max(), y.min(), y.max())
                    A = array([x, ones(x.size)])
                    w = linalg.lstsq(A.T, y)[0]  # obtaining the parameters
                    y_line = w[0] * x_line + w[1]  # regression line
                    #---
                    #from scipy import stats
                    #slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
                    #print "slope, intercept, r_value, p_value, std_err"
                    #print slope, intercept, r_value, p_value, std_err
                    #y_line = slope*x_line + intercept
                    from pylab import plot
                    plot(x_line, y_line, 'r-')
                    plt.savefig(outputFileScatter)
                    y_hat = w[0] * data[rgb2].data + w[1]
                    print("y_hat.shape: ", y_hat.shape)

                    # set clear sky to 0
                    y_hat[ind_clear] = 0
                    y_hat = ma.asarray(y_hat)
                    y_hat.mask = (y_hat == 9999.9) | (y_hat <= 0.0001)

                    from trollimage.colormap import RainRate
                    colormap = rainbow
                    min_data = 0.0
                    #max_data=y_hat.max()
                    max_data = 8
                    colormap.set_range(min_data, max_data)
                    #colormap = RainRate
                    in_msg.colormap[rgb] = colormap
                    units = 'mm/h'
                    img = trollimage(y_hat, mode="L")
                    img.colorize(in_msg.colormap[rgb])
                    in_msg.colormap[rgb] = colormap.reverse()
                    PIL_image = img.pil_image()
                    outputFile = outputDir + format_name(
                        'fit_%(area)s_%Y%m%d%H%M_%(rgb)s_precip.png',
                        data.time_slot,
                        area=area,
                        rgb=rgb,
                        sat_nr=data.number)
                    #PIL_image.save(outputFile)

                ## add coasts, borders, and rivers, database is heree
                ## http://www.soest.hawaii.edu/pwessel/gshhs/index.html
                ## possible resolutions
                ## f  full resolution: Original (full) data resolution.
                ## h  high resolution: About 80 % reduction in size and quality.
                ## i  intermediate resolution: Another ~80 % reduction.
                ## l  low resolution: Another ~80 % reduction.
                ## c  crude resolution: Another ~80 % reduction.
                if in_msg.add_rivers:
                    if in_msg.verbose:
                        print("    add rivers to image (resolution=" +
                              resolution + ")")
                    cw.add_rivers(PIL_image,
                                  area_tuple,
                                  outline='blue',
                                  resolution=resolution,
                                  outline_opacity=127,
                                  width=0.5,
                                  level=5)  #
                    if in_msg.verbose:
                        print("    add lakes to image (resolution=" +
                              resolution + ")")
                    cw.add_coastlines(PIL_image,
                                      area_tuple,
                                      outline='blue',
                                      resolution=resolution,
                                      outline_opacity=127,
                                      width=0.5,
                                      level=2)  #, outline_opacity=0
                if in_msg.add_borders:
                    if in_msg.verbose:
                        print("    add coastlines to image (resolution=" +
                              resolution + ")")
                    cw.add_coastlines(PIL_image,
                                      area_tuple,
                                      outline=(255, 0, 0),
                                      resolution=resolution,
                                      width=1)  #, outline_opacity=0
                    if in_msg.verbose:
                        print("    add borders to image (resolution=" +
                              resolution + ")")
                    cw.add_borders(PIL_image,
                                   area_tuple,
                                   outline=(255, 0, 0),
                                   resolution=resolution,
                                   width=1)  #, outline_opacity=0

                #if area.find("EuropeCanary") != -1 or area.find("ccs4") != -1:
                dc = DecoratorAGG(PIL_image)

                # add title to image
                if in_msg.add_title:
                    PIL_image = add_title(PIL_image, rgb, int(data.number),
                                          dateS, hourS, minS, area, dc,
                                          in_msg.font_file, in_msg.verbose)

                # add MeteoSwiss and Pytroll logo
                if in_msg.add_logos:
                    if in_msg.verbose:
                        print('... add logos')
                    dc.align_right()
                    if in_msg.add_colorscale:
                        dc.write_vertically()
                    dc.add_logo("../logos/meteoSwiss3.jpg", height=60.0)
                    dc.add_logo("../logos/pytroll3.jpg", height=60.0)

                # add colorscale
                if in_msg.add_colorscale and in_msg.colormap[rgb] is not None:

                    dc.align_right()
                    dc.write_vertically()
                    font_scale = aggdraw.Font(
                        "black",
                        "/usr/share/fonts/truetype/ttf-dejavu/DejaVuSerif-Bold.ttf",
                        size=16)

                    # get tick marks
                    tick_marks = 20  # default
                    minor_tick_marks = 5  # default
                    if rgb in list(in_msg.tick_marks.keys()):
                        tick_marks = in_msg.tick_marks[rgb]
                    if rgb in list(in_msg.minor_tick_marks.keys()):
                        minor_tick_marks = in_msg.minor_tick_marks[rgb]
                    if rgb.find(
                            "-"
                    ) != -1:  # for channel differences use tickmarks of 1
                        tick_marks = 1
                        minor_tick_marks = 1

                    tick_marks = 2  # default
                    minor_tick_marks = 1  # default

                    if in_msg.verbose:
                        print('... add colorscale')
                    dc.add_scale(in_msg.colormap[rgb],
                                 extend=True,
                                 tick_marks=tick_marks,
                                 minor_tick_marks=minor_tick_marks,
                                 font=font_scale,
                                 line_opacity=100)  #, unit='T / K'

                ## test to plot a wind barb
                #import matplotlib.pyplot as plt
                #ax = plt.axes(PIL_image)
                #ax.barbs(0, 0, 20, 20, length=8, pivot='middle', barbcolor='red')
                #ax.barbs(8, 46, 20, 20, length=8, pivot='middle', barbcolor='red')

                # check if output directory exists, if not create it
                path = dirname(outputFile)
                if not exists(path):
                    if in_msg.verbose:
                        print('... create output directory: ' + path)
                    makedirs(path)

                # save file
                if in_msg.verbose:
                    print('... save final file :' + outputFile)
                PIL_image.save(outputFile,
                               optimize=True)  # optimize -> minimize file size

                if in_msg.compress_to_8bit:
                    if in_msg.verbose:
                        print('... compress to 8 bit image: display ' +
                              outputFile.replace(".png", "-fs8.png") + ' &')
                    subprocess.call("/usr/bin/pngquant -force 256 " +
                                    outputFile + " 2>&1 &",
                                    shell=True)  # 256 == "number of colors"

                #if in_msg.verbose:
                #   print "    add coastlines to "+outputFile

                ## alternative: reopen image and modify it (takes longer due to additional reading and saving)
                #cw.add_rivers_to_file(img, area_tuple, level=5, outline='blue', width=0.5, outline_opacity=127)
                #cw.add_coastlines_to_file(outputFile, obj_area, resolution=resolution, level=4)
                #cw.add_borders_to_file(outputFile, obj_area, outline=outline, resolution=resolution)

                # copy to another place
                if in_msg.scpOutput:
                    if in_msg.verbose:
                        print("... secure copy " + outputFile + " to " +
                              in_msg.scpOutputDir)
                    subprocess.call("scp " + in_msg.scpID + " " + outputFile +
                                    " " + in_msg.scpOutputDir + " 2>&1 &",
                                    shell=True)
                    if in_msg.compress_to_8bit:
                        if in_msg.verbose:
                            print("... secure copy " +
                                  outputFile.replace(".png", "-fs8.png") +
                                  " to " + in_msg.scpOutputDir)
                            subprocess.call(
                                "scp " + in_msg.scpID + " " +
                                outputFile.replace(".png", "-fs8.png") + " " +
                                in_msg.scpOutputDir + " 2>&1 &",
                                shell=True)

                if rgb not in RGBs_done:
                    RGBs_done.append(rgb)

        ## start postprocessing
        if area in in_msg.postprocessing_areas:
            postprocessing(in_msg, global_data.time_slot, data.number, area)

    if in_msg.verbose:
        print(" ")

    return RGBs_done
예제 #10
0
def plot_forecast_area(ttt, model, outputDir, current_labels = None, t_stop=None, BackgroundFile=None, ForeGroundRGBFile=None, labels_dir = '/opt/users/'+getpass.getuser()+'/PyTroll/scripts/labels/', in_msg = None):
    verbose = True
    if t_stop is None:
        t_stop = ttt
    
    ylabel = "area"

    while ttt <= t_stop:
        yearS, monthS, dayS, hourS, minS = string_date(ttt)
        if verbose:
            print("******** read cell properties from shelve")
        
        if current_labels is None:
              yearS, monthS, dayS, hourS, minS = string_date(ttt)
              filename = 'Labels_%s.shelve'%(yearS+monthS+dayS+hourS+minS)
              myShelve = shelve.open(filename)
              labels_all = deepcopy(myShelve['labels'])
        else:
              labels_all = deepcopy(current_labels)
        if verbose:
            print(labels_all)
        
        unique_labels = np.unique(labels_all[labels_all>0])
        if verbose:
            print(("... cells with unique labels: ", unique_labels))
                
        forecasted_labels = {}
        forecasted_areas = []    
        at_least_one_cell = False        

        if verbose:
            print("*** computing history backward (", labels_dir, ")")

        for interesting_cell in unique_labels:

              forecasted_labels["ID"+str(interesting_cell)]=[]
              
              # calculate backward history for 1 hour and save it in labels_dir
              ind, area, displacement, time, center = history_backward(ttt,  interesting_cell, True, in_msg, ttt-timedelta(hours = 1), labels_dir=labels_dir) #-timedelta(minutes = 10))
              #                                                        current time, cell_id, backward?   time_stop
              if area is None or len(area)<=1:  
                  if verbose:
                        print("new cell or cell with COM outside domain")
                  continue
              at_least_one_cell = True 
                 
              if len(area)<=3:
                    # if history is too short, use linear extrapolation
                    t, y = future_properties(time, area, ylabel, "linear")
              else:
                    t, y = future_properties(time, area, ylabel, model)
              
              if False:
                    ind1, area1, displacement1, time1, center = history_backward(ttt, interesting_cell, False, ttt+timedelta(hours=1), labels_dir=labels_dir)
                    print("******** computed history forward")
            
                    t2 = time1 #[::-1]
                    y2 = area1 #[::-1]
            
            
              nx,ny = labels_all.shape
              #if verbose:
              #    print(nx,ny)
      
              label_cell = np.zeros(labels_all.shape)
              label_cell[labels_all==interesting_cell] = 1
              #pickle.dump(label_cell, open("test_label.p", "wb" ) )
              #quit()
              dt = 0
              if False:
                  figure_labels(label_cell, outputDir, ttt, dt, area_plot="ccs4", add_name = "_ID"+str(interesting_cell), verbose=verbose)
      
              area_current = sum(sum(label_cell))
      
              forecasted_areas.append(area_current)
      
              indx = np.where(t==ttt)[0] + 1
      
              if verbose:
                    print("*** compute displacement ")

              if displacement.shape[1]==2:
                    if len(displacement) == 0:
                        dx = 0
                        dy = 0
                    else:
                        try:
                            dx = int(round(displacement[:,0].mean()))
                            dy = int(round(displacement[:,1].mean()))
                        except ValueError:
                            print("VALUE ERROR")
                            print(displacement)
                            quit()
                    print("    computed displacement dx, dy = ", dx, dy)
      
              else:
                    print("wrong displacement")
                    quit()
      
              labels_in_time={}
              
              index_stop = 12
              
              
              print(("*** calculate forecasts for cell ID"+str(interesting_cell)))
              if verbose:
                  print("index   time    area  growth")
                  print("----------------------------")

              for i in range(13):
                  
                  dt += 5
                  #if verbose:
                  #    print("... for time ", dt ,", index ", indx + i)

                  if indx+i >= len(y):
                      index_stop = deepcopy(i)
                      break
                  else:    
                      area_new  = y[indx+i]
                      area_prev = y[indx+i-1]

                  #if verbose:
                  #    print("area px that will be grown ", area_current)
                  #    print("area forecasted ", area_new)
                  #    print("area forecasted prev ", area_prev)

                  ###growth = sqrt(float(area_new)/float(area_current))
                  
                  if area_new < 0 or len(area_new)==0 or len(area_prev)==0:
                      if verbose:
                          print("the cell is predicted to disappear")
                      index_stop = deepcopy(i)
                      break
                  
                  growth = sqrt(float(area_new)/float(area_prev))
                  #if verbose:
                  #    print("growing by ", growth)
                  #    print("dx ", dx)
                  #    print("dy ", dy)

                  if verbose:
                      print((indx + i, dt, area_new, growth)) 

                  #figure_labels(label_cell, outputDir, ttt, dt, area_plot="ccs4", add_name = "before")

                  shifted_label = resize_array(label_cell, dx, dy, nx, ny)

                  #figure_labels(shifted_label, outputDir, ttt, dt, area_plot="ccs4", add_name = "before_shifted")
                  #quit()
                  if verbose:
                      print(("   after shift ", sum(sum(shifted_label))))
                  
                  if sum(sum(shifted_label))==0: #the cell is outside the domain
                      break
                  
                  #center of mass before resizing
                  center_before = ndimage.measurements.center_of_mass(shifted_label)
                  center_before = np.rint(center_before)        
                  #if verbose:
                  #    print("   after shift ", sum(sum(shifted_label)))

                  resized_label = scipy.misc.imresize(shifted_label,float(growth),'nearest')
      
                  resized_label[resized_label >0] = 1
                          
                  temp_label = np.zeros((nx,ny))

                  #after resizing, the array is larger/smaller than nx,ny --> create new array that contains all the label region                  
                  if resized_label.shape[0]<nx:
                      temp_label[0:resized_label.shape[0],0:resized_label.shape[1]] = deepcopy(resized_label)
                  else:
                      x_start = max(min(np.nonzero(resized_label)[0])-1,0)
                      y_start = max(min(np.nonzero(resized_label)[1])-1,0)      
                      temp_label[0:min(nx,resized_label.shape[0]-x_start),0:min(ny,resized_label.shape[1]-y_start)] = deepcopy(resized_label[x_start:min(x_start+nx,resized_label.shape[0]),y_start:min(y_start+ny,resized_label.shape[1])])            
                  
                  #if verbose:
                  #    print(np.unique(temp_label))
                  #    print("   after resize ", sum(sum(temp_label)))

                  #figure_labels(resized_label, outputDir, ttt, dt, area_plot="ccs4", add_name = "before_shifted_resized")
      
                  #center of mass after resizing
                  center_after = ndimage.measurements.center_of_mass(temp_label)
                  center_after = np.rint(center_after)         
      
                  dx_new,dy_new = center_before - center_after
      
                  shifted_label = resize_array(temp_label,dx_new,dy_new, nx, ny)
                  #if verbose:
                  #    print("   after shift2 ", sum(sum(shifted_label)))
                  label_cell = np.zeros((nx,ny))

                  label_cell[0:,0:] = shifted_label[0:nx,0:ny]
      
                  if label_cell.shape[0] != nx or label_cell.shape[1] != ny:
                        print("incorrect size")
                        quit()
                  
                  forecasted_labels["ID"+str(interesting_cell)].append(deepcopy(label_cell))
                  
                  
                  #indx+=1
      
                  label_cell = shifted_label #????????????????????????????????????
      
                  area_current = sum(sum(label_cell))
                  if verbose:
                      print(("end ", area_current))
                  forecasted_areas.append(area_current)
                  #add check to make sure the area you produced is more or less correct
      
      
              t_temp = deepcopy(ttt)
              forecasted_time = []
      
              for gg in range(len(forecasted_areas)):
                  forecasted_time.append(t_temp)
                  t_temp+=timedelta(minutes = 5)
      
              """
              if verbose:
                print("******** produce images")

              if False:
                  t_composite = deepcopy(ttt)
                  for i in range(min(len(y),index_stop)):
          
                      yearSf, monthSf, daySf, hourSf, minSf = string_date(t_composite)
                      contour_file = outputDir + "Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+"_Forc"+hourSf+minSf+"_ID"+str(interesting_cell)+".png"    
                      type_image = "_HRV"
                      #background_file = "/data/COALITION2/PicturesSatellite//"+yearS+"-"+monthS+"-"+dayS+"/"+yearS+"-"+monthS+"-"+dayS+type_image+"_"+"ccs4"+"/MSG"+type_image+"-"+"ccs4"+"_"+yearS[2:]+monthS+dayS+hourS+minS+".png"
                      background_file = "/data/COALITION2/PicturesSatellite/LEL_results_wind/"+yearS+"-"+monthS+"-"+dayS+"/RGB-HRV_dam/"+yearS+monthS+dayS+"_"+hourS+minS+"*.png"            
                      out_file1 = create_dir( outputDir+"/Contours/")+"Obs"+hourS+minS+"_Forc"+hourSf+minSf+"_ID"+str(interesting_cell)+".png"
                      if verbose:
                          print("... create composite "+contour_file+" "+background_file+" "+out_file1)
                      #subprocess.call("/usr/bin/composite "+contour_file+" "+background_file+" "+out_file1, shell=True)
                      if verbose:
                          print("... saved composite: display ", out_file1, " &")
                      t_composite+=timedelta(minutes=5)
              """
              """
              if False:
                  fig, ax = plt.subplots()
                  ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
                  ax.plot_date(t, y, 'o',label="Fit and extrapolation")
                  ax.plot_date(forecasted_time, forecasted_areas, '*',label="forecasted")
                  ax.plot_date(t2, y2, '*', label="Observations")
                  #ax.set_xlim([t[0]-timedelta(minutes = 5), t2[-1]+timedelta(minutes = 5)])
                  ax.set_ylabel("area")
                  ax.legend(loc="best");
                  fig.savefig(yearS+monthS+dayS+"_"+hourS+minS+"_AreaInTime"+"ID"+str(interesting_cell)+".png")
                  plt.close( fig)    
              """      

        t_composite = deepcopy(ttt)
        
        # merge coalition2 file with 
        if ForeGroundRGBFile is None:
            currentRGB_im_filename = "/opt/users/"+getpass.getuser()+"/PyTroll/scripts/Mecikalski/cosmo/Channels/indicators_in_time/RGB_dam/"+yearS+monthS+dayS+"_"+hourS+minS+"*ccs4.png"
        else:
            currentRGB_im_filename = ForeGroundRGBFile
        
        currentRGB_im = glob.glob(currentRGB_im_filename)
        if len(currentRGB_im)<1:  
            print("No file found:", currentRGB_im_filename)

        # get background file 
        if BackgroundFile is None:
            background_im_filename = '/data/COALITION2/PicturesSatellite/LEL_results_wind/'+yearS+'-'+monthS+'-'+dayS+'/RGB-HRV_dam/'+yearS+monthS+dayS+'_'+hourS+minS+'*.png'
        else:
            background_im_filename = BackgroundFile
        background_im = glob.glob(background_im_filename)

        if len(background_im)>0:
            im = plt.imread(background_im[0])
            back_exists = True
        else:
            back_exists = False
        #img1 = Image.imread(currentRGB_im[0])

        obj_area = get_area_def("ccs4")
        fig,ax = prepare_figure(obj_area)
        if in_msg.nrt == False:
              if back_exists:
                  plt.imshow(np.flipud(im))   
              else:
                  # now read the data we would like to forecast
                  global_data = GeostationaryFactory.create_scene(in_msg.sat_str(), in_msg.sat_nr_str(), "seviri", ttt)
                  #global_data_RGBforecast = GeostationaryFactory.create_scene(in_msg.sat, str(10), "seviri", time_slot)
      
                  # area we would like to read
                  area2load = "EuropeCanary95" #"ccs4" #c2"#"ccs4" #in_windshift.ObjArea
                  area_loaded = get_area_def(area2load )#(in_windshift.areaExtraction)  
  
                  # load product, global_data is changed in this step!
                  area_loaded = load_products(global_data, ['IR_108'], in_msg, area_loaded ) 
                  data = global_data.project("ccs4")                  
                  plt.imshow(np.flipud(data['IR_108'].data),cmap = pylab.gray())
        
        # background file form function argument or default
        if BackgroundFile is None:
            background_im_filename = '/data/COALITION2/PicturesSatellite/LEL_results_wind/'+yearS+'-'+monthS+'-'+dayS+'/RGB-HRV_dam/'+yearS+monthS+dayS+'_'+hourS+minS+'*.png'
        else:
            if verbose:
                print("... BackgroundFile ", BackgroundFile)
            background_im_filename = BackgroundFile
            
        # searching background file (wildcards are possible)
        background_im = glob.glob(background_im_filename)
        if len(background_im) == 0:
            print("*** Error in plot_forecast_area (test_forecast.py)")
            print("    no background file found: ", background_im_filename)
            quit()
        elif len(background_im) > 1:
            print("*** Warning in plot_forecast_area (test_forecast.py)")
            print("    several background files found: ", background_im)

        # read background file
        im = plt.imread(background_im[0])
        
        #img1 = Image.imread(currentRGB_im[0])
        obj_area = get_area_def("ccs4")
        fig,ax = prepare_figure(obj_area)
        #plt.imshow(np.flipud(im))   

        # plot contour lines for all cells

        if at_least_one_cell:      
              time_wanted_minutes = [5,20,40,60] 
              time_wanted = []
              color_wanted = []
              vmax = 70
              
              for t_want in time_wanted_minutes:
                  time_wanted.append((t_want-5)/5)
                  tmp = (mpl.cm.Blues(float(t_want)/vmax))
                  tmp1 = [tmp]
                  color_wanted.append(tmp1)
              
              all_labels_in_time = np.zeros((nx,ny))
              
              for i in range(len(time_wanted)-1,-1,-1):
                  ind_time = time_wanted [i]
                  
                  for key, forc_labels in forecasted_labels.items():  #forecasted_labels["ID"+str(interesting_cell)]=[]  
                      
                      if len(forc_labels)>ind_time:
                          #plt.contour(np.flipud(forc_labels[ind_time]),[0.5],colors = color_wanted_cont[i]) #colors='w') #
                          
                          all_labels_in_time[forc_labels[ind_time]>0] = time_wanted_minutes[i]                     
              
              forc_labels_tmp = np.ma.masked_where(all_labels_in_time==0,all_labels_in_time)
              plt.contourf(np.flipud(forc_labels_tmp), cmap="Blues", vmin=0, vmax=vmax)    
              
              
              if False:    
                    for i in range(len(time_wanted)):
                        
                        ind_time = time_wanted [i]
                        
                        for key, forc_labels in forecasted_labels.items():  #forecasted_labels["ID"+str(interesting_cell)]=[]  
                            
                            if len(forc_labels)>ind_time:
                                plt.contour(np.flipud(forc_labels[ind_time]),[0.5],colors = color_wanted[i]) #colors='w') #
        else:
            print("*** Warning, no COALITION2 cell detected ")
            print("    produce empty figure ...")
        
        
        PIL_image = fig2img ( fig )
        
        standardOutputName = in_msg.standardOutputName.replace('%y%m%d%H%M',strftime('%y%m%d%H%M',ttt.timetuple()))
        
        #PIL_image.paste(img1, (0, 0), img1)
        if in_msg is None:
            PIL_image.save(create_dir(outputDir)+"Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+".png")
            path = (outputDir)+yearS+monthS+dayS+hourS+minS+"Forecast.png"
        else:

            # dic_figure={}
            # if in_msg.nrt == True:
            #     dic_figure['rgb']= 'Forecast' #'C2rgbForecastTMP-IR-108'
            # else:
            #     dic_figure['rgb']= 'Forecast-C2rgb'
            # dic_figure['area']='ccs4'
            # PIL_image.save(create_dir(outputFile)+standardOutputName%dic_figure)
            # path = (outputFile)+standardOutputName%dic_figure
            # if in_msg.nrt == False:
            #     dic_figure={}
            #     dic_figure['rgb']= 'C2rgb-Forecast-HRV' #'C2rgbForecastTMP-IR-108'
            #     dic_figure['area']='ccs4'
            #     path_output = (outputFile)+standardOutputName%dic_figure
            #     print ("creating composite: ",currentRGB_im[0],"+",path)
        #        subprocess.call("/usr/bin/composite "+currentRGB_im[0]+" "+path+" "+path_output, shell=True)
        
        #print ("... display ",path_output," &")

            #dic_figure={}
            #dic_figure['rgb']= 'Forecast' #'C2rgbForecastTMP-IR-108'
            #dic_figure['area']='ccs4'
            outputFile = format_name(create_dir(outputDir)+in_msg.outputFile, ttt, rgb='Forecast', area='ccs4', sat_nr=int(in_msg.sat_nr))
            #PIL_image.save(create_dir(outputDir)+in_msg.outputFile%dic_figure)
            PIL_image.save(outputFile)
            #path = (outputDir)+in_msg.outputFile%dic_figure
            path = outputFile

        print("... display ",path," &")

        plt.close( fig)                             
        if True:
            if verbose:
                print("path foreground", currentRGB_im[0])
            
            if in_msg is None:
                path_composite = (outputFile)+yearS+monthS+dayS+"_Obs"+hourS+minS+"Forecast_composite.png"     
            else:
                # dic_figure={}
                # dic_figure['rgb']='C2rgb-Forecast-HRV'
                # dic_figure['area']='ccs4'
                # path_composite = (outputFile)+standardOutputName%dic_figure
                #dic_figure = {}
                #dic_figure['rgb'] = "_HRV" #'IR-108'
                #dic_figure['area']='ccs4'
                #path_IR108 = (outputFile)+standardOutputName%dic_figure

                #dic_figure={}
                #dic_figure['rgb'] = 'C2rgbForecast-IR-108'
                #dic_figure['area'] = 'ccs4'
                #path_composite = (outputDir) + in_msg.outputFile%dic_figure
                path_composite = format_name( outputDir+in_msg.outputFile, ttt, rgb='C2rgbForecast-IR-108', area='ccs4', sat_nr=int(in_msg.sat_nr))
                #dic_figure = {}
                #dic_figure['rgb'] = 'IR-108'
                #dic_figure['area']='ccs4'
                #path_IR108 = (outputDir) + in_msg.outputFile%dic_figure
                path_IR108 = format_name( outputDir+in_msg.outputFile, ttt, rgb='IR-108', area='ccs4', sat_nr=int(in_msg.sat_nr))

                
            if in_msg.nrt == True:
                if verbose:
                    print("---starting post processing")
                #if area in in_msg.postprocessing_areas:
                in_msg.postprocessing_composite = deepcopy(in_msg.postprocessing_composite2)

                postprocessing(in_msg, ttt, in_msg.sat_nr, "ccs4")
            #print ("... display",path_composite,"&")
            if in_msg.scpOutput and in_msg.nrt == True and False: #not necessary because already done within postprocessing
                print("... secure copy "+path_composite+ " to "+in_msg.scpOutputDir) #
                subprocess.call("scp "+in_msg.scpID+" "+path_composite  +" "+in_msg.scpOutputDir+" 2>&1 &", shell=True)    #BackgroundFile   #
        
        if False:
            for i in range(12):    
                  contour_files = glob.glob(outputDir + "Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+"_Forc"+hourSf+minSf+"_ID*.png")
                  if verbose:
                            print(("Files found: ",contour_files))
                  if len(contour_files)>0:
                      background_file = "/data/COALITION2/PicturesSatellite/LEL_results_wind/"+yearS+"-"+monthS+"-"+dayS+"/RGB-HRV_dam/"+yearS+monthS+dayS+"_"+hourS+minS+"*.png"
                      out_file1 = create_dir( outputDir+"/Contours/")+"Obs"+hourS+minS+"_Forc"+hourSf+minSf+".png"
                  t_composite+=timedelta(minutes=5)  
  
        ttt += timedelta(minutes = 5)
예제 #11
0
        PIL_image = HRWstreamplot(
            u2d, v2d, obj_area, HRW_data.interpol_method,
            color_mode='speed')  # , legend=True, legend_loc=3

        #outputFile = output_dir+'/MSG_stream'+detailed_char+'-'+area+'_'+yearS[2:]+monthS+dayS+hourS+minS
        rgb_str = 'stream' + detailed_char
        title = layer + ' ' + detailed_str + ' High Resolution Winds streamplot'  # [white v. weak, green weak, yellow med., red strong]

    else:
        print("*** Error in plot_hrw.py")
        print("    unknown plot_mode")
        quit()

    output_dir = format_name(in_msg.outputDir,
                             time_slot,
                             area=area,
                             rgb=rgb_str,
                             sat='MSG',
                             sat_nr=10)
    outputFile = output_dir + '/' + format_name(in_msg.outputFile,
                                                time_slot,
                                                area=area,
                                                rgb=rgb_str,
                                                sat='MSG',
                                                sat_nr=10)
    if not exists(output_dir):
        print('... create output directory: ' + output_dir)
        makedirs(output_dir)

    # create decorator
    dc = DecoratorAGG(PIL_image)
    draw = ImageDraw.Draw(PIL_image)
예제 #12
0
def n_file_composite(composite,
                     satellite,
                     sat_nr,
                     time_slot,
                     area,
                     outDir,
                     outFile,
                     scpOutput=False,
                     scpOutDir="/tmp/",
                     scpID=None,
                     scpProducts=[],
                     scpOutDir2="/tmp/",
                     scpID2=None,
                     scpProducts2=[],
                     ftpUpload=False,
                     ftpProducts=[],
                     ftpServer=None,
                     ftpUser=None,
                     ftpPassword=None,
                     bits_per_pixel=8,
                     compress_to_8bit=False,
                     composites_done=[],
                     verbose=True):

    n_rgb = composite.count('-') + 1

    # create smaller required composites
    if n_rgb - 1 >= 2:
        rgb_list = composite.split("-", n_rgb - 2)
        n_file_composite(rgb_list[-1],
                         satellite,
                         sat_nr,
                         time_slot,
                         area,
                         outDir,
                         outFile,
                         scpOutput=scpOutput,
                         scpOutDir=scpOutDir,
                         scpID=scpID,
                         scpProducts=scpProducts,
                         scpOutDir2=scpOutDir2,
                         scpID2=scpID2,
                         scpProducts2=scpProducts2,
                         bits_per_pixel=bits_per_pixel,
                         compress_to_8bit=compress_to_8bit,
                         verbose=verbose)

    print("    create ", n_rgb, " file composite ", composite)

    if satellite == "meteosat":
        sat = 'MSG'
    else:
        sat = satellite

    # get the filename of the last two files to compose
    file_list = get_file_list(composite,
                              sat,
                              sat_nr,
                              time_slot,
                              area,
                              outDir,
                              outFile,
                              n=2)
    if file_list is None:  # if not all files are found
        return composites_done  # return [] as error marker

    # get result filename
    comp_file = get_comp_filename(composite, sat_nr, time_slot, area, outDir,
                                  outFile)
    comp_dir = dirname(comp_file)
    if not exists(comp_dir):
        if verbose:
            print('... create output directory: ' + comp_dir)
        makedirs(comp_dir)

    command = "/usr/bin/composite -depth " + str(
        bits_per_pixel
    ) + " " + file_list[0] + " " + file_list[1] + " " + comp_file
    if verbose:
        print("    " + command)
    subprocess.call(command, shell=True)  #+" 2>&1 &"
    # check if file is produced
    if isfile(comp_file):
        composites_done.append(composite)

    if scpOutput:
        if (composite in scpProducts) or ('all' in [
                x.lower() for x in scpProducts if type(x) == str
        ]):
            scpOutputDir = format_name(scpOutDir,
                                       time_slot,
                                       area=area,
                                       rgb=composite,
                                       sat=sat,
                                       sat_nr=sat_nr)
            if verbose:
                print("/usr/bin/scp " + scpID + " " + comp_file + " " +
                      scpOutputDir + " 2>&1 &")
            subprocess.call("/usr/bin/scp " + scpID + " " + comp_file + " " +
                            scpOutputDir + " 2>&1 &",
                            shell=True)

    if scpOutput and (scpID2 is not None) and (scpOutDir2 is not None):
        if (composite in scpProducts2) or ('all' in [
                x.lower() for x in scpProducts2 if type(x) == str
        ]):
            scpOutputDir2 = format_name(scpOutDir2,
                                        time_slot,
                                        area=area,
                                        rgb=composite,
                                        sat=sat,
                                        sat_nr=sat_nr)
            if compress_to_8bit:
                if verbose:
                    print("... secure copy " +
                          comp_file.replace(".png", "-fs8.png") + " to " +
                          scpOutputDir2)
                subprocess.call("scp " + scpID2 + " " +
                                comp_file.replace(".png", "-fs8.png") + " " +
                                scpOutputDir2 + " 2>&1 &",
                                shell=True)
            else:
                if verbose:
                    print("... secure copy " + comp_file + " to " +
                          scpOutputDir2)
                subprocess.call("scp " + scpID2 + " " + comp_file + " " +
                                scpOutputDir2 + " 2>&1 &",
                                shell=True)

    if ftpUpload:
        print("... ftpUpload requested")
        if ftpServer == None or ftpUser == None or ftpPassword == None:
            print("*** ERROR, ftp input is not complete")
            print("ftp_server=", ftpServer)
            print("ftp_user="******"ftp_password="******"curl -T " + comp_file + " " + ftpServer + " --user " + ftpUser + ":" + ftpPassword
                print(command_line)
                subprocess.call(command_line, shell=True)

    return composites_done
예제 #13
0
                                   river_color=in_msg.river_color,
                                   resolution=in_msg.resolution,
                                   verbose=in_msg.verbose)

        #if area.find("EuropeCanary") != -1 or area.find("ccs4") != -1:
        dc = DecoratorAGG(PIL_image)

        # add title to image
        if in_msg.add_title:
            add_title(PIL_image, rgb, int(data.number), dateS, hourS, minS,
                      area, dc, in_msg.font_file, in_msg.verbose)

        # create output filename
        outputDir = format_name(in_msg.outputDir,
                                data.time_slot,
                                area=area,
                                rgb=rgb,
                                sat_nr=data.number)
        outputFile = outputDir + format_name(
            in_msg.outputFile + 'p' + str(it * delta_t).zfill(2) + '.png',
            data.time_slot,
            area=area,
            rgb=rgb,
            sat_nr=data.number)

        if not exists(outputDir):
            print('... create output directory: ' + outputDir)
            makedirs(outputDir)

        # save file
        if in_msg.verbose:
        show_image(prop,
                   chn + '_cloudy',
                   save_png,
                   colors=colors,
                   min_data=min_data,
                   max_data=max_data,
                   title=chn + '_cloudy')

    if show_chn_PC and verbose:
        show_image(data[chn + "_PC"].data,
                   chn + '_PC',
                   save_png,
                   colors=colors,
                   min_data=min_data,
                   max_data=max_data,
                   title=chn + '_PC')

    if save_data:
        from my_msg_module import format_name
        ncOutputFile = data.time_slot.strftime("./test_rad_PLAX.nc")
        ncOutputFile = format_name('%(msg)s_%(area)s_%Y%m%d%H%M_rad_PLAX.nc',
                                   time_slot,
                                   area=area,
                                   sat_nr=data.sat_nr())
        ncOutputFile = "/data/COALITION2/tmp/nc_PLAX/" + ncOutputFile.replace(
            "-", "")
        print "... save reprojected data: ncview " + ncOutputFile + " &"
        #data.save(ncOutputFile, to_format="netcdf4", compression=False)
        data.save(ncOutputFile, band_axis=2,
                  concatenate_bands=False)  # netCDF4 is default
예제 #15
0
def postprocessing(in_msg, time_slot, sat_nr, area):

    if hasattr(in_msg, 'postprocessing_composite'):
        if isinstance(in_msg.postprocessing_composite, dict):
            if area in in_msg.postprocessing_composite:
                postprocessing_composite = in_msg.postprocessing_composite[
                    area]
            else:
                postprocessing_composite = []
        else:
            postprocessing_composite = in_msg.postprocessing_composite

        if in_msg.verbose:
            print("==================================")
            print("*** start post processing for area: ", area, ', time: ',
                  str(time_slot))
            print("... desired composites: ", postprocessing_composite)

        composites_done = []

        for composite in postprocessing_composite:

            print("... creating composite: ", composite)
            composites_done = n_file_composite(
                composite,
                in_msg.sat,
                sat_nr,
                time_slot,
                area,
                in_msg.outputDir,
                in_msg.outputFile,
                scpOutput=in_msg.scpOutput,
                scpOutDir=in_msg.scpOutputDir,
                scpID=in_msg.scpID,
                scpProducts=in_msg.scpProducts,
                scpOutDir2=in_msg.scpOutputDir2,
                scpID2=in_msg.scpID2,
                scpProducts2=in_msg.scpProducts2,
                ftpUpload=in_msg.ftpUpload,
                ftpProducts=in_msg.ftpProducts,
                ftpServer=in_msg.ftpServer,
                ftpUser=in_msg.ftpUser,
                ftpPassword=in_msg.ftpPassword,
                composites_done=composites_done,
                verbose=in_msg.verbose)

        if in_msg.verbose:
            if len(composites_done) > 0:
                print("... produced composites: ")
                for comp in composites_done:
                    print("   ", comp)
            else:
                print("*** Warning, no composites produced ")

    # ----------------------------------------------
    if in_msg.verbose:
        print("*** start montage_pictures for area: ", area)

    montage_done = []
    if hasattr(in_msg, 'postprocessing_montage'):

        if isinstance(in_msg.postprocessing_montage, dict):
            if area in in_msg.postprocessing_montage:
                postprocessing_montage = in_msg.postprocessing_montage[area]
            else:
                postprocessing_montage = []
        else:
            postprocessing_montage = in_msg.postprocessing_montage

        if in_msg.verbose:
            print("")
            print("*** start post processing for area: ", area, ', time: ',
                  str(time_slot))
            print("... desired montages: ", postprocessing_montage)

        for montage in postprocessing_montage:
            if len(montage) == 0:
                continue

            print("... creating montage: ", montage)
            # sleep_str = " && sleep 1 "
            sleep_str = " "

            n_pics = len(montage)
            if n_pics == 2:
                tile = "2x1"
            if n_pics == 3:
                tile = "3x1"
            if n_pics == 4:
                tile = "2x2"
            if n_pics == 5:
                tile = "5x1"
            if n_pics == 6:
                tile = "3x2"
            if n_pics == 8:
                tile = "4x2"
            if n_pics == 9:
                tile = "3x3"
            if n_pics == 10:
                tile = "5x2"
            if n_pics == 12:
                tile = "4x3"

            files = ""
            outfile = ""
            files_exist = True
            nr_files_exist = 0
            files_complete = True

            for mfile in montage:

                print('    analyse first filename', mfile)
                rgb = mfile.split("_")[1]
                outputDir = format_name(in_msg.outputDir,
                                        time_slot,
                                        rgb=rgb,
                                        area=area)

                next_file = outputDir + "/" + format_name(
                    mfile + '-' + area + '_%y%m%d%H%M.png',
                    time_slot,
                    area=area)
                if not isfile(next_file):
                    files_complete = False
                    print("*** Warning, can not find " + mfile + " file: " +
                          next_file)
                    if area in [
                            "ccs4", "EuropeCanaryS95", "odysseyS25",
                            "euroHDready", "SeviriDiskFull00S4"
                    ]:
                        # produce image with placeholder for missing product
                        files += " " + "/opt/users/common/logos/missing_product_textbox_" + area + ".png"
                        outfile += mfile[mfile.index("_") + 1:] + "-"
                    else:
                        print("*** ERROR, can not find " + mfile + " file: " +
                              next_file)
                        print("*** skip montage: ", montage)
                        files_exist = False
                else:
                    files += " " + outputDir + "/" + format_name(
                        mfile + '-' + area + '_%y%m%d%H%M.png',
                        time_slot,
                        area=area)
                    nr_files_exist += 1
                    outfile += mfile[mfile.index("_") + 1:] + "-"

            outputDir = format_name(in_msg.outputDir,
                                    time_slot,
                                    rgb=outfile[:-1],
                                    area=area)
            if not exists(outputDir):
                if in_msg.verbose:
                    print('... create output directory: ' + outputDir)
                makedirs(outputDir)

            montage_name = deepcopy(outfile[:-1])
            outfile = outputDir + "/" + format_name(
                "MSG_" + outfile[:-1] + '-' + area + '_%y%m%d%H%M.png',
                time_slot,
                area=area)

            #filename1 = format_name(mfile+'-'+area+'_%y%m%d%H%M.png', time_slot, area=area)
            #file1 = outputDir + filename1
            #filename2 =  format_name(montage[1]+'-'+area+'_%y%m%d%H%M.png', time_slot, area=area)
            #file2 = outputDir + filename2

            #m1 =  montage[1][montage[1].index("_")+1:]  # get rid all strings before "_" (filename convection of SAT live)
            #outfilename = format_name(montage[0]+'-'+m1+"-"+area+'_%y%m%d%H%M.png', time_slot, area=area)
            #outfile = outputDir + outfilename

            if files_exist and nr_files_exist > 0:
                if in_msg.verbose:
                    print("/usr/bin/montage -tile " + tile +
                          " -geometry +0+0 " + files + " " + outfile +
                          " 2>&1 " + sleep_str)
                subprocess.call("/usr/bin/montage -tile " + tile +
                                " -geometry +0+0 " + files + " " + outfile +
                                " 2>&1 " + sleep_str,
                                shell=True)

                if hasattr(in_msg, 'resize_montage'):
                    if in_msg.resize_montage != 100:
                        if in_msg.verbose:
                            print("/usr/bin/convert -resize " +
                                  str(in_msg.resize_montage) + "% " + outfile +
                                  " tmp.png 2>&1 " + sleep_str)
                        subprocess.call("/usr/bin/convert -resize " +
                                        str(in_msg.resize_montage) + "% " +
                                        outfile + " tmp.png; mv tmp.png " +
                                        outfile + " 2>&1 ",
                                        shell=True)

                # check if file is produced
                if isfile(outfile) and files_complete:
                    montage_done.append(montage)

                if in_msg.scpOutput:
                    if (montage in in_msg.scpProducts) or ('all' in [
                            x.lower()
                            for x in in_msg.scpProducts if type(x) == str
                    ]):
                        scpOutputDir = format_name(in_msg.scpOutputDir,
                                                   time_slot,
                                                   area=area,
                                                   rgb=montage_name,
                                                   sat=in_msg.sat,
                                                   sat_nr=sat_nr)
                        if in_msg.verbose:
                            print("    /usr/bin/scp " + in_msg.scpID + " " +
                                  outfile + " " + scpOutputDir + " 2>&1 &")
                        subprocess.call("/usr/bin/scp " + in_msg.scpID + " " +
                                        outfile + " " + scpOutputDir +
                                        " 2>&1 &",
                                        shell=True)

                if in_msg.scpOutput and (in_msg.scpID2
                                         is not None) and (in_msg.scpOutputDir2
                                                           is not None):
                    if (montage in in_msg.scpProducts2) or ('all' in [
                            x.lower()
                            for x in in_msg.scpProducts2 if type(x) == str
                    ]):
                        scpOutputDir2 = format_name(in_msg.scpOutputDir2,
                                                    time_slot,
                                                    area=area,
                                                    rgb=montage_name,
                                                    sat=in_msg.sat,
                                                    sat_nr=sat_nr)
                        if in_msg.compress_to_8bit:
                            if in_msg.verbose:
                                print("    scp " + in_msg.scpID2 + " " +
                                      outfile.replace(".png", "-fs8.png") +
                                      " " + scpOutputDir2 + " 2>&1 &")
                            subprocess.call(
                                "scp " + in_msg.scpID2 + " " +
                                outfile.replace(".png", "-fs8.png") + " " +
                                scpOutputDir2 + " 2>&1 &",
                                shell=True)
                        else:
                            if in_msg.verbose:
                                print("    scp " + in_msg.scpID2 + " " +
                                      outfile + " " + scpOutputDir2 +
                                      " 2>&1 &")
                            subprocess.call("scp " + in_msg.scpID2 + " " +
                                            outfile + " " + scpOutputDir2 +
                                            " 2>&1 &",
                                            shell=True)

        if in_msg.verbose:
            if len(montage_done) > 0:
                print("... produced montages: ")
                for montage in montage_done:
                    print("   ", montage)
            else:
                if len(postprocessing_montage) > 0:
                    if len(postprocessing_montage[0]) > 0:
                        print("*** Warning, no montages produced (requested ",
                              postprocessing_montage, ")")

    return composites_done, montage_done
def plot_msg_minus_cosmo(in_msg):

    # do statistics for the last full hour (minutes=0, seconds=0)
    in_msg.datetime = datetime(in_msg.datetime.year, in_msg.datetime.month,
                               in_msg.datetime.day, in_msg.datetime.hour, 0, 0)

    area_loaded = choose_area_loaded_msg(in_msg.sat, in_msg.sat_nr,
                                         in_msg.datetime)

    # define contour write for coasts, borders, rivers
    cw = ContourWriterAGG(in_msg.mapDir)

    # check if input data is complete
    if in_msg.verbose:
        print("*** check input data for ", in_msg.sat_str())
    RGBs = check_input(in_msg,
                       in_msg.sat_str(layout="%(sat)s") + in_msg.sat_nr_str(),
                       in_msg.datetime)
    # in_msg.sat_nr might be changed to backup satellite

    if in_msg.verbose:
        print('*** Create plots for ')
        print('    Satellite/Sensor: ' + in_msg.sat_str())
        print('    Satellite number: ' + in_msg.sat_nr_str() + ' // ' +
              str(in_msg.sat_nr))
        print('    Satellite instrument: ' + in_msg.instrument)
        print('    Date/Time:        ' + str(in_msg.datetime))
        print('    RGBs:            ', in_msg.RGBs)
        print('    Area:            ', in_msg.areas)
        print('    reader level:    ', in_msg.reader_level)

    # define satellite data object
    #global_data = GeostationaryFactory.create_scene(in_msg.sat, in_msg.sat_nr_str(), "seviri", in_msg.datetime)
    global_data = GeostationaryFactory.create_scene(in_msg.sat_str(),
                                                    in_msg.sat_nr_str(),
                                                    in_msg.instrument,
                                                    in_msg.datetime)
    # global_data = GeostationaryFactory.create_scene("msg-ot", "", "Overshooting_Tops", in_msg.datetime)

    if len(RGBs) == 0 and len(in_msg.postprocessing_areas) == 0:
        return RGBs

    if in_msg.verbose:
        print(
            "*** load satellite channels for " + in_msg.sat_str() +
            in_msg.sat_nr_str() + " ", global_data.fullname)

    # initialize processed RGBs
    RGBs_done = []

    # -------------------------------------------------------------------
    # load reflectivities, brightness temperatures, NWC-SAF products ...
    # -------------------------------------------------------------------
    area_loaded = load_products(global_data, RGBs, in_msg, area_loaded)

    cosmo_input_file = "input_cosmo_cronjob.py"
    print("... read COSMO input file: ", cosmo_input_file)
    in_cosmo = parse_commandline_and_read_inputfile(
        input_file=cosmo_input_file)

    # add composite
    in_msg.scpOutput = True
    in_msg.resize_montage = 70
    in_msg.postprocessing_montage = [[
        "MSG_IR-108cpc", "COSMO_SYNMSG-BT-CL-IR10.8",
        "MSG_IR-108-COSMO-minus-MSGpc"
    ]]
    in_msg.scpProducts = [[
        "MSG_IR-108cpc", "COSMO_SYNMSG-BT-CL-IR10.8",
        "MSG_IR-108-COSMO-minus-MSGpc"
    ]]
    #in_msg.scpProducts = ["all"]

    # define satellite data object
    cosmo_data = GeostationaryFactory.create_scene(in_cosmo.sat_str(),
                                                   in_cosmo.sat_nr_str(),
                                                   in_cosmo.instrument,
                                                   in_cosmo.datetime)

    area_loaded_cosmo = load_products(cosmo_data, ['SYNMSG_BT_CL_IR10.8'],
                                      in_cosmo, area_loaded)

    # preprojecting the data to another area
    # --------------------------------------
    if len(RGBs) > 0:
        for area in in_msg.areas:
            print("")
            obj_area = get_area_def(area)

            if area != 'ccs4':
                print("*** WARNING, diff MSG-COSMO only implemented for ccs4")
                continue

            # reproject data to new area
            print(area_loaded)

            if obj_area == area_loaded:
                if in_msg.verbose:
                    print("*** Use data for the area loaded: ", area)
                #obj_area = area_loaded
                data = global_data
                resolution = 'l'
            else:
                if in_msg.verbose:
                    print("*** Reproject data to area: ", area,
                          "(org projection: ", area_loaded.name, ")")
                obj_area = get_area_def(area)
                # PROJECT data to new area
                data = global_data.project(area, precompute=True)
                resolution = 'i'

            if in_msg.parallax_correction:
                loaded_products = [chn.name for chn in data.loaded_channels()]

                if 'CTH' not in loaded_products:
                    print("*** Error in plot_msg (" +
                          inspect.getfile(inspect.currentframe()) + ")")
                    print(
                        "    Cloud Top Height is needed for parallax correction "
                    )
                    print(
                        "    either load CTH or specify the estimation of the CTH in the input file (load 10.8 in this case)"
                    )
                    quit()

                if in_msg.verbose:
                    print(
                        "    perform parallax correction for loaded channels: ",
                        loaded_products)

                data = data.parallax_corr(fill=in_msg.parallax_gapfilling,
                                          estimate_cth=in_msg.estimate_cth,
                                          replace=True)

            # save reprojected data
            if area in in_msg.save_reprojected_data:
                save_reprojected_data(data, area, in_msg)

            # apply a mask to the data (switched off at the moment)
            if False:
                mask_data(data, area)

            # save average values
            if in_msg.save_statistics:

                mean_array = zeros(len(RGBs))
                #statisticFile = '/data/COALITION2/database/meteosat/ccs4/'+yearS+'/'+monthS+'/'+dayS+'/MSG_'+area+'_'+yearS[2:]+monthS+dayS+'.txt'
                statisticFile = './' + yearS + '-' + monthS + '-' + dayS + '/MSG_' + area + '_' + yearS[
                    2:] + monthS + dayS + '.txt'
                if in_msg.verbose:
                    print("*** write statistics (average values) to " +
                          statisticFile)
                f1 = open(statisticFile, 'a')  # mode append
                i_rgb = 0
                for rgb in RGBs:
                    if rgb in products.MSG_color:
                        mean_array[i_rgb] = data[rgb.replace("c",
                                                             "")].data.mean()
                        i_rgb = i_rgb + 1

                # create string to write
                str2write = dateS + ' ' + hourS + ' : ' + minS + ' UTC  '
                for mm in mean_array:
                    str2write = str2write + ' ' + "%7.2f" % mm
                str2write = str2write + "\n"
                f1.write(str2write)
                f1.close()

            # creating plots/images
            if in_msg.make_plots:

                # choose map resolution
                in_msg.resolution = choose_map_resolution(
                    area, in_msg.mapResolution)

                # define area
                proj4_string = obj_area.proj4_string
                # e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
                area_extent = obj_area.area_extent
                # e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
                area_tuple = (proj4_string, area_extent)

                RGBs = ['IR_108-COSMO-minus-MSG']

                print(data['IR_108'].data.shape)
                print(cosmo_data['SYNMSG_BT_CL_IR10.8'].data.shape)
                diff_MSG_COSMO = cosmo_data['SYNMSG_BT_CL_IR10.8'].data - data[
                    'IR_108'].data
                HRV_enhance_str = ''

                # add IR difference as "channel object" to satellite regional "data" object
                data.channels.append(
                    Channel(name=RGBs[0],
                            wavelength_range=[0., 0., 0.],
                            resolution=data['IR_108'].resolution,
                            data=diff_MSG_COSMO))

                for rgb in RGBs:

                    if not check_loaded_channels(rgb, data):
                        continue

                    PIL_image = create_PIL_image(rgb,
                                                 data,
                                                 in_msg,
                                                 obj_area=obj_area)
                    # !!! in_msg.colorbar[rgb] is initialized inside (give attention to rgbs) !!!

                    add_borders_and_rivers(PIL_image,
                                           cw,
                                           area_tuple,
                                           add_borders=in_msg.add_borders,
                                           border_color=in_msg.border_color,
                                           add_rivers=in_msg.add_rivers,
                                           river_color=in_msg.river_color,
                                           resolution=in_msg.resolution,
                                           verbose=in_msg.verbose)

                    # indicate mask
                    if in_msg.indicate_mask:
                        PIL_image = indicate_mask(rgb, PIL_image, data,
                                                  in_msg.verbose)

                    #if area.find("EuropeCanary") != -1 or area.find("ccs4") != -1:
                    dc = DecoratorAGG(PIL_image)

                    # add title to image
                    if in_msg.add_title:
                        add_title(PIL_image,
                                  in_msg.title,
                                  HRV_enhance_str + rgb,
                                  in_msg.sat_str(),
                                  data.sat_nr(),
                                  in_msg.datetime,
                                  area,
                                  dc,
                                  in_msg.font_file,
                                  in_msg.verbose,
                                  title_color=in_msg.title_color,
                                  title_y_line_nr=in_msg.title_y_line_nr
                                  )  # !!! needs change

                    # add MeteoSwiss and Pytroll logo
                    if in_msg.add_logos:
                        if in_msg.verbose:
                            print('... add logos')
                        dc.align_right()
                        if in_msg.add_colorscale:
                            dc.write_vertically()
                        if PIL_image.mode != 'L':
                            height = 60  # height=60.0 normal resolution
                            dc.add_logo(in_msg.logos_dir + "/pytroll3.jpg",
                                        height=height)  # height=60.0
                            dc.add_logo(in_msg.logos_dir + "/meteoSwiss3.jpg",
                                        height=height)
                            dc.add_logo(
                                in_msg.logos_dir +
                                "/EUMETSAT_logo2_tiny_white_square.png",
                                height=height)  # height=60.0

                    # add colorscale
                    if in_msg.add_colorscale and in_msg.colormap[rgb] != None:
                        if rgb in products.MSG_color:
                            unit = data[rgb.replace("c", "")].info['units']
                        #elif rgb in products.MSG or rgb in products.NWCSAF or rgb in products.HSAF:
                        #   unit = data[rgb].info['units']
                        else:
                            unit = None
                            loaded_channels = [
                                chn.name for chn in data.loaded_channels()
                            ]
                            if rgb in loaded_channels:
                                if hasattr(data[rgb], 'info'):
                                    print("    hasattr(data[rgb], 'info')",
                                          list(data[rgb].info.keys()))
                                    if 'units' in list(data[rgb].info.keys()):
                                        print(
                                            "'units' in data[rgb].info.keys()")
                                        unit = data[rgb].info['units']
                        print("... units = ", unit)
                        add_colorscale(dc, rgb, in_msg, unit=unit)

                    if in_msg.parallax_correction:
                        parallax_correction_str = 'pc'
                    else:
                        parallax_correction_str = ''
                    rgb += parallax_correction_str

                    # create output filename
                    outputDir = format_name(
                        in_msg.outputDir,
                        data.time_slot,
                        area=area,
                        rgb=rgb,
                        sat=data.satname,
                        sat_nr=data.sat_nr())  # !!! needs change
                    outputFile = outputDir + "/" + format_name(
                        in_msg.outputFile,
                        data.time_slot,
                        area=area,
                        rgb=rgb,
                        sat=data.satname,
                        sat_nr=data.sat_nr())  # !!! needs change

                    # check if output directory exists, if not create it
                    path = dirname(outputFile)
                    if not exists(path):
                        if in_msg.verbose:
                            print('... create output directory: ' + path)
                        makedirs(path)

                    # save file
                    if exists(outputFile) and not in_msg.overwrite:
                        if stat(outputFile).st_size > 0:
                            print('... outputFile ' + outputFile +
                                  ' already exists (keep old file)')
                        else:
                            print(
                                '*** Warning, outputFile' + outputFile +
                                ' already exists, but is empty (overwrite file)'
                            )
                            PIL_image.save(outputFile, optimize=True
                                           )  # optimize -> minimize file size
                            chmod(
                                outputFile, 0o777
                            )  ## FOR PYTHON3: 0o664  # give access read/write access to group members
                    else:
                        if in_msg.verbose:
                            print('... save final file: ' + outputFile)
                        PIL_image.save(
                            outputFile,
                            optimize=True)  # optimize -> minimize file size
                        chmod(
                            outputFile, 0o777
                        )  ## FOR PYTHON3: 0o664  # give access read/write access to group members

                    if in_msg.compress_to_8bit:
                        if in_msg.verbose:
                            print('... compress to 8 bit image: display ' +
                                  outputFile.replace(".png", "-fs8.png") +
                                  ' &')
                        subprocess.call(
                            "/usr/bin/pngquant -force 256 " + outputFile +
                            " 2>&1 &",
                            shell=True)  # 256 == "number of colors"

                    #if in_msg.verbose:
                    #   print "    add coastlines to "+outputFile
                    ## alternative: reopen image and modify it (takes longer due to additional reading and saving)
                    #cw.add_rivers_to_file(img, area_tuple, level=5, outline='blue', width=0.5, outline_opacity=127)
                    #cw.add_coastlines_to_file(outputFile, obj_area, resolution=resolution, level=4)
                    #cw.add_borders_to_file(outputFile, obj_area, outline=outline, resolution=resolution)

                    # secure copy file to another place
                    if in_msg.scpOutput:
                        if (rgb in in_msg.scpProducts) or ('all' in [
                                x.lower()
                                for x in in_msg.scpProducts if type(x) == str
                        ]):
                            scpOutputDir = format_name(in_msg.scpOutputDir,
                                                       data.time_slot,
                                                       area=area,
                                                       rgb=rgb,
                                                       sat=data.satname,
                                                       sat_nr=data.sat_nr())
                            if in_msg.compress_to_8bit:
                                if in_msg.verbose:
                                    print("... secure copy " +
                                          outputFile.replace(
                                              ".png", "-fs8.png") + " to " +
                                          scpOutputDir)
                                subprocess.call(
                                    "scp " + in_msg.scpID + " " +
                                    outputFile.replace(".png", "-fs8.png") +
                                    " " + scpOutputDir + " 2>&1 &",
                                    shell=True)
                            else:
                                if in_msg.verbose:
                                    print("... secure copy " + outputFile +
                                          " to " + scpOutputDir)
                                subprocess.call("scp " + in_msg.scpID + " " +
                                                outputFile + " " +
                                                scpOutputDir + " 2>&1 &",
                                                shell=True)

                    if in_msg.scpOutput and in_msg.scpID2 != None and in_msg.scpOutputDir2 != None:
                        if (rgb in in_msg.scpProducts2) or ('all' in [
                                x.lower()
                                for x in in_msg.scpProducts2 if type(x) == str
                        ]):
                            scpOutputDir2 = format_name(in_msg.scpOutputDir2,
                                                        data.time_slot,
                                                        area=area,
                                                        rgb=rgb,
                                                        sat=data.satname,
                                                        sat_nr=data.sat_nr())
                            if in_msg.compress_to_8bit:
                                if in_msg.verbose:
                                    print("... secure copy " +
                                          outputFile.replace(
                                              ".png", "-fs8.png") + " to " +
                                          scpOutputDir2)
                                subprocess.call(
                                    "scp " + in_msg.scpID2 + " " +
                                    outputFile.replace(".png", "-fs8.png") +
                                    " " + scpOutputDir2 + " 2>&1 &",
                                    shell=True)
                            else:
                                if in_msg.verbose:
                                    print("... secure copy " + outputFile +
                                          " to " + scpOutputDir2)
                                subprocess.call("scp " + in_msg.scpID2 + " " +
                                                outputFile + " " +
                                                scpOutputDir2 + " 2>&1 &",
                                                shell=True)

                    if 'ninjotif' in in_msg.outputFormats:
                        ninjotif_file = format_name(outputDir + '/' +
                                                    in_msg.ninjotifFilename,
                                                    data.time_slot,
                                                    sat_nr=data.sat_nr(),
                                                    RSS=in_msg.RSS,
                                                    area=area,
                                                    rgb=rgb)
                        from plot_coalition2 import pilimage2geoimage
                        GEO_image = pilimage2geoimage(PIL_image, obj_area,
                                                      data.time_slot)
                        GEO_image.save(ninjotif_file,
                                       fformat='mpop.imageo.formats.ninjotiff',
                                       ninjo_product_name=rgb,
                                       chan_id=products.ninjo_chan_id[
                                           rgb.replace("_", "-") + "_" + area],
                                       nbits=8)
                        chmod(ninjotif_file, 0o777)
                        print(("... save ninjotif image: display ",
                               ninjotif_file, " &"))

                    if rgb not in RGBs_done:
                        RGBs_done.append(rgb)

        ## start postprocessing
        for area in in_msg.postprocessing_areas:
            postprocessing(in_msg, global_data.time_slot, int(data.sat_nr()),
                           area)

    if in_msg.verbose:
        print(" ")

    return RGBs_done
예제 #17
0
def plot_msg(in_msg):


   # get date of the last SEVIRI observation
   if in_msg.datetime == None:
      in_msg.get_last_SEVIRI_date()

   yearS = str(in_msg.datetime.year)
   #yearS = yearS[2:]
   monthS = "%02d" % in_msg.datetime.month
   dayS   = "%02d" % in_msg.datetime.day
   hourS  = "%02d" % in_msg.datetime.hour
   minS   = "%02d" % in_msg.datetime.minute

   dateS=yearS+'-'+monthS+'-'+dayS
   timeS=hourS+'-'+minS

   if in_msg.sat_nr==None:
      in_msg.sat_nr=choose_msg(in_msg.datetime,in_msg.RSS)

   if in_msg.datetime.year > 2012:
      if in_msg.sat_nr == 8:
         area_loaded = get_area_def("EuropeCanary35")
      elif in_msg.sat_nr ==  9: # rapid scan service satellite
         area_loaded = get_area_def("EuropeCanary95")  
      elif in_msg.sat_nr == 10: # default satellite
         area_loaded = get_area_def("met09globeFull")  # full disk service, like EUMETSATs NWC-SAF products
      elif in_msg.sat_nr == 0: # fake satellite for reprojected ccs4 data in netCDF
         area_loaded = get_area_def("ccs4")  # 
         #area_loaded = get_area_def("EuropeCanary")
         #area_loaded = get_area_def("alps")  # new projection of SAM
      else:
         print("*** Error, unknown satellite number ", in_msg.sat_nr)
         area_loaded = get_area_def("hsaf")  # 
   else:
      if in_msg.sat_nr == 8:
         area_loaded = get_area_def("EuropeCanary95") 
      elif in_msg.sat_nr ==  9: # default satellite
         area_loaded = get_area_def("EuropeCanary")

   # define contour write for coasts, borders, rivers
   cw = ContourWriterAGG(in_msg.mapDir)

   if type(in_msg.sat_nr) is int:
      sat_nr_str = str(in_msg.sat_nr).zfill(2)
   elif type(in_msg.sat_nr) is str:
      sat_nr_str = in_msg.sat_nr
   else:
      print("*** Waring, unknown type of sat_nr", type(in_msg.sat_nr))
      sat_nr_str = in_msg.sat_nr

   if in_msg.verbose:
      print('*** Create plots for ')
      print('    Satellite/Sensor: '+in_msg.sat + '  ' + sat_nr_str)
      print('    Date/Time:        '+dateS +' '+hourS+':'+minS+'UTC')
      print('    RGBs:            ', in_msg.RGBs)
      print('    Area:            ', in_msg.areas)


   # check if input data is complete 
   if in_msg.verbose:
      print("*** check input data")
   RGBs = check_input(in_msg, in_msg.sat+sat_nr_str, in_msg.datetime)
   if len(RGBs) != len(in_msg.RGBs):
      print("*** Warning, input not complete.")
      print("*** Warning, process only: ", RGBs)

   # define satellite data object
   global_data = GeostationaryFactory.create_scene(in_msg.sat, sat_nr_str, "seviri", in_msg.datetime)
   # print "type(global_data) ", type(global_data)   # <class 'mpop.scene.SatelliteInstrumentScene'>
   # print "dir(global_data)", dir(global_data)  [..., '__init__', ... 'area', 'area_def', 'area_id', 'channel_list', 'channels', 
   #      'channels_to_load', 'check_channels', 'fullname', 'get_area', 'image', 'info', 'instrument_name', 'lat', 'load', 'loaded_channels', 
   #      'lon', 'number', 'orbit', 'project', 'remove_attribute', 'satname', 'save', 'set_area', 'time_slot', 'unload', 'variant']
   
   ## define satellite data object one scan before
   #if in_msg.RSS:
   #   scan_time =  5 # min
   #else:
   #   scan_time = 15 # min
   scan_time = 15 # min
   datetime_m1 =  in_msg.datetime - timedelta(minutes=scan_time)
   global_data_m1 = GeostationaryFactory.create_scene(in_msg.sat, sat_nr_str, "seviri", datetime_m1)

   if len(RGBs) == 0:
      return RGBs

   if in_msg.verbose:
      print("*** load satellite channels for "+in_msg.sat+sat_nr_str+" ", global_data.fullname)

   # initialize processed RGBs
   RGBs_done=[]

   # load reflectivities, brightness temperatures, NWC-SAF products ...
   print("*** read ", str(in_msg.datetime))
   area_loaded = load_products(global_data,    RGBs, in_msg, area_loaded)
   #print "*** read ", str(datetime_m1)
   #area_loaded = load_products(global_data_m1, RGBs, in_msg, area_loaded)

   # check if all prerequisites are loaded
   #rgb_complete = []
   #for rgb in RGBs:
   #   all_loaded = True
   #   if rgb in products.RGBs_buildin or rgb in products.RGB_user:
   #      obj_image = get_image(global_data, rgb)
   #      for pre in obj_image.prerequisites:
   #         if pre not in loaded_channels:
   #            all_loaded = False
   #   elif rgb in products.MSG_color:
   #      if rgb.replace("c","") not in loaded_channels:
   #         all_loaded = False
   #   else:
   #      if rgb not in loaded_channels:
   #         all_loaded = False
   #   if all_loaded:
   #      rgb_complete.append(rgb)
   #print "rgb_complete", rgb_complete

   # preprojecting the data to another area 
   # --------------------------------------
   for area in in_msg.areas:
      print("")
      obj_area = get_area_def(area)

      # reproject data to new area 
      if obj_area == area_loaded: 
         if in_msg.verbose:
            print("*** Use data for the area loaded: ", area)
         #obj_area = area_loaded
         data    = global_data
         data_m1 = global_data_m1
         resolution='l'
      else:
         if in_msg.verbose:    
            print("*** Reproject data to area: ", area, "(org projection: ",  area_loaded.name, ")")     
         obj_area = get_area_def(area)
         # PROJECT data to new area 
         data    = global_data.project(area, precompute=True)
         data_m1 = global_data_m1.project(area, precompute=True)
         resolution='i'

      loaded_products = [chn.name for chn in data.loaded_channels()]
      print(loaded_products)
      #loaded_products_m1 = [chn.name for chn in data_m1.loaded_channels()]
      #print loaded_products_m1

      #for prod in loaded_products:
      #   print "xxx ", prod 
      #   print data_m1[prod]
      #   data[prod] =  data[prod] - data_m1[prod] # 

      # save reprojected data
      if area in in_msg.save_reprojected_data:
         save_reprojected_data(data, area, in_msg)

      # apply a mask to the data (switched off at the moment)
      if False:
         mask_data(data, area)

      # save average values 
      if in_msg.save_statistics:

         mean_array = zeros(len(RGBs))
         #statisticFile = '/data/COALITION2/database/meteosat/ccs4/'+yearS+'/'+monthS+'/'+dayS+'/MSG_'+area+'_'+yearS[2:]+monthS+dayS+'.txt'
         statisticFile = './'+yearS+'-'+monthS+'-'+dayS+'/MSG_'+area+'_'+yearS[2:]+monthS+dayS+'.txt'
         if in_msg.verbose:
            print("*** write statistics (average values) to "+statisticFile)
         f1 = open(statisticFile,'a')   # mode append
         i_rgb=0
         for rgb in RGBs:
            if rgb in products.MSG_color:
               mean_array[i_rgb]=data[rgb.replace("c","")].data.mean()
               i_rgb=i_rgb+1

         # create string to write
         str2write = dateS +' '+hourS+' : '+minS+' UTC  '
         for mm in mean_array:
            str2write = str2write+' '+ "%7.2f" % mm
         str2write = str2write+"\n"
         f1.write(str2write)
         f1.close()

      # creating plots/images 
      if in_msg.make_plots:
      
         # choose map resolution 
         resolution = choose_map_resolution(area, in_msg.mapResolution)

         # define area
         proj4_string = obj_area.proj4_string            
         # e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
         area_extent = obj_area.area_extent              
         # e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
         area_tuple = (proj4_string, area_extent)

         for rgb in RGBs:
            PIL_image = create_PIL_image(rgb, data, in_msg)   # !!! in_msg.colorbar[rgb] is initialized inside (give attention to rgbs) !!!

            if in_msg.add_rivers:
               if in_msg.verbose:
                  print("    add rivers to image (resolution="+resolution+")")
               cw.add_rivers(PIL_image, area_tuple, outline='blue', resolution=resolution, outline_opacity=127, width=0.5, level=5) # 
               if in_msg.verbose:
                  print("    add lakes to image (resolution="+resolution+")")
               cw.add_coastlines(PIL_image, area_tuple, outline='blue', resolution=resolution, outline_opacity=127, width=0.5, level=2)  #, outline_opacity=0
            if in_msg.add_borders:
               if in_msg.verbose:
                  print("    add coastlines to image (resolution="+resolution+")")
               cw.add_coastlines(PIL_image, area_tuple, outline=(255, 0, 0), resolution=resolution, width=1)  #, outline_opacity=0
               if in_msg.verbose:
                  print("    add borders to image (resolution="+resolution+")")
               cw.add_borders(PIL_image, area_tuple, outline=(255, 0, 0), resolution=resolution, width=1)       #, outline_opacity=0 
   
            #if area.find("EuropeCanary") != -1 or area.find("ccs4") != -1:
            dc = DecoratorAGG(PIL_image)
   
            # add title to image
            if in_msg.add_title:
               add_title(PIL_image, rgb, int(data.number), dateS, hourS, minS, area, dc, in_msg.font_file, in_msg.verbose )

            # add MeteoSwiss and Pytroll logo
            if in_msg.add_logos:
               if in_msg.verbose:
                  print('... add logos')
               dc.align_right()
               if in_msg.add_colorscale:
                  dc.write_vertically()
               dc.add_logo("../logos/meteoSwiss3.jpg",height=60.0)
               dc.add_logo("../logos/pytroll3.jpg",height=60.0)
   
            # add colorscale
            if in_msg.add_colorscale and in_msg.colormap[rgb] != None:
               add_colorscale(dc, rgb, in_msg)

            # create output filename
            outputDir =              format_name(in_msg.outputDir,  data.time_slot, area=area, rgb=rgb, sat_nr=data.number)
            outputFile = outputDir + format_name(in_msg.outputFile, data.time_slot, area=area, rgb=rgb, sat_nr=data.number)
   
            # check if output directory exists, if not create it
            path= dirname(outputFile)
            if not exists(path):
               if in_msg.verbose:
                  print('... create output directory: ' + path)
               makedirs(path)
   
            # save file
            if in_msg.verbose:
               print('... save final file :' + outputFile)
            PIL_image.save(outputFile, optimize=True)  # optimize -> minimize file size
   
            if in_msg.compress_to_8bit:
               if in_msg.verbose:
                  print('... compress to 8 bit image: display '+outputFile.replace(".png","-fs8.png")+' &')
               subprocess.call("/usr/bin/pngquant -force 256 "+outputFile+" 2>&1 &", shell=True) # 256 == "number of colors"
   
            #if in_msg.verbose:
            #   print "    add coastlines to "+outputFile   
            ## alternative: reopen image and modify it (takes longer due to additional reading and saving)
            #cw.add_rivers_to_file(img, area_tuple, level=5, outline='blue', width=0.5, outline_opacity=127)
            #cw.add_coastlines_to_file(outputFile, obj_area, resolution=resolution, level=4)
            #cw.add_borders_to_file(outputFile, obj_area, outline=outline, resolution=resolution)
    
            # copy to another place
            if in_msg.scpOutput:
               if in_msg.verbose:
                  print("... secure copy "+outputFile+ " to "+in_msg.scpOutputDir)
               subprocess.call("scp "+in_msg.scpID+" "+outputFile+" "+in_msg.scpOutputDir+" 2>&1 &", shell=True)
               if in_msg.compress_to_8bit:
                  if in_msg.verbose:
                     print("... secure copy "+outputFile.replace(".png","-fs8.png")+ " to "+in_msg.scpOutputDir)
                     subprocess.call("scp "+in_msg.scpID+" "+outputFile.replace(".png","-fs8.png")+" "+in_msg.scpOutputDir+" 2>&1 &", shell=True)
   
            if rgb not in RGBs_done:
               RGBs_done.append(rgb)
   
      ## start postprocessing
      if area in in_msg.postprocessing_areas:
         postprocessing(in_msg, global_data.time_slot, data.number, area)

   if in_msg.verbose:
      print(" ")

   return RGBs_done