示例#1
0
 def _initialize_secondary(self):
     """Helper method to initializes sources from secondary source file"""
     units = PointSourceContainer.units
     try:
         secondary_sources = open(self.secondary_file_name)
     except IOError as error:
         err_string = "Unable to open file {0}".format(self.secondary_file_name)
         raise IOError(err_string)
         
     secondary_list = []
     
     headers = secondary_sources.readline()
     line = secondary_sources.readline()
     while line!="":
         line_list = line.split(',')
         name=line_list[0]
         parent = line_list[1]
         parentSource = getattr(self,parent)
         lat=float(line_list[2])
         lon=float(line_list[3])
         emissions = float(line_list[4])
         yearly_emissions = {2014: Units.SciVal(emissions, units),
                             2015: Units.SciVal(emissions, units),
                             2016: Units.SciVal(emissions, units)
                             }
         height = float(line_list[5])
         source = PST.PointSource(name,lat,lon,name,emissions,yearly_emissions,height)
         setattr(self,name,source)
         secondary_list.append(source)
         parentSource.secondary.append(source)
         line=secondary_sources.readline()
     secondary_sources.close()
     return secondary_list
示例#2
0
 def _initialize_sources(self):
     """Helper method to initialize sources from the main csv file"""
     units = PointSourceContainer.units
     source_list = []
     if not os.path.exists(self.file_name):
         raise IOError("The file %s does not exists" % self.file_name)
     
     try:
         source_file = open(self.file_name)
     except:
         print "Exception Raised: see message"
         raise
     file_heading = source_file.readline()
     next_line = source_file.readline()
     while next_line != "":
         temp = next_line.split(',')
         name = temp[0]
         loc = temp[1]
         lon = float(temp[2])
         lat = float(temp[3])
         emissions_2014 = float(temp[4])
         emissions_2015 = float(temp[5])
         emissions_2016 = float(temp[6])
         average_emissions = (emissions_2014 + emissions_2015 + emissions_2016)*(1./3.)
         yearly_emissions = {2014: Units.SciVal(emissions_2014, units),
                             2015: Units.SciVal(emissions_2015, units),
                             2016: Units.SciVal(emissions_2016, units)
                             }
         var_name = temp[7]
         height = float(temp[8])
         monthly_factors = map(lambda x: float(x),temp[9:21])
         full_name = '{0}, {1}'.format(name,loc)
         source = PST.PointSource(var_name,lat,lon,full_name,average_emissions,yearly_emissions,height)
         source.monthly_factors = monthly_factors
         setattr(self,var_name,source)
         source_list.append(source)
         next_line = source_file.readline()
     
     source_file.close()
     return source_list
示例#3
0
    def wind_arrow(overpass, lat=None, lon=None, size=0.02,
                                sources=['MERRA', 'ECMWF', 'GEM', 'Average']):
        """Creates a string which draws wind arrows in final KML file"""
                        
        
        arrows = ""
        input_lat = str(lat) if lat else "None"
        input_lon = str(lon) if lon else "None"
        
        if overpass.MERRA==overpass.ECMWF:
            w = PST.AllWinds(overpass)
            
            try:
                w.get_winds()
                wind_source = w
            
            except:
                wind_source = overpass

        else:
            # use winds from overpass instance
            wind_source = overpass
            
        MERRA_updated = wind_source.MERRA
        MERRA_beginning = wind_source.MERRA_beginning
        GEM = wind_source.GEM
        ECMWF_old = wind_source.ECMWF_old
        ECMWF = wind_source.ECMWF
        avg = wind_source.Average
        avg_old = wind_source.Average_old
        avg_beginning = wind_source.Average_beginning
        
        if ECMWF.speed>0.001:
            ECMWF_old = PST.Wind((0,0),0)
            avg_old = PST.Wind((0,0),0)
            
        source_map = {  'MERRA':(MERRA_updated, 'MERRA', 'MERRA'),
                        'ECMWF':(ECMWF,"ECMWF","ECMWF"),
                        'Average':(avg, "Average", "Average"),
                        'GEM':(GEM, 'GEM', 'GEM')
                    }
        winds = [source_map[src] for src in sources]
        wind_files = set([])
        
        lat = overpass.lat if lat==None else lat
        lon = overpass.lon if lon==None else lon

        # try:
            # gem_new = ReadWinds.get_gem_highres(overpass.time)
        # except IOError:
            # pass
        # except:
            # raise
        # else:
            # winds.append((gem_new,'GEM Forecast','GEM'))
        
        # try:
            # ecmwf_new = ReadWinds.get_ecmwf_highres(overpass.time,True)[0]
        # except IOError:
            # pass
        # except:
            # raise
        # else:
            # winds.append((ecmwf_new,'ECMWF (0.3 degree)','ECMWF'))
        
        winds_to_show = [w for w in winds if w[0].speed>0.001]
        winds_to_show = sorted(winds_to_show, key=lambda l: l[0].speed, reverse=True)
            
        for index,(wind,wind_source,wind_arrow_source) in enumerate(winds_to_show):
            speed = wind.speed
            north = lat + speed*size
            south = lat - speed*size
            east = lon + speed*size
            west = lon - speed*size
            heading = -90 - wind.bearing
            arrow = Formats.Arrow_Format.format(windsource=wind_source,
                                                description=str(wind),
                                                N=north, S=south,
                                                E=east, W=west,
                                                rotation=heading,
                                                arrow=wind_arrow_source,
                                                draworder=index+20)
            wind_file = os.path.join(data.kml_images, 
                                            'arrow_%s.png' % wind_arrow_source)
            wind_files.add(wind_file)
            arrows += arrow
        
        return Overlay(arrows, list(wind_files))
示例#4
0
 def _Initialize(self, csv_name):
     """Creates Overpass objects for each overpass, labelled as {PointSource short name}{YYYYMMDD}
     for example, Gavin20160207"""
     
     if not os.path.exists(csv_name):
         raise IOError("File %s does not exist" % csv_name)
     try:
         csv=open(csv_name,'r')
     except:
         print "Could not open %s: See message" % csv_name
         raise
     header=csv.readline()
     line=csv.readline()
     list = line.split(',')
     overpass_list = []
     while line !="":
         try:
             key = list[0]
             source = Sources.Sources[key]
         except AttributeError:
             raise AttributeError("Attribute {0} does not exist in module Sources.\nCurrent line of csv: {1}; in csv {2}".format(key, line, self.source_file))
             
         if len(list)==18:
             try:
                 date = map(int,list[1].split('-'))
                 year,month,day,hour,minute = date
                 dt = datetime.datetime(year,month,day,hour,minute)
                 time = PST.Time(dt,source)
                 merra_vector = map(float,list[2:4])
                 ecmwf_old_vector = map(float,list[4:6])
                 ecmwf_vector = (0,0)
                 gem_vector = map(float,list[6:8])
                 avg_vector = map(float,list[8:10])
                 surf_vector = map(float,list[10:12])
                 stability_class = list[12]
                 a = float(list[13].strip('\n'))
                 stability_corrected = list[14]
                 a_corrected = float(list[15])
                 lite = list[16]
                 full = list[17].strip('\n').strip('\r')
             except IndexError:
                 raise
             except Exception as exc:
                 print "An unecpexted exception was raised: see exception"
                 raise
         
         else:
             try:
                 date = map(int,list[1].split('-'))
                 year,month,day,hour,minute = date
                 dt = datetime.datetime(year,month,day,hour,minute)
                 time = PST.Time(dt,source)
                 merra_beginning = map(float,list[2:4])
                 merra_middle = map(float, list[4:6]) 
                 merra_interp = map(float, list[6:8])
                 ecmwf_old_vector = map(float,list[8:10])
                 ecmwf_vector = map(float,list[10:12])
                 gem_vector = map(float,list[12:14])
                 avg_vector = map(float,list[14:16])
                 surf_vector = map(float,list[16:18])
                 stability_class = list[18]
                 a = float(list[19].strip('\n'))
                 stability_corrected = list[20]
                 a_corrected = float(list[21])
                 lite = list[22]
                 full = list[23].strip('\n').strip('\r')
             except IndexError:
                 raise
             except Exception as exc:
                 print "An unecpexted exception was raised: see exception"
                 raise
         
         try:
             overpass = PST.Overpass(time)
             overpass.height = source.height
             overpass.MERRA_beginning = PST.Wind(merra_beginning,source.height)
             overpass.MERRA = PST.Wind(merra_middle, source.height)
             overpass.MERRA_interp = PST.Wind(merra_interp, source.height)
             overpass.ECMWF = PST.Wind(ecmwf_vector,source.height)
             overpass.ECMWF_old = PST.Wind(ecmwf_old_vector, source.height)
             overpass.GEM = PST.Wind(gem_vector,source.height)
             overpass.Average_beginning = 0.5*(overpass.MERRA_beginning + overpass.ECMWF)
             overpass.Average = 0.5*(overpass.MERRA + overpass.ECMWF)
             overpass.Average_interp = 0.5*(overpass.MERRA_interp + overpass.ECMWF)
             overpass.Average_old = 0.5*(overpass.MERRA + overpass.ECMWF_old)
             overpass.Average_hybrid = PST.Wind.construct(overpass.Average.speed, overpass.Average_beginning.bearing, overpass.height)
             overpass.surface = PST.Wind(surf_vector,source.height)
             overpass.a_elevated = PST.Stability(overpass.Average.speed,0.).a
             overpass.a_old = a
             overpass.a = a_corrected
             overpass.stability_old = stability_class
             overpass.stability = stability_corrected
             overpass.source = source
             overpass.FullFile = full
             overpass.LiteFile = lite
             try:
                 obs_mode = full.split('/')[-1].split('_')[1][-2:]
             except:
                 obs_mode = ''
             overpass.observation_mode = obs_mode
             overpass_name = source.short + time.strf8
             
             setattr(self, overpass_name, overpass)
             self.all.append(overpass)
             Sources.Sources[key].Overpasses.append(overpass)
             overpass_list.append(overpass)
             line=csv.readline()
             list=line.split(',')
             
             overpass.secondary = source.secondary
         except:
             raise
     csv.close()
     return overpass_list
示例#5
0
def Model(overpass,
          f_plume=0.10,
          f_background=0.01,
          offset=3000.,
          y_max_positive=50.e3,
          y_max_negative=50.e3,
          y_min_negative=0.,
          y_min_positive=0.,
          direction='y',
          wind_adjustment=0.,
          wind_sources=['Average'],
          smooth=False,
          surface_stability=True,
          stability="new",
          temporal_factors=False,
          bias_correction='corrected',
          LocalBackground=PlumeModel.InBackground,
          LocalInPlume=PlumeModel.InPlume,
          co2_source='xco2',
          custom_wind=None,
          snr_strong_co2_min=None,
          chi_squared_max=None,
          albedo_min=None,
          albedo_max=None,
          outcome_flags={1, 2},
          surface_pressure_max=None,
          surface_pressure_min=None,
          background_average=None,
          secondary_sources=False,
          fixed_secondary_sources=[],
          x_max=75.e3,
          scatter_plot=False,
          force_winds=None,
          units=Units.output_units,
          sza_adjustments=True,
          weighted=False,
          uncertainty=True):
    """Computes model enhancements for an overpass, and compares them to observed data.
    
    The behaviour is specified by the many default arguments. This function
    does everything needed including calculating model enhancements, but
    delegates the actual comparison to the ModelFunctions module. See
    the keyword list document for an explanation of the optional arguments
    """
    ## First all the default arguments are dealt with, and then move on to actually classifying data and fitting model

    # Args to pass to full_file.quality(i,**kwargs) method
    quality_args = {
        'chi_squared_max': chi_squared_max,
        'snr_strong_co2_min': snr_strong_co2_min,
        'albedo_min': albedo_min,
        'albedo_max': albedo_max,
        'outcome_flags': outcome_flags,
        'surface_pressure_min': surface_pressure_min,
        'surface_pressure_max': surface_pressure_max
    }

    bg_kwargs = {
        'background_factor': f_background,
        'ymax_positive': y_max_positive,
        'ymax_negative': y_max_negative,
        'ymin_negative': y_min_negative,
        'ymin_positive': y_min_positive,
        'offset': offset,
        'sign': direction
    }

    plume_kwargs = {'xmax': x_max, 'plume_factor': f_plume}
    if secondary_sources == False:
        secondary = []
        secondary_sources = False
    elif secondary_sources == True:
        secondary = overpass.source.secondary
        secondary_sources = True
    elif hasattr(secondary_sources, "__iter__"):
        secondary = secondary_sources
        secondary_sources = True
    else:
        raise TypeError("Invalid argument '{}' for secondary_sources.\
        Must be True, False, or iterable collection of PointSources".format(
            secondary_sources))

    # bias correction:
    if bias_correction not in File.allowed_bias_correction:
        raise ValueError(
            "bias correction must be one of {0}; given {1}".format(
                ', '.join(File.allowed_bias_correction), bias_correction))

    if not (co2_source == 'xco2' or co2_source == 'co2_column'):
        raise ValueError(
            "co2_source must be one of 'xco2' or 'co2_column' not '{0}'".
            format(co2_source))
    co2 = "{0}_{1}".format(bias_correction, co2_source)
    if smooth:
        co2 = 'smoothed_' + co2

    # get emissions for the overpass and make sure they're in g/s
    F = overpass.get_emissions(temporal_factors=temporal_factors)
    F.convert(Units._model_units)

    # atmospheric stability parameter
    if surface_stability == True:
        if stability == "new":
            a = overpass.a
        elif stability == "old":
            a = overpass.a_old
    elif surface_stability == False:
        a = overpass.a_elevated
    else:
        a = surface_stability

    secondary_emissions = [
        second.get_emissions(overpass)(Units._model_units)
        for second in secondary
    ]
    all_emissions = [F(units)] + [em(units) for em in secondary_emissions]
    total_emissions = F(units)
    for second in secondary_emissions:
        total_emissions += second(units)
    emissions_info = ', '.join([str(emi) for emi in all_emissions])
    sources_info = ', '.join(
        [src for src in [overpass.short] + [s.short for s in secondary]])

    print "Using reported emissions:", emissions_info
    print "For sources:", sources_info
    print "Total emissions:", total_emissions
    print "Using atmospheric stability parameter a={0}".format(a)

    print "Opening and reading full file"
    full_file = File.full(overpass)

    # force_winds is an override for forcing the model to run with a given list of Wind instances.
    # Useful for running the model with many wind adjustments
    if force_winds is None:
        all_winds = PST.AllWinds(overpass)
        try:
            WindSources, wind_labels = all_winds.parse(wind_sources)
            WindSources = [wnd.rotate(wind_adjustment) for wnd in WindSources]
        except Exception as exc:
            WindSources, wind_labels = [], []
            print "Exception raised and ignored:", exc

        try:
            custom_winds, custom_labels = all_winds.add_custom(custom_wind)
        except ValueError as ve:
            print "custom_wind value was not valid. See message:", ve
        except Exception as sxc:
            print "Unexpected error occured:", sxc
        else:
            WindSources.extend(custom_winds)
            wind_labels.extend(custom_labels)
    else:
        WindSources = force_winds
        wind_labels = [str(wind) for wind in force_winds]

    if sza_adjustments:
        print "Assigning a sign to the sensor zenith angle"
        full_file.sign_zenith_angle(overpass)

    # filtered_data has the same fields as full_file but only the points that pass the quality filter
    filtered_data = full_file.filter(**quality_args)

    print "Classifying points"
    in_plume_objects = []
    background_objects = []
    model_enhancement_lists = []
    model_alpha_lists = []
    fixed_enhancement_lists = []

    for wind in WindSources:
        # set wind.height to the weighted (by emissions) average height
        height_list = [overpass.height
                       ] + [source.height for source in secondary]
        mean_height = numpy.average(height_list, weights=all_emissions)
        wind.height = mean_height

        plume_data = File.File()
        background_data = File.File()
        model_enhancements = []  # will be column vector [ [V1], [V2], ... ]
        model_alpha = []  # will be matrix A from math docs
        fixed_enhancements = []  # list of enhancements from fixed sources

        # we already have a, F determined
        u = wind.speed

        # the offset (x,y) in wind basis components from the source to the center plume (highest enhancement)
        x_offset, y_offset = filtered_data.get_offset(overpass, wind)

        # same as above offset but for each secondary source
        secondary_offsets = filtered_data.get_secondary_offset(
            overpass, wind, secondary_sources=secondary)

        # emissons of all the fixed secondary sources
        fixed_emissions = [
            fixed.get_emissions(overpass, temporal_factors=temporal_factors)(
                Units._model_units) for fixed in fixed_secondary_sources
        ]

        for i in range(len(filtered_data)):
            coordinate = Geometry.CoordGeom(wind)
            x, y = coordinate.coord_to_wind_basis(
                overpass.lat, overpass.lon,
                filtered_data.retrieval_latitude[i],
                filtered_data.retrieval_longitude[i])
            dist = Geometry.CoordGeom.cartesian_distance((x, y),
                                                         (x_offset, y_offset))

            sza = Geometry.SZA(filtered_data, wind)

            # check if point is in background or in plume, including secondary sources
            in_background = PlumeModel.InBackground(x, y, dist, u, F, a,
                                                    **bg_kwargs)
            in_plume = PlumeModel.InPlume(x, y, u, F, a, **plume_kwargs)
            if secondary_sources:
                for ind, secondary_src in enumerate(secondary):
                    SZA_secondary = Geometry.SZA(filtered_data, wind)
                    xs, ys = Geometry.CoordGeom(wind).coord_to_wind_basis(
                        secondary_src.lat, secondary_src.lon,
                        filtered_data.retrieval_latitude[i],
                        filtered_data.retrieval_longitude[i])
                    secondary_dist = Geometry.CoordGeom.cartesian_distance(
                        (xs, ys), secondary_offsets[ind])

                    in_secondary_plume = PlumeModel.InPlume(
                        xs, ys, u, 1., a, **plume_kwargs)
                    in_secondary_background = PlumeModel.InBackground(
                        xs, ys, secondary_dist, u, 1., a, **bg_kwargs)

                    # logic for when to consider it in-plume/background with secondary sources
                    in_plume = in_plume or in_secondary_plume
                    in_background = in_background and in_secondary_background

            if in_background:
                background_data.append(filtered_data, i)

            if in_plume:
                total_enhancement = 0.  #enhancements from all sources
                plume_data.append(filtered_data, i)
                # get enhancement from main source, then add secondary sources
                if sza_adjustments:
                    main_enhancement = sza.V(x, y, u, F, a, i)
                else:
                    main_enhancement = PlumeModel.V(x, y, u, F, a)
                alpha = [main_enhancement / F]
                total_enhancement += main_enhancement

                for ind, secondary_src in enumerate(secondary):
                    xs, ys = Geometry.CoordGeom(wind).coord_to_wind_basis(
                        secondary_src.lat, secondary_src.lon,
                        filtered_data.retrieval_latitude[i],
                        filtered_data.retrieval_longitude[i])
                    F_secondary = secondary_emissions[ind]

                    if sza_adjustments:
                        secondary_enhancement = sza.V(xs, ys, u, F_secondary,
                                                      a, i)
                    else:
                        secondary_enhancement = PlumeModel.V(
                            xs, ys, u, F_secondary, a)

                    total_enhancement += secondary_enhancement
                    alpha.append(secondary_enhancement / F_secondary)

                model_enhancements.append([total_enhancement])
                model_alpha.append(alpha)

                fixed_enhancement = 0.
                for ind, fixed in enumerate(fixed_secondary_sources):
                    xs, ys = Geometry.CoordGeom(wind).coord_to_wind_basis(
                        fixed.lat, fixed.lon,
                        filtered_data.retrieval_latitude[i],
                        filtered_data.retrieval_longitude[i])
                    F_fixed = fixed_emissions[ind]

                    if sza:
                        source_enhancement = sza.V(xs, ys, u, F_fixed, a, i)
                    else:
                        source_enhancement = PlumeModel.V(
                            xs, ys, u, F_fixed, a)
                    fixed_enhancement += source_enhancement

                fixed_enhancements.append(fixed_enhancement)

        in_plume_objects.append(plume_data)
        background_objects.append(background_data)
        model_enhancement_lists.append(numpy.array(model_enhancements))
        model_alpha_lists.append(numpy.array(model_alpha))
        fixed_enhancement_lists.append(numpy.array(fixed_enhancements))

    all_results = []
    for (k, wind) in enumerate(WindSources):
        print ''
        print wind_labels[k]

        defaults = ModelFunctions.results_defaults()
        defaults.co2_attribute = co2
        defaults.co2_name = co2_source
        defaults.output_units = units
        in_plume_objects[k].xco2_uncert = numpy.array(
            in_plume_objects[k].xco2_uncert)

        try:
            results = ModelFunctions.interpret_results(
                in_plume_objects[k],
                background_objects[k],
                model_enhancement_lists[k],
                model_alpha_lists[k],
                fixed_enhancement_lists[k],
                overpass,
                wind,
                defaults,
                float(total_emissions),
                weights=weighted,
                background_average=background_average,
                uncertainty=uncertainty)
            # results is tuple (scale factor, estimated emissions, number of plume points, correlation)

        except:
            results = defaults.null_value
            # print "An exception was raised: see message following"
            # print exc
            raise

        all_results.append(results)

        if secondary_sources:
            print '\nEmissions are ordered as', ', '.join(
                [overpass.short] + map(lambda s: s.short, secondary))
        print ''

    # make scatter plot if it's asked for, using last wind source
    if scatter_plot:
        if type(scatter_plot) != str:
            raise TypeError(
                "Keyword scatter_plot must be a string corresponding to a valid path. Given '{0}'"
                .format(scatter_plot))

        try:
            sc_location = ModelFunctions.make_scatter_plot(
                numpy.array((model_enhancement_lists[k]) /
                            numpy.mean(background_objects[k].k)).flatten(),
                in_plume_objects[k][co2], scatter_plot)
        except:
            raise
        else:
            print "Scatter plot made and saved as {0}".format(scatter_plot)

    return all_results
示例#6
0
def get_new_ecmwf(time,
                  interp=True,
                  return_stability=True,
                  return_surface=False):
    """Reads 0.75 degree 6h ECMWF data"""
    source = time.source
    print "\nGathering ECMWF data..."
    grib_file = _new_ecmwf_fmt
    step = _new_ecmwf_step
    year = time.year
    month = time.month
    day = time.day
    hour = time.hour
    minute = time.minute
    tlon = source.lon % 360
    tlat = source.lat

    hour_minus = int(step * (time.decimaltime // step))
    delta_minus = dt.timedelta(hours=(hour_minus - hour), minutes=-minute)
    hour_plus = hour_minus + step
    delta_plus = dt.timedelta(hours=hour_plus - hour, minutes=-minute)

    time_minus = PST.Time(time.datetimeobj + delta_minus, source)
    time_plus = PST.Time(time.datetimeobj + delta_plus, source)

    decimal_hour = hour + minute / 60.

    file_minus = time_minus.strftime(grib_file)
    file_plus = time_plus.strftime(grib_file)

    time_closest = time.round(step * 3600)
    file_closest = time_closest.strftime(grib_file)

    if interp:
        files = [file_minus, file_plus]
        hours = [hour_minus, hour_plus]
    else:
        files = [file_closest]
        hours = [decimal_hour]

    u_interp = []
    v_interp = []
    usurf_interp = []
    vsurf_interp = []
    cloud_interp = []

    for file_name in files:
        try:
            grib = pygrib.open(file_name)
        except IOError:
            raise IOError("Could not open file {0}".format(file_name))
        except:
            print "Unexpected Error follows:"
            raise

        u_list = grib.select(name='U component of wind')
        v_list = grib.select(name='V component of wind')
        Psurface = grib.select(name='Surface pressure')[0].values
        cloud = grib.select(name="Total cloud cover")[0].values
        u_surf_list = grib.select(name="10 metre U wind component")[0].values
        v_surf_list = grib.select(name="10 metre V wind component")[0].values

        A = u_list[0].pv[:61] / 100.
        B = u_list[1].pv[61:]
        lat_ax = u_list[0].latlons()[0][:, 0]
        lon_ax = u_list[0].latlons()[1][0, :]
        lat0 = lat_ax[0]
        lat1 = lat_ax[1]
        lon0 = lon_ax[0]
        lon1 = lon_ax[1]
        dlat = lat1 - lat0
        dlon = lon1 - lon0
        lat_row = int((tlat - lat0) // dlat)
        lon_col = int((tlon - lon0) // dlon)
        lat_used = lat_ax[lat_row]
        lon_used = lon_ax[lon_col]
        lat_error = lat_used - tlat
        lon_error = lon_used - tlon

        # check if lat, lon are too far from the expected lat, lon
        if lat_error > 1. or lon_error > 1.:
            raise ValueError(
                "Lat/lon disagree by more than the resolution: Errors are ({0}, {1})"
                .format(lat_error, lon_error))
        # print "Using value at ({0}, {1})".format(lat_row,lon_col)
        stack = source.height
        H = 7000.
        p0 = Psurface[lat_row, lon_col] / 100.
        H_list = []
        i = 0
        prev = 0
        while i < 60:
            A0 = A[i]
            A1 = A[i + 1]
            B0 = B[i]
            B1 = B[i + 1]
            Pk0 = A0 + B0 * p0
            Pk1 = A1 + B1 * p0
            Pk = 0.5 * (Pk0 + Pk1)
            z = H * math.log(p0 / Pk)
            if z < stack:
                h1 = z
                h2 = prev
                break
            i += 1
            prev = z
        else:
            h1 = z
            h2 = stack
            i -= 1

        u1 = u_list[i].values[lat_row, lon_col]
        u2 = u_list[i - 1].values[lat_row, lon_col]
        v1 = v_list[i].values[lat_row, lon_col]
        v2 = v_list[i - 1].values[lat_row, lon_col]

        u = numpy.interp(stack, [h1, h2], [u1, u2])
        u_interp.append(u)
        v = numpy.interp(stack, [h1, h2], [v1, v2])
        v_interp.append(v)

        cloud_fraction = cloud[lat_row, lon_col]
        cloud_interp.append(cloud_fraction)

        u_surf = u_surf_list[lat_row, lon_col]
        usurf_interp.append(u_surf)
        v_surf = v_surf_list[lat_row, lon_col]
        vsurf_interp.append(v_surf)

    u_final = numpy.interp(decimal_hour, hours, u_interp)
    v_final = numpy.interp(decimal_hour, hours, v_interp)

    usurf_final = numpy.interp(decimal_hour, hours, usurf_interp)
    vsurf_final = numpy.interp(decimal_hour, hours, vsurf_interp)

    cloud_final = numpy.interp(decimal_hour, hours, cloud_interp)
    surface_wind = PST.Wind((usurf_final, vsurf_final), 0.)
    stability = PST.Stability(surface_wind.speed, cloud_final)
    return_values = [PST.Wind((u_final, v_final), stack)]
    if return_stability:
        return_values.append(stability)
    if return_surface:
        return_values.append(surface_wind)

    return return_values
示例#7
0
def find(sources, lat_threshold=_lat_threshold, lon_threshold=_lon_threshold, min_date=None, max_date=None, download=True, min_points=0):
        """searches all OCO-2 lite files for soundings close to each source in sources.
        
        Writes a text file summarizing the overpass (number of observations,
        time, file names, etc), and writes overpass data to a CSV file.
        
        option download will download ECMWF data for the overpass date if we
        don't already have that data downloaded
        """
        print "Starting search for %d sources" % len(sources)
        print "Lat threshold: %s" % lat_threshold
        print "Lon threshold: %s" % lon_threshold
        print "Minimum points found: %d" % min_points
        print "Sources being searched:"
        for src in sources:
            print src.short
        print ""
        
        if min_date==None and max_date==None:
            year_list = ['2014','2015','2016']
        
        elif min_date==None:
            year_list = map(str, range(2014,max_date.year+1))
        
        elif max_date == None:
            year_list = map(str, range(min_date.year, 2017))
        
        else:
            year_list = map(str, range(min_date.year, max_date.year + 1))
        
        if min_date==None:
            min_date = dt.datetime(2014,1,1,0,0)
        if max_date==None:
            max_date = dt.datetime(2016,12,31,23,59)

        fnames = [_summary_file_loc.format(source.short) for source in sources]
        for s in fnames:
            dir = os.path.dirname(s)
            if not os.path.exists(dir):
                os.mkdir(dir)
        
        # use 'with' context manager so if this crashes or is stopped you don't lose all the data!
        with open(_csv_save_loc,'w') as csv_output:
            csv_output.write(PST.Overpass.header)
            with nested(*[open(nm,"w") for nm in fnames]) as open_files:
                summary_files = {source.short: open_files[i] for i,source in enumerate(sources)}
            
                for name,file in summary_files.items():
                    file.write('Overpasses for %s' % name)
                    file.write('Fields are ID, Date [Year Month Day Hour Minute Second Millisecond], Number of soundings found, Filename\n')
                
                for year in year_list:
                    print year
                    for lite_name in os.listdir(lite_dir.format(year)):
                        date = dt.datetime.strptime(lite_name.split('_')[2],'%Y%m%d')
                        if min_date<=date<=max_date:
                            # dict of name:[# nadir, # glint, # target, # other] overpasses
                            all_overpasses={source.short:[0,0,0,0] for source in sources}
                            lite_file = os.path.join(lite_dir.format(year),lite_name)
                            lite_data = File.lite(lite_file)
                            print lite_file
                            lats = lite_data.latitude
                            lons = lite_data.longitude
                            times = lite_data.date
                            lite_id = lite_data.sounding_id
                            mode = lite_data.observation_mode
                            
                            found = []
                            close_indices = []
                            
                            for k in range(len(lite_data)):
                                sounding_lat = lats[k]
                                sounding_lon = lons[k]
                                for source in sources:
                                    key = source.short
                                    lat = source.lat
                                    lon = source.lon
                                    dlat = abs(sounding_lat-lat) % 360.
                                    dlon = abs(sounding_lon-lon) % 360.
                                    if dlat<=lat_threshold and dlon<=lon_threshold:
                                        if not key in found:
                                            close_indices.append(k)
                                            found.append(key)
                                            print("Found overpass for %s: dlat=%f, dlon=%f" % (source.short,dlat, dlon))
                                        else:
                                            pass
                                        if mode[k]=="ND":
                                            all_overpasses[key][0]+=1
                                        elif mode[k]=="GL":
                                            all_overpasses[key][1]+=1
                                        elif mode[k]=="TG":
                                            all_overpasses[key][2]+=1
                                        else:
                                            all_overpasses[key][3]+=1
                            
                            for (i,source_name) in zip(close_indices,found):
                                print "Processing %s overpass" % source_name
                                nadir,glint,tg,other = all_overpasses[source_name]
                                total_points = nadir+glint+tg+other
                                if total_points<min_points:
                                    print "Only {0} points".format(total_points)
                                else:
                                    id = lite_id[i]
                                    id_info = 'ID: {0}'.format(id)
                                    type_info = 'Nadir: {0}, Glint: {1}, Target: {2}, Transition: {3}'.format(nadir,glint,tg,other)
                                    source = Sources.Sources[source_name]
                                    time = PST.Time(None,source,time_string=str(times[i]))
                                    
                                    if download:
                                        ECMWF.download(time)
                                    
                                    full = lite_data.full_file(i)
 
                                    overpass = PST.Overpass.new(source, time, full, lite_file)
                                    file_info = 'File: {0}, {1}'.format(lite_file, full)
                                    overpass_info=', '.join([id_info,str(times[i]),type_info,file_info])
                                    if full!="":
                                        csv_output.write(overpass.write())
                                        summary_file = summary_files[source_name]
                                        summary_file.write(overpass_info+'\n')
                
            print("Done")
        
        print("Saved overpass information to {0}".format(_csv_save_loc))
        return _csv_save_loc
示例#8
0
def get_merra(time, print_time=False, action='middle'):
    """Reads 3h average MERRA data"""
    Source = time.source
    print "\nGathering MERRA data..."

    file_times = [
        "01:30", "04:30", "07:30", "10:30", "13:30", "16:30", "19:30", "22:30"
    ]

    filename = time.strftime(_merra_fmt)

    t_lon = Source.lon
    t_lat = Source.lat

    if action == 'middle':
        time_indices = [int((time.decimaltime) // _merra_step)]
        hours = [time.decimaltime]

    elif action == 'beginning':
        time_indices = [int((time.decimaltime - 1.5) // _merra_step)]
        hours = [time.decimaltime]

    elif action == 'interpolate':
        tfloor = int((time.decimaltime - 1.5) // _merra_step)
        time_indices = [tfloor, tfloor + 1]
        hours = [(_merra_step * h + 1.5) for h in time_indices]

    else:
        raise ValueError('Invalid option "%s" for "action"' % action)

    if print_time:
        ftimes = [file_times[i] for i in time_indices]
        print "Time {0}; using file for time {1}".format(
            time.datetimeobj, ', '.join(ftimes))

    if not os.path.exists(filename):
        raise IOError("File {0} does not exist".format(filename))
    u_interp = []
    v_interp = []
    for time_index in time_indices:
        try:
            merra = netcdf.open(filename)
        except IOError:
            raise
        except:
            print "Unexpected Error Encountered:"
            raise

        U = merra.U
        V = merra.V
        lat0 = U.lat[0]
        lon0 = U.lon[0]
        lat1 = U.lat[1]
        lon1 = U.lon[1]
        dlat = lat1 - lat0
        dlon = lon1 - lon0
        lat_row = int((t_lat - lat0) // dlat)
        lon_col = int((t_lon - lon0) // dlon)
        lat_used = U.lat[lat_row]
        lon_used = U.lon[lon_col]
        lat_error = lat_used - t_lat
        lon_error = lon_used - t_lon

        if lat_error > dlat or lon_error > dlon:
            raise ValueError(
                "Lat/lon disagree by more than the resolution: Errors are ({0}, {1})"
                .format(lat_error, lon_error))

        U = U[time_index, :, lat_row, lon_col]
        V = V[time_index, :, lat_row, lon_col]
        Heights = merra.H[time_index, :, lat_row, lon_col]
        stack = Source.height
        H_list = list(Heights)
        i = 0
        while i < len(H_list) and H_list[i] > stack:
            i += 1
        if i == len(H_list):
            i = -1
            u = U[i]
            v = V[i]
    #       print("Using height {0}".format(H_list[i]))
        else:
            h1 = H_list[i]
            h2 = H_list[i - 1]
            u1 = U[i]
            u2 = U[i - 1]
            v1 = V[i]
            v2 = V[i - 1]
            #         print("Interpolating between heights {0} and {1} for a stack height of {2}".format(h1,h2,stack))
            u = numpy.interp(stack, [h1, h2], [u1, u2])
            v = numpy.interp(stack, [h1, h2], [v1, v2])
        u_interp.append(u)
        v_interp.append(v)
    if len(u_interp) == 1:
        u_final = u_interp[0]
        v_final = v_interp[0]
    else:
        u_final = numpy.interp(time.decimaltime, hours, u_interp)
        v_final = numpy.interp(time.decimaltime, hours, v_interp)
    return PST.Wind((u_final, v_final), stack)
示例#9
0
def get_surface(time, interp=True):
    """Reads just surface wind from 0.75 degree 6h ECMWF data"""
    source = time.source
    print "\nGathering ECMWF data..."
    grib_file = _new_ecmwf_fmt
    step = _new_ecmwf_step
    year = time.year
    month = time.month
    day = time.day
    hour = time.hour
    minute = time.minute
    tlon = source.lon % 360
    tlat = source.lat

    hour_minus = int(step * (time.decimaltime // step))
    delta_minus = dt.timedelta(hours=(hour_minus - hour), minutes=-minute)
    hour_plus = hour_minus + step
    delta_plus = dt.timedelta(hours=hour_plus - hour, minutes=-minute)

    time_minus = time + delta_minus
    time_plus = time + delta_plus

    decimal_hour = hour + minute / 60.

    file_minus = time_minus.strftime(grib_file)
    file_plus = time_plus.strftime(grib_file)

    time_closest = time.round(step * 3600)
    file_closest = time_closest.strftime(grib_file)

    if interp:
        files = [file_minus, file_plus]
        hours = [hour_minus, hour_plus]
    else:
        files = [file_closest]
        hours = [decimal_hour]

    usurf_interp = []
    vsurf_interp = []

    for file_name in files:
        try:
            grib = pygrib.open(file_name)
        except IOError:
            raise IOError("Could not open file {0}".format(file_name))
        except:
            print "Unexpected Error follows:"
            raise

        u_surf_dataset = grib.select(name="10 metre U wind component")[0]
        u_surf_list = u_surf_dataset.values
        v_surf_list = grib.select(name="10 metre V wind component")[0].values

        lat_ax = u_surf_dataset.latlons()[0][:, 0]
        lon_ax = u_surf_dataset.latlons()[1][0, :]
        lat0 = lat_ax[0]
        lat1 = lat_ax[1]
        lon0 = lon_ax[0]
        lon1 = lon_ax[1]
        dlat = lat1 - lat0
        dlon = lon1 - lon0
        lat_row = int((tlat - lat0) // dlat)
        lon_col = int((tlon - lon0) // dlon)
        lat_used = lat_ax[lat_row]
        lon_used = lon_ax[lon_col]
        lat_error = lat_used - tlat
        lon_error = lon_used - tlon
        if lat_error > 1. or lon_error > 1.:
            raise ValueError(
                "Lat/lon disagree by more than the resolution: Errors are ({0}, {1})"
                .format(lat_error, lon_error))

        u_surf = u_surf_list[lat_row, lon_col]
        v_surf = v_surf_list[lat_row, lon_col]
        usurf_interp.append(u_surf)
        vsurf_interp.append(v_surf)
    u = numpy.interp(decimal_hour, hours, usurf_interp)
    v = numpy.interp(decimal_hour, hours, vsurf_interp)

    return PST.Wind((u, v), 0.)
示例#10
0
def get_ecmwf(time,
              surface=True,
              interp=True,
              print_time=False,
              return_stability=True):
    """Reads 1 degree 3h interpolated ECMWF data"""
    source = time.source
    print "\nGathering ECMWF data..."
    step = _ecmwf_step
    time_step = 3600 * step
    year = time.year
    month = time.month
    day = time.day
    hour = time.hour
    minute = time.minute
    decimal_hour = hour + float(minute) / 60.

    time_closest = time.round(time_step)
    file_closest = time_closest.strftime(_ecmwf_fmt)

    hour_minus = int(step * (time.decimaltime // step))
    delta_minus = dt.timedelta(hours=(hour_minus - hour), minutes=-minute)

    hour_plus = hour_minus + step
    delta_plus = dt.timedelta(hours=hour_plus - hour, minutes=-minute)

    time_minus = time + delta_minus
    time_plus = time + delta_plus

    file_minus = time_minus.strftime(_ecmwf_fmt)
    file_plus = time_plus.strftime(_ecmwf_fmt)

    if interp:
        files = [file_minus, file_plus]
        hours = [hour_minus, hour_plus]
    else:
        files = [file_closest]
        hours = [decimal_hour]

    u_interp = []
    v_interp = []
    uSurface_interp = []
    vSurface_interp = []
    if print_time:
        print "Time {0}; using files {1}".format(time.datetimeobj, files)

    for file_name in files:
        try:
            grib = pygrib.open(file_name)
        except IOError:
            raise IOError("Could not open file {0}".format(file_name))
        except:
            print "Unexpected Error follows:"
            raise

        u_list = grib.select(name='U component of wind')
        v_list = grib.select(name='V component of wind')
        Psurface = grib.select(name='Surface pressure')[0].values
        cloud = grib[304].values

        A = u_list[0].pv[:61] / 100.
        B = u_list[1].pv[61:]
        lat_ax = u_list[0].latlons()[0][:, 0]
        lon_ax = u_list[0].latlons()[1][0, :]
        lat0 = lat_ax[0]
        lat1 = lat_ax[1]
        lon0 = lon_ax[0]
        lon1 = lon_ax[1]
        dlat = lat1 - lat0
        dlon = lon1 - lon0
        lat_row = int((source.lat - lat0) // dlat)
        lon_col = int((source.lon - lon0) // dlon)
        lat_used = lat_ax[lat_row]
        lon_used = lon_ax[lon_col]
        lat_error = lat_used - source.lat
        lon_error = lon_used - source.lon
        if lat_error > 1. or lon_error > 1.:
            raise ValueError(
                "Lat/lon disagree by more than the resolution: Errors are ({0}, {1})"
                .format(lat_error, lon_error))

        stack = source.height
        H = 7000.
        p0 = Psurface[lat_row, lon_col] / 100.
        H_list = []
        i = 0
        prev = 0
        while i < 60:
            A0 = A[i]
            A1 = A[i + 1]
            B0 = B[i]
            B1 = B[i + 1]
            Pk0 = A0 + B0 * p0
            Pk1 = A1 + B1 * p0
            Pk = 0.5 * (Pk0 + Pk1)
            z = H * math.log(p0 / Pk)
            if z < stack:
                h1 = z
                h2 = prev
                break
            i += 1
            prev = z
        i = min(59, i)
        if i == 59:
            h1 = stack
            h2 = stack
        u1 = u_list[i].values[lat_row, lon_col]
        u2 = u_list[i - 1].values[lat_row, lon_col]
        v1 = v_list[i].values[lat_row, lon_col]
        v2 = v_list[i - 1].values[lat_row, lon_col]
        #        print("Interpolating between heights {0} and {1} for a stack height of {2}".format(h1,h2,stack))
        u = numpy.interp(stack, [h1, h2], [u1, u2])
        v = numpy.interp(stack, [h1, h2], [v1, v2])
        uSurface = grib.select(
            name="10 metre U wind component")[0].values[lat_row, lon_col]
        vSurface = grib.select(
            name="10 metre V wind component")[0].values[lat_row, lon_col]
        u_interp.append(u)
        v_interp.append(v)
        uSurface_interp.append(uSurface)
        vSurface_interp.append(vSurface)
    cloudFraction = cloud[lat_row, lon_col]
    u_final = numpy.interp(decimal_hour, hours, u_interp)
    v_final = numpy.interp(decimal_hour, hours, v_interp)
    uSurface_final = numpy.interp(decimal_hour, hours, uSurface_interp)
    vSurface_final = numpy.interp(decimal_hour, hours, vSurface_interp)
    if surface:
        stability = PST.Stability(
            PST.Wind((uSurface_final, vSurface_final), stack).speed,
            cloudFraction)
    else:
        stability = PST.Stability(
            PST.Wind((u_final, v_final), stack).speed, cloudFraction)

    if return_stability:
        return [PST.Wind((u_final, v_final), stack), stability]
    else:
        return PST.Wind((u_final, v_final), stack)
示例#11
0
def get_gem_highres(time):
    """Reads 1h Gem Forecast data"""
    print "\nGathering GEM data..."

    t_lon = 360. + time.lon if time.lon < 0. else time.lon
    t_lat = time.lat

    year = time.year
    month = time.month
    day = time.day
    hour = time.hour
    minute = time.minute
    decimal_hour = time.decimaltime

    hour_minus = time.hour
    hour_plus = time.hour + 1

    time_minus = PST.Time(dt.datetime(year, month, day, hour_minus, 0),
                          time.source)
    time_plus = PST.Time(time_minus.datetimeobj + dt.timedelta(hours=1),
                         time.source)

    hour_string_minus = '00' if time_minus.hour < 12 else '12'
    hour_string_plus = '00' if time_plus.hour < 12 else '12'

    file_minus = time_minus.strftime(_gemh_fmt).replace("AM", "00").replace(
        "PM", "12")
    file_plus = time_plus.strftime(_gemh_fmt).replace("AM", "00").replace(
        "PM", "12")

    if (not os.path.isfile(file_minus)) and (not os.path.isfile(file_plus)):
        err = IOError(
            "Neither file on either side of the sounding time exists. Unable to read wind data"
        )
        raise err

    elif (not os.path.isfile(file_minus)) and os.path.isfile(file_plus):
        print(
            "File before in time does not exist, but file after in time does; using only time after sounding"
        )
        files = [file_plus, file_plus]

    elif (not os.path.isfile(file_plus)) and os.path.isfile(file_minus):
        print(
            "File after in time does not exist, but file before in time does; using only time before sounding"
        )
        files = [file_minus, file_minus]

    else:
        files = [file_minus, file_plus]

    u_interp = []
    v_interp = []
    for fname in files:
        try:
            rpn = fstd.open(fname)
        except IOError as e:
            print "Unable to open file {0}".format(fname)
            raise e

        try:
            U = rpn.UU
            V = rpn.VV
            GZ = rpn.GZ
        except AttributeError as error:
            print("Unable to read all fields -- see exception raised")
            print error

        # <Var 'GZ'> has 160 levels, but U, V only have 80. In tests opening files,
        # <LogHybrid> from GZ has extra levels compared to it from U and V; test to make sure of this

        if not numpy.array_equal(U.axes[2][:], GZ.axes[2][1::2]):
            raise ValueError(
                "<LogHybrid> from GZ and UU, VV variables don't match up. Look closer at this file"
            )
            return PST.Wind((0, 0), 0)

        lats = U.lat[:]
        lons = U.lon[:]
        lat_row = 0
        lon_col = 0
        while lats[lat_row] < t_lat:
            lat_row += 1
        while lons[lon_col] < t_lon:
            lon_col += 1

        lat_row -= 1
        lon_col -= 1
        used_lat = lats[lat_row]
        used_lon = lons[lon_col]

        lat_error = abs(t_lat - used_lat)
        lon_error = abs(t_lon - used_lon)

        if lat_error > abs(lats[1] - lats[0]) or lon_error > abs(lons[1] -
                                                                 lons[0]):
            print "Source position :", (t_lat, t_lon)
            print "Calculated position:", (used_lat, used_lon)
            raise ValueError(
                "Rounding error: latitude or longitude was off by more than their resolution"
            )

        U = U[0, 0, :, lat_row, lon_col]
        V = V[0, 0, :, lat_row, lon_col]
        GZ = GZ[0, 0, 1::2, lat_row, lon_col]
        i = 0
        while i < len(GZ):
            if GZ[i] < time.height:
                h1 = GZ[i]
                h2 = GZ[i - 1]
                break
            i += 1
        else:
            h1 = h2 = time.height
        u1 = U[i] / 3.6
        u2 = U[i - 1] / 3.6
        v1 = V[i] / 3.6
        v2 = V[i - 1] / 3.6
        u = numpy.interp(time.height, [h1, h2], [u1, u2])
        v = numpy.interp(time.height, [h1, h2], [v1, v2])
        u_interp.append(u)
        v_interp.append(v)
    u, v = map(
        lambda x: numpy.interp(decimal_hour, [hour_minus, hour_plus], x),
        [u_interp, v_interp])
    return PST.Wind((u, v), time.height)
示例#12
0
def get_gem(time, interp=True, print_time=False):
    """Reads 6h GEM data"""
    print "\nGathering GEM data..."

    t_lon = time.lon % 360
    t_lat = time.lat
    year = time.year
    month = time.month
    day = time.day
    hour = time.hour
    minute = time.minute
    step = _gem_step
    time_step = 3600 * step

    decimal_hour = hour + minute / 60.

    hour_minus = int(6 * (time.decimaltime // 6))
    delta_minus = dt.timedelta(hours=hour_minus - hour, minutes=-minute)
    hour_plus = hour_minus + 6
    delta_plus = dt.timedelta(hours=hour_plus - hour, minutes=-minute)

    time_minus = time + delta_minus
    time_plus = time + delta_plus

    file_minus = time_minus.strftime(_gem_fmt)
    file_plus = time_plus.strftime(_gem_fmt)

    time_closest = time.round(time_step)
    file_closest = time_closest.strftime(_gem_fmt)

    if interp:
        files = [file_minus, file_plus]
        hours = [hour_minus, hour_plus]
    else:
        files = [file_closest]
        hours = [decimal_hour]

    u_interp = []
    v_interp = []

    if print_time:
        print "Time {0}; using files {1}".format(time.datetimeobj, files)

    for fname in files:
        if not os.path.exists(fname):
            raise IOError("File does not exists {0}".format(fname))

        try:
            rpn = fstd.open(fname)
        except IOError:
            raise IOError("Could not open {0} for GEM data\n".format(fname))

        try:
            U = rpn.UU
            V = rpn.VV
            Heights = rpn.GZ
            lats = U.lat[:]
            lons = U.lon[:]
        except AttributeError:
            raise AttributeError(
                "Could not read U, V, GZ, lat, and lon from file {0}".format(
                    fname))
        lat_row = 0
        lon_col = 0
        while lats[lat_row] < t_lat:
            lat_row += 1
        while lons[lon_col] < t_lon:
            lon_col += 1
        lat_row -= 1
        lon_col -= 1
        used_lat = lats[lat_row]
        used_lon = lons[lon_col]
        lat_error = t_lat - used_lat
        lon_error = t_lon - used_lon
        if lat_error > 1. or lon_error > 1.:
            raise ValueError("Position error: ({0},{1})".format(
                lat_error, lon_error))

        U = U[0, 0, :, lat_row, lon_col]
        V = V[0, 0, :, lat_row, lon_col]
        Heights = Heights[0, 0, :, lat_row, lon_col]
        stack = time.height
        H_list = list(Heights)
        i = 0
        while i < len(H_list) and H_list[i] > stack:
            i += 1
        i = min(i, len(H_list) - 1)
        h1 = H_list[i]
        h2 = H_list[i - 1]
        u1 = U[i] / 3.6
        u2 = U[i - 1] / 3.6
        v1 = V[i] / 3.6
        v2 = V[i - 1] / 3.6
        u = numpy.interp(stack, [h1, h2], [u1, u2])
        v = numpy.interp(stack, [h1, h2], [v1, v2])
        u_interp.append(u)
        v_interp.append(v)
    u = numpy.interp(decimal_hour, hours, u_interp)
    v = numpy.interp(decimal_hour, hours, v_interp)
    return PST.Wind((u, v), stack)
示例#13
0
def get_ecmwf_highres(time, surface=True):
    """Reads 0.3 degree ECMWF data"""
    print "Gathering ECMWF data..."
    source = time.source

    step = 1
    year = time.year
    month = time.month
    day = time.day
    hour = time.hour
    minute = time.minute

    hour_minus = hour
    hour_plus = hour + 1
    time_minus = PST.Time(dt.datetime(year, month, day, hour_minus, 0), source)
    time_plus = PST.Time(time_minus.datetimeobj + dt.timedelta(hours=1),
                         source)

    file_minus = time_minus.strftime(_ecmwfh_fmt)
    file_plus = time_plus.strftime(_ecmwfh_fmt)

    u_time_interpolate = []
    v_time_interpolate = []
    u_surface_interp = []
    v_surface_interp = []
    cloud_fractions = []

    lat_resolution, lon_resolution = (0.3, 0.3)

    for file_name in [file_minus, file_plus]:
        if not os.path.isfile(file_name):
            raise IOError('File {0} does not exist'.format(file_name))

        try:
            ecmwf = pygrib.open(file_name)
        except IOError:
            print("IOError: Could not open file {0}".format(file_name))
            raise
        except Excption as exc:
            print "Unexpected error occured"
            raise exc

        try:
            UU = ecmwf.select(name='U component of wind')
            VV = ecmwf.select(name='V component of wind')
            P_surface = ecmwf.select(name='Surface pressure')[0]
            P_surface.expand_grid(0)
            P_surface = numpy.reshape(P_surface.values[:55800], (150, 372))
            cloud_fraction = ecmwf.select(name="Total cloud cover")[0].values
            U_surface = ecmwf.select(
                name="10 metre U wind component")[0].values[:-1]
            V_surface = ecmwf.select(
                name="10 metre V wind component")[0].values[:-1]
        except AttributeError:
            raise AttributeError(
                "File {0} is missing some expected attributes".format(
                    file_name))

        # This data is missing part of the last row ( 33.3 to 33.0 degrees latitude for -109.8 to -64.8 degrees longitude)
        # so ignore the last row of the data (last 151 values)
        U = []
        V = []
        levels = len(UU)
        half_levels = levels + 1
        try:
            for level in range(len(UU)):
                UU[level].expand_grid(0)
                U.append(numpy.reshape((UU[level].values)[:55800], (150, 372)))

                VV[level].expand_grid(0)
                V.append(numpy.reshape((VV[level].values)[:55800], (150, 372)))

            latitudes = numpy.reshape(UU[0].latitudes[:55800], (150, 372))[:,
                                                                           0]
            longitudes = numpy.reshape(UU[0].longitudes[:55800],
                                       (150, 372))[0] - 360.
        except:
            raise IndexError(
                "Problem reshaping U, V, lat, lon arrays in file {0}".format(
                    file_name))

        U = numpy.array(U)
        V = numpy.array(V)

        A = UU[0].pv[:half_levels]
        A /= 100.
        B = VV[0].pv[half_levels:]

        lat_row = int((latitudes[0] - time.lat) // lat_resolution)
        lon_col = int((time.lon - longitudes[0]) // lon_resolution)

        lat_check = abs(latitudes[lat_row] - time.lat) < 0.3
        lon_check = abs(longitudes[lon_col] - time.lon) < 0.3

        coord = (latitudes[lat_row], longitudes[lon_col])

        if not lat_check or not lon_check:
            raise ValueError(
                "Latitude and longitude points are further than the resolution away from the source. Source position ({0},{1}); calculated as ({2},{3})"
                .format(time.lat, time.lon, latitudes[lat_row],
                        longitudes[lon_col]))

        p0 = P_surface[lat_row, lon_col]
        H = 7000.
        z_prev = 0.
        k = 0
        while k < (levels - 1):
            A0 = A[k]
            A1 = A[k + 1]
            A0 = A[k]
            A1 = A[k + 1]
            B0 = B[k]
            B1 = B[k + 1]
            Pk0 = A0 + B0 * p0
            Pk1 = A1 + B1 * p0
            Pk = 0.5 * (Pk0 + Pk1)
            z = H * math.log(p0 / Pk)
            if z < time.height:
                h1 = z
                h2 = z_prev
                break
            k += 1
            z_prev = z
        else:
            h1 = h2 = time.height
        u1 = U[k][lat_row, lon_col]
        u2 = U[k - 1][lat_row, lon_col]
        v1 = V[k][lat_row, lon_col]
        v2 = V[k - 1][lat_row, lon_col]
        u_surface = U_surface[lat_row, lon_col]
        v_surface = V_surface[lat_row, lon_col]

        u_interpolated = numpy.interp(time.height, [h1, h2], [u1, u2])
        v_interpolated = numpy.interp(time.height, [h1, h2], [v1, v2])

        u_time_interpolate.append(u_interpolated)
        v_time_interpolate.append(v_interpolated)
        u_surface_interp.append(u_surface)
        v_surface_interp.append(v_surface)

        cloud_fractions.append(cloud_fraction[lat_row, lon_col])

    u = numpy.interp(time.decimaltime, [hour_minus, hour_plus],
                     u_time_interpolate)
    v = numpy.interp(time.decimaltime, [hour_minus, hour_plus],
                     v_time_interpolate)

    # print u,v
    u_surface = numpy.interp(time.decimaltime, [hour_minus, hour_plus],
                             u_surface_interp)
    v_surface = numpy.interp(time.decimaltime, [hour_minus, hour_plus],
                             v_surface_interp)

    surface_wind = PST.Wind(
        (u_surface, v_surface), 10.) if surface else PST.Wind(
            (u, v), time.height)
    wind = PST.Wind((u, v), time.height)

    if surface:
        stability = PST.Stability(surface_wind.speed, min(cloud_fractions))
    else:
        stability = PST.Stability(wind.speed, min(cloud_fractions))
    return (wind, stability)
示例#14
0
            pro = node.probability_vector[current_tag]
        else:
            pro = 0
        if pro < μ_min:  # 判断为非法序列
            count_intrusion += 1
        print(pro)
    if (count_intrusion / N) > lambda_min:
        print("该序列:%s异常!" % sequence, "异常度:", count_intrusion / N)
    else:
        print("该序列:%s正常。" % sequence, "异常度:", count_intrusion / N)


if __name__ == '__main__':
    txt = 'pst_data.txt'
    pkl = 'pst_result.pkl'
    tree = PST.gen_tree(txt)
    PST.draw_pst(tree)
    sequence_list = ["abcab", "bcabbbca"]
    print(
        '----------------------------------计算异常度-------------------------------'
    )
    for sequence in sequence_list:
        detection_sequence(sequence, tree)

    # test operatecode mapping
    source_code_list = ['login', 'trans1', 'trans2', 'trans3', 'logout']
    ope_code_dict = Properties("opecode_mapper.properties").getProperties()
    str_transed = ""
    for source_code in source_code_list:
        if ope_code_dict.get(source_code):
            str_transed += ope_code_dict.get(source_code)