Example #1
0
    def __init__(self, filename):

        self.dataset = cdf.Dataset(filename)
        self.variables = self.dataset.variables  # For brevity

        # Get the variable of interest (e.g. mean temperature,
        # precipitation etc) from the variables OrderedDict
        var_name = [k for k in self.dataset.variables][3]
        # The main variable of the dataset (e.g. mean temperature)
        self.variable = self.variables[var_name]

        # Get the start and end date of the time series (as datetime objects):
        self.startdate = cdf.num2date(
            self.variables['time'][0],
            units=self.variables['time'].units,
            calendar=self.variables['time'].calendar,
        )
        self.enddate = cdf.num2date(
            self.variables['time'][-1],
            units=self.variables['time'].units,
            calendar=self.variables['time'].calendar,
        )

        # Calc number of days in time series
        self.ndays = (
            self.enddate - self.startdate + datetime.timedelta(1)).days
        assert(self.ndays == len(self.variables['time']))

        # Grid size in degrees
        self.gridsize = abs(
            (self.variables['latitude'][0] - self.variables['latitude'][1]))
        # Calculate the minimum latitude and longitude of the lower left (SW)
        # corner of the grid
        self.minlat = self.variables['latitude'][0] - (self.gridsize / 2.0)
        self.minlon = self.variables['longitude'][0] - (self.gridsize / 2.0)
def gen_data(time, times, maskedArray):
    timeUnits = getUnits(time)
    start = None
    if timeUnits:
        start = (netCDF.num2date(times[0], time.units, calendar="standard")).isoformat()
    else:
        start = "".join(times[0])

    # =========================================================================
    # if np.isnan(max) or np.isnan(min) or np.isnan(std) or np.isnan(mean) or np.isnan(median):
    #   output = {}
    #   g.graphError = "no valid data available to use"
    # else:
    #   output['global'] = {'mean': mean, 'median': median,'std': std, 'min': min, 'max': max, 'time': start}
    # =========================================================================

    output = {}
    output["data"] = {}
    data = []
    # print len(time)
    for i, row in enumerate(maskedArray):
        # print i
        if timeUnits:
            if i < len(time):
                date = netCDF.num2date(time[i], time.units, calendar="standard").isoformat()
        else:
            date = "".join(times[i])
        mean = getMean(row)

        if np.isnan(mean):
            pass
        else:
            data.append(mean)
    return data
Example #3
0
    def runTest(self):
        # Get the real dates
        # skip this until cftime pull request #55 is in a released
        # version (1.0.1?). Otherwise, fix for issue #808 breaks this
        if parse_version(cftime.__version__) >= parse_version('1.0.1'):
            dates = []
            for file in self.files:
                f = Dataset(file)
                t = f.variables['time']
                dates.extend(num2date(t[:], t.units, t.calendar))
                f.close()

        # Compare with the MF dates
        f = MFDataset(self.files,check=True)
        t = f.variables['time']
        mfdates = num2date(t[:], t.units, t.calendar)

        T = MFTime(t)
        assert_equal(len(T), len(t))
        assert_equal(T.shape, t.shape)
        assert_equal(T.dimensions, t.dimensions)
        assert_equal(T.typecode(), t.typecode())
        # skip this until cftime pull request #55 is in a released
        # version (1.0.1?). Otherwise, fix for issue #808 breaks this
        if parse_version(cftime.__version__) >= parse_version('1.0.1'):
            assert_array_equal(num2date(T[:], T.units, T.calendar), dates)
        assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366)
        f.close()
Example #4
0
def getdates(f):
    """ Returns the years from a filename and directory path

    Parameters
    ----------
    string : name of file including path

    Returns
    -------
    string of start year
    string of end year
    """
    nc = Dataset(f, 'r')
    time = nc.variables['time_bnds'][:].squeeze()
    nc_time = nc.variables['time']
    try:
        cal = nc_time.calendar
    except:
        cal = 'standard'
    start = nc_time[:][0]
    end = nc_time[:][-1]
    start = num2date(start, nc_time.units, cal)
    end = num2date(end, nc_time.units, cal)
    start = start.year
    end = end.year
    return start, end
Example #5
0
   def IsAdiacent(self,Test) :
      #print stc,ttc
      if self.ClimatologicalField :
         import netCDF4
         import datetime
         #print self.TimeCells,Test.TimeCells
         stc=netCDF4.num2date(self.TimeCells,units='hours since 1900-01-01 00:00:00',calendar='standard')
         ttc=netCDF4.num2date(Test.TimeCells,units='hours since 1900-01-01 00:00:00',calendar='standard')
         print >>sys.stderr, 'WARNING 11 : leap year for climatology...'
         #print 'stc',stc
         #print 'ttc',ttc
         #if (stc[0].year == ttc[0].year and stc[1].year == ttc[1].year ) : 
            #stc[1].year=ttc[0].year

         if stc[-1][1].month == 2 and stc[-1][1].day == 29 : il_day=28
         else : il_day=stc[-1][1].day
         nstc=datetime.datetime(ttc[0][0].year,stc[-1][1].month,il_day,stc[-1][1].hour,stc[-1][1].minute)
         #print nstc,stc[0][0].year,ttc[-1][1].month,ttc[-1][1].day

         if ttc[-1][1].month == 2 and ttc[-1][1].day == 29 : il_day=28
         else : il_day=ttc[-1][1].day
         nttc=datetime.datetime(stc[0][0].year,ttc[-1][1].month,il_day,ttc[-1][1].hour,ttc[-1][1].minute)

         #print 'nstc',nstc
         #print 'nttc',nttc
#         if stc[0][0].year <= ttc[0][0].year and ( nstc==ttc[0][0] or nttc==stc[0][0] ) :
         if ( nstc==ttc[0][0] and stc[0][0].year <= ttc[0][0].year ) or ( nttc==stc[0][0] and ttc[0][0].year <= stc[0][0].year ) :
         #if self.TimeCells[0]+Test.TimeCells[1]-self.TimeCells[0]==Test.TimeCells[0] :
            #print 'vero',stc[0][0].year,ttc[0][0].year
            return True
      else :
         if self.TimeCells[-1] == Test.TimeCells[0] or self.TimeCells[0] == Test.TimeCells[1] :
            return True
      return False 
Example #6
0
def get_monthly_time_slices(ncvar_time):
    '''
    Based on an input NetCDF4 time variable returns calendar appropriate monthly slices
    '''
    assert 'calendar' in ncvar_time.ncattrs(), "Time variable does not have a defined calendar"
    cal = ncvar_time.calendar

    assert 'units' in ncvar_time.ncattrs(), "Time variable must have 'unit' attribute"
    units = ncvar_time.units

    assert len(ncvar_time.dimensions) == 1, "Time varaible must be single dimension"

    slices = []

    d_start = num2date(ncvar_time[0], units, cal)
    current_month = d_start.month
    t_start = 0
    for i, val in enumerate(ncvar_time):
        d = num2date(val, units, cal)
        if d.month != current_month:
            slices.append(slice(t_start, i))
            t_start = i
            current_month = d.month
    slices.append(slice(t_start, i+1))

    return slices
Example #7
0
    def get_time_from_dim(cls, time_var):
        """Get min/max from a NetCDF time variable and convert to datetime"""
        ndim = len(time_var.shape)
        if ndim == 0:
            ret_val = time_var.item()
            res = ret_val, ret_val
        elif ndim == 1:
            # NetCDF Users' Guide states that when time is a coordinate variable,
            # it should be monotonically increasing or decreasing with no
            # repeated variables. Therefore, first and last elements for a
            # vector should correspond to start and end time or end and start
            # time respectively. See Section 2.3.1 of the NUG
            res = time_var[0], time_var[-1]
        else:
            # FIXME: handle multidimensional time variables.  Perhaps
            # take the first and last element of time variable in the first
            # dimension and then take the min and max of the resulting values
            return None, None

        # if not > 1d, return the min and max elements found
        min_elem, max_elem = np.min(res), np.max(res)
        if hasattr(time_var, 'calendar'):
            num2date([min_elem, max_elem], time_var.units,
                      time_var.calendar)
            return num2date([min_elem, max_elem], time_var.units,
                            time_var.calendar)
        else:
            return num2date([min_elem, max_elem], time_var.units)
Example #8
0
def doflow_fb(first_frame, second_frame, winSize = (5,5), filter_len = 10, sig_min = 150, n_iter = 40, levels = 1):
    im0 = copy.deepcopy(first_frame.fields['IR_filt']['data'])
    im0[np.where(im0 < sig_min)] = sig_min
    im1 = copy.deepcopy(second_frame.fields['IR_filt']['data'])
    im1[np.where(im1 < sig_min)] = sig_min
    sim0 = (im0 - im0.min())*(im0.max()/(im0.max()-im0.min()))
    sim1 = (im1 - im1.min())*(im1.max()/(im1.max()-im1.min()))
    
    u, v = get_optic_flow_fb(sim0[0], 
                          sim1[0],
                          winSize = winSize[0], n_iter=n_iter, levels=levels)
    t1 = netCDF4.num2date(second_frame.axes['time']['data'][0], units = second_frame.axes['time']['units'])
    t0 = netCDF4.num2date(first_frame.axes['time']['data'][0], units = first_frame.axes['time']['units'])
    dt = (t1-t0).seconds
    dx = np.expand_dims(np.gradient(second_frame.fields['x']['data'])[1], 0)
    dy = np.expand_dims(np.gradient(second_frame.fields['y']['data'])[0], 0)
    u_fld = {'data' : dt * ndimage.median_filter(u.reshape([1,u.shape[0], u.shape[1]]),filter_len)/dx,
                                        'units' :'pixels',
                                        'standard_name' : 'disp',
                                        'long name' : 'todo'}

    v_fld = {'data' : dt * ndimage.median_filter( v.reshape([1,v.shape[0], v.shape[1]]),filter_len)/dy,
                                        'units' :'pixels',
                                        'standard_name' : 'disp',
                                        'long name' : 'todo'}

    return u_fld, v_fld
Example #9
0
def are_time_axis_the_same(filenames):
    # print "inside get times func"
    # print filenames
    times = {}
    for key in filenames:
        # print filenames[key]
        times[key] = getCoordinateVariable(netCDF.Dataset(filenames[key], "r+"), "Time")

    keys = times.keys()

    # if (len(times[keys[0]]) != len(times[keys[1]]) ):
    #   pass
    # return False

    # else:
    time_range = len(times[keys[0]]) if len(times[keys[0]]) > len(times[keys[1]]) else len(times[keys[1]])
    # print "using range %d" % time_range
    # print len(times[keys[0]])
    # print len(times[keys[1]])
    for x in range(time_range):
        time1 = datetime.datetime.strptime(
            netCDF.num2date(times[keys[0]][x], times[keys[0]].units, calendar="standard").isoformat(),
            "%Y-%m-%dT%H:%M:%S",
        )
        time2 = datetime.datetime.strptime(
            netCDF.num2date(times[keys[1]][x], times[keys[1]].units, calendar="standard").isoformat(),
            "%Y-%m-%dT%H:%M:%S",
        )
        # print time1, time2
        # print times[keys[0]][x] , times[keys[1]][x]
        dif = time1 - time2
        # print dif
        if dif > timedelta.min:
            return False
    return True
Example #10
0
    def _init_fields(self, nc_dataset):
        nc_vars = nc_dataset.variables
        lons = nc_vars["lon"][:]
        lats = nc_vars["lat"][:]


        if lons.ndim == 1:
            lats2d, lons2d = np.meshgrid(lats, lons)
        elif lons.ndim == 2:
            lats2d, lons2d = lats, lons
        else:
            raise NotImplementedError("Cannot handle {}-dimensional coordinates".format(lons.ndim))


        self.lons2d, self.lats2d = lons2d, lats2d

        self.times_var = nc_vars["time"]
        self.times_num = nc_vars["time"][:]

        if hasattr(self.times_var, "calendar"):
            self.times = num2date(self.times_num, self.times_var.units, self.times_var.calendar)
        else:
            self.times = num2date(self.times_num, self.times_var.units)


        if not self.lazy:

            self.var_data = nc_vars[self.var_name][:]
            if nc_vars[self.var_name].shape[1:] != self.lons2d.shape:
                print("nc_vars[self.var_name].shape = {}".format(nc_vars[self.var_name].shape))
                self.var_data = np.transpose(self.var_data, axes=[0, 2, 1])


        x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
        self.kdtree = cKDTree(list(zip(x_in, y_in, z_in)))
Example #11
0
def get_pasap_plot_title(dset,
    varname = 'hr24_prcp',
    timestep= 0,
    ):
    """ Given an open pydap object, and some extra information, return a nice
        plot title.
    """
    header = "PASAP: Dynamical Seasonal Outlooks for the Pacific."
    subheader1 = "Outlook based on POAMA 1.5 CGCM adjusted for historical skill"
    subheader2 = "Experimental outlook for demonstration and research only"
    time_var = dset['time']

    if 'units' in time_var.attributes.keys():
        time_units = time_var.attributes['units']
    else:
        time_units = ''
    if 'units' in dset[varname].attributes.keys():
        units = dset[varname].attributes['units']
    else:
        units = ''
    valid_time = datetime.datetime.strftime(
        num2date(time_var[timestep],time_units),"%Y%m%d")
    start_date = datetime.datetime.strftime(
        num2date(dset['init_date'][0],time_units),"%Y%m%d")

    period_label = str(dset['time_label'][timestep])
    titlestring = header + '\n' \
                  + subheader1 + '\n'  \
                  + subheader2 + '\n'  \
                  + "Variable: " + varname + ' (' + units + ')' + '\n' \
                  + 'Model initialised ' + start_date + '\n' \
                  # + 'Forecast period: ' + period_label 

    return titlestring
Example #12
0
def read_nc(infile, varname, dimension=-1, is_time=0):
	'''Read a variable from a netCDF file

	Input:
		input file path
		variable name
		dimension: if < 0, read in all dimensions of the variable; if >= 0, only read in the [dimension]th of the variable (index starts from 0). For example, if the first dimension of the variable is time, and if dimension=2, then only reads in the 3rd time step.
		is_time: if the desired variable is time (1 for time; 0 for not time). If it is time, return an array of datetime object

	Return:
		var: a numpy array of
	''' 
	from netCDF4 import Dataset
	from netCDF4 import num2date

	nc = Dataset(infile, 'r')
	if is_time==0:  # if not time variable
		if dimension<0:
			var = nc.variables[varname][:]
		else:
			var = nc.variables[varname][dimension]
	if is_time==1:  # if time variable
		time = nc.variables[varname]
		if hasattr(time, 'calendar'):  # if time variable has 'calendar' attribute
			if dimension<0:
				var = num2date(time[:], time.units, time.calendar)
			else:
				var = num2date(time[dimension], time.units, time.calendar)
		else:  # if time variable does not have 'calendar' attribute
			if dimension<0:
				var = num2date(time[:], time.units)
			else:
				var = num2date(time[dimension], time.units)
	nc.close()
	return var
Example #13
0
def show_tbounds(t):
    
    print 'Start date: ', num2date(t[0],t.units)
    try:
        print 'End date: ', num2date(t[-1],t.units)
    except IndexError:
        print num2date(t[0],t.units)
Example #14
0
def get_event(id):
    '''
    Object response for the GET(id) request.  This response is NOT cached.
    '''
    try:
        #set up all the contaners.
        data = {}
        asset_id = ""
        #create uframe instance, and fetch the data.
        uframe_obj = UFrameEventsCollection()
        payload = uframe_obj.to_json(id)
        data = payload.json()
        if payload.status_code != 200:
            return  jsonify({ "events" : payload.json()}), payload.status_code

        try:
            data['class'] = data.pop('@class')
            data['startDate'] = num2date(float(data['startDate'])/1000, units='seconds since 1970-01-01 00:00:00', calendar='gregorian')
            data['endDate'] = num2date(float(data['endDate'])/1000, units='seconds since 1970-01-01 00:00:00', calendar='gregorian')
        except (KeyError, TypeError):
            pass
        return jsonify(**data)

    except requests.exceptions.ConnectionError as e:
        error = "Error: Cannot connect to uframe.  %s" % e
        print error
        return make_response(error, 500)
Example #15
0
    def check_time_extents(self, ds):
        """
        Check that the values of time_coverage_start/time_coverage_end approximately match the data.
        """
        if not (hasattr(ds, 'time_coverage_start') and hasattr(ds, 'time_coverage_end')):
            return

        # Parse the ISO 8601 formatted dates
        try:
            t_min = dateparse(ds.time_coverage_start)
            t_max = dateparse(ds.time_coverage_end)
        except:
            return Result(BaseCheck.MEDIUM,
                          False,
                          'time_coverage_extents_match',
                          ['time_coverage attributes are not formatted properly. Use the ISO 8601:2004 date format, preferably the extended format.'])

        timevar = cfutil.get_time_variable(ds)

        if not timevar:
            return Result(BaseCheck.MEDIUM,
                          False,
                          'time_coverage_extents_match',
                          ['Could not find time variable to test extent of time_coverage_start/time_coverage_end, see CF-1.6 spec chapter 4.4'])

        # Time should be monotonically increasing, so we make that assumption here so we don't have to download THE ENTIRE ARRAY
        try:
            # num2date returns as naive date, but with time adjusted to UTC
            # we need to attach timezone information here, or the date
            # subtraction from t_min/t_max will assume that a naive timestamp is
            # in the same time zone and cause erroneous results.
            # Pendulum uses UTC by default, but we are being explicit here
            time0 = pendulum.instance(num2date(ds.variables[timevar][0],
                                      ds.variables[timevar].units), 'UTC')
            time1 = pendulum.instance(num2date(ds.variables[timevar][-1],
                                      ds.variables[timevar].units), 'UTC')
        except:
            return Result(BaseCheck.MEDIUM,
                          False,
                          'time_coverage_extents_match',
                          ['Failed to retrieve and convert times for variables %s.' % timevar])

        start_dt = abs(time0 - t_min)
        end_dt = abs(time1 - t_max)

        score = 2
        msgs = []
        if start_dt > timedelta(hours=1):
            msgs.append("Date time mismatch between time_coverage_start and actual "
                        "time values %s (time_coverage_start) != %s (time[0])" % (t_min.isoformat(), time0.isoformat()))
            score -= 1
        if end_dt > timedelta(hours=1):
            msgs.append("Date time mismatch between time_coverage_end and actual "
                        "time values %s (time_coverage_end) != %s (time[N])" % (t_max.isoformat(), time1.isoformat()))
            score -= 1

        return Result(BaseCheck.MEDIUM,
                      (score, 2),
                      'time_coverage_extents_match',
                      msgs)
Example #16
0
def _get_old_hiwrap_time(fname, ncFile, Good_Indices):
    """
    Pull the time from HIWRAP file and convert to AWOT useable.
    The time structure is odd here (to me) and is in
    seconds since last Sunday.
    The assumption that the data is the 4th 'field' in the filename
    is required to make this work.
    """
    # Pull out the date, convert the date to a datetime friendly string
    # Adds dashes between year, month, and day
    yyyymmdd = fname.split("_")[3]

    # Find the date for Sunday previous and
    # check this (should be = 6 for Sunday)
    startday = int(yyyymmdd[6:8]) - int(divmod(
        ncFile.variables['time'][0], 24 * 3600)[0])
    if datetime.date(ncFile.variables['year'][:],
                     int(yyyymmdd[4:6]), startday).weekday() != 6:
        print("Time could be incorrect, check file to see if time units "
              "are 'computer time (sec from last Sunday at 12 am)'")

    StartDate = yyyymmdd[0:4] + '-' + yyyymmdd[4:6] + '-' + str(startday)

    # Create the time array
    # Now convert the time array into a datetime instance
    dtHrs = num2date(ncFile.variables['time'][
        Good_Indices], 'seconds since ' + StartDate + '00:00:00+0:00')
    # Now convert this datetime instance into a number of seconds since Epoch
    TimeSec = date2num(dtHrs, common.EPOCH_UNITS)
    # Now once again convert this data into a datetime instance
    Time_unaware = num2date(TimeSec, common.EPOCH_UNITS)
    Time = {'data': Time_unaware, 'units': common.EPOCH_UNITS,
            'title': 'Time', 'full_name': 'Time (UTC)'}
    return Time
def plot_filtered_timeseries(pressure, pressure_units, pressure_time, time_units,
                             pressure_filtered, delay, figdir, n1, dayrissaga):
    # Very simple plot to show the evolution of the pressure
    hfmt = dates.DateFormatter('%d %B')
    time2plot = netCDF4.num2date(pressure_time, time_units)
    time2plot_filter = netCDF4.num2date(pressure_time - delay, time_units)
    fig = plt.figure(num=None, figsize=(14, 6))
    ax = fig.add_subplot(111)
    plt.plot(time2plot, pressure, 'k', lw=0.5, label='Raw signal')
    plt.plot(time2plot_filter[n1:], pressure_filtered[n1:], 'c', linewidth=2, zorder=2, label='Filtered signal')
    plt.axvline(x=dayrissaga, linewidth=3, color='r', alpha=0.5)
    plt.xlabel('Time')
    plt.ylabel(("Pressure\n (%s)" % (pressure_units)), ha='right', rotation=0)
    plt.legend()
    ax.xaxis.set_major_locator(dates.DayLocator())
    ax.xaxis.set_major_formatter(hfmt)
    plt.grid()
    plt.savefig(os.path.join(figdir, 'SantAntoni_timeseries_' + dayrissaga.strftime('%Y%m%d')))
    plt.close()

    fig = plt.figure(num=None, figsize=(14, 6))
    ax = fig.add_subplot(111)
    plt.plot(time2plot[n1 / 2:-n1 / 2], pressure[n1 / 2:-n1 / 2] - pressure_filtered[n1:], 'k', lw=0.5)
    plt.axvline(x=dayrissaga, linewidth=3, color='r', alpha=0.5)
    plt.xlabel('Time')
    plt.ylabel(("Pressure anomaly\n (%s)" % pressure_units), ha='right', rotation=0)
    ax.set_xlim(time2plot[0], time2plot[-1])
    ax.xaxis.set_major_locator(dates.DayLocator())
    ax.xaxis.set_major_formatter(hfmt)
    fig.autofmt_xdate()
    plt.grid()
    plt.savefig(os.path.join(figdir, 'SantAntoni_anomalies_' + dayrissaga.strftime('%Y%m%d')))
    plt.close()
def rcs_model(winlen, modfile):
	from grid_tools import trim_time_jandec
	from netCDF4 import num2date
	from netCDF4 import date2num
	from scipy import ndimage
	from netcdf_tools import ncextractall
	from convert import mmd_mmm

	#Extract the model data and clip to the required start and end months
	modnc = ncextractall(modfile)
	mdata = modnc['pr']
	mdata = mdata*86400. #convert to same units as obs
	mlon = modnc['lon']
	mlat = modnc['lat']
	mtime = modnc['time']
	
	time_u = modnc['time_units']
	if 'time_calendar' in modnc.keys(): 
	     cal = modnc['time_calendar']
	     mtime = num2date(mtime,units = time_u, calendar=cal)
	else: mtime = num2date(mtime,units = time_u)
	
	mdata, mtime = trim_time_jandec(mdata, mtime)
	
	mdata = mmd_mmm(mdata)
	mdata = ndimage.filters.uniform_filter(mdata,size=[winlen,1,1])
	
		#Trim first or last values if required as they are unrepresentative
	trim = int(winlen/2)
	mdata = mdata[trim:,:,:]
	if winlen % 2 == 0: trim = trim - 1
	mdata = mdata[:-trim,:,:]
	
	return, mdata, mlat, mlon
Example #19
0
def setupROMSfiles(loc, date, ff, tout, time_units, tstride=1):
    """
    setupROMSfiles()
    Kristen Thyng, March 2013

    Figures out necessary files to read in for track times and what
    model output indices within those files to use.

    Args:
        loc: File location. loc can be a thredds server web address, a single
         string of a file location, a list of strings of multiple file
         locations to be searched through.
        date: datetime format start date
        ff: Time direction. ff=1 forward, ff=-1 backward
        tout: Number of model outputs to use
        time_units: To convert to datetime
        tstride: Stride in time, in case want to use less model output than
         is available. Default is 1, using all output.

    Returns:
        * nc - NetCDF object for relevant files
        * tinds - Indices of outputs to use from fname files
    """

    # For thredds server where all information is available in one place
    # or for a single file
    if 'http' in loc or type(loc) == str:
        nc = netCDF.Dataset(loc)

    # This is for the case when we have a bunch of files to sort through
    else:
        # the globbing should happen ahead of time so this case looks
        # different than the single file case
        # files in fname are in chronological order
        nc = netCDF.MFDataset(loc)

    # Convert date to number
    # dates = netCDF.num2date(nc.variables['ocean_time'][:], time_units)
    # The calendar definition extends dates to before the year 1582 for use
    # with idealized simulations without meaningful dates.
    if 'time' in nc.variables:
        dates = netCDF.num2date(nc.variables['time'][:], time_units,
                                calendar='proleptic_gregorian')
    elif 'ocean_time' in nc.variables:
        dates = netCDF.num2date(nc.variables['ocean_time'][:], time_units,
                                calendar='proleptic_gregorian')
    # time index with time value just below date (relative to file ifile)
    istart = find(dates <= date)[-1]

    # Select indices
    if ff == 1:
        # indices of model outputs desired
        tinds = range(istart, istart+tout, tstride)
    else:  # backward in time
        # have to shift istart since there are now new indices behind since
        # going backward
        tinds = range(istart, istart-tout, -tstride)

    return nc, tinds
Example #20
0
def get_date(filename):
    try:
        radar = pyart.io.read(filename)
        t = (num2date(radar.time['data'][0], radar.time['units']), num2date(radar.time['data'][-1], radar.time['units']))
    except:
        #return start time and end time being the same
        t = (num2date(0, 'seconds since 2000-01-01T00:00:00Z'), num2date(0, 'seconds since 2000-01-01T00:00:00Z'))
    return t
Example #21
0
def datetime_from_grid(grid, epoch=False):
    """ Return a datetime for the volume start in a Grid. """
    if epoch:
        dtrad = num2date(grid.time['data'][0], grid.time['units'])
        epnum = date2num(dtrad, EPOCH_UNITS)
        return num2date(epnum, EPOCH_UNITS)
    else:
        return num2date(grid.time['data'][0], grid.time['units'])
Example #22
0
def start_stop(nc,tvar):
    ncv = nc.variables
    time_var = ncv[tvar]
    first = netCDF4.num2date(time_var[0],time_var.units)
    last = netCDF4.num2date(time_var[-1],time_var.units)

    print first.strftime('%Y-%b-%d %H:%M')
    print last.strftime('%Y-%b-%d %H:%M')
Example #23
0
def datetimes_from_radar(radar, epoch=False):
    """ Return an array of datetimes for the rays in a Radar. """
    if epoch:
        dtrad = num2date(radar.time['data'][:], radar.time['units'])
        epnum = date2num(dtrad, EPOCH_UNITS)
        return num2date(epnum, EPOCH_UNITS)
    else:
        return num2date(radar.time['data'][:], radar.time['units'])
Example #24
0
def cf_time(units):

	try:
		netCDF4.num2date(0, units)
	except:
		return False
	else:
		return True
Example #25
0
 def gettimebounds(self, var=None, **kwargs):
     assert var in self.nc.variables
     time = self.gettimevar(var)
     if "units" in kwargs:
         bounds = (netCDF4.num2date(np.min(time), units=u), netCDF4.num2date(np.max(time), units=u))
     else:
         bounds = (np.min(time.dates), np.max(time.dates))
     return bounds
Example #26
0
 def time(self):
     if self.time_units is not None:
         dt = num2date(self.dataset[self.dimensions['time']][:],
                       self.time_units, self.calendar)
         dt -= num2date(0, self.time_units, self.calendar)
         return list(map(timedelta.total_seconds, dt))
     else:
         return self.dataset[self.dimensions['time']][:]
Example #27
0
def show_ncfile_tbounds(filename,tvar='time'):
    
    t = Dataset(filename).variables[tvar]
    print('Start date: ', num2date(t[0],t.units))
    try:
        print('End date: ', num2date(t[-1],t.units))
    except IndexError:
        print(num2date(t[0],t.units))
Example #28
0
def datetime_from_radar(radar, epoch=False):
    """ Return a datetime for the first ray in a Radar. """
    if epoch:
        dtrad = num2date(radar.time['data'][0], radar.time['units'])
        epnum = date2num(dtrad, EPOCH_UNITS)
        return num2date(epnum, EPOCH_UNITS)
    else:
        return num2date(radar.time['data'][0], radar.time['units'])
Example #29
0
def PrintMetaRecord(myin,myinm) :
   print myin
   #print myinm.StandardName
   #print myinm.created
   #print myinm.celltype
   #print myinm.LonCells[0],myinm.LatCells[0],'-',myinm.LonCells[-1],myinm.LatCells[-1]
   t_start=netCDF4.num2date(myinm.TimeCells[0],units='hours since 1900-01-01 00:00:00',calendar='standard')
   t_end=netCDF4.num2date(myinm.TimeCells[1],units='hours since 1900-01-01 00:00:00',calendar='standard')
   print t_start,'-',t_end
Example #30
0
 def time2year(self, t):
     """
     convert time to year
     """
     time = self.ds.variables["time"]
     if type(t) == np.int:
         return num2date(t, time.units).year
     else:
         return np.asarray([y.year for y in np.asarray(num2date(t, time.units))])
def storm_motion_deltas_algorithm(REFlev, REFlev1, big_storm, zero_z_trigger, storm_to_track, year, month, day, hour, start_min, duration, calibration, station, Bunkers_s, Bunkers_m, track_dis=10, GR_mins=1.0):
    #Set vector perpendicular to FFD Z gradient
    #storm_relative_dir = storm_relative_dir
    #Set storm motion
    Bunkers_m = Bunkers_m
    #Set ZDR Threshold for outlining arcs
    #zdrlev = [zdrlev]
    #Set KDP Threshold for finding KDP feet
    #kdplev = [kdplev]
    #Set reflectivity thresholds for storm tracking algorithm
    REFlev = [REFlev]
    REFlev1 = [REFlev1]
    #Set storm size threshold that triggers subdivision of big storms
    big_storm = big_storm #km^2
    Outer_r = 30 #km
    Inner_r = 6 #km
    #Set trigger to ignore strangely-formatted files right before 00Z
    #Pre-SAILS #: 17
    #SAILS #: 25
    zero_z_trigger = zero_z_trigger
    storm_to_track = storm_to_track
    zdr_outlines = []
    #Here, set the initial time of the archived radar loop you want.
    #Our specified time
    dt = datetime(year,month, day, hour, start_min)
    station = station
    end_dt = dt + timedelta(hours=duration)

    #Set up nexrad interface
    conn = nexradaws.NexradAwsInterface()
    scans = conn.get_avail_scans_in_range(dt,end_dt,station)
    results = conn.download(scans, 'RadarFolder')

    #Setting counters for figures and Pandas indices
    f = 27
    n = 1
    storm_index = 0
    scan_index = 0
    tracking_index = 0
    #Create geod object for later distance and area calculations
    g = Geod(ellps='sphere')
    #Open the placefile
    f = open("DELTA_dev"+station+str(dt.year)+str(dt.month)+str(dt.day)+str(dt.hour)+str(dt.minute)+"_Placefile.txt", "w+")
    f.write("Title: Storm Motion Deltas Placefile \n")
    f.write("Refresh: 8 \n \n")

    #Load ML algorithm
#    forest_loaded = pickle.load(open('BestRandomForest.pkl', 'rb'))
#    forest_loaded_col = pickle.load(open('BestRandomForestColumnsLEN200.pkl', 'rb'))

    #Actual algorithm code starts here
    #Create a list for the lists of arc outlines
    zdr_out_list = []
    tracks_dataframe = []
    for i,scan in enumerate(results.iter_success(),start=1):
    #Local file option:
        #Loop over all files in the dataset and pull out each 0.5 degree tilt for analysis
        try:
            radar1 = scan.open_pyart()
        except:
            print('bad radar file')
            continue
        #Local file option
        print('File Reading')
        #Make sure the file isn't a strange format
        if radar1.nsweeps > zero_z_trigger:
            continue
            
        for i in range(radar1.nsweeps):
            print('in loop')
            print(radar1.nsweeps)
            try:
                radar4 = radar1.extract_sweeps([i])
            except:
                print('bad file')
            #Checking to make sure the tilt in question has all needed data and is the right elevation
            if ((np.mean(radar4.elevation['data']) < .65) and (np.max(np.asarray(radar4.fields['differential_reflectivity']['data'])) != np.min(np.asarray(radar4.fields['differential_reflectivity']['data'])))):
                n = n+1

                #Calling ungridded_section; Pulling apart radar sweeps and creating ungridded data arrays
                [radar,n,range_2d,ungrid_lons,ungrid_lats] = quality_control_arcalg(radar4,n,calibration)

                time_start = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
                object_number=0.0
                month = time_start.month
                if month < 10:
                    month = '0'+str(month)
                hour = time_start.hour
                if hour < 10:
                    hour = '0'+str(hour)
                minute = time_start.minute
                if minute < 10:
                    minute = '0'+str(minute)
                day = time_start.day
                if day < 10:
                    day = '0'+str(day)
                time_beg = time_start - timedelta(minutes=0.1)
                time_end = time_start + timedelta(minutes=GR_mins)
                sec_beg = time_beg.second
                sec_end = time_end.second
                min_beg = time_beg.minute
                min_end = time_end.minute
                h_beg = time_beg.hour
                h_end = time_end.hour
                d_beg = time_beg.day
                d_end = time_end.day
                if sec_beg < 10:
                    sec_beg = '0'+str(sec_beg)
                if sec_end < 10:
                    sec_end = '0'+str(sec_end)
                if min_beg < 10:
                    min_beg = '0'+str(min_beg)
                if min_end < 10:
                    min_end = '0'+str(min_end)
                if h_beg < 10:
                    h_beg = '0'+str(h_beg)
                if h_end < 10:
                    h_end = '0'+str(h_end)
                if d_beg < 10:
                    d_beg = '0'+str(d_beg)
                if d_end < 10:
                    d_end = '0'+str(d_end)

                #Calling kdp_section; Using NWS method, creating ungridded, smoothed KDP field
                kdp_nwsdict = kdp_genesis(radar)

                #Add field to radar
                radar.add_field('KDP', kdp_nwsdict)
                kdp_ungridded_nws = radar.fields['KDP']['data']


                #Calling grid_section; Now let's grid the data on a ~250 m x 250 m grid
                [REF,KDP,CC,ZDRmasked1,REFmasked,KDPmasked,rlons,rlats,rlons_2d,rlats_2d,cenlat,cenlon] = gridding_arcalg(radar)

                #Calling gradient_section; Determining gradient direction and masking some Zhh and Zdr grid fields
                #[grad_mag,grad_ffd,ZDRmasked] = grad_mask_arcalg(REFmasked,REF,storm_relative_dir,ZDRmasked1,CC)

                #Let's create a field for inferred hail
                #Commenting out for the moment
#                 REF_Hail = np.copy(REFmasked)
#                 REF_Hail1 = ma.masked_where(ZDRmasked1 > 1.0, REF_Hail)
#                 REF_Hail2 = ma.masked_where(CC > 1.0, REF_Hail1)
#                 REF_Hail2 = ma.filled(REF_Hail2, fill_value = 1)

                #Let's set up the map projection!
                crs = ccrs.LambertConformal(central_longitude=-100.0, central_latitude=45.0)

                #Set up our array of latitude and longitude values and transform our data to the desired projection.
                tlatlons = crs.transform_points(ccrs.LambertConformal(central_longitude=265, central_latitude=25, standard_parallels=(25.,25.)),rlons[0,:,:],rlats[0,:,:])
                tlons = tlatlons[:,:,0]
                tlats = tlatlons[:,:,1]

                #Limit the extent of the map area, must convert to proper coords.
                LL = (cenlon-1.0,cenlat-1.0,ccrs.PlateCarree())
                UR = (cenlon+1.0,cenlat+1.0,ccrs.PlateCarree())
                print(LL)

                #Get data to plot state and province boundaries
                states_provinces = cfeature.NaturalEarthFeature(
                        category='cultural',
                        name='admin_1_states_provinces_lakes',
                        scale='50m',
                        facecolor='none')
                #Make sure these shapefiles are in the same directory as the script
                #fname = 'cb_2016_us_county_20m/cb_2016_us_county_20m.shp'
                #fname2 = 'cb_2016_us_state_20m/cb_2016_us_state_20m.shp'
                #counties = ShapelyFeature(Reader(fname).geometries(),ccrs.PlateCarree(), facecolor = 'none', edgecolor = 'black')
                #states = ShapelyFeature(Reader(fname2).geometries(),ccrs.PlateCarree(), facecolor = 'none', edgecolor = 'black')

                #Create a figure and plot up the initial data and contours for the algorithm
                fig=plt.figure(n,figsize=(30.,25.))
                ax = plt.subplot(111,projection=ccrs.PlateCarree())
                ax.coastlines('50m',edgecolor='black',linewidth=0.75)
                #ax.add_feature(counties, edgecolor = 'black', linewidth = 0.5)
                #ax.add_feature(states, edgecolor = 'black', linewidth = 1.5)
                ax.set_extent([LL[0],UR[0],LL[1],UR[1]])
                REFlevels = np.arange(20,73,2)

                #Options for Z backgrounds/contours
                #refp = ax.pcolormesh(ungrid_lons, ungrid_lats, ref_c, cmap=plt.cm.gist_ncar, vmin = 10, vmax = 73)
                #refp = ax.pcolormesh(ungrid_lons, ungrid_lats, ref_ungridded_base, cmap='HomeyerRainbow', vmin = 10, vmax = 73)
                #refp = ax.pcolormesh(rlons_2d, rlats_2d, REFrmasked, cmap=pyart.graph.cm_colorblind.HomeyerRainbow, vmin = 10, vmax = 73)
                refp2 = ax.contour(rlons_2d, rlats_2d, REFmasked, [40], colors='grey', linewidths=5, zorder=1)
                #refp3 = ax.contour(rlons_2d, rlats_2d, REFmasked, [45], color='r')
                #plt.contourf(rlons_2d, rlats_2d, ZDR_sum_stuff, depth_levels, cmap=plt.cm.viridis)

                #Option to have a ZDR background instead of Z:
                #zdrp = ax.pcolormesh(ungrid_lons, ungrid_lats, zdr_c, cmap=plt.cm.nipy_spectral, vmin = -2, vmax = 6)

                #Storm tracking algorithm starts here
                #Reflectivity smoothed for storm tracker
                smoothed_ref = ndi.gaussian_filter(REFmasked, sigma = 3, order = 0)
                #1st Z contour plotted
                refc = ax.contour(rlons[0,:,:],rlats[0,:,:],smoothed_ref,REFlev, alpha=.01)

                #Set up projection for area calculations
                proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
                           pyproj.Proj(init='epsg:3857'))

                #Main part of storm tracking algorithm starts by looping through all contours looking for Z centroids
                #This method for breaking contours into polygons based on this stack overflow tutorial:
                #https://gis.stackexchange.com/questions/99917/converting-matplotlib-contour-objects-to-shapely-objects
                #Calling stormid_section
                [storm_ids,max_lons_c,max_lats_c,ref_areas,storm_index, alg_speeds, alg_directions] = storm_objects_new(refc,proj,REFlev,REFlev1,big_storm,smoothed_ref,ax,rlons,rlats,storm_index,tracking_index,scan_index,tracks_dataframe, track_dis, time_start)

                #Setup tracking index for storm of interest
                tracking_ind=np.where(np.asarray(storm_ids)==storm_to_track)[0]
                max_lons_c = np.asarray(max_lons_c)
                max_lats_c = np.asarray(max_lats_c)
                ref_areas = np.asarray(ref_areas)
                #Create the ZDR and KDP contours which will later be broken into polygons
#                 if np.max(ZDRmasked) > zdrlev:
#                     zdrc = ax.contour(rlons[0,:,:],rlats[0,:,:],ZDRmasked,zdrlev,linewidths = 2, colors='purple', alpha = .5)
#                 else:
#                     zdrc=[]
#                 if np.max(KDPmasked) > kdplev:
#                     kdpc = ax.contour(rlons[0,:,:],rlats[0,:,:],KDPmasked,kdplev,linewidths = 2, colors='green', alpha = 0.01)
#                 else:
#                     kdpc=[]
#                 if np.max(REF_Hail2) > 50.0:
#                     hailc = ax.contour(rlons[0,:,:],rlats[0,:,:],REF_Hail2,[50],linewidths = 4, colors='pink', alpha = 0.01)
#                 else:
#                     hailc=[]
#                 if np.max(REFmasked) > 35.0:
#                     zhhc = ax.contour(rlons[0,:,:],rlats[0,:,:],REFmasked,[35.0],linewidths = 3,colors='orange', alpha = 0.8)
#                 else:
#                     zhhc=[]
                plt.contour(ungrid_lons, ungrid_lats, range_2d, [73000], linewidths=7, colors='r')
                plt.savefig('testfig.png')
                print('Testfig Saved')

                if len(max_lons_c) > 0:
                    #Calling zdr_arc_section; Create ZDR arc objects using a similar method as employed in making the storm objects
#                     [zdr_storm_lon,zdr_storm_lat,zdr_dist,zdr_forw,zdr_back,zdr_areas,zdr_centroid_lon,zdr_centroid_lat,zdr_mean,zdr_cc_mean,zdr_max,zdr_masks,zdr_outlines,ax,f] = zdrarc(zdrc,ZDRmasked,CC,REF,grad_ffd,grad_mag,KDP,forest_loaded,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,zdrlev,proj,storm_relative_dir,Outer_r,Inner_r,tracking_ind)


                    #Calling hail_section; Identify Hail core objects in a similar way to the ZDR arc objects
#                     [hail_areas,hail_centroid_lon,hail_centroid_lat,hail_storm_lon,hail_storm_lat,ax,f] = hail_objects(hailc,REF_Hail2,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,proj)


                    #Calling zhh_section; Identify 35dBz storm area in a similar way to the ZDR arc objects
#                     [zhh_areas,zhh_centroid_lon,zhh_centroid_lat,zhh_storm_lon,zhh_storm_lat,zhh_max,zhh_core_avg] = zhh_objects(zhhc,REFmasked,rlons,rlats,max_lons_c,max_lats_c,proj)


#                     #Calling kdpfoot_section; Identify KDP foot objects in a similar way to the ZDR arc objects
#                     [kdp_areas,kdp_centroid_lon,kdp_centroid_lat,kdp_storm_lon,kdp_storm_lat,kdp_max,ax,f] = kdp_objects(kdpc,KDPmasked,ax,f,time_start,month,d_beg,h_beg,min_beg,sec_beg,d_end,h_end,min_end,sec_end,rlons,rlats,max_lons_c,max_lats_c,kdplev,proj)


                    #Consolidating the arc objects associated with each storm:
#                     zdr_areas_arr = np.zeros((len(zdr_areas)))
#                     zdr_max_arr = np.zeros((len(zdr_max)))
#                     zdr_mean_arr = np.zeros((len(zdr_mean)))                    
#                     for i in range(len(zdr_areas)):
#                         zdr_areas_arr[i] = zdr_areas[i].magnitude
#                         zdr_max_arr[i] = zdr_max[i]
#                         zdr_mean_arr[i] = zdr_mean[i]
#                     zdr_centroid_lons = np.asarray(zdr_centroid_lon)
#                     zdr_centroid_lats = np.asarray(zdr_centroid_lat)
#                     zdr_con_areas = []
#                     zdr_con_maxes = []
#                     zdr_con_means = []
#                     zdr_con_centroid_lon = []
#                     zdr_con_centroid_lat = []
#                     zdr_con_max_lon = []
#                     zdr_con_max_lat = []
#                     zdr_con_storm_lon = []
#                     zdr_con_storm_lat = []
#                     zdr_con_masks = []
#                     zdr_con_dev = []
#                     zdr_con_10max = []
#                     zdr_con_mode = []
#                     zdr_con_median = []
#                     zdr_masks = np.asarray(zdr_masks)

#                     #Consolidate KDP objects as well
#                     kdp_areas_arr = np.zeros((len(kdp_areas)))
#                     kdp_max_arr = np.zeros((len(kdp_max)))
#                     for i in range(len(kdp_areas)):
#                         kdp_areas_arr[i] = kdp_areas[i].magnitude
#                         kdp_max_arr[i] = kdp_max[i]
#                     kdp_centroid_lons = np.asarray(kdp_centroid_lon)
#                     kdp_centroid_lats = np.asarray(kdp_centroid_lat)
#                     kdp_con_areas = []
#                     kdp_con_maxes = []
#                     kdp_con_centroid_lon = []
#                     kdp_con_centroid_lat = []
#                     kdp_con_max_lon = []
#                     kdp_con_max_lat = []
#                     kdp_con_storm_lon = []
#                     kdp_con_storm_lat = []

                    #Consolidate Hail objects as well
#                     hail_areas_arr = np.zeros((len(hail_areas)))
#                     for i in range(len(hail_areas)):
#                         hail_areas_arr[i] = hail_areas[i].magnitude
#                     hail_centroid_lons = np.asarray(hail_centroid_lon)
#                     hail_centroid_lats = np.asarray(hail_centroid_lat)
#                     hail_con_areas = []
#                     hail_con_centroid_lon = []
#                     hail_con_centroid_lat = []
#                     hail_con_storm_lon = []
#                     hail_con_storm_lat = []

                    #Consolidate Zhh objects as well
#                     zhh_areas_arr = np.zeros((len(zhh_areas)))
#                     zhh_max_arr = np.zeros((len(zhh_max)))
#                     zhh_core_avg_arr = np.zeros((len(zhh_core_avg)))
#                     for i in range(len(zhh_areas)):
#                         zhh_areas_arr[i] = zhh_areas[i].magnitude
#                         zhh_max_arr[i] = zhh_max[i]
#                         zhh_core_avg_arr[i] = zhh_core_avg[i]
#                     zhh_centroid_lons = np.asarray(zhh_centroid_lon)
#                     zhh_centroid_lats = np.asarray(zhh_centroid_lat)
#                     zhh_con_areas = []
#                     zhh_con_maxes = []
#                     zhh_con_core_avg = []
#                     zhh_con_centroid_lon = []
#                     zhh_con_centroid_lat = []
#                     zhh_con_max_lon = []
#                     zhh_con_max_lat = []
#                     zhh_con_storm_lon = []
#                     zhh_con_storm_lat = []

#                     for i in enumerate(max_lons_c):
#                         try:
#                             #Find the arc objects associated with this storm:
#                             zdr_objects_lons = zdr_centroid_lons[np.where(zdr_storm_lon == max_lons_c[i[0]])]
#                             zdr_objects_lats = zdr_centroid_lats[np.where(zdr_storm_lon == max_lons_c[i[0]])]

#                             #Get the sum of their areas
#                             zdr_con_areas.append(np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             #print("consolidated area", np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             zdr_con_maxes.append(np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             #print("consolidated max", np.max(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             zdr_con_means.append(np.mean(zdr_mean_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             #print("consolidated mean", np.mean(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             zdr_con_max_lon.append(rlons_2d[np.where(ZDRmasked==np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))])
#                             zdr_con_max_lat.append(rlats_2d[np.where(ZDRmasked==np.max(zdr_max_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))])

#                             #Find the actual centroids
#                             weighted_lons = zdr_objects_lons * zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]
#                             zdr_con_centroid_lon.append(np.sum(weighted_lons) / np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             weighted_lats = zdr_objects_lats * zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]
#                             zdr_con_centroid_lat.append(np.sum(weighted_lats) / np.sum(zdr_areas_arr[np.where(zdr_storm_lon == max_lons_c[i[0]])]))
#                             zdr_con_storm_lon.append(max_lons_c[i[0]])
#                             zdr_con_storm_lat.append(max_lats_c[i[0]])
#                             zdr_con_masks.append(np.sum(zdr_masks[np.where(zdr_storm_lon == max_lons_c[i[0]])],axis=0, dtype=bool))
#                             mask_con = np.sum(zdr_masks[np.where(zdr_storm_lon == max_lons_c[i[0]])], axis=0, dtype=bool)
#                             zdr_con_dev.append(np.std(ZDRmasked[mask_con]))
#                             ZDRsorted = np.sort(ZDRmasked[mask_con])[::-1]
#                             zdr_con_10max.append(np.mean(ZDRsorted[0:10]))
#                             zdr_con_mode.append(stats.mode(ZDRmasked[mask_con]))
#                             zdr_con_median.append(np.median(ZDRmasked[mask_con]))
#                         except:
#                             zdr_con_maxes.append(0)
#                             zdr_con_means.append(0)
#                             zdr_con_centroid_lon.append(0)
#                             zdr_con_centroid_lat.append(0)
#                             zdr_con_max_lon.append(0)
#                             zdr_con_max_lat.append(0)
#                             zdr_con_storm_lon.append(max_lons_c[i[0]])
#                             zdr_con_storm_lat.append(max_lats_c[i[0]])
#                             zdr_con_masks.append(0)
#                             zdr_con_dev.append(0)
#                             zdr_con_10max.append(0)
#                             zdr_con_mode.append(0)
#                             zdr_con_median.append(0)

#                         try:
#                             #Find the kdp objects associated with this storm:
#                             kdp_objects_lons = kdp_centroid_lons[np.where(kdp_storm_lon == max_lons_c[i[0]])]
#                             kdp_objects_lats = kdp_centroid_lats[np.where(kdp_storm_lon == max_lons_c[i[0]])]

#                             #Get the sum of their areas
#                             kdp_con_areas.append(np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
#                             kdp_con_maxes.append(np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
#                             kdp_con_max_lon.append(rlons_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
#                             kdp_con_max_lat.append(rlats_2d[np.where(KDPmasked==np.max(kdp_max_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))])
#                             #Find the actual centroids
#                             weighted_lons_kdp = kdp_objects_lons * kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]
#                             kdp_con_centroid_lon.append(np.sum(weighted_lons_kdp) / np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
#                             weighted_lats_kdp = kdp_objects_lats * kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]
#                             kdp_con_centroid_lat.append(np.sum(weighted_lats_kdp) / np.sum(kdp_areas_arr[np.where(kdp_storm_lon == max_lons_c[i[0]])]))
#                             kdp_con_storm_lon.append(max_lons_c[i[0]])
#                             kdp_con_storm_lat.append(max_lats_c[i[0]])
#                         except:
#                             kdp_con_maxes.append(0)
#                             kdp_con_max_lon.append(0)
#                             kdp_con_max_lat.append(0)
#                             kdp_con_centroid_lon.append(0)
#                             kdp_con_centroid_lat.append(0)
#                             kdp_con_storm_lon.append(0)
#                             kdp_con_storm_lat.append(0)

#                         try:
#                             #Find the hail core objects associated with this storm:
#                             hail_objects_lons = hail_centroid_lons[np.where(hail_storm_lon == max_lons_c[i[0]])]
#                             hail_objects_lats = hail_centroid_lats[np.where(hail_storm_lon == max_lons_c[i[0]])]
#                             #Get the sum of their areas
#                             hail_con_areas.append(np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
#                             #Find the actual centroids
#                             weighted_lons_hail = hail_objects_lons * hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]
#                             hail_con_centroid_lon.append(np.sum(weighted_lons_hail) / np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
#                             weighted_lats_hail = hail_objects_lats * hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]
#                             hail_con_centroid_lat.append(np.sum(weighted_lats_hail) / np.sum(hail_areas_arr[np.where(hail_storm_lon == max_lons_c[i[0]])]))
#                             hail_con_storm_lon.append(max_lons_c[i[0]])
#                             hail_con_storm_lat.append(max_lats_c[i[0]])
#                         except:
#                             hail_con_centroid_lon.append(0)
#                             hail_con_centroid_lat.append(0)
#                             hail_con_storm_lon.append(0)
#                             hail_con_storm_lat.append(0)

#                         try:
#                             #Find the zhh objects associated with this storm:
#                             zhh_objects_lons = zhh_centroid_lons[np.where(zhh_storm_lon == max_lons_c[i[0]])]
#                             zhh_objects_lats = zhh_centroid_lats[np.where(zhh_storm_lon == max_lons_c[i[0]])]
#                             #Get the sum of their areas
#                             zhh_con_areas.append(np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
#                             zhh_con_maxes.append(np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
#                             zhh_con_core_avg.append(np.max(zhh_core_avg_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
#                             zhh_con_max_lon.append(rlons_2d[np.where(REFmasked==np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))])
#                             zhh_con_max_lat.append(rlats_2d[np.where(REFmasked==np.max(zhh_max_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))])
#                             #Find the actual centroids
#                             weighted_lons_zhh = zhh_objects_lons * zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]
#                             zhh_con_centroid_lon.append(np.sum(weighted_lons_zhh) / np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
#                             weighted_lats_zhh = zhh_objects_lats * zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]
#                             zhh_con_centroid_lat.append(np.sum(weighted_lats_zhh) / np.sum(zhh_areas_arr[np.where(zhh_storm_lon == max_lons_c[i[0]])]))
#                             zhh_con_storm_lon.append(max_lons_c[i[0]])
#                             zhh_con_storm_lat.append(max_lats_c[i[0]])
#                         except:
#                             zhh_con_maxes.append(0)
#                             zhh_con_core_avg.append(0)
#                             zhh_con_max_lon.append(0)
#                             zhh_con_max_lat.append(0)
#                             zhh_con_centroid_lon.append(0)
#                             zhh_con_centroid_lat.append(0)
#                             zhh_con_storm_lon.append(0)
#                             zhh_con_storm_lat.append(0)

                        #Calculate KDP-ZDR separation
        #             kdp_con_centroid_lons1 = np.asarray(kdp_con_centroid_lon)
        #             kdp_con_centroid_lats1 = np.asarray(kdp_con_centroid_lat)
        #             zdr_con_centroid_lons1 = np.asarray(zdr_con_centroid_lon)
        #             zdr_con_centroid_lats1 = np.asarray(zdr_con_centroid_lat)
        #             #Eliminate consolidated arcs smaller than a specified area
        #             area = 2 #km*2
        #             zdr_con_areas_arr = np.asarray(zdr_con_areas)
        #             zdr_con_centroid_lats = zdr_con_centroid_lats1[zdr_con_areas_arr > area]
        #             zdr_con_centroid_lons = zdr_con_centroid_lons1[zdr_con_areas_arr > area]
        #             kdp_con_centroid_lats = kdp_con_centroid_lats1[zdr_con_areas_arr > area]
        #             kdp_con_centroid_lons = kdp_con_centroid_lons1[zdr_con_areas_arr > area]
        #             zdr_con_max_lons1 = np.asarray(zdr_con_max_lon)[zdr_con_areas_arr > area]
        #             zdr_con_max_lats1 = np.asarray(zdr_con_max_lat)[zdr_con_areas_arr > area]
        #             kdp_con_max_lons1 = np.asarray(kdp_con_max_lon)[zdr_con_areas_arr > area]
        #             kdp_con_max_lats1 = np.asarray(kdp_con_max_lat)[zdr_con_areas_arr > area]
        #             zdr_con_max1 = np.asarray(zdr_con_maxes)[zdr_con_areas_arr > area]
        #             zdr_con_areas1 = zdr_con_areas_arr[zdr_con_areas_arr > area]
#                     kdp_con_centroid_lat = np.asarray(kdp_con_centroid_lat)
#                     kdp_con_centroid_lon = np.asarray(kdp_con_centroid_lon)
#                     zdr_con_centroid_lat = np.asarray(zdr_con_centroid_lat)
#                     zdr_con_centroid_lon = np.asarray(zdr_con_centroid_lon)

#                     kdp_inds = np.where(kdp_con_centroid_lat*zdr_con_centroid_lat > 0)
#                     distance_kdp_zdr = g.inv(kdp_con_centroid_lon[kdp_inds], kdp_con_centroid_lat[kdp_inds], zdr_con_centroid_lon[kdp_inds], zdr_con_centroid_lat[kdp_inds])
#                     dist_kdp_zdr = distance_kdp_zdr[2] / 1000.
#                     #Now make an array for the distances which will have the same shape as the lats to prevent errors
#                     shaped_dist = np.zeros((np.shape(zdr_con_areas)))
#                     shaped_dist[kdp_inds] = dist_kdp_zdr

#                     #Get separation angle for KDP-ZDR centroids
#                     back_k = distance_kdp_zdr[1]
#                     for i in range(back_k.shape[0]):
#                         if distance_kdp_zdr[1][i] < 0:
#                             back_k[i] = distance_kdp_zdr[1][i] + 360

#                     forw_k = np.abs(back_k - storm_relative_dir)
#                     rawangle_k = back_k - storm_relative_dir
#                     #Account for weird angles
#                     for i in range(back_k.shape[0]):
#                         if forw_k[i] > 180:
#                             forw_k[i] = 360 - forw_k[i]
#                             rawangle_k[i] = (360-forw_k[i])*(-1)

#                     rawangle_k = rawangle_k*(-1)

#                     #Now make an array for the distances which will have the same shape as the lats to prevent errors
#                     shaped_ang = np.zeros((np.shape(zdr_con_areas)))
#                     shaped_ang[kdp_inds] = rawangle_k
#                     shaped_ang = (180-np.abs(shaped_ang))*(shaped_ang/np.abs(shaped_ang))

#                     new_angle_all = shaped_ang + storm_relative_dir

#                     shaped_ang = (new_angle_all - Bunkers_m)* (-1)

#                     shaped_ang = 180 - shaped_ang

                    ###Now let's consolidate everything to fit the Pandas dataframe!
#                     p_zdr_areas = []
#                     p_zdr_maxes = []
#                     p_zdr_means = []
#                     p_zdr_devs = []
#                     p_zdr_10max = []
#                     p_zdr_mode = []
#                     p_zdr_median = []
#                    # p_hail_areas = []
#                     p_zhh_areas = []
#                     p_zhh_maxes = []
#                     p_zhh_core_avgs = []
#                     p_separations = []
#                     p_sp_angle = []
#                     for storm in enumerate(max_lons_c):
#                         matching_ind = np.flatnonzero(np.isclose(max_lons_c[storm[0]], zdr_con_storm_lon, rtol=1e-05))
#                         if matching_ind.shape[0] > 0:
#                             p_zdr_areas.append((zdr_con_areas[matching_ind[0]]))
#                             p_zdr_maxes.append((zdr_con_maxes[matching_ind[0]]))
#                             p_zdr_means.append((zdr_con_means[matching_ind[0]]))
#                             p_zdr_devs.append((zdr_con_dev[matching_ind[0]]))
#                             p_zdr_10max.append((zdr_con_10max[matching_ind[0]]))
#                             p_zdr_mode.append((zdr_con_mode[matching_ind[0]]))
#                             p_zdr_median.append((zdr_con_median[matching_ind[0]]))
#                             p_separations.append((shaped_dist[matching_ind[0]]))
#                             p_sp_angle.append((shaped_ang[matching_ind[0]]))
#                         else:
#                             p_zdr_areas.append((0))
#                             p_zdr_maxes.append((0))
#                             p_zdr_means.append((0))
#                             p_zdr_devs.append((0))
#                             p_zdr_10max.append((0))
#                             p_zdr_mode.append((0))
#                             p_zdr_median.append((0))
#                             p_separations.append((0))
#                             p_sp_angle.append((0))

#                         matching_ind_hail = np.flatnonzero(np.isclose(max_lons_c[storm[0]], hail_con_storm_lon, rtol=1e-05))
#                         if matching_ind_hail.shape[0] > 0:
#                             p_hail_areas.append((hail_con_areas[matching_ind_hail[0]]))
#                         else:
#                             p_hail_areas.append((0))

#                         matching_ind_zhh = np.flatnonzero(np.isclose(max_lons_c[storm[0]],zhh_con_storm_lon, rtol=1e-05))
#                         if matching_ind_zhh.shape[0] > 0:
#                             p_zhh_maxes.append((zhh_con_maxes[matching_ind_zhh[0]]))
#                             p_zhh_areas.append((zhh_con_areas[matching_ind_zhh[0]]))
#                             p_zhh_core_avgs.append((zhh_con_core_avg[matching_ind_zhh[0]]))
#                         else:
#                             p_zhh_areas.append((0))
#                             p_zhh_maxes.append((0))
#                             p_zhh_core_avgs.append((0))

                    #Now start plotting stuff!
#                     if np.asarray(zdr_centroid_lon).shape[0] > 0:
#                         ax.scatter(zdr_centroid_lon, zdr_centroid_lat, marker = '*', s = 100, color = 'black', zorder = 10, transform=ccrs.PlateCarree())
#                     if np.asarray(kdp_centroid_lon).shape[0] > 0:
#                         ax.scatter(kdp_centroid_lon, kdp_centroid_lat, marker = '^', s = 100, color = 'black', zorder = 10, transform=ccrs.PlateCarree())
                    #Uncomment to print all object areas
                    #for i in enumerate(zdr_areas):
                    #    plt.text(zdr_centroid_lon[i[0]]+.016, zdr_centroid_lat[i[0]]+.016, "%.2f km^2" %(zdr_areas[i[0]].magnitude), size = 23)
                        #plt.text(zdr_centroid_lon[i[0]]+.016, zdr_centroid_lat[i[0]]+.016, "%.2f km^2 / %.2f km / %.2f dB" %(zdr_areas[i[0]].magnitude, zdr_dist[i[0]], zdr_forw[i[0]]), size = 23)
                        #plt.annotate(zdr_areas[i[0]], (zdr_centroid_lon[i[0]],zdr_centroid_lat[i[0]]))
                    #ax.contourf(rlons[0,:,:],rlats[0,:,:],KDPmasked,KDPlevels1,linewide = .01, colors ='b', alpha = .5)
                    #plt.tight_layout()
                    #plt.savefig('ZDRarcannotated.png')
                    storm_times = []
                    for l in range(len(max_lons_c)):
                        storm_times.append((time_start))
                    tracking_index = tracking_index + 1
                    #Get storm motion deltas:
                    u_B, v_B = wind_components(Bunkers_s*units('m/s'), Bunkers_m*units('degree'))
                    u_alg, v_alg = wind_components(alg_speeds*units('m/s'), alg_directions*units('degree'))
                    print(u_B, v_B, 'Bunkers motion components')
                    print(u_alg, v_alg, 'Observed motion components')
                    u_diff = u_alg-u_B
                    v_diff = v_alg-v_B
                    motion_delta = np.sqrt(u_diff**2 + v_diff**2).magnitude
                #If there are no storms, set everything to empty arrays!
                else:
                    storm_ids = []
                    storm_ids = []
                    alg_speeds = []
                    alg_directions = []
                    motion_delta = []
                    max_lons_c = []
                    max_lats_c = []
                    storm_times = time_start
                #Now record all data in a Pandas dataframe.
                new_cells = pd.DataFrame({
                    'scan': scan_index,
                    'storm_id' : storm_ids,
                    'storm speed' : alg_speeds,
                    'storm_direction' : alg_directions,
                    'motion_deltas' : motion_delta,
                    'storm_id1' : storm_ids,
                    'storm_lon' : max_lons_c,
                    'storm_lat' : max_lats_c,
                    'times' : storm_times
                })
                new_cells.set_index(['scan', 'storm_id'], inplace=True)
                if scan_index == 0:
                    tracks_dataframe = new_cells
                else:
                    tracks_dataframe = tracks_dataframe.append(new_cells)
                n = n+1
                scan_index = scan_index + 1

                #Plot the consolidated stuff!
                #Write some text objects for the ZDR arc attributes to add to the placefile
                f.write('TimeRange: '+str(time_start.year)+'-'+str(month)+'-'+str(d_beg)+'T'+str(h_beg)+':'+str(min_beg)+':'+str(sec_beg)+'Z '+str(time_start.year)+'-'+str(month)+'-'+str(d_end)+'T'+str(h_end)+':'+str(min_end)+':'+str(sec_end)+'Z')
                f.write('\n')
                f.write("Color: 139 000 000 \n")
                f.write('Font: 1, 30, 1,"Arial" \n')
                for y in range(len(max_lats_c)):
                    #f.write('Text: '+str(max_lats_c[y])+','+str(max_lons_c[y])+', 1, "X"," Arc Area: '+str(p_zdr_areas[y])+'\\n Arc Mean: '+str(p_zdr_means[y])+'\\n KDP-ZDR Separation: '+str(p_separations[y])+'\\n Separation Angle: '+str(p_sp_angle[y])+'" \n')
                    f.write('Text: '+str(max_lats_c[y])+','+str(max_lons_c[y])+', 1, "X"," Storm Speed: %.2f m/s \\n Storm Direction: %.2f deg \\n Motion Delta: %.2f m/s \n' %(alg_speeds[y], alg_directions[y], motion_delta[y]))



                title_plot = plt.title(station+' Radar Reflectivity, ZDR, and KDP '+str(time_start.year)+'-'+str(time_start.month)+'-'+str(time_start.day)+
                                           ' '+str(hour)+':'+str(minute)+' UTC', size = 25)

#                 try:
#                     plt.plot([zdr_con_centroid_lon[kdp_inds], kdp_con_centroid_lon[kdp_inds]], [zdr_con_centroid_lat[kdp_inds],kdp_con_centroid_lat[kdp_inds]], color = 'k', linewidth = 5, transform=ccrs.PlateCarree())
#                 except:
#                     print('Separation Angle Failure')

                ref_centroid_lon = max_lons_c
                ref_centroid_lat = max_lats_c
                if len(max_lons_c) > 0:
                    ax.scatter(max_lons_c,max_lats_c, marker = "o", color = 'k', s = 500, alpha = .6)
                    for i in enumerate(ref_centroid_lon): 
                        plt.text(ref_centroid_lon[i[0]]+.016, ref_centroid_lat[i[0]]+.016, "storm_id: %.1f" %(storm_ids[i[0]]), size = 25)
                #Comment out this line if not plotting tornado tracks
                #plt.plot([start_torlons, end_torlons], [start_torlats, end_torlats], color = 'purple', linewidth = 5, transform=ccrs.PlateCarree())
                #Add legend stuff
                zdr_outline = mlines.Line2D([], [], color='blue', linewidth = 5, linestyle = 'solid', label='ZDR Arc Outline(Area/Max)')
                kdp_outline = mlines.Line2D([], [], color='green', linewidth = 5,linestyle = 'solid', label='"KDP Foot" Outline')
                separation_vector = mlines.Line2D([], [], color='black', linewidth = 5,linestyle = 'solid', label='KDP/ZDR Centroid Separation Vector (Red Text=Distance)')
                #tor_track = mlines.Line2D([], [], color='purple', linewidth = 5,linestyle = 'solid', label='Tornado Tracks')
                elevation = mlines.Line2D([], [], color='grey', linewidth = 5,linestyle = 'solid', label='Height AGL (m)')

                plt.legend(handles=[zdr_outline, kdp_outline, separation_vector, elevation], loc = 3, fontsize = 25)
                alt_levs = [1000, 2000]
                plt.savefig('Machine_Learning/DELTA_dev'+station+str(time_start.year)+str(time_start.month)+str(day)+str(hour)+str(minute)+'.png')
                print('Figure Saved')
                plt.close()
                zdr_out_list.append(zdr_outlines)
                #except:
                #    traceback.print_exc()
                #    continue
    f.close()
    plt.show()
    print('Fin')
    #export_csv = tracks_dataframe.to_csv(r'C:\Users\Nick\Downloads\tracksdataframe.csv',index=None,header=True)
    return tracks_dataframe
Example #32
0
def grid_displacement_pc(grid1, grid2, field, level, return_value='pixels'):
    """
    Calculate the grid displacement using phase correlation.

    See:
    http://en.wikipedia.org/wiki/Phase_correlation

    Implementation inspired by Christoph Gohlke:
    http://www.lfd.uci.edu/~gohlke/code/imreg.py.html

    Note that the grid must have the same dimensions in x and y and assumed to
    have constant spacing in these dimensions.

    Parameters
    ----------
    grid1, grid2 : Grid
        Py-ART Grid objects separated in time and square in x/y.
    field : string
        Field to calculate advection from. Field must be in both grid1
        and grid2.
    level : integer
        The vertical (z) level of the grid to use in the calculation.
    return_value : str, optional
        'pixels', 'distance' or 'velocity'. Distance in pixels (default)
        or meters or velocity vector in m/s.

    Returns
    -------
    displacement : two-tuple
         Calculated displacement in units of y and x. Value returned in
         integers if pixels, otherwise floats.

    """
    # create copies of the data
    field_data1 = grid1.fields[field]['data'][level].copy()
    field_data2 = grid2.fields[field]['data'][level].copy()

    # replace fill values with valid_min or minimum value in array
    if 'valid_min' in grid1.fields[field]:
        min_value1 = grid1.fields[field]['valid_min']
    else:
        min_value1 = field_data1.min()
    field_data1 = np.ma.filled(field_data1, min_value1)

    if 'valid_min' in grid2.fields[field]:
        min_value2 = grid2.fields[field]['valid_min']
    else:
        min_value2 = field_data2.min()
    field_data2 = np.ma.filled(field_data2, min_value2)

    # discrete fast fourier transformation and complex conjugation of field 2
    image1fft = np.fft.fft2(field_data1)
    image2fft = np.conjugate(np.fft.fft2(field_data2))

    # inverse fourier transformation of product -> equal to cross correlation
    imageccor = np.real(np.fft.ifft2((image1fft*image2fft)))

    # shift the zero-frequency component to the center of the spectrum
    imageccorshift = np.fft.fftshift(imageccor)

    # determine the distance of the maximum from the center
    # find the peak in the correlation
    row, col = field_data1.shape
    yshift, xshift = np.unravel_index(np.argmax(imageccorshift), (row, col))
    yshift -= int(row/2)
    xshift -= int(col/2)

    dx = grid1.x['data'][1] - grid1.x['data'][0]
    dy = grid1.y['data'][1] - grid1.y['data'][0]
    x_movement = xshift * dx
    y_movement = yshift * dy

    if return_value == 'pixels':
        displacement = (yshift, xshift)
    elif return_value == 'distance':
        displacement = (y_movement, x_movement)
    elif return_value == 'velocity':
        t1 = num2date(grid1.time['data'][0], grid1.time['units'])
        t2 = num2date(grid2.time['data'][0], grid2.time['units'])
        dt = (t2 - t1).total_seconds()
        u = x_movement/dt
        v = y_movement/dt
        displacement = (v, u)
    else:
        displacement = (yshift, xshift)
    return displacement
Example #33
0
def in_out_water(netCDFfile, var_name=None):

    out_file = []

    for fn in netCDFfile:
        ds = Dataset(fn, 'a')

        nc_vars = ds.variables
        to_add = []
        if var_name:
            to_add.append(var_name)
        else:
            for v in nc_vars:
                if "TIME" in nc_vars[v].dimensions:
                    #print (vars[v].dimensions)
                    if v != 'TIME':
                        to_add.append(v)
            # remove any anx variables from the list
            for v in nc_vars:
                if 'ancillary_variables' in nc_vars[v].ncattrs():
                    remove = nc_vars[v].getncattr('ancillary_variables').split(' ')
                    print("remove ", remove)
                    for r in remove:
                        to_add.remove(r)

        time_var = nc_vars["TIME"]
        time = num2date(time_var[:], units=time_var.units, calendar=time_var.calendar)

        time_deploy = parser.parse(ds.time_deployment_start, ignoretz=True)
        time_recovery = parser.parse(ds.time_deployment_end, ignoretz=True)

        print('file', fn)
        print('deployment time', time_deploy)

        print('var to add', to_add)

        # create a mask for the time range
        mask = (time <= time_deploy) | (time >= time_recovery)
        count = -1
        for v in to_add:
            if v in nc_vars:
                print("var", v, ' dimensions ', nc_vars[v].dimensions)

                ncVarOut = nc_vars[v + "_quality_control"]
                ncVarOut[mask] = 6

                # create a qc variable just for this test flags
                if v + "_quality_control_io" in ds.variables:
                    ncVarOut = ds.variables[v + "_quality_control_io"]
                    ncVarOut[:] = 0
                else:
                    ncVarOut = ds.createVariable(v + "_quality_control_io", "i1", nc_vars[v].dimensions, fill_value=99, zlib=True)  # fill_value=0 otherwise defaults to max
                    nc_vars[v].ancillary_variables = nc_vars[v].ancillary_variables + " " + v + "_quality_control_io"

                ncVarOut[:] = 0
                ncVarOut.long_name = "quality flag for " + nc_vars[v].long_name
                try:
                    ncVarOut.standard_name = nc_vars[v].standard_name + " status_flag"
                except AttributeError:
                    pass

                ncVarOut.quality_control_conventions = "IMOS standard flags"
                ncVarOut.flag_values = np.array([0, 1, 2, 3, 4, 6, 7, 9], dtype=np.int8)
                ncVarOut.flag_meanings = 'unknown good_data probably_good_data probably_bad_data bad_data not_deployed interpolated missing_value'
                ncVarOut.comment = 'data flagged not deployed (6) when out of water'

                ncVarOut[mask] = 6
                # calculate the number of points marked as bad_data
                marked = np.zeros_like(ncVarOut)
                marked[mask] = 1
                count = sum(marked)

        ds.file_version = "Level 1 - Quality Controlled Data"
        if count > 0:
            # update the history attribute
            try:
                hist = ds.history + "\n"
            except AttributeError:
                hist = ""

            ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + ' :  ' + ' marked ' + str(int(count)))

        ds.close()

        out_file.append(fn)

    return out_file
Example #34
0
 def test_cf_datetime(self):
     import netCDF4 as nc4
     for num_dates, units in [
         (np.arange(10), 'days since 2000-01-01'),
         (np.arange(10).reshape(2, 5), 'days since 2000-01-01'),
         (12300 + np.arange(5), 'hours since 1680-01-01 00:00:00'),
             # here we add a couple minor formatting errors to test
             # the robustness of the parsing algorithm.
         (12300 + np.arange(5), 'hour since 1680-01-01  00:00:00'),
         (12300 + np.arange(5), u'Hour  since 1680-01-01 00:00:00'),
         (12300 + np.arange(5), ' Hour  since  1680-01-01 00:00:00 '),
         (10, 'days since 2000-01-01'),
         ([10], 'daYs  since 2000-01-01'),
         ([[10]], 'days since 2000-01-01'),
         ([10, 10], 'days since 2000-01-01'),
         (np.array(10), 'days since 2000-01-01'),
         (0, 'days since 1000-01-01'),
         ([0], 'days since 1000-01-01'),
         ([[0]], 'days since 1000-01-01'),
         (np.arange(2), 'days since 1000-01-01'),
         (np.arange(0, 100000, 20000), 'days since 1900-01-01'),
         (17093352.0, 'hours since 1-1-1 00:00:0.0'),
         ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'),
         (0, 'milliseconds since 2000-01-01T00:00:00'),
         (0, 'microseconds since 2000-01-01T00:00:00'),
     ]:
         for calendar in ['standard', 'gregorian', 'proleptic_gregorian']:
             expected = _ensure_naive_tz(
                 nc4.num2date(num_dates, units, calendar))
             print(num_dates, units, calendar)
             with warnings.catch_warnings():
                 warnings.filterwarnings('ignore',
                                         'Unable to decode time axis')
                 actual = conventions.decode_cf_datetime(
                     num_dates, units, calendar)
             if (isinstance(actual, np.ndarray)
                     and np.issubdtype(actual.dtype, np.datetime64)):
                 # self.assertEqual(actual.dtype.kind, 'M')
                 # For some reason, numpy 1.8 does not compare ns precision
                 # datetime64 arrays as equal to arrays of datetime objects,
                 # but it works for us precision. Thus, convert to us
                 # precision for the actual array equal comparison...
                 actual_cmp = actual.astype('M8[us]')
             else:
                 actual_cmp = actual
             self.assertArrayEqual(expected, actual_cmp)
             encoded, _, _ = conventions.encode_cf_datetime(
                 actual, units, calendar)
             if '1-1-1' not in units:
                 # pandas parses this date very strangely, so the original
                 # units/encoding cannot be preserved in this case:
                 # (Pdb) pd.to_datetime('1-1-1 00:00:0.0')
                 # Timestamp('2001-01-01 00:00:00')
                 self.assertArrayEqual(num_dates, np.around(encoded, 1))
                 if (hasattr(num_dates, 'ndim') and num_dates.ndim == 1
                         and '1000' not in units):
                     # verify that wrapping with a pandas.Index works
                     # note that it *does not* currently work to even put
                     # non-datetime64 compatible dates into a pandas.Index :(
                     encoded, _, _ = conventions.encode_cf_datetime(
                         pd.Index(actual), units, calendar)
                     self.assertArrayEqual(num_dates, np.around(encoded, 1))
Example #35
0
def load_history(filename,
                 start_time=datetime(1, 1, 1),
                 end_time=datetime(1, 1, 1),
                 grid=None,
                 epoch=default_epoch,
                 url=_url,
                 load_data=False):
    """
    Download soda data and save into local file

    Parameters
    ----------
    filename: string
        name of output file
    start_time: datetime
        starting date to load soda data
    end_time: datetime
        ending date for loading soda data
    grid: seapy.model.grid, optional
        if specified, only load SODA data that covers the grid
    epoch: datetime, optional
        reference time for new file
    url: string, optional
        URL to load SODA data from
    load_data: bool, optional
        If true (default) actually load the data. If false, it
        displays the information needed to load the data using ncks

    Returns
    -------
    None
    """
    # Load the grid
    grid = asgrid(grid)

    # Open the soda data
    soda = netCDF4.Dataset(url)

    # Figure out the time records that are required
    soda_time = netCDF4.num2date(soda.variables["time"][:],
                                 soda.variables["time"].units)

    time_list = np.where(
        np.logical_and(soda_time >= start_time, soda_time <= end_time))
    if not any(time_list):
        raise Exception("Cannot find valid times")

    # Get the latitude and longitude ranges
    minlat = np.min(grid.lat_rho) - 0.5
    maxlat = np.max(grid.lat_rho) + 0.5
    minlon = np.min(grid.lon_rho) - 0.5
    maxlon = np.max(grid.lon_rho) + 0.5
    soda_lon = soda.variables["lon"][:]
    soda_lat = soda.variables["lat"][:]

    # Ensure same convention
    if not grid.east():
        soda_lon[soda_lon > 180] -= 360

    latlist = np.where(np.logical_and(soda_lat >= minlat, soda_lat <= maxlat))
    lonlist = np.where(np.logical_and(soda_lon >= minlon, soda_lon <= maxlon))
    if not np.any(latlist) or not np.any(lonlist):
        raise Exception("Bounds not found")

    # Build the history file
    if load_data:
        his = ncgen.create_zlevel(filename,
                                  len(latlist[0]),
                                  len(lonlist[0]),
                                  len(soda.variables["lev"][:]),
                                  epoch,
                                  "soda history from " + url,
                                  dims=1)

        # Write out the data
        his.variables["lat"][:] = soda_lat[latlist]
        his.variables["lon"][:] = soda_lon[lonlist]
        his.variables["depth"][:] = soda.variables["lev"]
        his.variables["time"][:] = netCDF4.date2num(
            soda_time[time_list], his.variables["time"].units)
    # Loop over the variables
    sodavars = {"ssh": 3, "u": 4, "v": 4, "temp": 4, "salt": 4}
    hisvars = {
        "ssh": "zeta",
        "u": "u",
        "v": "v",
        "temp": "temp",
        "salt": "salt"
    }

    if not load_data:
        print(
            "ncks -v {:s} -d time,{:d},{:d} -d lat,{:d},{:d} -d lon,{:d},{:d} {:s} {:s}"
            .format(",".join(sodavars.keys()), time_list[0][0],
                    time_list[0][-1], latlist[0][0], latlist[0][-1],
                    lonlist[0][0], lonlist[0][-1], _url, filename))
    else:
        for rn, recs in enumerate(chunker(time_list[0], _maxrecs)):
            print("{:s}-{:s}: ".format(
                soda_time[recs[0]].strftime("%m/%d/%Y"),
                soda_time[recs[-1]].strftime("%m/%d/%Y")),
                  end='',
                  flush=True)
            for var in sodavars:
                print("{:s} ".format(var), end='', flush=True)
                hisrange = np.arange(rn * _maxrecs,
                                     (rn * _maxrecs) + len(recs))
                if sodavars[var] == 3:
                    his.variables[hisvars[var]][hisrange, :, :] = \
                        np.ma.array(
                        soda.variables[var][recs, latlist[0], lonlist[0]]). \
                        filled(fill_value=9.99E10)
                else:
                    his.variables[hisvars[var]][hisrange, :, :, :] = \
                        soda.variables[var][recs, :, latlist[0],
                                            lonlist[0]].filled(fill_value=9.99E10)
            his.sync()
            print("", flush=True)
    pass
    cblabel = 'temperature 2 m [C]' #'3h acumm precipitation [mm]' #
    time_units = 'seconds since 2010-01-01 00:00:00'

    # map data
    members = 3
    treshhold = 0.01   # mm / h
    # grd_data = np.where(grd_data < treshhold, np.nan, grd_data)

    clevs, cmaps = make_cmap(temp()) #[0], precip1()


    for date, grid, obs in zip(dates, grd_data, obs_data):

        obs_dict['colors'] = obs

        title = num2date(date, time_units)
        print('Plotting date: {}'.format(title) )

        print('--> observed:  minimum value: {:.2f}, maximum value: {:.2f}'.format(obs.min(), obs.max()))
        for m in range(members):

            grid_m = grid[...,m]
            print('--> member: {}, minimum value: {:.2f}, maximum value: {:.2f}'.format(m+1, grid_m.min(), grid_m.max()))
            mapa = mu.get_mapa(lon_lat[0], lon_lat[1], proj=proj_dict)

            fig = mu.graphgtmap(grd_x, grd_y, grid_m, mapa, cblabel, title,
                                clevs, cmaps, asp,  calpha=0.9, stn_dict=obs_dict,
                                fig_dpi=300)

            fname = 'map_' + str(title).replace(' ', '_') + '_member:'+str(m+1)
            fig.savefig(os.path.join('outputs/plots', domain, var_key, fname + '.png'))
Example #37
0
        time_var_name = name
        break
time_var = data.variables[time_var_name]

lat_var = data.variables['lat']
lon_var = data.variables['lon']

# Get actual data values and remove any size 1 dimensions
lat = lat_var[:].squeeze()
lon = lon_var[:].squeeze()
height = height_var[0, 0, :, :].squeeze()
u_wind = u_wind_var[0, 0, :, :].squeeze() * units('m/s')
v_wind = v_wind_var[0, 0, :, :].squeeze() * units('m/s')

# Convert number of hours since the reference time into an actual date
time = num2date(time_var[:].squeeze(), time_var.units)

# Combine 1D latitude and longitudes into a 2D grid of locations
lon_2d, lat_2d = np.meshgrid(lon, lat)

# Smooth height data
height = ndimage.gaussian_filter(height, sigma=1.5, order=0)

# Set up some constants based on our projection, including the Coriolis parameter and
# grid spacing, converting lon/lat spacing to Cartesian
f = mpcalc.coriolis_parameter(np.deg2rad(lat_2d)).to('1/s')
dx, dy = mpcalc.lat_lon_grid_deltas(lon_2d, lat_2d)

# In MetPy 0.5, geostrophic_wind() assumes the order of the dimensions is (X, Y),
# so we need to transpose from the input data, which are ordered lat (y), lon (x).
# Once we get the components,transpose again so they match our original data.
Example #38
0
scT1fd_in = Dataset('shT1_osh_test.nc')

#%% sagehen timming
scT1time = scT1fd_in.variables['time'][:]  # get values
t_unitST1 = scT1fd_in.variables[
    'time'].units  # get unit  "days since 1950-01-01T00:00:00Z"

try:

    t_cal = scT1fd_in.variables['time'].calendar

except AttributeError:  # Attribute doesn't exist

    t_cal = u"gregorian"  # or standard

tvalueScT1 = num2date(scT1time, units=t_unitST1, calendar=t_cal)
scT1date_in = [i.strftime("%Y-%m-%d %H:%M") for i in tvalueScT1]

#%% make new nc file
new_fc_sc = Dataset("forcing_scT1_open_calib.nc",
                    'w',
                    format='NETCDF3_CLASSIC')
# define dimensions
hru = new_fc_sc.createDimension('hru', hru_num)
time = new_fc_sc.createDimension('time', None)
# define variables
hruid = new_fc_sc.createVariable('hruId', np.int32, ('hru', ))
lat = new_fc_sc.createVariable('latitude', np.float64, ('hru', ))
lon = new_fc_sc.createVariable('longitude', np.float64, ('hru', ))
ds = new_fc_sc.createVariable('data_step', np.float64)
times = new_fc_sc.createVariable('time', np.float64, ('time', ))
f = cf.read(datafile)
temperature = f.select('sea_water_temperature')
lon = temperature[0].coord('lon').array
lat = temperature[0].coord('lat').array
time = temperature[0].coord('time').array
temperatureQC = temperature[0].ancillary_variables[0].array
temperature = temperature[0].array
cf.close_files()

temperature = np.ma.masked_where(temperatureQC != 1, temperature)
temperature = np.ma.masked_where(temperature > 32., temperature)
temperature = np.ma.masked_where(temperature < 18., temperature)

print temperature.min()
with netCDF4.Dataset(datafile) as nc:
    time2plot = netCDF4.num2date(nc.variables['time'][:], nc.variables['time'].units)

temperature = np.ma.masked_where(lon<coordinates[0], temperature)
lon = np.ma.masked_where(lon<coordinates[0], lon)
lat = np.ma.masked_where(lon<coordinates[0], lat)
time = np.ma.masked_where(lon<coordinates[0], time)
lon, lat = m(lon, lat)

fig = plt.figure(figsize=(15, 6))

ax = fig.add_subplot(122)
monthformat = dates.DateFormatter('%B')
monthticks  = dates.MonthLocator(interval=2)  # every month
dayxticks  = dates.DayLocator()  # every month
ax.xaxis.set_major_locator(monthticks)
ax.xaxis.set_major_formatter(monthformat)
Example #40
0
from efa_xray.assimilation.ensrf import EnSRF

# directory where the ensemble of all times is
infile = '/home/disk/hot/stangen/Documents/GEFS/ensembles' + \
            '/2017090600/2017090600_21mem_10days.nc'

# only variable I am using is 500mb height
vrbls=['Z500','T500','RH500','U500','V500','Z700','T700','RH700','U700','V700', \
       'Z850','T850','RH850','U850','V850','Z925','T925','RH925','U925','V925', \
       'Z1000','T1000','RH1000','U1000','V1000','T2M','RH2M','U10M','V10M', \
       'PWAT','MSLP','P6HR']

# loading/accessing the netcdf data
with Dataset(infile, 'r') as ncdata:
    times = ncdata.variables['time']
    ftimes = num2date(times[:], times.units)
    lats = ncdata.variables['lat'][:]
    lons = ncdata.variables['lon'][:]
    mems = ncdata.variables['ens'][:]
    #print(ncdata.variables)

    # storing the variable data in a dict (state?)
    allvars = {}
    for var in vrbls:
        allvars[var] = (['validtime', 'y', 'x',
                         'mem'], ncdata.variables[var][:])
lonarr, latarr = np.meshgrid(lons, lats)

# Package into an EnsembleState object knowing the state and metadata
statecls = EnsembleState.from_vardict(
    allvars, {
Example #41
0
 def load_time_axis(path):
     with netCDF4.Dataset(path) as dataset:
         var = dataset.variables["time"]
         values = netCDF4.num2date(var[:], units=var.units)
     return np.array(values, dtype='datetime64[s]')
Example #42
0
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.colors
from matplotlib import ticker
import datetime as dt
import netCDF4 as nc

file_path = "/g/data/w35/mm3972/cable/EucFACE/EucFACE_run/met/"
famb = "EucFACE_met_amb.nc"

amb = nc.Dataset(os.path.join(file_path,famb), 'r')

step_2_sec = 30.*60.

df_amb              = pd.DataFrame(amb.variables['Rainf'][:,0], columns=['Rainf'])
df_amb['dates']     = nc.num2date(amb.variables['time'][:],amb.variables['time'].units)
df_amb              = df_amb.set_index('dates')

df_amb              = df_amb*step_2_sec
df_amb              = df_amb.resample("A").agg('sum')
#df_amb              = df_amb.drop(df_amb.index[len(df_amb)-1])
df_amb.index        = df_amb.index.strftime('%Y')

print(df_amb)
fig, ax = plt.subplots()

ax.scatter(df_amb.index, df_amb.values)

for i, txt in enumerate(df_amb.values):
    ax.annotate(txt, (df_amb.index[i], df_amb.values[i]))
for style, info in style_dict.items():  #,'cpd','tas'
    for scenario in ['Plus20-Artificial-v1'
                     ]:  #'Plus20-Future','Plus15-Future',
        state_files = sorted(
            glob.glob(working_path + scenario + '/' + style + '/' + style +
                      '_*_state.nc'))
        for state_file in state_files:
            print(state_file)

            states = info['states']
            nc = da.read_nc(state_file)

            if 'calendar' in nc['time'].attrs.keys():
                datevar = num2date(nc['time'].values,
                                   units=nc['time'].units,
                                   calendar=nc['time'].calendar)
            else:
                datevar = num2date(nc['time'].values, units=nc['time'].units)
            month = np.array([date.month for date in datevar])

            if 'stateCount' not in globals():
                stateCount = da.Dataset({})
                for state in states:
                    stateCount[state] = da.DimArray(
                        np.zeros([4, len(nc.lat), len(nc.lon)]),
                        axes=[seasons.keys(), nc.lat, nc.lon],
                        dims=['season', 'lat', 'lon'])
                    stateCount[state + '_possible_days'] = da.DimArray(
                        np.zeros([4, len(nc.lat), len(nc.lon)]),
                        axes=[seasons.keys(), nc.lat, nc.lon],
Example #44
0
    def plot_imd_radar(fpath, ele, R, plotpath):
        # Plotting Options Set in Function
        # ele = 0 #Elevation
        # R = 50#  #Min and Mac Range from Radar in km

        # Read Files and Setup Plotting
        radar = pyart.io.read(fpath)
        display = pyart.graph.RadarDisplay(radar)

        # set the figure title and show
        instrument_name = (radar.metadata['instrument_name'])[0:7]
        time_start = netCDF4.num2date(radar.time['data'][0],
                                      radar.time['units'])
        time_text = ' ' + time_start.strftime('%Y-%m-%d %H:%M:%SZ')

        elevation = radar.fixed_angle['data'][ele]
        ele_text = '%0.1f' % (elevation)

        title = instrument_name + time_text + ' Elevation %.1f' % (elevation)

        if np.round(elevation, 0) > 9.99:
            ele_folder = 'ele' + ele_text[0] + ele_text[1] + '_' + ele_text[
                3] + '/'
            ele_name = 'ele' + ele_text[0] + ele_text[1] + '_' + ele_text[3]
        else:
            ele_folder = 'ele' + ele_text[0] + '_' + ele_text[2] + '/'
            ele_name = 'ele' + ele_text[0] + '_' + ele_text[2]

        plot_file_name = instrument_name + '_' + time_start.strftime(
            '%Y-%m-%dT%H%M%SZ') + ele_name + '.png'

        loc_folder = instrument_name + '/'

        R_folder = str(R) + 'km' + '/'

        fig_save_file_path = plotpath + loc_folder + ele_folder + R_folder

        if os.path.isfile(fig_save_file_path + plot_file_name):
            print(fig_save_file_path + plot_file_name + ' exists')
        else:

            # Figure Options
            width = 30  # in inches
            height = 10  # in inches

            fig = plt.figure(figsize=(width, height))
            nrows = 2
            ncols = 5

            ax1 = fig.add_subplot(nrows, ncols, 1)
            display.plot('dBZ',
                         ele,
                         ax=ax1,
                         vmin=-32,
                         vmax=16.,
                         title='Horizontal Reflectivity',
                         colorbar_label=radar.fields['dBZ']['units'],
                         axislabels=('',
                                     'North South distance from radar (km)'))
            display.set_limits((-R, R), (-R, R), ax=ax1)

            ax2 = fig.add_subplot(nrows, ncols, 2)
            display.plot('ZDR',
                         ele,
                         ax=ax2,
                         vmin=-2,
                         vmax=10.,
                         title='Differential Reflectivity',
                         colorbar_label=radar.fields['ZDR']['units'],
                         axislabels=('', ''),
                         cmap='pyart_RefDiff')
            display.set_limits((-R, R), (-R, R), ax=ax2)

            ax3 = fig.add_subplot(nrows, ncols, 3)
            display.plot('RhoHV',
                         ele,
                         ax=ax3,
                         vmin=0,
                         vmax=1.,
                         title='Cross Correlation Ratio',
                         colorbar_label=radar.fields['RhoHV']['units'],
                         axislabels=('', ''),
                         cmap='pyart_RefDiff')
            display.set_limits((-R, R), (-R, R), ax=ax3)

            ax4 = fig.add_subplot(nrows, ncols, 4)
            display.plot('KDP',
                         ele,
                         ax=ax4,
                         vmin=-2,
                         vmax=5.,
                         title='Specific Differential Phase',
                         colorbar_label=radar.fields['KDP']['units'],
                         axislabels=('', ''),
                         cmap='pyart_Theodore16')
            display.set_limits((-R, R), (-R, R), ax=ax4)

            ax5 = fig.add_subplot(nrows, ncols, 5)
            display.plot('PhiDP',
                         ele,
                         ax=ax5,
                         vmin=-180,
                         vmax=180.,
                         title='Differential Phase',
                         colorbar_label=radar.fields['PhiDP']['units'],
                         axislabels=('', ''),
                         cmap='pyart_Wild25')
            display.set_limits((-R, R), (-R, R), ax=ax5)

            ax6 = fig.add_subplot(nrows, ncols, 6)
            display.plot('V',
                         ele,
                         ax=ax6,
                         vmin=-16,
                         vmax=16.,
                         title='Doppler Velocity',
                         colorbar_label=radar.fields['V']['units'],
                         axislabels=('East West distance from radar (km)',
                                     'North South distance from radar (km)'),
                         cmap='pyart_BuDRd18')
            display.set_limits((-R, R), (-R, R), ax=ax6)

            ax7 = fig.add_subplot(nrows, ncols, 7)
            display.plot('W',
                         ele,
                         ax=ax7,
                         vmin=0,
                         vmax=5.,
                         title='Spectrum Width',
                         colorbar_label=radar.fields['W']['units'],
                         axislabels=('East West distance from radar (km)', ''),
                         cmap='pyart_NWS_SPW')
            display.set_limits((-R, R), (-R, R), ax=ax7)

            ax8 = fig.add_subplot(nrows, ncols, 8)
            display.plot('SNR',
                         ele,
                         ax=ax8,
                         title='SNR',
                         colorbar_label=radar.fields['SNR']['units'],
                         axislabels=('East West distance from radar (km)', ''),
                         cmap='pyart_Carbone17')
            display.set_limits((-R, R), (-R, R), ax=ax8)

            ax9 = fig.add_subplot(nrows, ncols, 9)
            display.plot('SQI',
                         ele,
                         ax=ax9,
                         title='SQI',
                         colorbar_label=radar.fields['SQI']['units'],
                         axislabels=('East West distance from radar (km)', ''),
                         cmap='pyart_Carbone17')
            display.set_limits((-R, R), (-R, R), ax=ax9)

            ax10 = fig.add_subplot(nrows, ncols, 10)
            display.plot('uPhiDP',
                         ele,
                         ax=ax10,
                         vmin=-360,
                         vmax=360.,
                         title='Unfiltered Differential Phase',
                         colorbar_label=radar.fields['uPhiDP']['units'],
                         axislabels=('East West distance from radar (km)', ''),
                         cmap='pyart_Wild25')
            display.set_limits((-R, R), (-R, R), ax=ax10)

            display.plot_cross_hair(3.)
            plt.suptitle(title, fontsize=24)

            if not os.path.exists(fig_save_file_path):
                os.makedirs(fig_save_file_path)

            plt.savefig(fig_save_file_path + plot_file_name, dpi=100)

            plt.close('all')

            del radar
            del display
Example #45
0
def write_pcr_timeseries(lookup_table,
                         srcVar,
                         trgFolder,
                         trgPrefix,
                         logger,
                         slope=1.,
                         offset=0.,
                         year=False,
                         remove_missings=True):
    # if necessary, make trgPrefix maximum of 8 characters
    if len(trgPrefix) > 8:
        trgPrefix = trgPrefix[0:8]
    # prepare a counter for the number of files with non-missing values
    count = 0
    timeList = []
    # establish the points in the lookup_table where the dates of interest are located
    startDate = dt.datetime(year, 1, 1, 0, 0)
    endDate = dt.datetime(year, 12, 31, 0, 0)
    allDates = lookup_table['time']
    idxs = where(
        logical_and(array(allDates) >= startDate,
                    array(allDates) <= endDate))[0]

    # start with an empty string to compare with
    srcFile = ''
    for idx in idxs:
        if logical_and(lookup_table[srcVar][idx] != srcFile,
                       lookup_table[srcVar][idx] != ''):
            # read a new source netCDF file
            if srcFile != '':
                # if we are not reading the first time, then close the previous file
                nc_src.close()
            # set the new source file and open it and read x and y-axis and time and the variable of interest
            srcFile = lookup_table[srcVar][idx]
            nc_src = nc4.Dataset(srcFile, 'r')
            # read axis, try 'lat' otherwise try 'latitude'
            try:
                y = nc_src.variables['lat'][:]
            except:
                y = nc_src.variables['latitude'][:]

            try:
                x = nc_src.variables['lon'][:]
            except:
                x = nc_src.variables['longitude'][:]
            # read time axis and convert to time objects
            time = nc_src.variables['time']
            timeObjRaw = nc4.num2date(time[:],
                                      units=time.units,
                                      calendar=time.calendar)
            # UUGGGHHH: this seems to be a bug. calendar other than gregorian give other objects than
            # datetime.datetime objects. Here we convert to gregorian numbers, then back to date objects
            timeObj = []
            for t in timeObjRaw:
                timeObj.append(dt.datetime.strptime(t.strftime('%Y%j'),
                                                    '%Y%j'))


#            timeObj = nc4.num2date(nc4.date2num(timeObj, units=time.units, calendar='gregorian'), units=time.units, calendar='gregorian')
# now loop over all time steps, check the date and write valid dates to a list, write time series to PCRaster maps
            for n in range(len(timeObj)):
                # remove any hours or minutes data from curTime
                timeObj[n] = timeObj[n].replace(hour=0)
                timeObj[n] = timeObj[n].replace(minute=0)
            # Read the variable of interest
            nc_var = nc_src.variables[srcVar]

        #### Correct file is read, now read in position ####
        # look up the position in the current file, where the current time step is located
        curTime = allDates[idx]
        location = where(array(timeObj) == curTime)[0]
        if len(location) == 1:
            # in case of a difference in the used calendar, there may be one or more missing days in a year.
            # These are accounted for by ignoring dates that are not found inside the input data files
            curGrid = nc_var[location[0], :, :]
            if remove_missings:
                if hasattr(curGrid, 'mask'):
                    curMask = curGrid.mask
                    curGrid = curGrid.data
                    curGrid[curMask] = nan
                # check if grid contains valid values
                pixloc = where(isfinite(curGrid))  # != nc_var._FillValue)[0]
                if len(pixloc) > 0:
                    # there are valid values, so write to a PCRaster map and add time to list
                    timeList.append(curTime)
                    count += 1
                    below_thousand = count % 1000
                    above_thousand = count / 1000
                    pcraster_file = str(trgPrefix + '%0' +
                                        str(8 - len(trgPrefix)) +
                                        '.f.%03.f') % (above_thousand,
                                                       below_thousand)
                    pcraster_path = os.path.join(trgFolder, pcraster_file)
                    logger.debug('Writing "' + srcVar + '" on date ' +
                                 curTime.strftime('%Y-%m-%d') + ' to ' +
                                 pcraster_path)
                    # prepare grid for writing to file
                    trgGrid = curGrid * slope + offset
                    trgGrid[isnan(trgGrid)] = nc_var._FillValue
                    # write grid to PCRaster file
                    if y[1] - y[0] > 0.:
                        # the y-axis is ascending, flip the y axis and the target grid!
                        writeMap(pcraster_path, 'PCRaster', x, flipud(y),
                                 flipud(trgGrid), float(nc_var._FillValue))
                    else:
                        writeMap(pcraster_path, 'PCRaster', x, y, trgGrid,
                                 float(nc_var._FillValue))
        else:
            logger.debug('"' + srcVar + '" not available on date ' +
                         curTime.strftime('%Y-%m-%d'))
    nc_src.close()
    return timeList
ds_cf = netCDF4.Dataset(data_dir + 'ecmwf_iri_' + calendar.month_abbr[target_month] + '2017_cf.nc') #Load the pf file
ds_pf = netCDF4.Dataset(data_dir + 'ecmwf_iri_' + calendar.month_abbr[target_month] + '2017_pf.nc') #Load the cf file

# You may want to uncomment the following lines
# to print out the file and variable information
#print(ds_pf.file_format)
#print(ds_pf.dimensions.keys())
#print(ds_pf.variables)

# Read in the variables from cf
prec_hd    = ds_cf.variables['hdate'][:]
prec_lead  = ds_cf.variables['L'][:]
prec_lat   = ds_cf.variables['Y'][:]
prec_lon   = ds_cf.variables['X'][:]
prec_temp  = ds_cf.variables['S']
prec_start = netCDF4.num2date(prec_temp[:],prec_temp.units)

if ec_lat.all() != prec_lat.all():
    print('Latitude files are not the same')
if ec_lon.all() != prec_lon.all():
    print('Longitude files are not the same')
# Read in the array from pf's total precip ('tp') variable
arr_pf = ds_pf.variables['tp'][:]
# Read in the cf's total precip ('tp') variable
arr_cf = ds_cf.variables['tp'][:]

arr_shp = arr_pf.shape

# Create a new array to accommodate the 10 pf and 1 cf members
arr_comb = np.empty([arr_shp[0], arr_shp[1],arr_shp[2], arr_shp[3]+1, arr_shp[4], arr_shp[5]])
# Populate arr_comb with pf and cf data
Example #47
0
def time_file_index(
        base_dir,
        time_var,
        begin_date=datetime(2013, 1, 1, 0, 0),
        end_date=datetime(2013, 1, 5, 0, 0),
        filenames=None,
):
    """Index a set of files and generate a pandas data frame of time (index) and
    filename. 

    Parameters
    ----------
    base_dir : str
        Index a set of files and generate a pandas data frame of time (index) and 
        filename.

    time_var : str
        The title of the time variable in the netCDF files.

    begin_date : datetime.datetime, optional
        The minimum date from which to include files.

    end_date : datetime.datetime, optional
        The maximum date from which to include files.

    filenames : list, default: None
        A list of filenames, if you found them some other way.

    Returns
    -------
    pandas.Dataframe
        A pandas dataframe with a DateTimeIndex constructed from the files,
        and columns of filename and t_index. The latter represents the
        numerical index value of the time slice in the named file.
    """

    if filenames is None:
        filenames = find_files(base_dir, begin_date, end_date)

    nfiles = len(filenames)

    # Build a tuple with dates and filenames the file contains for every file in the index
    time_file = []

    for i in trange(nfiles):

        with netCDF4.Dataset(filenames[i]) as netcdf:
            # extract the time, turn it into a date

            time_dat = netcdf.variables[time_var]
            times = np.array(time_dat)

            # some have calendar, some don't
            try:
                times = netCDF4.num2date(times, time_dat.units,
                                         time_dat.calendar)
            except:
                times = netCDF4.num2date(times, time_dat.units)

        for j in range(len(times)):
            time_file.append((times[j], filenames[i], j))

    result = pd.DataFrame(time_file, columns=["date", "file", "t_index"])
    result = result.set_index("date")

    # check for duplicates
    dupes = result.index.duplicated(keep="first")

    if dupes.sum() > 0:
        #       print('Found duplicate times, using first one found.')
        result = result[~dupes]

    return result
Example #48
0
    tStart = parse(time_deployment_start, ignoretz=True)
    tEnd = parse(time_deployment_end, ignoretz=True)

    tStartnum = date2num(tStart,
                         units=ncTime[0].units,
                         calendar=ncTime[0].calendar)
    tEndnum = date2num(tEnd,
                       units=ncTime[0].units,
                       calendar=ncTime[0].calendar)

    maTime = ma.array(ncTime[0][:])
    msk = (maTime < tStartnum) | (maTime > tEndnum)
    maTime.mask = msk

    dates = num2date(maTime.compressed(),
                     units=ncTime[0].units,
                     calendar=ncTime[0].calendar)

    # create a new filename
    # from:
    # IMOS_<Facility-Code>_<Data-Code>_<Start-date>_<Platform-Code>_FV<File-Version>_ <Product-Type>_END-<End-date>_C-<Creation_date>_<PARTX>.nc
    # to:
    # OS_[PlatformCode]_[DeploymentCode]_[DataMode]_[PARTX].nc

    # TODO: what to do with <Data-Code> with a reduced number of variables

    fileName = os.path.basename(fn)
    splitParts = fileName.split(
        "_")  # get the last path item (the file name), split by _

    tStartMaksed = dates[0]
Example #49
0
    def fix_dataset(self, fn_out, species, fcinit):
        # read vertical coordinates
        hyn, hyam, hybm = load_vert_coord(CAMS_LEVELDEV_STR)
        with Dataset(fn_out, 'a', format='NETCDF4_CLASSIC') as nc:
            t_obj = num2date(nc.variables['time'][:],
                             nc.variables['time'].units)
            t_unit = 'hours since {:%Y-%m-%dT%H:%M:%S}'.format(fcinit)
            nc.variables['time'][:] = date2num(t_obj, t_unit)
            nc.variables['time'].setncattr('units', t_unit)
            nc.variables['time'].setncattr('standard_name', 'time')

            nc.variables['latitude'].setncattr('standard_name', 'latitude')
            nc.variables['longitude'].setncattr('standard_name', 'longitude')

            if species == 'AIR_PRESSURE':
                ps_ = nc.variables['lnsp'][:]  # TODO use species definition
                # calculate air_pressure
                p_ = (hyam[np.newaxis, :, np.newaxis, np.newaxis] +
                      hybm[np.newaxis, :, np.newaxis, np.newaxis] *
                      np.exp(ps_[:, np.newaxis, :, :]))
                # create 'level' variable
                nc.createDimension('level', hyn.size)
                v_lev = nc.createVariable('level', np.int32, ('level', ))
                v_lev[:] = np.arange(1, 61, 1, dtype='int32')
                # write air_pressure to file
                nc.createVariable(
                    'P', np.float32,
                    ('time', 'level', 'latitude', 'longitude'),
                    zlib=True, complevel=6, shuffle=True, fletcher32=True)
                nc.variables['P'][:] = p_.astype('float32')
                # set air_pressure attributes
                nc.variables['P'].setncattr('standard_name', 'air_pressure')
                nc.variables['P'].setncattr('units', 'Pa')

            self.set_standard_name(nc, species)

            v_hy = nc.createVariable('hybrid', np.float32, ('level', ))
            v_hy[:] = hyn
            v_hy.setncattr('standard_name',
                           'atmosphere_hybrid_sigma_pressure_coordinate')
            v_hy.setncattr('units', 'sigma')
            v_hy.setncattr('positive', 'down')
            v_hy.setncattr('formula', 'p(time, level, lat, lon) = '
                           'ap(level) + b(level) * exp(lnsp(time, lat, lon))')
            v_hy.setncattr('formula_terms', 'ap: hyam b: hybm lnsp:lnsp')

            v_am = nc.createVariable('hyam', np.float32, ('level', ))
            v_am[:] = hyam
            v_am.setncattr('units', 'Pa')
            v_am.setncattr('standard_name', 'atmosphere_pressure_coordinate')

            v_bm = nc.createVariable('hybm', np.float32, ('level', ))
            v_bm[:] = hybm
            v_bm.setncattr('units', '1')
            v_bm.setncattr('standard_name',
                           'atmosphere_hybrid_height_coordinate')

            nc.variables['level'].setncattr('standard_name',
                                            'model_level_number')

            for var in ['time', 'latitude', 'longitude']:
                nc.variables[var].setncattr('standard_name', var)
Example #50
0
# do some finagling with the start times in the data files
for ff in os.listdir(setup.Data_Dir):
    if ff.endswith("filelist.txt"):
        fn = ff

f = file(os.path.join(setup.Data_Dir, fn))
flist = []
for line in f:
    name = os.path.join(setup.Data_Dir, line)
    flist.append(name[:-1])  # Gonzo cat version
    # flist.append(name[:-1])   # laptop current version
Time_Map = []
for fn in flist:
    d = nc4.Dataset(fn)
    t = d['time']
    file_start_time = nc4.num2date(t[0], units=t.units)
    Time_Map.append((file_start_time, fn))

### WIND
# do some finagling with the start times in the data files
for ff in os.listdir(setup.Data_DirW):
    if ff.endswith("filelist.txt"):
        fn = ff
# fn = os.path.join(setup.Data_Dir,'SoCal_filelist.txt')
f = file(os.path.join(setup.Data_DirW, fn))
flist = []
for line in f:
    name = os.path.join(setup.Data_DirW, line)
    flist.append(name[:-1])  # Gonzo cat version
    # flist.append(name[:-1])   # laptop current version
Time_Map_W = []
Example #51
0
Vqc = ADCP200.variables['VCUR_quality_control']
Uqc = ADCP200.variables['UCUR_quality_control']
Dqc = ADCP200.variables['DEPTH_quality_control']

#To regrid for Depth_interp_final = (np.arange(Depth_f,Depth_i,-Depth_dz))
Depth_i = 0
Depth_f = 145
Depth_dz = 5

# In[20]:

#block 4: Convert the ADCP time series dataset (Matlab to python time series: yyyy/mm/DD hh/mm/SS)

units = ADCP200.variables['TIME'].units
calendar = ADCP200.variables['TIME'].calendar
times = num2date(ADCP200.variables['TIME'][:], units=units, calendar=calendar)
times1 = mdates.date2num(times)

# In[21]:

#block 5: Create variables with ADCP dataset

#basic data
depth_ADCP = DEPTH[:, 0, 0]
depth_Bin = Wheight[:]
data_u = U[:, :, :, :]
data_v = V[:, :, :, :]
v2d = data_v[:, :, 0, 0]
u2d = data_u[:, :, 0, 0]

#quality controle data
def seasonal_climatology_cice(directory, out_file):

    # Starting and ending months (1-based) for each season
    start_month = [12, 3, 6, 9]
    end_month = [2, 5, 8, 11]
    # Starting and ending days of the month (1-based) for each season
    # Assume no leap years, we'll fix this later if we need
    start_day = [1, 1, 1, 1]
    end_day = [28, 31, 31, 30]

    # Find the first output file
    for file in listdir(directory):
        if file.startswith('iceh.') and file.endswith('.nc'):
            # Read the grid
            id = Dataset(directory + file, 'r')
            lon = id.variables['TLON'][:, :]
            lat = id.variables['TLAT'][:, :]
            id.close()
            break

    # Set up arrays to integrate seasonal climatology of aice and hi
    seasonal_aice = ma.empty([4, size(lon, 0), size(lon, 1)])
    seasonal_hi = ma.empty([4, size(lon, 0), size(lon, 1)])
    seasonal_aice[:, :, :] = 0.0
    seasonal_hi[:, :, :] = 0.0
    # Also integrate number of days in each season
    ndays = zeros(4)
    # Total number of days for a final check
    total_days = 0

    # Loop over files
    for file in listdir(directory):
        if file.startswith('iceh.') and file.endswith('.nc'):
            print 'Processing ' + directory + file
            # Read aice and hi
            id = Dataset(directory + file, 'r')
            aice = id.variables['aice'][0, :, :]
            hi = id.variables['hi'][0, :, :]
            # Read the time value and convert to Date object
            time_id = id.variables['time']
            time = num2date(time_id[0],
                            units=time_id.units,
                            calendar=time_id.calendar.lower())
            id.close()
            total_days += 5
            # 5-day averages marked with next day's date
            year = time.year
            month = time.month
            day = time.day
            # Get the season of this day
            if month in [12, 1, 2]:
                # DJF
                season = 0
            elif month in [3, 4, 5]:
                # MAM
                season = 1
            elif month in [6, 7, 8]:
                # JJA
                season = 2
            elif month in [9, 10, 11]:
                # SON
                season = 3
            # Check for leap years
            leap_year = False
            if mod(year, 4) == 0:
                leap_year = True
                if mod(year, 100) == 0:
                    leap_year = False
                    if mod(year, 400) == 0:
                        leap_year = True
            # Update last day in February
            if leap_year:
                end_day[0] = 29
            else:
                end_day[0] = 28
            if month == start_month[season]:
                # We are in the first month of the season
                if day - 5 < start_day[season]:
                    # Spills over into the previous season
                    prev_season = mod(season - 1, 4)
                    # How many days does it spill over by?
                    spill_days = start_day[season] - day + 5
                    # Should be between 1 and 5
                    if spill_days < 1 or spill_days > 5:
                        print 'Problem: spill_days is ' + str(spill_days)
                        print 'Timestep ' + str(t + 1)
                        print 'Year ' + str(year)
                        print 'Month ' + str(month + 1)
                        print 'Day ' + str(day)
                        return
                    # Split between previous season and this season
                    seasonal_aice[prev_season, :, :] += aice * spill_days
                    seasonal_hi[prev_season, :, :] += hi * spill_days
                    ndays[prev_season] += spill_days
                    seasonal_aice[season, :, :] += aice * (5 - spill_days)
                    seasonal_hi[season, :, :] += hi * (5 - spill_days)
                    ndays[season] += 5 - spill_days
                else:
                    # Entirely within the season
                    seasonal_aice[season, :, :] += aice * 5
                    seasonal_hi[season, :, :] += hi * 5
                    ndays[season] += 5
            else:
                # Not in the first month of the season
                # The 5 days will be entirely within the season
                seasonal_aice[season, :, :] += aice * 5
                seasonal_hi[season, :, :] += hi * 5
                ndays[season] += 5
    # Convert from sums to averages
    for season in range(4):
        seasonal_aice[
            season, :, :] = seasonal_aice[season, :, :] / ndays[season]
        seasonal_hi[season, :, :] = seasonal_hi[season, :, :] / ndays[season]
    if sum(ndays) != total_days:
        print 'Problem: files have ' + str(
            total_days) + ' days, but climatology has ' + str(sum(ndays))
        return

    # Write to file
    print 'Writing ' + out_file
    id = Dataset(out_file, 'w')
    id.createDimension('ni', size(lon, 1))
    id.createDimension('nj', size(lon, 0))
    id.createDimension('time', 4)
    id.createVariable('TLON', 'f8', ('nj', 'ni'))
    id.variables['TLON'].long_name = 'T grid center longitude'
    id.variables['TLON'].units = 'degrees_east'
    id.variables['TLON'][:, :] = lon
    id.createVariable('TLAT', 'f8', ('nj', 'ni'))
    id.variables['TLAT'].long_name = 'T grid center latitude'
    id.variables['TLAT'].units = 'degrees_north'
    id.variables['TLAT'][:, :] = lat
    id.createVariable('time', 'f8', ('time'))
    id.variables['time'].units = 'season'
    id.variables['time'].long_name = 'DJF, MAM, JJA, SON'
    id.variables['time'][:] = arange(1, 4 + 1)
    id.createVariable('aice', 'f8', ('time', 'nj', 'ni'))
    id.variables['aice'].units = '1'
    id.variables['aice'][:, :, :] = seasonal_aice
    id.createVariable('hi', 'f8', ('time', 'nj', 'ni'))
    id.variables['hi'].units = 'm'
    id.variables['hi'][:, :, :] = seasonal_hi
    id.close()
Example #53
0
timestep = 'time'

## Extracting longitude and latitude of all mesh nodes ##
lon = nc['x'][:]
lat = nc['y'][:]

## Extracting element description details (list of 3 vertices specified for each triangle denoted by index number)from the output file ##
nv = nc['element'][:, :] - 1

## Extracting time step information from the output file ##
time_var = nc['time']
## startdate specifies start time of ADCIRC computations ##
startdate = time_var.units

## Converting the time step information into datetime objects ##
dtime = netCDF4.num2date(time_var[:], startdate)

## Converting datetime objects to string format - YYMMDDHHMMSS ##
a = []
for j in dtime:
    a.append(j.strftime("%Y%m%d%H%M%S"))

## RUNNING TIME ##
print 'Finished organizing input information after %d seconds' % (time.time() -
                                                                  time0)


## FUNCTION TO CHECK IF VERTEX FALLS WITHIN A SPECIFIED LONG/LAT BOX##
def vertexcheck(LatN, LatS, LongE, LongW, vertex):
    if lon[vertex] < LongE and lon[vertex] > LongW and lat[
            vertex] > LatS and lat[vertex] < LatN:
Example #54
0
def read_era5(domain, times, pres=True):
    #Open ERA5 netcdf files and extract variables needed for a range of times
    # and given spatial domain

    ref = dt.datetime(1900, 1, 1, 0, 0, 0)
    if len(times) > 1:
        date_list = date_seq(times, "hours", 1)
    else:
        date_list = times
    formatted_dates = [format_dates(x) for x in date_list]
    unique_dates = np.unique(formatted_dates)
    time_hours = np.empty(len(date_list))
    for t in np.arange(0, len(date_list)):
        time_hours[t] = (date_list[t] - ref).total_seconds() / (3600)
    if (date_list[0].day == 1) & (date_list[0].hour < 3):
        fc_unique_dates = np.insert(
            unique_dates, 0, format_dates(date_list[0] - dt.timedelta(1)))
    else:
        fc_unique_dates = np.copy(unique_dates)

    #Get time-invariant pressure and spatial info
    no_p, p, p_ind = get_pressure(100)
    p = p[p_ind]
    lon, lat = get_lat_lon()
    lon_ind = np.where((lon >= domain[2]) & (lon <= domain[3]))[0]
    lat_ind = np.where((lat >= domain[0]) & (lat <= domain[1]))[0]
    lon = lon[lon_ind]
    lat = lat[lat_ind]
    terrain = reform_terrain(lon, lat)
    sfc_lon, sfc_lat = get_lat_lon_sfc()
    sfc_lon_ind = np.where((sfc_lon >= domain[2]) & (sfc_lon <= domain[3]))[0]
    sfc_lat_ind = np.where((sfc_lat >= domain[0]) & (sfc_lat <= domain[1]))[0]
    sfc_lon = sfc_lon[sfc_lon_ind]
    sfc_lat = sfc_lat[sfc_lat_ind]

    #Initialise arrays for each variable
    if pres:
        ta = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
        dp = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
        hur = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
        hgt = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
        ua = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
        va = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
        wap = np.empty((len(date_list), no_p, len(lat_ind), len(lon_ind)))
    uas = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
    vas = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
    ps = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
    cp = np.zeros(ps.shape) * np.nan
    cape = np.zeros(ps.shape) * np.nan
    wg10 = np.zeros(ps.shape) * np.nan

    tas = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))
    ta2d = np.empty((len(date_list), len(sfc_lat_ind), len(sfc_lon_ind)))

    for date in unique_dates:

        #Load ERA-Interim reanalysis files
        if pres:
            ta_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/t/"+date[0:4]+\
             "/t_era5_aus_"+date+"*.nc")[0])
            z_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/z/"+date[0:4]+\
             "/z_era5_aus_"+date+"*.nc")[0])
            ua_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/u/"+date[0:4]+\
             "/u_era5_aus_"+date+"*.nc")[0])
            va_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/v/"+date[0:4]+\
             "/v_era5_aus_"+date+"*.nc")[0])
            hur_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/pressure/r/"+date[0:4]+\
             "/r_era5_aus_"+date+"*.nc")[0])

        uas_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/u10/"+date[0:4]+\
         "/u10_era5_global_"+date+"*.nc")[0])
        vas_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/v10/"+date[0:4]+\
         "/v10_era5_global_"+date+"*.nc")[0])
        ta2d_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/d2m/"+date[0:4]+\
         "/d2m_era5_global_"+date+"*.nc")[0])
        tas_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/t2m/"+date[0:4]+\
         "/t2m_era5_global_"+date+"*.nc")[0])
        ps_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/sp/"+date[0:4]+\
         "/sp_era5_global_"+date+"*.nc")[0])
        cp_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/cp/"+date[0:4]+\
         "/cp_era5_global_"+date+"*.nc")[0])
        wg10_file = nc.Dataset(glob.glob("/g/data/ub4/era5/netcdf/surface/fg10/"+date[0:4]+\
         "/fg10_era5_global_"+date+"*.nc")[0])

        #Get times to load in from file
        if pres:
            times = ta_file["time"][:]
            time_ind = [
                np.where(x == times)[0][0] for x in time_hours if (x in times)
            ]
            date_ind = np.where(np.array(formatted_dates) == date)[0]
        else:
            times = uas_file["time"][:]
            time_ind = [
                np.where(x == times)[0][0] for x in time_hours if (x in times)
            ]
            date_ind = np.where(np.array(formatted_dates) == date)[0]

        #Get times to load in from forecast files (wg10 and cp)
        fc_times = cp_file["time"][:]
        fc_time_ind = [
            np.where(x == fc_times)[0][0] for x in time_hours
            if (x in fc_times)
        ]

        #Load analysis data
        if pres:
            ta[date_ind, :, :, :] = ta_file["t"][time_ind, p_ind, lat_ind,
                                                 lon_ind] - 273.15
            #wap[date_ind,:,:,:] = wap_file["wap"][time_ind,p_ind,lat_ind,lon_ind]
            ua[date_ind, :, :, :] = ua_file["u"][time_ind, p_ind, lat_ind,
                                                 lon_ind]
            va[date_ind, :, :, :] = va_file["v"][time_ind, p_ind, lat_ind,
                                                 lon_ind]
            hgt[date_ind, :, :, :] = z_file["z"][time_ind, p_ind, lat_ind,
                                                 lon_ind] / 9.8
            hur[date_ind, :, :, :] = hur_file["r"][time_ind, p_ind, lat_ind,
                                                   lon_ind]
            hur[hur < 0] = 0
            hur[hur > 100] = 100
            dp[date_ind, :, :, :] = get_dp(ta[date_ind, :, :, :],
                                           hur[date_ind, :, :, :])
        uas[date_ind, :, :] = uas_file["u10"][time_ind, sfc_lat_ind,
                                              sfc_lon_ind]
        vas[date_ind, :, :] = vas_file["v10"][time_ind, sfc_lat_ind,
                                              sfc_lon_ind]
        tas[date_ind, :, :] = tas_file["t2m"][time_ind, sfc_lat_ind,
                                              sfc_lon_ind] - 273.15
        ta2d[date_ind, :, :] = ta2d_file["d2m"][time_ind, sfc_lat_ind,
                                                sfc_lon_ind] - 273.15
        ps[date_ind, :, :] = ps_file["sp"][time_ind, sfc_lat_ind,
                                           sfc_lon_ind] / 100
        fc_date_ind = np.in1d(
            date_list,
            nc.num2date(cp_file["time"][fc_time_ind], cp_file["time"].units))
        cp[fc_date_ind, :, :] = cp_file["cp"][fc_time_ind, sfc_lat_ind,
                                              sfc_lon_ind]
        wg10[fc_date_ind, :, :] = wg10_file["fg10"][fc_time_ind, sfc_lat_ind,
                                                    sfc_lon_ind]

        if pres:
            ta_file.close()
            z_file.close()
            ua_file.close()
            va_file.close()
            hur_file.close()
        uas_file.close()
        vas_file.close()
        tas_file.close()
        ta2d_file.close()
        ps_file.close()

    if pres:
        p = np.flip(p)
        ta = np.flip(ta, axis=1)
        dp = np.flip(dp, axis=1)
        hur = np.flip(hur, axis=1)
        hgt = np.flip(hgt, axis=1)
        ua = np.flip(ua, axis=1)
        va = np.flip(va, axis=1)
        return [
            ta, dp, hur, hgt, terrain, p, ps, ua, va, uas, vas, tas, ta2d, cp,
            wg10, cape, lon, lat, date_list
        ]
    else:
        return [
            ps, uas, vas, tas, ta2d, cp, wg10, cape, sfc_lon, sfc_lat,
            date_list
        ]
Example #55
0
def date_range(start=None, end=None, periods=None, freq='D', tz=None,
               normalize=False, name=None, closed=None, calendar='standard',
               **kwargs):
    ''' Return a fixed frequency datetime index, with day (calendar) as the
    default frequency

    Parameters
    ----------
    start : string or datetime-like, default None
        Left bound for generating dates
    end : string or datetime-like, default None
        Right bound for generating dates
    periods : integer or None, default None
        If None, must specify start and end
    freq : string or DateOffset, default 'D' (calendar daily)
        Frequency strings can have multiples, e.g. '5H'
    tz : string or None
        Time zone name for returning localized DatetimeIndex, for example
        Asia/Hong_Kong
    normalize : bool, default False
        Normalize start/end dates to midnight before generating date range
    name : str, default None
        Name of the resulting index
    closed : string or None, default None
        Make the interval closed with respect to the given frequency to
        the 'left', 'right', or both sides (None)
    calendar : string
        Describes the calendar used in the time calculations. Default is a the
        standard calendar (with leap years)

    Notes
    -----
    2 of start, end, or periods must be specified
    To learn more about the frequency strings, please see `this link
    <http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases>`__.

    Returns
    -------
    rng : DatetimeIndex
    '''
    if calendar in ['standard', 'gregorian', 'propoleptic_gregorian']:
        return pd.date_range(start=start, end=end, periods=periods,
                             freq=freq, tz=tz, normalize=normalize, name=name,
                             closed=closed, **kwargs)
    else:
        # start and end are give
        if (start is not None) and (end is not None) and (periods is None):
            steps, units = decode_freq(freq)
            start_num, end_num = date2num(
                pd.to_datetime([start, end]).to_pydatetime(),
                units, calendar=calendar)
            periods = int((end_num - start_num) / steps) + 1

            times = num2date(
                np.linspace(start_num, end_num, periods,
                            endpoint=True,
                            dtype=np.float128), units, calendar)
            index = xr.CFTimeIndex(times).to_datetimeindex()
            return index

        else:
            raise NotImplementedError(
                'Specified arguments are not valid for this calendar')
def do_grid_map_gates_to_grid(radar_fname):
    import pyart
    from matplotlib import pyplot as plt
    from netCDF4 import num2date, date2num
    import numpy as np
    from time import time
    import os
    md = '/lcrc/group/earthscience/radar/nexrad/chicago_floods/'
    try:
        radar = pyart.io.read(radar_fname)
        rain_z = radar.fields['reflectivity']['data'].copy()
        z_lin = 10.0**(radar.fields['reflectivity']['data'] / 10.)
        rain_z = (z_lin / 300.0)**(1. / 1.4)  #Z=300 R1.4
        radar.add_field_like('reflectivity',
                             'rain_z',
                             rain_z,
                             replace_existing=True)
        radar.fields['rain_z']['units'] = 'mm/h'
        radar.fields['rain_z']['standard_name'] = 'rainfall_rate'
        radar.fields['rain_z']['long_name'] = 'rainfall_rate_from_z'
        radar.fields['rain_z']['valid_min'] = 0
        radar.fields['rain_z']['valid_max'] = 500
        grid = pyart.map.grid_from_radars(
            (radar, ),
            grid_shape=(1, 501, 501),
            grid_limits=((0, 0), (-50000, 50000), (-50000, 50000)),
            fields=radar.fields.keys(),
            gridding_algo="map_gates_to_grid",
            weighting_function='BARNES')
        dts = num2date(grid.axes['time']['data'], grid.axes['time']['units'])
        sstr = dts[0].strftime('%Y%m%d_%H%M%S')
        pyart.io.write_grid(md + 'grid_250_' + sstr + '.nc', grid)
        myd = pyart.graph.RadarMapDisplay(radar)
        fig = plt.figure(figsize=[18, 10])
        myd.plot_ppi_map('rain_z',
                         vmin=0,
                         vmax=100,
                         resolution='h',
                         max_lat=41.8,
                         min_lat=41.25,
                         min_lon=-88.3,
                         max_lon=-87.5)
        m = myd.basemap
        m.drawparallels(np.linspace(41, 42, 9), labels=[1, 0, 0, 0])
        m.drawmeridians(np.linspace(-88.4, -87, 8), labels=[0, 0, 0, 1])
        m.drawrivers()
        m.drawcounties()
        m.drawstates()
        m.drawmapscale(-88.,
                       41.55,
                       -88.,
                       41.55,
                       10,
                       barstyle='fancy',
                       fontcolor='k',
                       fillcolor1='b',
                       fillcolor2='k')
        myd.plot_point(-87.9706,
                       41.6815,
                       label_text='Argonne Lab',
                       label_offset=(0.0, 0.0))
        plt.savefig(md + 'radar_' + sstr + '.png')
        plt.close(fig)
        fig = plt.figure(figsize=[15, 15])
        max_lat = 43
        min_lat = 41.5
        min_lon = -88.3
        max_lon = -87.5
        display = pyart.graph.GridMapDisplay(grid)
        display.plot_basemap(lat_lines=np.arange(min_lat, max_lat, .1),
                             lon_lines=np.arange(min_lon, max_lon, .1),
                             resolution='h')
        display.plot_grid('rain_z', vmin=0, vmax=100)
        xcf, ycf = display.basemap(-87.9706, 41.6815)
        display.basemap.plot(xcf, ycf, 'ro')
        plt.text(xcf + 2000., ycf + 2000., 'Argonne Lab')
        display.basemap.drawcounties()
        display.basemap.drawrivers()

        display.basemap.drawmapscale(-88.,
                                     41.55,
                                     -88.,
                                     41.55,
                                     10,
                                     barstyle='fancy',
                                     fontcolor='k',
                                     fillcolor1='b',
                                     fillcolor2='k')
        display.plot_colorbar()
        plt.savefig(md + 'mapped_250_' + sstr + '.png')
        plt.close(fig)
        del (radar)
        del (grid)
    except:
        pass
    return 0
Example #57
0
def parse(file):

    hdr = True
    dataLine = 0
    name = []
    number_samples_read = 0
    nVars = 0
    data = []
    ts = []

    filepath = file[0]

    with open(filepath, 'r', errors='ignore') as fp:
        line = fp.readline()
        matchObj = re.match(first_line_expr, line)
        if not matchObj:
            print("Not a Starmon-mini DAT file !")
            return None

        cnt = 1
        while line:
            #print("Line {}: {} : {}".format(cnt, dataLine, line.strip()))
            if hdr:
                if line[0] != '#':
                    hdr = False
                else:
                    matchObj = re.match(soft_version_expr, line)
                    if matchObj:
                        #print("soft_version_expr:matchObj.group() : ", matchObj.group())
                        #print("soft_version_expr:matchObj.group(1) : ", matchObj.group(1))
                        software = matchObj.group(1)

                    matchObj = re.match(first_line_expr, line)
                    if matchObj:
                        #print("first_line_expr:matchObj.group() : ", matchObj.group())
                        #print("first_line_expr:matchObj.group(1) : ", matchObj.group(1))
                        file_created = matchObj.group(1)

                    matchObj = re.match(recorder_expr, line)
                    if matchObj:
                        #print("recorder_expr:matchObj.group() : ", matchObj.group())
                        #print("recorder_expr:matchObj.group(1) : ", matchObj.group(1))
                        #print("recorder_expr:matchObj.group(2) : ", matchObj.group(2))
                        #print("recorder_expr:matchObj.group(3) : ", matchObj.group(3))
                        instrument_model = matchObj.group(2)
                        instrument_serial_number = matchObj.group(3)

                    matchObj = re.match(series_expr, line)
                    if matchObj:
                        #print("series_expr:matchObj.group() : ", matchObj.group())
                        #print("series_expr:matchObj.group(1) : ", matchObj.group(1))
                        #print("series_expr:matchObj.group(2) : ", matchObj.group(2))
                        #print("series_expr:matchObj.group(3) : ", matchObj.group(3))
                        nameN = matchObj.group(1)
                        ncVarName = matchObj.group(2)
                        if ncVarName in nameMap:
                            ncVarName = nameMap[ncVarName]
                        unit = matchObj.group(3)
                        if unit in unitMap:
                            unit = unitMap[unit]

                        name.insert(nVars, {
                            'col': int(nameN),
                            'var_name': ncVarName,
                            'unit': unit
                        })

            if not hdr:
                lineSplit = line.split('\t')
                #print(lineSplit)
                splitVarNo = 0
                d = np.zeros(len(name))
                d.fill(np.nan)
                t = datetime.strptime(lineSplit[1], '%d/%m/%Y %H:%M:%S')
                ts.append(t)
                #print("timestamp %s" % ts)
                for v in name:
                    #print("{} : {}".format(splitVarNo, v))
                    d[splitVarNo] = float(lineSplit[v['col'] + 2])
                    splitVarNo = splitVarNo + 1
                data.append(d)
                #print(t, d)
                number_samples_read = number_samples_read + 1

                dataLine = dataLine + 1

            line = fp.readline()
            cnt += 1

    # trim data
    print("samplesRead %d data shape %s" % (number_samples_read, len(name)))

    #
    # build the netCDF file
    #

    ncTimeFormat = "%Y-%m-%dT%H:%M:%SZ"

    outputName = filepath + ".nc"

    print("output file : %s" % outputName)

    ncOut = Dataset(outputName, 'w', format='NETCDF4')

    ncOut.instrument = 'StarODDI - ' + instrument_model
    ncOut.instrument_model = instrument_model
    ncOut.instrument_serial_number = instrument_serial_number

    #     TIME:axis = "T";
    #     TIME:calendar = "gregorian";
    #     TIME:long_name = "time";
    #     TIME:units = "days since 1950-01-01 00:00:00 UTC";

    tDim = ncOut.createDimension("TIME", number_samples_read)
    ncTimesOut = ncOut.createVariable("TIME", "d", ("TIME", ), zlib=True)
    ncTimesOut.long_name = "time"
    ncTimesOut.units = "days since 1950-01-01 00:00:00 UTC"
    ncTimesOut.calendar = "gregorian"
    ncTimesOut.axis = "T"
    ncTimesOut[:] = date2num(ts,
                             units=ncTimesOut.units,
                             calendar=ncTimesOut.calendar)

    i = 0
    for v in name:
        print("Variable %s : unit %s" % (v['var_name'], v['unit']))
        varName = v['var_name']
        ncVarOut = ncOut.createVariable(
            varName, "f4", ("TIME", ), fill_value=np.nan,
            zlib=True)  # fill_value=nan otherwise defaults to max
        if v['unit']:
            ncVarOut.units = v['unit']
        ncVarOut[:] = np.array([d[i] for d in data])

        i += 1

    ncOut.setncattr(
        "time_coverage_start",
        num2date(ncTimesOut[0],
                 units=ncTimesOut.units,
                 calendar=ncTimesOut.calendar).strftime(ncTimeFormat))
    ncOut.setncattr(
        "time_coverage_end",
        num2date(ncTimesOut[-1],
                 units=ncTimesOut.units,
                 calendar=ncTimesOut.calendar).strftime(ncTimeFormat))
    ncOut.setncattr("date_created", datetime.utcnow().strftime(ncTimeFormat))
    ncOut.setncattr(
        "history",
        datetime.utcnow().strftime("%Y-%m-%d") + " created from file " +
        filepath + " source file created " + file_created + " by software  " +
        software.replace("\t", " "))

    ncOut.close()

    return outputName
Example #58
0
def main(f, t, T, y, Y, x, X, var='', z=0, Z=0, Source='usn-ncom'):

    try:
        url = sources[Source]
    except:
        url = Source

    source = Source

    if source == 'GLBu0.08':
        Vers = get_vers(t)
        url = url + Vers
    elif source == 'GLBv0.08':
        Vers = 'expt_93.0'
        url = url + Vers
    # Parse time

    if t is not None:
        try:  # Date and time is given
            startTime = datetime.datetime.strptime(t, '%Y-%m-%d %H:%M:%S')
            endTime = datetime.datetime.strptime(T, '%Y-%m-%d %H:%M:%S')
        except:  # Only date is given
            try:
                startTime = datetime.datetime.strptime(t, '%Y-%m-%d')
                endTime = datetime.datetime.strptime(T, '%Y-%m-%d')
            except:
                print('Could not parse time: ' + str(t))

    # Get metadata from URL
    d = Dataset(url, 'r')

    if f is None:
        print('==================================================')
        print(d)
    print('==================================================')

    # Get dimensions and their vectors
    for dimName in d.dimensions:
        print(dimName)
        dimvals = d.variables[dimName][:]
        if dimName == 'time':
            dimvals = num2date(dimvals, units=d.variables[dimName].units)
            if t is None:  # Return first time steps, if time not given
                startTime = dimvals[-2]
                endTime = dimvals[-1]
            if startTime > dimvals[-1] or endTime < dimvals[0]:
                sys.exit('Requested time span (%s - %s) not covered '
                         'by dataset (%s - %s)' %
                         (startTime, endTime, dimvals[0], dimvals[-1]))
            timeIndexStart = bisect.bisect_left(dimvals, startTime)
            timeIndexEnd = bisect.bisect_right(dimvals, endTime)
            timeIndexStart = np.maximum(timeIndexStart, 0)
            timeIndexEnd = np.minimum(timeIndexEnd, len(dimvals) - 1)
        if dimName == 'depth':
            print(str(dimvals))
            hasDepth = True
        else:
            print('\t' + str(dimvals[0]) + ' (min)')
            print('\t' + str(dimvals[-1]) + ' (max)')

        if dimName == 'lon':
            if x is None:
                x = dimvals[0]
                X = dimvals[1]
        if dimName == 'lat':
            if y is None:
                y = dimvals[0]
                Y = dimvals[1]

    # Print available parameters
    print('\nParameters (CF standard name):')
    for varName in d.variables:
        try:
            standard_name = d.variables[varName].standard_name
            print('\t' + varName + ' (' + standard_name + ')')
        except:
            print('\t' + varName)

    if var != '':
        varCommand = '-v ' + ','.join(var)
    else:
        varCommand = ''

    d.close()

    # Suggest command, if filename is not given
    if f is not None:
        # Subsetting in time and space
        subset = varCommand+' -d lon,%.2f,%.2f -d lat,%.2f,%.2f -d time,%i,%i' % \
            (x, X, y, Y, timeIndexStart, timeIndexEnd)
        if 'hasDepth' in locals():
            z = np.float(z)
            Z = np.float(Z)
            subset = subset + ' -d depth,%.2f,%.2f ' % (z, Z)

        out = ' -o out.nc '
        ncoCommand = 'ncks ' + subset + out + url + ' --overwrite'
        ncoCommand = ncoCommand + ' -o ' + f
        print(ncoCommand)
        subprocess.call(ncoCommand, shell=True)
        #subprocess.call('ncdump out.nc -v lon|tail -3', shell=True)
        #subprocess.call('ncdump out.nc -v lat|tail -3', shell=True)
    else:
        templateCommand = '%s -source %s -t \'%s\' -T \'%s\' ' \
                          ' -x %s -X %s -y %s -Y %s -f %s' % \
                          (sys.argv[0], source, startTime, endTime,
                           x, X, y, Y, 'out.nc')
        templateCommand = templateCommand + varCommand
        if 'hasDepth' in locals():
            templateCommand = templateCommand + \
                ' -z %s -Z %s ' % (z, Z)

        print('---------------------------------------------------------')
        print('Template command for data download (cut, paste and edit):')
        print(templateCommand)
        print('=========================================================')
Example #59
0
def scenarios():
    """
    Handle get and push requests for list of all finished scenarios and submit
    a new scenario, respectively.
    """
    if request.method == 'GET':

        scenarios = Scenario.objects

        # this is for the first three scenarios only
        if app.config['DEBUG'] and len(scenarios) < 3:
            for loop_counter in range(3):
                _init_dev_db(app.config['BASE_PARAMETER_NC'], loop_counter)

                scenarios = Scenario.objects

        return jsonify(scenarios=scenarios)

    else:
        BASE_PARAMETER_NC = app.config['BASE_PARAMETER_NC']

        # assemble parts of a new scenario record
        vegmap_json = request.json['veg_map_by_hru']

        name = request.json['name']

        time_received = datetime.datetime.now()

        new_scenario = Scenario(
            name=name,
            time_received=time_received,
        )

        scenario_runner = new_scenario.initialize_runner(BASE_PARAMETER_NC)

        # using vegmap sent by client, update coverage type variables
        scenario_runner.update_cov_type(vegmap_json['bare_ground'], 0)
        scenario_runner.update_cov_type(vegmap_json['grasses'], 1)
        scenario_runner.update_cov_type(vegmap_json['shrubs'], 2)
        scenario_runner.update_cov_type(vegmap_json['trees'], 3)
        scenario_runner.update_cov_type(vegmap_json['conifers'], 4)

        # commit changes to coverage type in preparation for scenario run
        scenario_runner.finalize_run()

        # now that changes to scenario_file are committed, we update Scenario
        new_scenario.veg_map_by_hru =\
            get_veg_map_by_hru(scenario_runner.scenario_file)

        new_scenario.save()

        modelserver_run = scenario_runner.run(
            auth_host=app.config['AUTH_HOST'],
            model_host=app.config['MODEL_HOST'],
            app_username=app.config['APP_USERNAME'],
            app_password=app.config['APP_PASSWORD'])

        time_finished = datetime.datetime.now()
        new_scenario.time_finished = time_finished

        # extract URLs pointing to input/output resources stored on modelserver
        resources = modelserver_run.resources

        control =\
            filter(lambda x: 'control' == x.resource_type, resources
                   ).pop().resource_url
        parameter =\
            filter(lambda x: 'param' == x.resource_type, resources
                   ).pop().resource_url
        data =\
            filter(lambda x: 'data' == x.resource_type, resources
                   ).pop().resource_url

        inputs = Inputs(control=control, parameter=parameter, data=data)
        new_scenario.inputs = inputs

        statsvar =\
            filter(lambda x: 'statsvar' == x.resource_type, resources
                   ).pop().resource_url

        outputs = Outputs(statsvar=statsvar)
        new_scenario.outputs = outputs

        # extract hydrograph from the statsvar file and add to Scenario
        if not os.path.isdir('.tmp'):
            os.mkdir('.tmp')

        tmp_statsvar = os.path.join('.tmp', 'statsvar-' + str(uuid4()))
        urlretrieve(statsvar, tmp_statsvar)

        d = netCDF4.Dataset(tmp_statsvar, 'r')
        cfs = d['basin_cfs_1'][:]

        t = d.variables['time']

        # need to subtract 1...bug in generation of statsvar b/c t starts at 1
        dates = netCDF4.num2date(t[:] - 1, t.units)

        hydrograph = Hydrograph(time_array=dates, streamflow_array=cfs)
        new_scenario.hydrograph = hydrograph

        new_scenario.save()

        # clean up temporary statsvar netCDF
        d.close()
        os.remove(tmp_statsvar)

        return jsonify(scenario=new_scenario.to_json())
Example #60
0
def lookup_climate_files(srcFolder,
                         inputVars,
                         startDate,
                         endDate,
                         timestep=dt.timedelta(days=1),
                         srcFiles=[],
                         lookup_table={}):
    """
    Generates or appends a dictionary of dates and references to files for each date and variable of interest
    The function checks for each date and each variable, which file in the srcFolder contains the variable
    of interest at the date of interest
    """
    if len(lookup_table) == 0:
        # lookup_table is still empty. Fill lookup_table with dates and empty strings
        # make list of all dates
        dateList = []
        curDate = startDate
        while curDate <= endDate:
            dateList.append(curDate)
            curDate += timestep
        lookup_table['time'] = dateList
        # pre-fill the dictionary with lists of empty strings for each file
        for inputVar in inputVars:
            lookup_table[inputVar] = empty(len(dateList), 'S1000')

    # make list of all available files
    if len(srcFiles) == 0:
        srcFiles = recursive_glob(rootdir=srcFolder, suffix='.nc')
        srcFiles.sort()

    for srcFile in srcFiles:
        print 'Scanning ' + srcFile
        nc_src = nc4.Dataset(srcFile, 'r')
        time = nc_src.variables['time']
        try:
            timeObjRaw = nc4.num2date(time[:],
                                      units=time.units,
                                      calendar=time.calendar)
        except:
            timeObjRaw = nc4.num2date(time[:],
                                      units=time.units,
                                      calendar='gregorian')
        # UUGGGHHH: this seems to be a bug. calendar other than gregorian give other objects than
        # datetime.datetime objects. Here we convert to a year with a julian day string, then back to date objects
        timeObj = []
        for t in timeObjRaw:
            timeObj.append(dt.datetime.strptime(t.strftime('%Y%j'), '%Y%j'))
        #timeObj = nc4.num2date(nc4.date2num(timeObj, units=time.units, calendar='gregorian'), units=time.units, calendar='gregorian')
        # now loop over all time steps, check the date and write valid dates to a list, write time series to PCRaster maps
        for n in range(len(timeObj)):
            # remove any hours or minutes data from curTime
            timeObj[n] = timeObj[n].replace(hour=0)
            timeObj[n] = timeObj[n].replace(minute=0)

        # check for first and last date in timeObj
        firstDate = timeObj[0]
        lastDate = timeObj[-1]
        # find locations of firstDate and lastDate in our target time periods
        idxStart = where(array(lookup_table['time']) == firstDate)[0]
        idxEnd = where(array(lookup_table['time']) == lastDate)[0]
        if len(idxStart) == 1 or len(idxEnd == 1):
            if len(idxStart) == 0:
                idxStart = array([0])
            if len(idxEnd) == 0:
                idxEnd = len(dateList) - 1
            allVars = nc_src.variables.keys()
            # check which variables are present in file, check file
            for inputVar in inputVars:
                if inputVar in allVars:
                    lookup_table[inputVar][idxStart:idxEnd + 1] = srcFile
        nc_src.close()
    return lookup_table