def __init__(self, startStr = '201706272200', endStr = '201706280400', upscaleKm = 1, min_rainrate = 0.01): startTime = ti.timestring2datetime(startStr) endTime = ti.timestring2datetime(endStr) self.startStr = startStr self.endStr = endStr self.startTime = startTime self.endTime = endTime self.upscaleKm = upscaleKm self.min_rainrate = min_rainrate self._DATA = {} self.load_observation()
def produce_forecast(timeStartStr,leadTimeMin): produce_radar_observations = False produce_radar_extrapolation = False produce_cosmo_1 = True # parameters rainThreshold = 0.08 ## RADAR OBSERVATIONS if produce_radar_observations: print('Retrieve radar observations for ' + timeStartStr + ' + ' + str(leadTimeMin) + ' min') # start 5 min earlier to compute 10min accumulation at t0 startTimeToPass = ti.timestring2datetime(timeStartStr) - datetime.timedelta(minutes=5) startTimeToPassStr = ti.datetime2timestring(startTimeToPass) radar_observations_5min, r = nw.get_radar_observations(startTimeToPassStr, leadTimeMin+5, product='RZC') # aggregate to 10-min forecast (to match COSMO1 resolution) radar_observations_10min = nw.aggregate_in_time(radar_observations_5min,timeAccumMin=10,type='sum') ## EXTRAPOLATION FORECAST if produce_radar_extrapolation: print('Run radar extrapolation for ' + timeStartStr + ' + ' + str(leadTimeMin) + ' min') # produce 5-min radar extrapolation radar_extrapolation_5min, radar_mask_5min = nw.radar_extrapolation(timeStartStr,leadTimeMin, product='RZC') # aggregate to 10-min forecast (to match COSMO1 resolution) radar_extrapolation_10min = nw.aggregate_in_time(radar_extrapolation_5min,timeAccumMin=10,type='sum') radar_mask_10min = nw.aggregate_in_time(radar_mask_5min,timeAccumMin=10,type='mean') # add observation at t0 radar_extrapolation_10min = np.concatenate((radar_observations_10min[:,:,0,np.newaxis],radar_extrapolation_10min),axis=2) ## COSMO-1 FORECASTS if produce_cosmo_1: print('Retrive COSMO-1 for ' + timeStartStr + ' + ' + str(leadTimeMin) + ' min') cosmo1_10min = nw.get_cosmo1(timeStartStr, leadTimeMin) cosmo1_10min[cosmo1_10min <= rainThreshold] = np.nan
def get_radar_observations(timeStartStr, leadTimeMin, domainSize = 512, product = 'RZC', rainThreshold = 0.08): # Get datetime format startTime = ti.timestring2datetime(timeStartStr) endTime = startTime + datetime.timedelta(minutes=leadTimeMin) # Number of consecutive radar images to retrieve dt = 5 # minutes nimages = int((endTime - startTime).total_seconds()/60/dt) + 1 # Retrieve images radarStack = get_n_next_radar_image(timeStartStr, nimages, domainSize, product, rainThreshold ) # Store in Numpy array radar_observations = np.zeros((domainSize,domainSize,nimages)) radar_mask = np.zeros((domainSize,domainSize,nimages)) timestamps = [] currentTime = startTime for t in xrange(len(radarStack)): # radar_observations[:,:,t] = radarStack[t].rainrateNans.copy() if radarStack[t].war>-1: radar_observations[:,:,t] = radarStack[t].rainrate.copy() radar_mask[:,:,t] = radarStack[t].mask.copy() timestamps.append(radarStack[t].datetime) else: radar_observations[:,:,t] = np.zeros((domainSize,domainSize))*np.nan radar_mask[:,:,t] = np.ones((domainSize,domainSize)) timestamps.append(currentTime) currentTime = currentTime + datetime.timedelta(minutes=5) robject = radarStack[0] return radar_observations, radar_mask, timestamps, robject
def produce_radar_observation_with_accumulation(startValidTimeStr, endValidTimeStr, newAccumulationMin=10, domainSize=512, product='RZC',rainThreshold=0.08): # base accumulation is 5 min baseAccumMin = 5 accumFactor = int(newAccumulationMin/baseAccumMin) # datetime format startValidTime = ti.timestring2datetime(startValidTimeStr) endValidTime = ti.timestring2datetime(endValidTimeStr) leadTimeMin = int((endValidTime - startValidTime).total_seconds()/60) # start to compute the correct accumulation at t0 if newAccumulationMin>baseAccumMin: startTimeToPass = startValidTime - datetime.timedelta(minutes=newAccumulationMin-baseAccumMin) else: startTimeToPass = startValidTime startTimeToPassStr = ti.datetime2timestring(startTimeToPass) with np.errstate(invalid='ignore'): radar_observations_5min, radar_mask_5min, timestamps, r = nw.get_radar_observations(startTimeToPassStr, leadTimeMin+newAccumulationMin-baseAccumMin, product=product, rainThreshold = 0) # convert to mm for computing accumulations radar_observations_5min = radar_observations_5min/60*baseAccumMin # aggregate to new accumulation if newAccumulationMin>baseAccumMin: radar_observations_new = nw.aggregate_in_time(radar_observations_5min,timeAccumMin=newAccumulationMin,type='sum') radar_mask_new = nw.aggregate_in_time(radar_mask_5min,timeAccumMin=newAccumulationMin,type='nansum') radar_mask_new[radar_mask_new==0] = np.nan radar_mask_new[radar_mask_new>0] = 1 timestamps = timestamps[accumFactor-1::accumFactor] else: radar_observations_new = radar_observations_5min radar_mask_new = radar_mask_5min # convert to mm/h radar_observations_new = radar_observations_new/newAccumulationMin*60 # Apply rain threshold radar_observations_new[radar_observations_new<=rainThreshold] = 0 # [time,y,x] radar_observations_new = np.rollaxis(radar_observations_new,2,0) radar_mask_new = np.rollaxis(radar_mask_new,2,0) return radar_observations_new,radar_mask_new,timestamps
def __init__(self, startStr='201706272200', endStr='201706280400', upscaleKm=1, min_rainrate=0.01): startTime = ti.timestring2datetime(startStr) endTime = ti.timestring2datetime(endStr) self.startStr = startStr self.endStr = endStr self.startTime = startTime self.endTime = endTime self.upscaleKm = upscaleKm self.min_rainrate = min_rainrate self._DATA = {} self.load_observation()
def get_n_last_radar_image(timeStampStr, nimages, domainSize, product = 'RZC', rainThreshold = 0.08): timeStamp = ti.timestring2datetime(timeStampStr) radarStack = [] nextTimeStamp = timeStamp for n in xrange(nimages): nextTimeStampStr = ti.datetime2timestring(nextTimeStamp) if product == 'RZC': r = io.read_bin_image(nextTimeStampStr,fftDomainSize=domainSize,product=product, minR = rainThreshold) # get full extent of the radar image else: r = io.read_gif_image(nextTimeStampStr,fftDomainSize=domainSize,product=product, minR = rainThreshold) # get full extent of the radar image radarStack.insert(0,r) nextTimeStamp = nextTimeStamp - datetime.timedelta(minutes=5) return radarStack
def get_n_next_radar_image(timeStampStr, nimages, domainSize, product = 'RZC', rainThreshold = 0.08): timeStamp = ti.timestring2datetime(timeStampStr) radarStack = [] nextTimeStamp = timeStamp for n in xrange(nimages): nextTimeStampStr = ti.datetime2timestring(nextTimeStamp) if product=='RZC': r = io.read_bin_image(nextTimeStampStr,fftDomainSize=domainSize,product=product, minR = rainThreshold) else: r = io.read_gif_image(nextTimeStampStr,fftDomainSize=domainSize,product=product, minR = rainThreshold) radarStack.append(r) nextTimeStamp = nextTimeStamp + datetime.timedelta(minutes=5) return radarStack
plotSpectrum = args.spec extension = '.' + args.format product = args.product if (int(args.start) > int(args.end)): print('Time end should be after time start') sys.exit(1) if (int(args.start) < 198001010000) or (int(args.start) > 203001010000): print('Invalid -start or -end time arguments.') sys.exit(1) else: timeStartStr = args.start timeEndStr = args.end ### Get list of filenames timeStart = ti.timestring2datetime(timeStartStr) timeEnd = ti.timestring2datetime(timeEndStr) fileNameExpr = product + '_' + plotSpectrum + 'PS_*' fileList = io.get_files_period(timeStart, timeEnd, inBaseDir, fileNameExpr, tempResMin=5) # Check if there are file found if len(fileList) == 0: print('No files found to generate video. Check your input filestamps.') sys.exit() ######## Generate clip
def get_radar_extrapolation(startValidTimeStr,endValidTimeStr, newAccumulationMin=10, domainSize=512, rainThreshold=0.08, product='RZC', outBaseDir='/scratch/ned/data/'): # datetime format startValidTime = ti.timestring2datetime(startValidTimeStr) endValidTime = ti.timestring2datetime(endValidTimeStr) leadTimeMin = int((endValidTime - startValidTime).total_seconds()/60) # Check if the nc file already exists year = startValidTime.year yearStr = str(year)[2:4] julianDay = startValidTime.timetuple().tm_yday julianDayStr = '%03i' % julianDay yearJulianStr = yearStr + julianDayStr outDir = outBaseDir + startValidTime.strftime("%Y") + '/' + startValidTime.strftime("%y") + julianDayStr + '/' fcstName = 'radar-extrapolation_' + startValidTime.strftime("%Y%m%d%H%M") + '_' + str(int(leadTimeMin/60)) + 'hours' fcstFile = r'%s' % (outDir + fcstName + '.nc') # base accumulation is 5 min baseAccumMin = 5 accumFactor = int(newAccumulationMin/baseAccumMin) # if not produce forecasts if os.path.isfile(fcstFile) == False: # produce 5-min radar extrapolation radar_extrapolation_5min, timestamps_5min = nw.radar_extrapolation(startValidTimeStr,leadTimeMin,finalDomainSize=domainSize,product=product,rainThreshold=rainThreshold) if product=='RZC': r = io.read_bin_image(startValidTimeStr, fftDomainSize=domainSize, inBaseDir = '/scratch/ned/data/') else: r = io.read_gif_image(startValidTimeStr, fftDomainSize=domainSize, inBaseDir = '/scratch/ned/data/') if accumFactor>1: # convert to mm radar_extrapolation_5min = radar_extrapolation_5min/60*baseAccumMin # aggregate to new accumulation (to match COSMO1 resolution) radar_extrapolation_new = nw.aggregate_in_time(radar_extrapolation_5min,timeAccumMin=newAccumulationMin,type='sum') timestamps_new = timestamps_5min[accumFactor-1::accumFactor] # convert to mm/h radar_extrapolation_new = radar_extrapolation_new/newAccumulationMin*60 # get observations at t0 [mm/h] radar_observations_t0, _, _ = produce_radar_observation_with_accumulation(startValidTimeStr, startValidTimeStr, newAccumulationMin, domainSize=domainSize, product=product) radar_observations_t0 = np.squeeze(radar_observations_t0)[:,:,None] else: # no need to aggregate forecasts radar_extrapolation_new = radar_extrapolation_5min timestamps_new = timestamps_5min # get observations at t0 radar_observations_t0 = r.rainrate[:,:,None] # add observation for t0 radar_extrapolation_new = np.concatenate((radar_observations_t0,radar_extrapolation_new),axis=2) timestamps_new.insert(0,startValidTime) timestamps_new = np.array(timestamps_new) # [time,y,x] radar_extrapolation_new = np.rollaxis(radar_extrapolation_new,2,0) # save netcdf Xcoords = r.subXcoords Ycoords = r.subYcoords save_3darray_netcdf(fcstFile, radar_extrapolation_new, 'radar_extrapolation',\ timestamps_new,Xcoords,Ycoords) print('Read: ' + fcstFile) # Now read the NetCDF file radar_extrapolation, timestamps, Xcoords, Ycoords = load_3darray_netcdf(fcstFile) testplot=False if testplot: n=0 while n<100: n+=1 for t in xrange(timestamps.shape[0]): plt.clf() plt.imshow(radar_extrapolation[t,:,:],interpolation ='nearest',vmin=0,vmax=65, extent=[Xcoords.min(), Xcoords.max(), Ycoords.min(), Ycoords.max()]) plt.title(timestamps[t]) plt.pause(1) return radar_extrapolation, timestamps, Xcoords, Ycoords
def get_cosmoE10min(startValidTimeStr, endValidTimeStr, members = 'all', domainSize=512, outBaseDir='/scratch/ned/data/', cosmoBaseDir='/store/s83/tsm/EXP_TST/611/',rainThreshold=0.08,latencyTimeMin=100,lag=0,overwrite=False,useavailable=True): # datetime format startValidTime = ti.timestring2datetime(startValidTimeStr) endValidTime = ti.timestring2datetime(endValidTimeStr) timebounds = [startValidTime, endValidTime] # Check if the single nc file already exists if members=='all': year = startValidTime.year yearStr = str(year)[2:4] julianDay = startValidTime.timetuple().tm_yday julianDayStr = '%03i' % julianDay yearJulianStr = yearStr + julianDayStr outDir = outBaseDir + startValidTime.strftime("%Y") + '/' + startValidTime.strftime("%y") + julianDayStr + '/' fcstName = 'COSMOE10min_' + startValidTime.strftime("%Y%m%d%H%M") + '_' + endValidTime.strftime("%Y%m%d%H%M") + '_lag' + str(lag) + '_ltMin' + str(int(latencyTimeMin)) fcstFile = r'%s' % (outDir + fcstName + '.nc') else: fcstFile = 'donotsavethisfile' # if not load original forecasts if (not os.path.isfile(fcstFile)) or overwrite: print(fcstFile + ' not found.') analysis_not_found = True nloops = 0 while analysis_not_found and (nloops < 2): # Get most recent run (or second most recent) analysisTimeStr = rfe.find_nearest_forecast(startValidTimeStr, 'cosmo-e', lat_timeMin = latencyTimeMin, lag = lag) print('run time: ',analysisTimeStr) analysisTime = ti.timestring2datetime(analysisTimeStr) # list of EPS members to load if members == 'all': members = range(21) # Folder to the nc files outDir = cosmoBaseDir + 'FCST' + analysisTime.strftime("%y") + '/' + analysisTime.strftime("%y%m%d%H") + '_611/output/' if os.path.isdir(outDir) == False: print('Folder not found: ' + outDir) if useavailable: startValidTime -= datetime.timedelta(hours=12) startValidTimeStr = ti.datetime2timestring(startValidTime) nloops+=1 else: return None,None,None,None else: analysis_not_found=False if analysis_not_found: return None,None,None,None # Load individual EPS member and merge them in one array countm=0 for member in members: thisFcstFile = outDir + 'cosmo-e_TOT_PREC_' + str(member).zfill(3) + '.nc' if not os.path.isfile(thisFcstFile): print('File not found: ' + thisFcstFile) sys.exit() else: print('Read: ' + thisFcstFile) # read the NetCDF file # this_member, timestamps, Xcoords, Ycoords = load_3darray_netcdf_with_bounds(thisFcstFile, timebounds, domainSize) # need to fix this... this_member, timestamps, Xcoords, Ycoords = load_3darray_netcdf(thisFcstFile) # print(timebounds) # print(this_member.shape) # flip and merge together the members if member == members[0]: if domainSize>0: cosmoe_data = np.zeros((this_member.shape[0], len(members), domainSize, domainSize)) else: cosmoe_data = np.zeros((this_member.shape[0], len(members),this_member.shape[2],this_member.shape[3])) for i in xrange(this_member.shape[0]): # flip frames this_frame = np.flipud(this_member[i,0,:,:]) if domainSize>0: cosmoe_data[i,countm,:,:] = dt.extract_middle_domain(this_frame, domainSize, domainSize) else: cosmoe_data[i,countm,:,:] = this_frame del this_member countm+=1 # convert to mm/h cosmoe_data = cosmoe_data*6 # Get coordinates of reduced domain if domainSize>0: extent = dt.get_reduced_extent(Xcoords.shape[0], Ycoords.shape[0], domainSize, domainSize) Xmin = Xcoords[extent[0]] Ymin = Ycoords[extent[1]] Xmax = Xcoords[extent[2]] Ymax = Ycoords[extent[3]] extent = (Xmin, Xmax, Ymin, Ymax) Xcoords = np.arange(Xmin,Xmax,1000) Ycoords = np.arange(Ymin,Ymax,1000) else: extent = (Xcoords.min(), Xcoords.max(), Ycoords.min(), Ycoords.max()) # Extract timestamps idxKeep1 = timestamps >= timebounds[0] idxKeep2 = timestamps <= timebounds[1] idxKeep = np.logical_and(idxKeep1,idxKeep2) cosmoe_data = cosmoe_data[idxKeep,:,:,:] timestamps = timestamps[idxKeep] # and store it before loading it again if (not fcstFile=='donotsavethisfile'): # print(fcstFile,cosmoe_data.shape,timestamps.shape,Xcoords.shape,Ycoords.shape) save_4darray_netcdf(fcstFile, cosmoe_data, 'COSMO-E10min',\ timestamps,cosmoe_data.shape[1],Xcoords,Ycoords) else: print('Read: ' + fcstFile) # Now read the NetCDF file cosmoe_data, timestamps, members, Ycoords, Xcoords = load_4darray_netcdf(fcstFile) # Apply rain threshold cosmoe_data[cosmoe_data<=rainThreshold] = 0 testplot=False if testplot: nmember=0 n=0 while n<100: n+=1 for t in xrange(timestamps.shape[0]): plt.clf() plt.imshow(10*np.log10(cosmoe_data[t,nmember,:,:]),interpolation ='nearest',vmin=-12,vmax=20, extent=[Xcoords.min(), Xcoords.max(), Ycoords.min(), Ycoords.max()]) plt.title(timestamps[t]) plt.pause(1) return cosmoe_data, timestamps, Xcoords, Ycoords
def get_lagged_cosmo1(startValidTimeStr, endValidTimeStr, domainSize=512, leadTimeMin = 6*60, outBaseDir='/scratch/ned/data/', rainThreshold=0.08): # get all individual forecasts' starting times latencyTimeHr = 1#100/60 maxLeadTimeHr=33 fcstStarts,fcstMembers,fcstLeadTimeStartsMin,fcstLeadTimeStopsMin = rf1.get_lagged_ensemble_members(startValidTimeStr, endValidTimeStr, latencyTimeHr, maxLeadTimeHr) nmembers = len(fcstMembers) # if necessary, extract them with fieldextra filesOut = [] analysisTimes = [] for n in xrange(nmembers): print('----------------------------------------------------') print(fcstMembers[n] + ', analysis time: ' + fcstStarts[n].strftime("%Y%m%d%H%M%S") + ', ' + str(fcstLeadTimeStartsMin[n]/60) + ' to ' + str(fcstLeadTimeStopsMin[n]/60) + ' hours.') # Check if the nc file already exists year = fcstStarts[n].year yearStr = str(year)[2:4] julianDay = fcstStarts[n].timetuple().tm_yday julianDayStr = '%03i' % julianDay yearJulianStr = yearStr + julianDayStr outDir = outBaseDir + fcstStarts[n].strftime("%Y") + '/' + fcstStarts[n].strftime("%y") + julianDayStr + '/' it_exists = False LeadTimeToCheck = fcstLeadTimeStopsMin[n]/60 while (it_exists == False) and (LeadTimeToCheck <= maxLeadTimeHr): fcstName = fcstStarts[n].strftime("%Y%m%d%H%M") + '_' + str(int(LeadTimeToCheck)) + 'hours' fcstFile = r'%s' % (outDir + 'cosmo-1_TOT_PREC_' + fcstName + '.nc') if os.path.isfile(fcstFile) == False: LeadTimeToCheck += 1 else: it_exists = True break if it_exists == False: LeadTimeToGetHrs = np.minimum(maxLeadTimeHr, np.ceil(( fcstLeadTimeStopsMin[n]/60 )/12)*12) fcstFile = rf1.run_fieldextra_c1(fcstStarts[n].strftime("%Y%m%d%H%M"),LeadTimeToGetHrs*60,outBaseDir=outBaseDir) print('read: ' + fcstFile) # Now read the NetCDF file cosmo1_data, timestamps, Xcoords, Ycoords = load_3darray_netcdf(fcstFile) # exclude first time step (it's all NaNs!) cosmo1_data=cosmo1_data[1:,:,:] timestamps=timestamps[1:] # convert to mm/h cosmo1_data = cosmo1_data*6 # flip and extract middle domain new_data = np.zeros((cosmo1_data.shape[0],domainSize,domainSize)) for i in range(cosmo1_data.shape[0]): # flip frames cosmo1_data[i,:,:] = np.flipud(cosmo1_data[i,:,:]) # cut domain if domainSize>0: new_data[i,:,:] = dt.extract_middle_domain(cosmo1_data[i,:,:], domainSize, domainSize) else: new_data[i,:,:] = cosmo1_data[i,:,:] cosmo1_data = new_data # Apply rain threshold cosmo1_data[cosmo1_data<=rainThreshold] = 0 # Extract timestamps idxKeep1 = timestamps >= ti.timestring2datetime(startValidTimeStr) idxKeep2 = timestamps <= ti.timestring2datetime(endValidTimeStr) cosmo1_data = cosmo1_data[idxKeep1*idxKeep2,:,:] timestamps = timestamps[idxKeep1*idxKeep2] # Build 4D array with all members if n==0: cosmo1_lagged_data = np.zeros((cosmo1_data.shape[0],nmembers,cosmo1_data.shape[1],cosmo1_data.shape[2])) cosmo1_lagged_data[:,n,:,:] = cosmo1_data.copy() # Get coordinates of reduced domain if domainSize>0: extent = dt.get_reduced_extent(Xcoords.shape[0], Ycoords.shape[0], domainSize, domainSize) Xmin = Xcoords[extent[0]] Ymin = Ycoords[extent[1]] Xmax = Xcoords[extent[2]] Ymax = Ycoords[extent[3]] extent = (Xmin, Xmax, Ymin, Ymax) subXcoords = np.arange(Xmin,Xmax,1000) subYcoords = np.arange(Ymin,Ymax,1000) testplot=False if testplot: nmember=0 n=0 while n<100: n+=1 for t in xrange(timestamps.shape[0]): plt.clf() plt.imshow(cosmo1_lagged_data[t,nmember,:,:],interpolation ='nearest',vmin=0,vmax=65, extent=[subXcoords.min(), subXcoords.max(), subYcoords.min(), subYcoords.max()]) plt.title(timestamps[t]) plt.pause(1) return cosmo1_lagged_data, timestamps, subXcoords, subYcoords
def get_cosmoE(startValidTimeStr, endValidTimeStr, analysisTimeStr = [], domainSize=512, leadTimeMin = 24*60, outBaseDir='/scratch/ned/data/',rainThreshold=0.08, latencyTimeMin=100, overwrite = False): # Get most recent run if len(analysisTimeStr)==0: analysisTimeStr = rfe.find_nearest_forecast(startValidTimeStr, 'cosmo-e', lat_timeMin = latencyTimeMin) analysisTime = ti.timestring2datetime(analysisTimeStr) leadtimeHrs = int (( ti.timestring2datetime(endValidTimeStr) - analysisTime ).total_seconds()/3600 ) # # Check if the nc file already exists # year = analysisTime.year # yearStr = str(year)[2:4] # julianDay = analysisTime.timetuple().tm_yday # julianDayStr = '%03i' % julianDay # yearJulianStr = yearStr + julianDayStr # outDir = outBaseDir + analysisTime.strftime("%Y") + '/' + analysisTime.strftime("%y") + julianDayStr + '/' # # fcstName = analysisTime.strftime("%Y%m%d%H%M") + '_' + str(int(leadTimeMin/60)) + 'hours' # fcstName = analysisTimeStr + '_' + startValidTimeStr + '_' + endValidTimeStr # fcstFile = r'%s' % (outDir + 'cosmo-E_TOT_PREC_' + fcstName + '.nc') # # if not call fieldextra # if os.path.isfile(fcstFile) == False: # rfe.run_fieldextra_ce(analysisTimeStr,startValidTimeStr,endValidTimeStr,outBaseDir=outBaseDir) outFile = rfe.run_fieldextra_forecast(analysisTimeStr, leadtimeHrs, fieldName='TOT_PREC', outBaseDir = '/scratch/ned/data/', modelName='cosmo-e', deltaMin = 60, overwrite = overwrite) print('Read: ' + outFile) # Now read the NetCDF file cosmoe_data, timestamps, members, Ycoords, Xcoords = load_4darray_netcdf(outFile) # exclude first time step (it's all NaNs!) # cosmoe_data=cosmoe_data[1:,:,:,:] # timestamps=timestamps[1:] # flip and extract middle domain new_data = np.zeros((cosmoe_data.shape[0], cosmoe_data.shape[1],domainSize,domainSize)) for i in xrange(cosmoe_data.shape[0]): for m in xrange(cosmoe_data.shape[1]): # flip frames cosmoe_data[i,m,:,:] = np.flipud(cosmoe_data[i,m,:,:]) # cut domain if domainSize>0: new_data[i,m,:,:] = dt.extract_middle_domain(cosmoe_data[i,m,:,:], domainSize, domainSize) else: new_data[i,m,:,:] = cosmoe_data[i,m,:,:] cosmoe_data = new_data # Apply rain threshold cosmoe_data[cosmoe_data<=rainThreshold] = 0 # Get coordinates of reduced domain if domainSize>0: extent = dt.get_reduced_extent(Xcoords.shape[0], Ycoords.shape[0], domainSize, domainSize) Xmin = Xcoords[extent[0]] Ymin = Ycoords[extent[1]] Xmax = Xcoords[extent[2]] Ymax = Ycoords[extent[3]] extent = (Xmin, Xmax, Ymin, Ymax) subXcoords = np.arange(Xmin,Xmax,1000) subYcoords = np.arange(Ymin,Ymax,1000) # Extract timestamps idxKeep1 = timestamps >= ti.timestring2datetime(startValidTimeStr) idxKeep2 = timestamps <= ti.timestring2datetime(endValidTimeStr) cosmoe_data = cosmoe_data[idxKeep1*idxKeep2,:,:,:] timestamps = timestamps[idxKeep1*idxKeep2] testplot=False if testplot: nmember=0 n=0 while n<100: n+=1 for t in xrange(timestamps.shape[0]): plt.clf() plt.imshow(cosmoe_data[t,nmember,:,:],interpolation ='nearest',vmin=0,vmax=65, extent=[subXcoords.min(), subXcoords.max(), subYcoords.min(), subYcoords.max()]) plt.title(timestamps[t]) plt.pause(1) return cosmoe_data, timestamps, subXcoords, subYcoords
def get_ensemble_radar_extrapolation(startValidTimeStr,endValidTimeStr, NumberMembers = 2, NumberLevels = 1, newAccumulationMin=10, rainThreshold = 0.08, domainSize=512, local_level = 0, seed = 42, product='RZC', outBaseDir='/scratch/ned/data/'): # datetime format startValidTime = ti.timestring2datetime(startValidTimeStr) endValidTime = ti.timestring2datetime(endValidTimeStr) leadTimeMin = int((endValidTime - startValidTime).total_seconds()/60) # Check if the nc file already exists year = startValidTime.year yearStr = str(year)[2:4] julianDay = startValidTime.timetuple().tm_yday julianDayStr = '%03i' % julianDay yearJulianStr = yearStr + julianDayStr outDir = outBaseDir + startValidTime.strftime("%Y") + '/' + startValidTime.strftime("%y") + julianDayStr + '/' fcstName = 'ensemble-radar-extrapolation_' + startValidTime.strftime("%Y%m%d%H%M") + '_' + str(int(leadTimeMin/60)) + 'hours' + '_' + str(NumberMembers) + 'members' + '_' + str(NumberLevels) + 'levels' + '_' + str(newAccumulationMin) + 'min' + '_' + product + '_' + str(domainSize) + 'km' + '_' + str(local_level) + 'local_seed' + str(seed) fcstNameMask = fcstName + '_mask' fcstFile = r'%s' % (outDir + fcstName + '.nc') fcstFileMask = r'%s' % (outDir + fcstNameMask + '.nc') # base accumulation is 5 min baseAccumMin = 5 accumFactor = int(newAccumulationMin/baseAccumMin) # if not produce forecasts if os.path.isfile(fcstFile) == False: # produce 5-min radar extrapolation radar_extrapolation_5min, timestamps_5min, radarMask_5min = nw.probabilistic_radar_extrapolation(startValidTimeStr,leadTimeMin,finalDomainSize=domainSize,NumberMembers=NumberMembers,NumberLevels=NumberLevels,product=product,local_level=local_level,seed=seed) if product=='RZC': r = io.read_bin_image(startValidTimeStr, fftDomainSize=domainSize, product=product, inBaseDir = '/scratch/ned/data/') else: r = io.read_gif_image(startValidTimeStr, fftDomainSize=domainSize, product=product, inBaseDir = '/scratch/ned/data/') # print(radarMask_5min.shape) if accumFactor>1: # convert to mm radar_extrapolation_5min = radar_extrapolation_5min/60*baseAccumMin # aggregate to new accumulation (to match COSMO1 resolution) for m in xrange(NumberMembers): radar_extrapolation_new_member = nw.aggregate_in_time(radar_extrapolation_5min[:,:,:,m],timeAccumMin=newAccumulationMin,type='sum') if m==0: radar_extrapolation_new = np.zeros((radar_extrapolation_new_member.shape[0],radar_extrapolation_new_member.shape[1],radar_extrapolation_new_member.shape[2],NumberMembers)) radar_extrapolation_new[:,:,:,m] = radar_extrapolation_new_member timestamps_new = timestamps_5min[accumFactor-1::accumFactor] radarMask_new = radarMask_5min[:,:,accumFactor-1::accumFactor] # print(radarMask_new.shape) # convert to mm/h radar_extrapolation_new = radar_extrapolation_new/newAccumulationMin*60 # get observations at t0 [mm/h] radar_observations_t0, _, _ = produce_radar_observation_with_accumulation(startValidTimeStr, startValidTimeStr, newAccumulationMin, domainSize, product=product) radar_observations_t0 = np.squeeze(radar_observations_t0) radarmask_t0 = r.mask.copy() radarmask_t0 = np.array(np.isnan(radarmask_t0),dtype=int) # print(radarmask_t0.shape) else: # no need to aggregate forecasts radar_extrapolation_new = radar_extrapolation_5min timestamps_new = timestamps_5min radarMask_new = radarMask_5min # get observations at t0 radar_observations_t0 = r.rainrate.copy() radarmask_t0 = r.mask.copy() radarmask_t0 = np.array(np.isnan(radarmask_t0),dtype=int) radar_observations_t0_allmembers = np.zeros((radar_observations_t0.shape[0],radar_observations_t0.shape[1],NumberMembers)) for m in xrange(NumberMembers): radar_observations_t0_allmembers[:,:,m] = radar_observations_t0.copy() radar_observations_t0_allmembers = radar_observations_t0_allmembers[:,:,None,:] radarmask_t0 = radarmask_t0[:,:,None] # print(radarmask_t0.shape) # add observation for t0 radar_extrapolation_new = np.concatenate((radar_observations_t0_allmembers,radar_extrapolation_new),axis=2) radarMask_new = np.concatenate((radarmask_t0,radarMask_new),axis=2) timestamps_new.insert(0,startValidTime) timestamps_new = np.array(timestamps_new) # [time,member,y,x] radar_extrapolation_new = np.rollaxis(radar_extrapolation_new,2,0) radar_extrapolation_new = np.rollaxis(radar_extrapolation_new,3,1) radarMask_new = np.rollaxis(radarMask_new,2,0) # Apply rain threshold radar_extrapolation_new[radar_extrapolation_new<=rainThreshold] = 0 # save netcdf Xcoords = r.subXcoords Ycoords = r.subYcoords save_4darray_netcdf(fcstFile, radar_extrapolation_new, 'ensemble_radar_extrapolation',\ timestamps_new,NumberMembers,Xcoords,Ycoords) save_3darray_netcdf(fcstFileMask, radarMask_new, 'ensemble_radar_extrapolation_mask',\ timestamps_new,Xcoords,Ycoords) print('Read: ' + fcstFile) # Now read the NetCDF file ensemble_radar_extrapolation, timestamps, members, Ycoords, Xcoords = load_4darray_netcdf(fcstFile) if os.path.isfile(fcstFileMask): ensemble_radar_extrapolation_mask, _, _, _ = load_3darray_netcdf(fcstFileMask) else: print('Radar mask file not found.') ensemble_radar_extrapolation_mask = np.zeros((timestamps.size,ensemble_radar_extrapolation.shape[2],ensemble_radar_extrapolation.shape[3]))*np.nan testplot=False if testplot: r = io.read_gif_image(startValidTimeStr) nmember=0 n=0 while n<100: n+=1 for t in xrange(timestamps.shape[0]): plt.clf() plt.imshow(ensemble_radar_extrapolation[t,nmember,:,:],interpolation ='nearest',norm=r.norm,cmap=r.cmap) plt.title(timestamps[t]) plt.pause(1) return ensemble_radar_extrapolation, timestamps, members, Xcoords, Ycoords, ensemble_radar_extrapolation_mask
# Parse input arguments parser = argparse.ArgumentParser(description='Create time periods to pass to the bash script for parallel radar data processing.') parser.add_argument('-start', default='201601010000', type=str,help='Starting date YYYYMMDDHHmmSS.') parser.add_argument('-end', default='201601310000', type=str,help='Starting date YYYYMMDDHHmmSS.') parser.add_argument('-n', default=-1, type=int,help='Number of periods.') parser.add_argument('-days', default=-1, type=int,help='Periods interval length [days].') parser.add_argument('-accum', default=5, type=int,help='Accumulation time of products.') parser.add_argument('-start0', default=1, type=int,help='Whether to start the periods at 0 time or args.accum later.') args = parser.parse_args() if (args.n == -1) & (args.days == -1): print('You have to define either the number of periods -p or the interval length in days -days') sys.exit(1) timeStart = ti.timestring2datetime(args.start) timeEnd = ti.timestring2datetime(args.end) # Compute total duration of the period totalDurationSec = (timeEnd - timeStart).total_seconds() timeDeltaAccum = datetime.timedelta(minutes = args.accum) ###### Get time stamps based on number of intervals (not tested) if (args.n != -1) & (args.days == -1): # Compute duration of the period intervals durationPeriodSec = int(totalDurationSec/args.n) if durationPeriodSec >= totalDurationSec: numberPeriods = 1 timeStampsArray = np.array([args.start,args.end]) else: timeStamps = []
advectionScheme = args.adv frameRate = args.frameRate product = args.product leadtime = args.leadtime timewindow = np.max((5,args.stack)) if (int(args.start) < 198001010000) or (int(args.start) > 203001010000): print('Invalid -start time arguments.') sys.exit(1) else: timeStartStr = args.start ######## Get dattime from timestamp timeStart = ti.timestring2datetime(timeStartStr) timeAccumMinStr = '%05i' % timeAccumMin timeAccum24hStr = '%05i' % (24*60) ######## GIS stuff # Limits of CCS4 domain Xmin = 255000 Xmax = 965000 Ymin = -160000 Ymax = 480000 allXcoords = np.arange(Xmin,Xmax+resKm*1000,resKm*1000) allYcoords = np.arange(Ymin,Ymax+resKm*1000,resKm*1000) # Shapefile filename fileNameShapefile = "/users/" + usrName + "/pyscripts/shapefiles/CHE_adm0.shp" proj4stringWGS84 = "+proj=longlat +ellps=WGS84 +datum=WGS84"
args = parser.parse_args() if (int(args.start) < 198001010000) or (int(args.start) > 203001010000): print('Invalid -start or -end time arguments.') sys.exit(1) else: timeStartStr = args.start timeEndStr = args.end product = args.product timeAccumMin = args.accum timeAccumMinStr = '%05i' % timeAccumMin username = args.username ################################ timeStart = ti.timestring2datetime(timeStartStr) timeEnd = ti.timestring2datetime(timeEndStr) ##### LOOP OVER FILES ############# tic = time.clock() timeLocal = timeStart while timeLocal <= timeEnd: year = timeLocal.year hour = timeLocal.hour minute = timeLocal.minute # Get Julian day julianDay = ti.get_julianday(timeLocal) # Create filename yearStr = str(year)[2:4] julianDayStr = '%03d' % julianDay
def get_cosmo1(startValidTimeStr, endValidTimeStr,domainSize=512, leadTimeMin = 12*60, outBaseDir='/scratch/ned/data/',rainThreshold=0.08): # Get most recent run analysisTimeStr = rf1.find_nearest_run_cosmo1(startValidTimeStr) analysisTime = ti.timestring2datetime(analysisTimeStr) # Check if the nc file already exists year = analysisTime.year yearStr = str(year)[2:4] julianDay = analysisTime.timetuple().tm_yday julianDayStr = '%03i' % julianDay yearJulianStr = yearStr + julianDayStr outDir = outBaseDir + analysisTime.strftime("%Y") + '/' + analysisTime.strftime("%y") + julianDayStr + '/' fcstName = analysisTime.strftime("%Y%m%d%H%M") + '_' + str(int(leadTimeMin/60)) + 'hours' fcstFile = r'%s' % (outDir + 'cosmo-1_TOT_PREC_' + fcstName + '.nc') # if not call fieldextra if os.path.isfile(fcstFile) == False: rf1.run_fieldextra_c1(analysisTimeStr,leadTimeMin,outBaseDir=outBaseDir) print('Read: ' + fcstFile) # Now read the NetCDF file cosmo1_data, timestamps, Xcoords, Ycoords = load_3darray_netcdf(fcstFile) # exclude first time step (it's all NaNs!) cosmo1_data=cosmo1_data[1:,:,:] timestamps=timestamps[1:] # convert to mm/h cosmo1_data = cosmo1_data*6 # flip and extract middle domain new_data = np.zeros((cosmo1_data.shape[0],domainSize,domainSize)) for i in range(cosmo1_data.shape[0]): # flip frames cosmo1_data[i,:,:] = np.flipud(cosmo1_data[i,:,:]) # cut domain if domainSize>0: new_data[i,:,:] = dt.extract_middle_domain(cosmo1_data[i,:,:], domainSize, domainSize) else: new_data[i,:,:] = cosmo1_data[i,:,:] cosmo1_data = new_data # Apply rain threshold cosmo1_data[cosmo1_data<=rainThreshold] = 0 # Get coordinates of reduced domain if domainSize>0: extent = dt.get_reduced_extent(Xcoords.shape[0], Ycoords.shape[0], domainSize, domainSize) Xmin = Xcoords[extent[0]] Ymin = Ycoords[extent[1]] Xmax = Xcoords[extent[2]] Ymax = Ycoords[extent[3]] extent = (Xmin, Xmax, Ymin, Ymax) subXcoords = np.arange(Xmin,Xmax,1000) subYcoords = np.arange(Ymin,Ymax,1000) # Extract timestamps idxKeep1 = timestamps >= ti.timestring2datetime(startValidTimeStr) idxKeep2 = timestamps <= ti.timestring2datetime(endValidTimeStr) cosmo1_data = cosmo1_data[idxKeep1*idxKeep2,:,:] timestamps = timestamps[idxKeep1*idxKeep2] testplot=False if testplot: n=0 while n<100: n+=1 for t in xrange(timestamps.shape[0]): plt.clf() plt.imshow(cosmo1_data[t,:,:],interpolation ='nearest',vmin=0,vmax=65, extent=[subXcoords.min(), subXcoords.max(), subYcoords.min(), subYcoords.max()]) plt.title(timestamps[t]) plt.pause(1) return cosmo1_data, timestamps, subXcoords, subYcoords