def rm_dupl_dates(dates, cXs, cYs, degs, chs, keys, daynos, tworecdt):
'''A function to subset event set to remove duplicate dates
read in and out dates, centroids, angles, cb outlines, and keys etc.
run evset_info first!
dates_d,cXs_d, cYs_d, degs_d, chs_d, keys_d, daynos_d, tworecdt_d =
rm_dupl_dates(dates,cXs, cYs, degs, chs, keys, daynos, tworecdt)
'''
# First find indices for dates as they appear for the first time only
inds = []
origdates = []
dcnt = 0
for dno in range(len(dates)):
thisdate = dates[dno]
if dcnt == 0:
inds.append(dno)
origdates.append(thisdate)
else:
tmpdts = np.asarray(origdates)
ix = my.ixdtimes(tmpdts, [thisdate[0]], [thisdate[1]],
[thisdate[2]], [thisdate[3]])
if len(ix) == 0:
inds.append(dno)
origdates.append(thisdate)
dcnt += 1
inds = np.asarray(inds)
origdates = np.asarray(origdates)
# Then use these inds to subset all inputs
dates_d = dates[inds]
cXs_d = cXs[inds]
cYs_d = cYs[inds]
degs_d = degs[inds]
chs_d = chs[inds]
keys_d = keys[inds]
daynos_d = daynos[inds]
tworecdt_d = tworecdt[inds]
# Remember that tworecdt just highlights the dates which have more than 1 entry
# so the subset sample will still have entries which have a "2" for tworecdt
# this just means that the original dataset has another entry for that day
# but it should have been removed for the subset
return dates_d, cXs_d, cYs_d, degs_d, chs_d, keys_d, daynos_d, tworecdt_d
_, idx = np.unique(tmp, return_index=True)
alldtime= alldtime[idx]
alldata=alldata[idx,:,:]
nsteps=len(alldtime)
sinds=[]
odts=[]
oinds=[]
print year1
print year2
for dt in range(nsteps):
thisdate=alldtime[dt]
if thisdate[0] >=year1 and thisdate[0] <=year2:
if thisdate[1] >=mon1 or thisdate[1] <=mon2:
ix = my.ixdtimes(smpdtime, [thisdate[0]], \
[thisdate[1]], [thisdate[2]], [0])
if len(ix) == 1:
sinds.append(dt)
elif len(ix) < 1:
odts.append(thisdate)
oinds.append(dt)
odts = np.asarray(odts)
oinds = np.asarray(oinds)
sinds = np.asarray(sinds)
print oinds
otherdata=alldata[oinds]
uvals=np.zeros((nlat,nlon),dtype=np.float32)
raindat=np.where(rdtime[:,1] == thismon)
rain_thmon = np.squeeze(rain[raindat, :, :])
rdtime_thmon = rdtime[raindat]
print 'Summing all rain for per TTT rain calc'
ndays=len(rdtime_thmon)
reg_all_sum = np.zeros((ndays), dtype=np.float32)
for st in range(ndays):
reg_all_sum[st] = np.ma.sum(rain_thmon[st, :, :])
tottot_allrain=np.nansum(reg_all_sum)
print 'Selecting TTTs from rain data'
tcnt=0
indices = []
for dt in range(nttt):
ix = my.ixdtimes(rdtime_thmon, [dates_ln[dt][0]], \
[dates_ln[dt][1]], [dates_ln[dt][2]], [0])
if len(ix) >= 1:
indices.append(ix)
tcnt+=1
indices = np.squeeze(np.asarray(indices))
if tcnt==0:
nottts=True
elif tcnt==1:
onlyone=True
else:
onlyone=False
nottts=False
if not nottts:
# Loop sample domain
for o in range(len(sample_dom)):
smp_dom = sample_dom[o]
print "Running for sample " + smp_dom
# Get sample
dates_d,cXs_d, cYs_d, degs_d, chs_d, keys_d, daynos_d, tworecdt_d = \
sset.sample_arche_cbs(sample,smp_dom, dates,cXs, cYs, degs, chs, keys, daynos, tworecdt)
# Find indices from var file
indices_m1 = []
for e in range(len(dates_d)):
date = dates_d[e]
ix = my.ixdtimes(dtime, [date[0]], [date[1]], [date[2]], [0])
if len(ix) >= 1:
indices_m1.append(ix)
indices_m1 = np.squeeze(np.asarray(indices_m1))
# Select these dates
olrsel = olrdata[indices_m1, :, :] # note this is going to give you in order of dates_d
# Count timesteps
nsteps=len(olrsel[:,0,0])
### Count number of events
count_sel = str(nsteps)
print "Total flagged CBs selected =" + str(count_sel)
'Sep', 'Oct', 'Nov', 'Dec'
]
else:
monthstr = [
'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'Jan', 'Feb', 'Mar',
'Apr', 'May', 'Jun', 'Jul'
]
monbegins = np.where(full_y_ar[:, 2] == 1)[0]
if not group_event:
indices = []
for e in range(len(edts)):
ix2 = my.ixdtimes(full_y_ar, [edts[e][0]],
[edts[e][1]], [edts[e][2]],
[edts[e][3]])
indices.append(ix2)
indices = np.squeeze(np.asarray(indices))
if timelonplot:
plt.figure(figsize=[5, 12])
plt.scatter(thesecXs, indices)
#,marker="o",edgecolour='face',size=20)
plt.xlim(7.5, 100)
plt.ylim(365, 0)
plt.yticks(monbegins, monthstr,
fontsize=13.0) # month labels
plt.xlabel('Centroid Longitude',
fontsize=12.0,
if (int(firstdate[1]) >= f_mon) or (int(firstdate[1]) <= l_mon):
thesekeys.append(k)
if from_event == 'first':
# If from first select the first date
if rm_samedates:
#first check if already in list
if ecnt == 0:
edts.append(firstdate)
chs.append(e.blobs[refkey]['ch'][e.trk[0]])
x, y = e.trkcX[0], e.trkcY[0]
cXs.append(x)
cYs.append(y)
else:
tmpedts = np.asarray(edts)
ix = my.ixdtimes(tmpedts, [firstdate[0]],
[firstdate[1]], [firstdate[2]],
[firstdate[3]])
if len(ix) == 0:
edts.append(firstdate)
chs.append(e.blobs[refkey]['ch'][e.trk[0]])
x, y = e.trkcX[0], e.trkcY[0]
cXs.append(x)
cYs.append(y)
else:
edts.append(firstdate)
chs.append(e.blobs[refkey]['ch'][e.trk[0]])
x, y = e.trkcX[0], e.trkcY[0]
cXs.append(x)
cYs.append(y)
elif from_event == 'all':
# if not from first loop all dates in event
edts = []
if under_of == 'under':
chs = []
ecnt = 1
for k in keys4rain:
e = s.events[k]
dts = s.blobs[key]['mbt'][e.ixflags]
for dt in range(len(dts)):
if ecnt == 1:
edts.append(dts[dt])
if under_of == 'under':
chs.append(e.blobs[key]['ch'][e.trk[dt]])
else:
tmpdt = np.asarray(edts)
# Check if it exists already
ix = my.ixdtimes(tmpdt, [dts[dt][0]], \
[dts[dt][1]], [dts[dt][2]], [0])
if len(ix) == 0:
edts.append(dts[dt])
if under_of == 'under':
chs.append(e.blobs[key]['ch'][e.trk[dt]])
ecnt += 1
edts = np.asarray(edts)
edts[:, 3] = 0
if under_of == 'under':
chs = np.asarray(chs)
print 'Selecting TTTs from rain data'
indices = []
if under_of == 'under':
chs_4rain = []
for edt in range(len(edts)):
for b in range(len(refmbt)):
date = refmbt[b]
mon = int(date[1])
cX = refmbs[b, 3]
cY = refmbs[b, 4]
deg = refmbs[b, 2]
if from_event == 'all':
edts.append(date)
cXs.append(cX)
cYs.append(cY)
degs.append(deg)
mons.append(mon)
elif from_event == 'first':
# print 'Checking if the date is the first day of an event'
ix = my.ixdtimes(ev_dts, [date[0]], \
[date[1]], [date[2]], [0])
if len(ix) == 1:
key = ev_keys[ix]
e = s.events[key[0]]
dts = s.blobs[refkey]['mbt'][e.ixflags]
if dts[0, 0] == date[0]:
if dts[0, 1] == date[1]:
if dts[0, 2] == date[2]:
# print 'it is the first date, so we keep it'
edts.append(date)
cXs.append(cX)
cYs.append(cY)
degs.append(deg)
mons.append(mon)
chs.append(e.blobs[refkey]['ch'][e.trk[0]])
def evset_info(s, refmbs, refmbt):
'''A function designed to output lots of information for a list of TTT events
dates, cXs, cYs, degs, chs, keys, daynos, tworecdt = evset_info(s,mbs,mbt)
refmbt # list of CB dates
cXs # centroid longitudes
cYs # centroid latitude
degs # angle of CB
chs # cb outline
keys # key - to link to synop event
daynos # day number in event
tworecdt # dates whcih show up more than once
Each item is a numpy array, with 1 value for each cloud band in the refmbs/refmbt list
So this should allow to get the full refmbt list, whilst also showing how it relates to the events
Note that I found the last date in refmbt did not feature as an event in synop for NOAA-CDR
This means it does not have a blob outline stored, so I have removed such cases from the event set
s - obtained by first opening syfile
refmbs and refmbt - obtained by first opening mbs and mbt files'''
# First get keys
ks = s.events.keys()
ks.sort() # all
refkey = s.mbskeys[0]
# Adjust hr time on dates
refmbt[:, 3] = 0
# Get list of all days info from synop
ev_dts = []
ev_keys = []
ev_cXs = []
ev_cYs = []
ev_chs = []
ev_dayno = []
for k in ks:
e = s.events[k]
dts = s.blobs[refkey]['mbt'][e.ixflags]
for dt in range(len(dts)):
ev_dts.append(dts[dt])
ev_keys.append(k)
x, y = e.trkcX[dt], e.trkcY[dt]
ev_cXs.append(x)
ev_cYs.append(y)
ev_chs.append(e.blobs[refkey]['ch'][e.trk[dt]])
ev_dayno.append(dt + 1)
ev_dts = np.asarray(ev_dts)
ev_dts[:, 3] = 0
ev_keys = np.asarray(ev_keys)
ev_cXs = np.asarray(ev_cXs)
ev_chs = np.asarray(ev_chs)
ev_dayno = np.asarray(ev_dayno)
### Get info for each flagged blob
dates = [] # dates
cXs = [] # centroid longitudes
cYs = [] # centroid latitude
degs = [] # angle of CB
chs = [] # cb outline
keys = [] # key - to link to synop event
daynos = [] # day number in event
tworecdt = [] # dates whcih show up more than once
for b in range(len(refmbt)):
date = refmbt[b]
cX = refmbs[b, 3]
cY = refmbs[b, 4]
deg = refmbs[b, 2]
# Find matching record in events list
ix = my.ixdtimes(ev_dts, [date[0]], [date[1]], [date[2]], [date[3]])
# if more than one date, highlight this in saved output
if len(ix) == 1:
# if just one matching date save match index
match = ix[0]
elif len(ix) > 1:
# if more than one matching date find the centroid which matches
todays_cXs = ev_cXs[ix]
index2 = np.where(todays_cXs == cX)[0]
if len(index2) != 1:
print 'Error - centroid not matching'
index2 = index2[0] # this just turns it from an array to integer
match = ix[index2]
elif len(ix) == 0:
# if no match disguard this one because it is not in synop - no ch
match = False
if match:
# Get info for that record
chs.append(ev_chs[match])
keys.append(ev_keys[match])
daynos.append(ev_dayno[match])
tworecdt.append(len(ix))
# And save info from refmbs
dates.append(date)
cXs.append(cX)
cYs.append(cY)
degs.append(deg)
dates = np.asarray(dates)
cXs = np.asarray(cXs)
cYs = np.asarray(cYs)
degs = np.asarray(degs)
chs = np.asarray(chs)
keys = np.asarray(keys)
daynos = np.asarray(daynos)
tworecdt = np.asarray(tworecdt)
return dates, cXs, cYs, degs, chs, keys, daynos, tworecdt
plt.scatter(smp_cXs,smp_degs,c='fuchsia',marker=mkrs[d],s=msiz[d],edgecolors='face',zorder=zorders[d])
elif group_event:
for k in ks:
e = s.events[k]
dts = s.blobs[refkey]['mbt'][e.ixflags]
cXs_ent = []
degs_ent = []
for dt in range(len(dts)):
x,y = e.trkcX[dt], e.trkcY[dt]
cXs_ent.append(x)
# Find angle
ix4 = my.ixdtimes(edts, [dts[dt][0]], [dts[dt][1]], [dts[dt][2]], [0])
if len(ix4)==1:
degs_ent.append(degs[ix4])
elif len(ix4)==0:
print 'Date from event not found in mbt...'
degs_ent.append(0)
elif len(ix4)>1:
print 'Date from event found more than once in mbt'
all_dates_event=edts[ix4]
print all_dates_event
all_cXs_event=cXs[ix4]
print all_cXs_event
print 'this cX='+str(x)
ix5=np.where(all_cXs_event==x)[0]
right_ind=ix4[ix5]
print 'right_ind='+str(right_ind)
# Get bias
bias = reg_mean - reg_ref_mean
yvals[cnt] = bias
# Finally getting intensity
dates_int, cXs_int, cYs_int, degs_int, chs_int, keys_int, daynos_int, tworecdt_int = \
sset.sel_cen_lon(int_ttt_wlon,int_ttt_elon,dates_per, cXs_per, cYs_per, \
degs_per, chs_per, keys_per, daynos_per, tworecdt_per)
n4int = len(dates_int)
print 'Selecting TTTs from rain data'
indices = []
for dt in range(n4int):
ix = my.ixdtimes(rdtime, [dates_int[dt][0]], \
[dates_int[dt][1]], [dates_int[dt][2]], [0])
if len(ix) >= 1:
indices.append(ix)
indices = np.squeeze(np.asarray(indices))
print 'Selecting rain on TTT days'
rainsel = rain[indices, :, :]
ttt_rain_dates = rdtime[indices]
# if under_of == 'under':
#
# print 'Selecting rain under TTTs'
# masked_rain = np.ma.zeros((n4int, nlat, nlon), dtype=np.float32)
# for rdt in range(n4int):
# chmask = my.poly2mask(rlon, rlat, chs_int[rdt])
