def getSpatialMeanCCCDataForMask(mask = None, path_to_ccc = 'data/ccc_data/aex/aex_p1sno',
startDate = None, endDate = None):
"""
returns a map {date => mean value over a mask}
"""
dt = timedelta(hours = 6)
dates = []
result = {}
for fName in os.listdir(path_to_ccc):
fPath = os.path.join( path_to_ccc, fName )
cccObj = champ_ccc(fPath)
theDate = datetime.strptime( fName.split('_')[-1][:-4], '%Y%m')
if endDate is not None and startDate is not None:
if endDate < theDate or theDate < startDate:
continue
dataStore = cccObj.charge_champs()
for field in dataStore:
if startDate is not None and endDate is not None:
if startDate > theDate:
continue
if endDate < theDate:
break
fieldData = field['field']
dates.append(theDate)
result[theDate] = np.mean(fieldData[mask == 1])
theDate += dt
dates.sort()
return dates, [result[d] for d in dates]
pass
def get_bedrock_depth_amno_180x172():
"""
Returns 2d field of the bedrock depth in meters
in amno domain
"""
path = "data/ccc_data/dpth_180x172"
file_obj = champ_ccc(fichier=path)
the_field = file_obj.charge_champs()[0]["field"]
return the_field
pass
def _func_for_each_process(file_path):
"""
Takes temporal mean of the file_path
"""
ccc_obj = champ_ccc(fichier = file_path)
records = ccc_obj.charge_champs()
current_fields = []
for the_record in records:
current_fields.append(the_record["field"])
return np.array(current_fields).mean(axis = 0)
def plot_fields_in_file(path = "data/ccc_data/aex/aex_p1gt/aex_p1gt_198001.ccc"):
cccObj = ccc.champ_ccc( fichier = path)
fields = cccObj.charge_champs()
basemap = polar_stereographic.basemap
x = polar_stereographic.xs
y = polar_stereographic.ys
for the_record in fields:
plt.figure()
basemap.pcolormesh(x, y, the_record["field"])
plt.colorbar()
plt.show(block = True)
pass
def getSpatialIntegralCCCDataForMask(mask = None, path_to_ccc = 'data/ccc_data/aex/aex_p1sno',
startDate = None, endDate = None):
"""
returns 2 lists of the same dimensions one contains dates and the other,
integral values over the mask corresponding to the date
"""
dt = timedelta(hours = 6)
dates = []
result = []
monthlyFileNames = os.listdir(path_to_ccc)
filePaths = map(lambda x: os.path.join(path_to_ccc, x), monthlyFileNames)
the_func = lambda x: datetime.strptime( x.split('_')[-1][:-4], '%Y%m')
monthlyDates = map(the_func, monthlyFileNames) # get dates for months
the_zip = zip(monthlyDates, filePaths)
dps = [DateAndPath(date = the_date, path = the_path) for the_date, the_path in the_zip]
dps.sort(key = lambda x: x.date)
for dp in dps:
fPath = dp.path
cccObj = champ_ccc(fPath)
theDate = dp.date
if endDate is not None and startDate is not None:
if endDate < theDate or theDate < startDate:
continue
dataStore = cccObj.charge_champs()
for field in dataStore:
if startDate is not None and endDate is not None:
if startDate > theDate:
continue
if endDate < theDate:
break
fieldData = field['field']
dates.append(theDate)
result.append(np.sum(fieldData[mask == 1]))
theDate += dt
return dates, np.array(result) * dt.seconds
pass
def get_mean_over_domain(the_id, stamp_year = 2001, mask = None):
dt = members.id_to_step[the_id]
timeToSpatialMean = {}
folderPath = data_folder_format % (the_id, the_id)
#get spatial mean
for fileName in os.listdir(folderPath):
d = get_date_from_name(name = fileName)
#skip files from the outside of interest dates
if d < start_date: continue
if d > end_date: continue
ccc_obj = champ_ccc(os.path.join(folderPath, fileName))
fields = ccc_obj.charge_champs()
for record in fields:
the_field = record["field"]
if mask is None:
timeToSpatialMean[d] = np.mean(the_field)
else:
timeToSpatialMean[d] = np.mean(the_field[mask == 1])
d += dt
#get mean for each day of a year
one_day = timedelta(days = 1)
stamp_start_date = datetime(stamp_year, 1, 1)
year_days = [stamp_start_date + i * one_day for i in xrange(365)]
day_to_datalist = {}
for yd in year_days:
day_to_datalist[yd] = []
for t, v in timeToSpatialMean.iteritems():
#skip leap day
if t.month == 2 and t.day == 29: continue
t1 = datetime(stamp_year, t.month, t.day)
day_to_datalist[t1].append(v)
for yd in year_days:
x = day_to_datalist[yd]
day_to_datalist[yd] = np.mean(x)
return year_days, [day_to_datalist[yd] for yd in year_days]
def calculate_mean_evap_integrated_over(mask = None, data_folder = "data/ccc_data/aex/aex_p1qfs"):
dt = timedelta( hours = 6 ) #data time step
data_for_months = [[] for i in xrange(12) ]
fields_for_months = [[] for i in xrange(12) ]
for the_file in os.listdir(data_folder):
ccc_obj = champ_ccc( fichier = os.path.join(data_folder, the_file) )
records = ccc_obj.charge_champs()
the_month = _get_month_from_filename(the_file)
#each record is a step in time
domain_mean_series = []
data_fields = []
for the_record in records:
data_fields.append( the_record["field"] )
datai = np.mean(the_record["field"][mask == 1]) #get mean for domain
domain_mean_series.append(datai)
data_for_months[the_month - 1].append(np.sum(domain_mean_series) * dt.seconds / 1000.0)
fields_for_months[the_month - 1].append( np.sum(data_fields, axis = 0) * dt.seconds / 1000.0 )
for i in xrange(12):
data_for_months[i] = np.mean(data_for_months[i])
fields_for_months[i] = np.mean(fields_for_months[i], axis = 0)
print np.sum(data_for_months)
plt.figure()
plt.plot(xrange(1,13), data_for_months)
plt.savefig("evap_qc_crcm4.png")
plt.figure()
b = polar_stereographic.basemap
[x, y] = b(polar_stereographic.lons, polar_stereographic.lats)
mean_annual_field = np.sum(fields_for_months, axis = 0)
b.pcolormesh(x,y,np.ma.masked_where(mask != 1, mean_annual_field))
print np.mean(mean_annual_field[mask == 1])
b.drawcoastlines()
plt.colorbar()
r = plot_utils.get_ranges(x[mask == 1], y[mask == 1])
plt.xlim(r[:2])
plt.ylim(r[2:])
plt.savefig("evap_qc_2d_crcm4.png")
def main():
basemap = Basemap(projection = 'npstere', #area_thresh = 10000,
lat_ts = 60, lat_0 = 60, lon_0 = -115,
boundinglat=10
)
x = polar_stereographic.lons
y = polar_stereographic.lats
x, y = basemap(x, y)
path = "/home/huziy/skynet3_exec1/crcm4_data/AN"
ccc_obj = champ_ccc(fichier=path)
records = ccc_obj.charge_champs()
fields_of_interest = []
for rec in records:
name = rec["ibuf"]["ibuf3"]
level = rec["ibuf"]["ibuf4"]
if name == "FCAN" and level < 5:
print rec["ibuf"]
field = rec["field"]
print field.min(), field.max()
fields_of_interest.append(field)
print rec["field"].shape
fig = plt.figure()
assert isinstance(fig, Figure)
#clevels = [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]
pData = 1.0 - np.sum(fields_of_interest, axis = 0)
pData = pData[10:190,10:182]
pData[np.abs( pData ) < 0.001] = 0.0
pData = maskoceans(polar_stereographic.lons, polar_stereographic.lats,pData,inlands=False)
print pData.min(), pData.max()
#pData = np.ma.masked_where((pData > 1) | (pData < 0), pData)
basemap.contourf(x,y,pData)
basemap.drawcoastlines()
plt.colorbar()
plt.xlim(x.min(), x.max())
plt.ylim(y.min(), y.max())
plt.show()
fig.savefig("bare_soil.png")
pass
def getTemporalMeanCCCDataForMask(mask=None, path_to_ccc='data/ccc_data/aex/aex_p1sno',
startDate=None, endDate=None,
months=None):
"""
months - is a list of month indices (i.e. 1,2,3,4 .. 12), over which
the mean is calculated
returns
"""
if not months: months = []
dt = timedelta(hours = 6)
dates = []
result = {}
for fName in os.listdir(path_to_ccc):
fPath = os.path.join( path_to_ccc, fName )
cccObj = champ_ccc(fPath)
theDate = datetime.strptime( fName.split('_')[-1][:-4], '%Y%m')
if endDate is not None and startDate is not None:
if endDate < theDate or theDate < startDate:
continue
dataStore = cccObj.charge_champs()
for field in dataStore:
if startDate is not None and endDate is not None:
if startDate > theDate:
continue
if endDate < theDate:
break
if theDate.month not in months:
continue
fieldData = field['field']
dates.append(theDate)
result[theDate] = fieldData[mask == 1]
theDate += dt
x = np.array(result.values())
return np.mean(x, axis = 0)
def get_mean(path = "data/ccc_data/aex/aex_p1gt", start_date = None, end_date = None):
result = None
count = 1.0
for fName in os.listdir(path):
the_date = fName.split("_")[-1].split(".")[0]
the_date = datetime.strptime(the_date, date_format)
if start_date is not None and end_date is not None:
if the_date < start_date or the_date > end_date:
continue
fPath = os.path.join(path, fName)
cccObj = ccc.champ_ccc( fichier = fPath)
fields = cccObj.charge_champs()
#calculate mean
for the_record in fields:
if result is None:
result = the_record["field"]
else:
count += 1.0
result = ( the_record["field"] + result * (count - 1) ) / count
return result
def main():
data_path = "data/permafrost/p1perma"
cccObj = champ_ccc(data_path)
pData = cccObj.charge_champs()[0]["field"]
