def read_continuous_csv(filename):
all_data = []
with open(filename, "r") as infile:
time = []
data = []
name = ""
for line in infile:
sline = line.split(",")
if sline[0] == "Year":
continue
else:
if name != "" and name != sline[1]:
all_data += [utils.Timeseries("Lake", time, data)]
time = []
data = []
# else just read in
name = sline[1]
time += [int(sline[0])]
data += [float(sline[2])]
# and last one
all_data += [utils.Timeseries("All", time, data)]
return all_data # read_continuous_csv
def obtain_timeseries(filename, cube_name, ts_name, index, is_era5=False):
# re-read the file to overwrite memory of what was done to this cube.
cube_list = iris.load(filename)
names = np.array([cube.name() for cube in cube_list])
selected_cube, = np.where(names == cube_name)
if len(selected_cube) == 0:
names = np.array([cube.var_name for cube in cube_list])
selected_cube, = np.where(names == cube_name)
cube = cube_list[selected_cube[0]]
cube.coord('latitude').guess_bounds()
cube.coord('longitude').guess_bounds()
nyears = np.ma.count(cube.data, axis=0)
# repeat so selection will work across all years
nyears = np.array([nyears for i in range(cube.data.shape[0])])
cube.data = np.ma.masked_where(nyears < THRESHOLD * cube.data.shape[0],
cube.data)
weight_areas = iris.analysis.cartography.area_weights(cube)
ts = cube.collapsed(['latitude', 'longitude'],
iris.analysis.MEAN,
weights=weight_areas)
times = GetYears(ts, is_era5=is_era5)
if index in ["TX90p", "TN90p", "TX10p", "TN10p"]:
ts = utils.Timeseries(ts_name, times, ts.data * 3.65)
else:
ts = utils.Timeseries(ts_name, times, ts.data)
# get coverage
cover_cube = copy.deepcopy(cube)
# replace data with 1
cover_cube.data.data[:] = 1
# and get timeseries
weight_areas = iris.analysis.cartography.area_weights(cover_cube)
cover_ts = cover_cube.collapsed(['latitude', 'longitude'],
iris.analysis.SUM,
weights=weight_areas)
# normalise - obtain max area possible with each gridbox = 1
norm_cube = copy.deepcopy(cover_cube)[0]
norm_cube.data.mask = False
weight_areas = iris.analysis.cartography.area_weights(norm_cube)
norm = norm_cube.collapsed(['latitude', 'longitude'],
iris.analysis.SUM,
weights=weight_areas)
# apply normalisation and convert to %. Also convert to %land.
cover_ts = (cover_ts / norm) * 100 / 0.3
cover_ts = utils.Timeseries("Coverage", GetYears(cover_ts), cover_ts.data)
return ts, cover_ts # obtain_timeseries
def read_hilo(filename):
indata = np.genfromtxt(filename, dtype=(float))
times = indata[:, 0]
high = utils.Timeseries("Highest", times, indata[:, 1])
low = utils.Timeseries("Lowest", times, indata[:, 2])
return high, low # read_hilo
def read_sie(filename):
indata = np.genfromtxt(filename, dtype=(float), skip_header=3)
arctic_max = utils.Timeseries("SIE", indata[:, 0], indata[:, 2])
arctic_min = utils.Timeseries("SIE", indata[:, 0], indata[:, 4])
antarctic_min = utils.Timeseries("SIE", indata[:, 0], indata[:, 6])
antarctic_max = utils.Timeseries("SIE", indata[:, 0], indata[:, 8])
return arctic_max, arctic_min, antarctic_min, antarctic_max # read_sie
def read_uaw_ts(filename, smooth=False, annual=False):
# IRIS doesn't like the Conventions attribute
ncfile = ncdf.Dataset(filename, 'r')
time = ncfile.variables["time"][:]
# time = YYYYMM
times = np.array(
[int(str(t)[:4]) + ((float(str(t)[4:]) - 1) / 12.) for t in time])
# smooth using 12 point boxcar
if smooth:
points = 12
merra = make_masked_smoothed_ts("MERRA-2", times,
ncfile.variables["MERRA2"][:], points)
# grasp = make_masked_smoothed_ts("GRASP", times, ncfile.variables["GRASP"][:], points)
# cera20c = make_masked_smoothed_ts("CERA20C", times, ncfile.variables["CERA20C"][:], points)
jra55 = make_masked_smoothed_ts("JRA-55", times,
ncfile.variables["JRA55"][:], points)
erai = make_masked_smoothed_ts("ERA-Interim", times,
ncfile.variables["ERAI"][:], points)
era5 = make_masked_smoothed_ts("ERA5", times,
ncfile.variables["ERA5"][:], points)
if annual:
merra = make_masked_annual_ts("MERRA-2", times,
ncfile.variables["MERRA2"][:])
# grasp = make_masked_annual_ts("GRASP", times, ncfile.variables["GRASP"][:])
# cera20c = make_masked_annual_ts("CERA20C", times, ncfile.variables["CERA20C"][:])
jra55 = make_masked_annual_ts("JRA-55", times,
ncfile.variables["JRA55"][:])
erai = make_masked_annual_ts("ERA-Interim", times,
ncfile.variables["ERAI"][:])
era5 = make_masked_annual_ts("ERA5", times,
ncfile.variables["ERA5"][:])
if not smooth and not annual:
merra = utils.Timeseries("MERRA-2", times,
ncfile.variables["MERRA2"][:])
# grasp = utils.Timeseries("GRASP", times, ncfile.variables["GRASP"][:])
# cera20c = utils.Timeseries("CERA20C", times, ncfile.variables["CERA20C"][:])
jra55 = utils.Timeseries("JRA-55", times, ncfile.variables["JRA55"][:])
erai = utils.Timeseries("ERA-Interim", times,
ncfile.variables["ERAI"][:])
era5 = utils.Timeseries("ERA5", times, ncfile.variables["ERA5"][:])
ncfile.close()
# set ERA-Interim linestyle
erai.ls = "--"
return era5, erai, merra, jra55 # read_uaw_ts
def read_ts(filename):
'''
Read timeseries
'''
indata = np.genfromtxt(filename, delimiter=',', dtype=(float))
years = indata[:, 0].astype(float)
anomalies = utils.Timeseries("Anomalies", years, indata[:, 1])
uncertainties = utils.Timeseries("Uncertainties", years, indata[:, 2])
return anomalies, uncertainties # read_ts
def read_ssu_csv(filename):
"""
Read user supplied CSV for LST into Timeseries object
"""
indata = np.genfromtxt(filename, delimiter=',', dtype=(float), \
skip_header=5, missing_values="", filling_values=-99.9)
indata = np.ma.masked_where(indata == -99.9, indata)
ssu2 = utils.Timeseries("NOAA", indata[:, 0], indata[:, 2])
ncar = utils.Timeseries("NCAR", indata[:, 0], indata[:, 4])
return ssu2, ncar # read_ssu_csv
def read_csv(filename):
"""
Read user supplied CSV for LST into Timeseries object
"""
indata = np.genfromtxt(filename, delimiter=',', dtype=(float), \
skip_header=5, missing_values="", filling_values=-99.9)
indata = np.ma.masked_where(indata == -99.9, indata)
raobcore = utils.Timeseries("RAOBCORE v1.7", indata[:, 0], indata[:, 1])
rich = utils.Timeseries("RICH v1.7", indata[:, 0], indata[:, 2])
ratpac = utils.Timeseries("RATPAC A2", indata[:, 0], indata[:, 3])
# unsw = utils.Timeseries("UNSW v1.0", indata[:, 0], indata[:, 4])
UAH = utils.Timeseries("UAH v6.0", indata[:, 0], indata[:, 4])
rss = utils.Timeseries("RSS v4.0", indata[:, 0], indata[:, 5])
noaa = utils.Timeseries("NOAA v4.1", indata[:, 0], indata[:, 6])
# era5 = utils.Timeseries("ERA5", indata[:, 0], indata[:, 8])
jra = utils.Timeseries("JRA-55", indata[:, 0], indata[:, 7])
merra = utils.Timeseries("MERRA-2", indata[:, 0], indata[:, 8])
# cmip = utils.Timeseries("CMIP5", indata[:, 0], indata[:, 11])
# ssu3 = utils.Timeseries("SSU-3", indata[:, 0], indata[:, 12])
return UAH, rss, ratpac, raobcore, rich, noaa, jra, merra # read_csv
def read_bartlett_gpp(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries object s
'''
raw_data = np.genfromtxt(filename, dtype=(float), skip_header=1, delimiter=",")
gpp = utils.Timeseries("Bartlett GPP", raw_data[:, 0], raw_data[:, 6])
gcc = utils.Timeseries("Bartlett GCC", raw_data[:, 7], raw_data[:, 8])
return gpp, gcc # read_bartlett_gpp
def read_bartlett_green(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries object s
'''
raw_data = np.genfromtxt(filename, dtype=(int))
start = utils.Timeseries("Greenup", raw_data[:, 0], raw_data[:, 1])
end = utils.Timeseries("Greendown", raw_data[:, 0], raw_data[:, 2])
diff = utils.Timeseries("Difference ", raw_data[:, 0], raw_data[:, 3])
return start, end, diff # read_bartlett_green
def read_dwd_fagus(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries object s
'''
raw_data = np.genfromtxt(filename, dtype=(int), skip_header=1, delimiter=",")
betula_fall = utils.Timeseries("B. pendula", raw_data[:, 0], raw_data[:, 2])
fagus_out = utils.Timeseries("F. sylvatica", raw_data[:, 0], raw_data[:, 4])
fagus_fall = utils.Timeseries("F. sylvatica", raw_data[:, 0], raw_data[:, 6])
return betula_fall, fagus_out, fagus_fall # read_dwd_fagus
def read_polar(filename):
"""
Read user supplied data for LST into Timeseries object
"""
indata = np.genfromtxt(filename, dtype=(float), skip_header=1)
year = indata[:, 0]
north = indata[:, 1]
south = indata[:, 2]
north = utils.Timeseries("North Pole", year, north)
south = utils.Timeseries("South Pole -15" + r'$^\circ$' + "C", year, south)
return north, south # read_polar
def read_time(filename):
"""
Read user supplied CSV for GLE into Timeseries object
"""
times = np.arange(1980, int(settings.YEAR) + 1, 1)
indata = np.genfromtxt(filename, delimiter=',', dtype=(float))
globe = utils.Timeseries("Globe", times, indata[:, 0])
NH = utils.Timeseries("N. Hemisphere", times, indata[:, 1])
SH = utils.Timeseries("S. Hemisphere", times, indata[:, 2])
SOI = utils.Timeseries("SOI", times, indata[:, 3])
return globe, NH, SH, SOI # read_time
def obtain_timeseries(filename, cube_name, ts_name):
# re-read the file to overwrite memory of what was done to this cube.
cube_list = iris.load(filename)
names = np.array([cube.name() for cube in cube_list])
selected_cube, = np.where(names == cube_name)
cube = cube_list[selected_cube]
cube.coord('latitude').guess_bounds()
cube.coord('longitude').guess_bounds()
nyears = np.ma.count(cube.data, axis=0)
# repeat so selection will work across all years
nyears = np.array([nyears for i in range(cube.data.shape[0])])
cube.data = np.ma.masked_where(nyears < THRESHOLD * cube.data.shape[0],
cube.data)
weight_areas = iris.analysis.cartography.area_weights(cube)
ts = cube.collapsed(['latitude', 'longitude'],
iris.analysis.MEAN,
weights=weight_areas)
ts = utils.Timeseries(ts_name, GetYears(ts), ts.data * 3.65)
return ts
def make_smoothed_ts(ts, points):
'''
Make a smoothed timeseries array.
:param Timeseries ts: input timeseries
:param int points: for boxcar
'''
smoothed = np.ma.zeros(len(ts.data))
smoothed.mask = np.zeros(smoothed.shape)
for d in range(len(ts.data)):
# if a running centred mean:
if d < points / 2.:
smoothed.mask[d] = 1
elif d > (len(ts.data) - points / 2.):
smoothed.mask[d] = 1
else:
smoothed[d] = np.mean(ts.data[d - points / 2:d + points / 2])
# if a running previous N-month mean
# if d < points:
# smoothed[d] = np.mean(ts.data[:d])
# elif d > (len(ts.data) - points):
# smoothed.mask[d] = 1
# else:
# smoothed[d] = np.mean(ts.data[d - points: d])
smoothed_ts = utils.Timeseries(ts.name, ts.times, smoothed)
return smoothed_ts # make_masked_smoothed_ts
def read_ts(filename, start, name, smooth=False):
'''
Read the timeseries data, and use the hard-coded start and end years
to make up the time axis for returning Timeseries objects.
:param str filename: file to read
:param int start: year of start of data
:param str name: name of Timeseries object
:param bool smooth: if True, then == value of smoothing
:returns: Timeseries object
'''
data = np.genfromtxt(filename, dtype=(float), skip_header=5)
# create the times
times = []
year = start
while year <= int(settings.YEAR):
times += [list(year + DECIMAL_MONTHS)]
year += 1
times = np.array([val for sublist in times for val in sublist])
assert len(times) == len(data)
# smooth if required
if smooth:
data = utils.boxcar(data, smooth)
ts = utils.Timeseries(name, times, data)
return ts # read_ts
def read_dwd_quercus(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries object s
'''
raw_data = np.genfromtxt(filename, dtype=(int), skip_header=3, delimiter=",", skip_footer=7, filling_values=-1)
raw_data = np.ma.masked_where(raw_data < 0, raw_data)
quercus_out = utils.Timeseries("Q. robur", raw_data[:, 0], raw_data[:, 8])
quercus_fall = utils.Timeseries("Q. robur", raw_data[:, 0], raw_data[:, 22])
return quercus_out, quercus_fall # read_dwd_betula
def read_winter_nao(DATALOC, years):
'''
Read the NAO data, returns Timeseries and smoothed version
'''
ts_list = []
smoothed = []
for y in years:
all_data = np.genfromtxt(DATALOC + "SLP_WinterNAOtimeseries_{}.txt".format(y), dtype=(float), skip_header=1)
days = all_data[:, 1].astype(int)
months = all_data[:, 0].astype(int)
years = np.array([y for i in days]).astype(int)
years[months < 12] = years[months < 12] + 1
times = np.array([dt.datetime(years[i], months[i], days[i]) for i in range(len(days))])
data = np.ma.masked_where(all_data[:, 2] <= -99.99, all_data[:, 2])
ts_list += [utils.Timeseries("{}/{}".format(y, int(y[2:])+1), times, data)]
smoothed += [np.ma.masked_where(all_data[:, 3] <= -99.99, all_data[:, 3])]
return ts_list, smoothed # read_winter_nao
def read_bartlett_duke(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries object s
'''
raw_data = np.genfromtxt(filename, dtype=(float), skip_header=1, delimiter=",")
days = raw_data[:, 0]
bartlett = utils.Timeseries("Bartlett GCC", days, raw_data[:, 10])
duke = utils.Timeseries("Duke GCC", days, raw_data[:, 12])
return bartlett, duke # read_bartlett_duke
def read_cei(filename):
indata = np.genfromtxt(DATALOC + filename,
delimiter=',',
encoding='latin-1')
return utils.Timeseries("CEI", indata[:, 0], indata[:, 1]) # read_cei
def read_swv(filename):
indata = np.genfromtxt(filename, dtype=(float), delimiter=",")
swv = utils.Timeseries("SWV", indata[:, 0], indata[:, 1])
return swv # read_SWV
def read_slr(filename):
indata = np.genfromtxt(filename, dtype=(float))
slr = utils.Timeseries("SLR", indata[:, 0], indata[:, 1])
return slr # read_SLR
def read_uk_csv(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries object s
'''
raw_data = np.genfromtxt(filename, dtype=(str), skip_header=6, skip_footer=13, delimiter=",")
alder = utils.Timeseries("A. glutinosa", raw_data[:, 0].astype(int), raw_data[:, 1].astype(int))
chestnut = utils.Timeseries("A. hippocastanum", raw_data[:, 0].astype(int), raw_data[:, 2].astype(int))
oak = utils.Timeseries("Q. robur (2000-{})".format(settings.YEAR[2:]), raw_data[:, 0].astype(int), raw_data[:, 3].astype(int))
beech = utils.Timeseries("F. sylvatica", raw_data[:, 0].astype(int), raw_data[:, 4].astype(int))
return alder, chestnut, oak, beech # read_uk_csv
def read_data(filename):
indata = np.genfromtxt(filename, skip_header=2, delimiter=",")
# process the years and months to give decimals
years = indata[:, 0]
months = indata[:, 1]
times = years + (months - 1) / 12.
# make timeseries objects
moderate = utils.Timeseries("Moderate", times, indata[:, 2])
severe = utils.Timeseries("Severe", times, indata[:, 3])
extreme = utils.Timeseries("Extreme", times, indata[:, 4])
return moderate, severe, extreme # read_data
def read_land(filename):
indata = np.genfromtxt(filename, dtype=(float), skip_header=1)
indata = np.ma.masked_where(indata == -99.99, indata)
ghcn = utils.Timeseries("GHCN", indata[:, 0], indata[:, 1])
# ghcn2 = utils.Timeseries("GHCNv2", indata[:, 0], indata[:, 2])
gpcc = utils.Timeseries("GPCC", indata[:, 0], indata[:, 2])
gpcp = utils.Timeseries("GPCPv23", indata[:, 0], indata[:, 3])
# cfsr = utils.Timeseries("CFSR", indata[:, 0], indata[:, 5])
# erai = utils.Timeseries("ERA-Interim", indata[:, 0], indata[:, 5])
# merra = utils.Timeseries("MERRA-2", indata[:, 0], indata[:, 6])
return ghcn, gpcc, gpcp # read_land
def read_ts(filename, name):
'''
Read the GRACE timeseries and return a Timeseries object
:param str filename: file to read
:param str name: name of timeseries
:returns: Timeseries object
'''
indata = np.genfromtxt(filename, dtype=(float))
if name == "GRACE":
data = np.ma.masked_where(indata[:, 1] == 998.0, indata[:, 1])
times = indata[:, 0].astype(str)
if name == "Model":
years = np.array([str(y)[:4] for y in times]).astype(float)
months = np.array([str(y)[4:6] for y in times]).astype(float)
time = years + (months - 1) / 12. + (1 / 24.
) # place in centre of month
# mask first 3 months (Matt Rodell email, 14-2-2019)
data = np.ma.array(indata[:, 2])
data.mask = np.zeros(data.shape[0])
data.mask[:3] = True
else:
time = times.astype(float)
return utils.Timeseries(name, time, data) # read_ts
def read_arct(filename):
""" Read the Arctic Temperatures"""
indata = np.genfromtxt(filename, delimiter=",", skip_header=1)
arct = utils.Timeseries("ARCT", indata[:, 0], indata[:, 1])
return arct # read_arct
def read_windermere_csv(filename):
'''
Read the timeseries data, and returning Timeseries objects.
:param str filename: file to read
:returns: Timeseries objects
'''
raw_data = np.genfromtxt(filename,
dtype=(int),
skip_header=1,
delimiter=",")
times = raw_data[:, 0]
north = utils.Timeseries("North Basin", times, raw_data[:, 1])
south = utils.Timeseries("South Basin", times, raw_data[:, 2])
return north, south # read_windermere_csv
def read_ts(filename):
'''
Read timeseries
'''
indata = np.genfromtxt(filename,
delimiter=',',
dtype=(float),
skip_header=1)
years = indata[:, 0].astype(float)
euro = utils.Timeseries("Lake", years, indata[:, 1])
africa = utils.Timeseries("Lake", years, indata[:, 2])
tibet = utils.Timeseries("Lake", years, indata[:, 3])
canada = utils.Timeseries("Lake", years, indata[:, 4])
return euro, africa, tibet, canada # read_ts
def read_csv(filename):
"""
Read user supplied CSV for LST into Timeseries object
"""
indata = np.genfromtxt(filename,
delimiter=",",
dtype=(float),
skip_header=1,
encoding="latin-1")
times = indata[:, 0]
data = indata[:, 1]
fit = indata[:, 2]
return utils.Timeseries("AT", times,
data), utils.Timeseries("AT", times, fit)