def _compute_M2_grids(ui):
logging.info('Computation of M2 grids started.')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
grace_path = os.path.join(project_dir, project_name, 'grace',
'single_solution')
grids_path = os.path.join(grace_path, 'grids')
internal_validation_path = os.path.join(project_dir, project_name,
'validation', 'internal')
# constant density
density = 1025.0
# which grid
if (ui.rb_id057.isChecked() or ui.rb_id100.isChecked()):
lon, lat, _, _ = np.genfromtxt(grid_file, skip_header=2, unpack=True)
_compute_buffer_grid(ui, lon, lat)
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1, '%Y-%m')[0].decode("utf-8")
grid_file_pm = os.path.join(grids_path, '%s-grid.txt' % (ymstr))
pointmass_file = os.path.join(internal_validation_path, 'grids',
'm2_%s-pointmass.txt' % (ymstr))
temp_dir = tempfile.mkdtemp()
tmp_grid = os.path.join(temp_dir, 'tmp.grid')
_compute_gfc2grid(ui, groops_bin, buffer_grid_file, tmp_grid, mjd1,
min_degree, max_degree, gauss)
try:
if ui.rb_id058.isChecked():
lon, lat, _, _ = np.genfromtxt(grid_file_pm,
skip_header=2,
unpack=True)
_compute_buffer_grid(ui, lon, lat)
_compute_gfc2grid(ui, groops_bin, buffer_grid_file, tmp_grid, mjd1,
min_degree, max_degree, gauss)
except:
continue
if not os.path.isfile(tmp_grid):
continue
lon, lat, h, area_i, tws = np.genfromtxt(tmp_grid,
skip_header=2,
unpack=True)
area = (area_i * defaults.EARTH_EQUATORIAL_RADIUS() *
defaults.EARTH_EQUATORIAL_RADIUS()) * 1e-6
kg = tws * (area * 1e6) * density
groops_interface.make_grid_file(groops_bin, pointmass_file, lon, lat,
h, area, *(kg, np.array(0)))
shutil.rmtree(temp_dir)
logging.info('Computing grid: %s', pointmass_file)
logging.info('Computation of M2 grids finished.')
def test_mjd2mjdmrange():
"""
Test if the function `mjd2mjdmrange` correctly determines the first and
last day of the given month from the Modifed Julian Date format in the
Modifed Julian date format.
"""
mjd = [14999.1, 51560.6, 58907.9]
mjd1_true = [14989, 51544, 58880]
mjd2_true = [15019, 51574, 58908]
mjd1_test, mjd2_test = date_functions.mjd2mjdmrange(mjd)
assert mjd1_test[0] == mjd1_true[0]
assert mjd1_test[1] == mjd1_true[1]
assert mjd1_test[2] == mjd1_true[2]
assert mjd2_test[0] == mjd2_true[0]
assert mjd2_test[1] == mjd2_true[1]
assert mjd2_test[2] == mjd2_true[2]
def _time_series(ui):
logging.info('Creating time series files from point mass grids...')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
corrections_path = os.path.join(project_dir, project_name, 'corrections')
# loop over correction types
for correction in ('a', 'gldas', 'ice5', 'ice6', 'ice6_grace', 'klemann',
'lsdm', 'promet', 'wghm', 'goco'):
# time series files
time_series_path = os.path.join(corrections_path, correction,
'time_series')
time_series_file = os.path.join(time_series_path,
'%s.txt' % correction)
with open(time_series_file, 'w') as f:
f.write('%25s %25s %25s\n' %
('time (mjd)', 'mass (kg)', 'sigma (kg)'))
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1,
'%Y-%m')[0].decode("utf-8")
pointmass_file = os.path.join(
corrections_path, correction, 'grids',
'%s_%s-pointmass.txt' % (correction, ymstr))
try:
lon, lat, h, area, x, sigma_x = np.genfromtxt(pointmass_file,
skip_header=2,
unpack=True)
kg = np.sum(x)
sigma_kg = np.sqrt(np.sum(sigma_x**2.0) / sigma_x.size)
with open(time_series_file, 'a+') as f:
f.write('%+25.16e %+25.16e %+25.16e\n' %
(mjd1, kg, sigma_kg))
except:
continue
logging.info('Creating time series file: %s_%s', correction, ymstr)
logging.info('Time series files created.')
def _time_series(ui):
logging.info('Creating time series files from point mass grids...')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
internal_validation_path = os.path.join(project_dir, project_name,
'validation', 'internal')
# loop over element types
for element in ('m1', 'm2', 'm1_star', 'm2_syn', 'm1_syn'):
# time series files
time_series_path = os.path.join(internal_validation_path,
'time_series')
time_series_file = os.path.join(time_series_path, '%s.txt' % element)
with open(time_series_file, 'w') as f:
f.write('%25s %25s %25s\n' %
('time (mjd)', 'mass (kg)', 'sigma (kg)'))
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1,
'%Y-%m')[0].decode("utf-8")
pointmass_file = os.path.join(
internal_validation_path, 'grids',
'%s_%s-pointmass.txt' % (element, ymstr))
try:
lon, lat, h, area, x, sigma_x = np.genfromtxt(pointmass_file,
skip_header=2,
unpack=True)
kg = np.sum(x)
sigma_kg = np.sqrt(np.sum(sigma_x**2.0) / sigma_x.size)
with open(time_series_file, 'a+') as f:
f.write('%+25.16e %+25.16e %+25.16e\n' %
(mjd1, kg, sigma_kg))
except:
continue
logging.info('Creating time series file: %s_%s', element, ymstr)
mjd, m1, _ = np.genfromtxt(os.path.join(time_series_path, 'm1.txt'),
skip_header=1,
unpack=True)
_, m2, _ = np.genfromtxt(os.path.join(time_series_path, 'm2.txt'),
skip_header=1,
unpack=True)
_, m2_syn, _ = np.genfromtxt(os.path.join(time_series_path, 'm2_syn.txt'),
skip_header=1,
unpack=True)
data = {}
data['iaf'] = m2 / m1
data['signal_loss'] = 100.0 * (1.0 - m2_syn / m2)
data['m2_syn_star'] = m2 * (1.0 + (1.0 - m2_syn / m2))
data['faf'] = data['m2_syn_star'] / m1
for key in ('iaf', 'signal_loss', 'm2_syn_star', 'faf'):
time_series_file = os.path.join(time_series_path, '%s.txt' % key)
with open(time_series_file, 'w') as f:
f.write('%25s %25s %25s\n' % ('time (mjd)', 'value', 'dummy'))
for jx in list(range(np.size(mjd))):
with open(time_series_file, 'a+') as f:
f.write('%+25.16e %+25.16e %+25.16e\n' %
(mjd[jx], data[key][jx], 0))
logging.info('Time series files created.')
def _compute_M1_syn_grids(ui):
logging.info('Computation of M1_syn grids started.')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
internal_validation_path = os.path.join(project_dir, project_name,
'validation', 'internal')
# global grid
grid_file_global = pkg_resources.resource_filename(
'picasso.data', 'grids/geographical_30-30.grid')
grid0 = np.array(np.genfromtxt(grid_file_global, skip_header=2), ndmin=2)
# indices
if (ui.rb_id057.isChecked() or ui.rb_id100.isChecked()):
lon, lat, _, _ = np.genfromtxt(grid_file, skip_header=2, unpack=True)
ix = []
for jx in list(range(np.size(lon))):
ix.append(
_return_index_closest_point(np.array([lon[jx], lat[jx]]),
grid0[:, 0:2]))
# constant density
density = 1025.0
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1, '%Y-%m')[0].decode("utf-8")
m1_file = os.path.join(internal_validation_path, 'grids',
'm1_%s-pointmass.txt' % (ymstr))
m2_file = os.path.join(internal_validation_path, 'grids',
'm2_%s-pointmass.txt' % (ymstr))
m1_star_file = os.path.join(internal_validation_path, 'grids',
'm1_star_%s-pointmass.txt' % (ymstr))
m2_syn_file = os.path.join(internal_validation_path, 'grids',
'm2_syn_%s-pointmass.txt' % (ymstr))
m1_syn_file = os.path.join(internal_validation_path, 'grids',
'm1_syn_%s-pointmass.txt' % (ymstr))
try:
lon1, lat1, h1, area1, x1, sigma_x1 = np.genfromtxt(m1_file,
skip_header=2,
unpack=True)
lon2, lat2, h2, area2, x2, sigma_x2 = np.genfromtxt(m2_file,
skip_header=2,
unpack=True)
except:
continue
kg1 = np.sum(x1)
kg2 = np.sum(x2)
IAF = kg2 / kg1
m1_star = kg1 * IAF
groops_interface.make_grid_file(groops_bin, m1_star_file, lon1, lat1,
h1, area1, *(x1 * IAF, np.array(0)))
if (ui.rb_id057.isChecked() or ui.rb_id100.isChecked()):
m1_star_i = np.zeros(grid0.shape[0])
m1_star_i[ix] = x1 * IAF / ((area1 * 1e6) * density)
else:
m1_star_i = griddata((lon1, lat1),
x1 * IAF / ((area1 * 1e6) * density),
(grid0[:, 0], grid0[:, 1]),
method='cubic')
temp_dir = tempfile.mkdtemp()
tmp_grid1 = os.path.join(temp_dir, 'tmp1.grid')
tmp_grid2 = os.path.join(temp_dir, 'tmp2.grid')
groops_interface.make_grid_file(groops_bin, tmp_grid1, grid0[:, 0],
grid0[:, 1], grid0[:, 2], grid0[:, 3],
*(m1_star_i, np.array(0)))
groops_interface.compute_grid_to_gfc_to_grid(groops_bin,
tmp_grid1,
tmp_grid2,
buffer_grid_file,
gauss=gauss)
if not os.path.isfile(tmp_grid2):
continue
lon, lat, h, area_i, tws = np.genfromtxt(tmp_grid2,
skip_header=2,
unpack=True)
area = (area_i * defaults.EARTH_EQUATORIAL_RADIUS() *
defaults.EARTH_EQUATORIAL_RADIUS()) * 1e-6
kg = tws * (area * 1e6) * density
kg2_syn = np.sum(kg)
groops_interface.make_grid_file(groops_bin, m2_syn_file, lon, lat, h,
area, *(kg, np.array(0)))
signal_loss = 100.0 * (1.0 - kg2_syn / kg2)
kg2_syn_star = kg2 * (1.0 + (1.0 - kg2_syn / kg2))
FAF = kg2_syn_star / kg1
kg1_syn = FAF * kg1
groops_interface.make_grid_file(groops_bin, m1_syn_file, lon1, lat1,
h1, area1, *(x1 * FAF, np.array(0)))
shutil.rmtree(temp_dir)
logging.info('Computing grid: %s', m1_syn_file)
logging.info('Computation of M1_syn grids finished.')
def _update_LS(parameters):
# global vars
global LS
mjd1, mjd2 = date_functions.mjd2mjdmrange(cg.current_mjd)
# return if already updated
LS_check = False
for ls_dict_i in cg.ga_solutions:
if len(ls_dict_i['parameters']) == len(parameters):
LS_check = np.all([
parameters[ix].value == ls_dict_i['parameters'][ix].value
for ix in range(len(parameters))
])
if LS_check:
LS = ls_dict_i['LS']
return
# ls dictionary
ls_dict = {}
ls_dict['parameters'] = parameters
# temporary directory
ls_dict['ga_id'] = str(uuid.uuid4())
normals_path = cg.current_normals_path + ls_dict['ga_id']
grid_file = cg.current_normals_path.replace('grace_', '').replace(
'normals', 'grids')
grid_file += ls_dict['ga_id'] + '-grid.txt'
# possible optimization candidates
pm_count = None
pm_lon = []
pm_lat = []
pm_depth = []
pm_magnitude = []
pm_grid_resolution = None
pm_grid_type = None
regularization = 0
# loop over parameters
for p in parameters:
if p.label == 'pm_count':
pm_count = np.round(p.value).astype(int)
elif p.label == 'pm_lon':
pm_lon.append(p.value)
elif p.label == 'pm_lat':
pm_lat.append(p.value)
elif p.label == 'pm_depth':
pm_depth.append(p.value)
elif p.label == 'pm_magnitude':
pm_magnitude.append(p.value)
elif p.label == 'regularization':
regularization = (10.0**p.value)
elif p.label == 'pm_grid_resolution':
pm_grid_resolution = p.value
elif p.label == 'pm_grid_type':
pm_grid_type = np.floor(p.value).astype(int)
pm_grid_type = 6 if pm_grid_type > 6 else pm_grid_type
# reassign values
if not pm_grid_type:
pm_grid_type = cg.current_pm_grid_type
if pm_grid_resolution is not None:
resolution = _map_grid_resolution(pm_grid_type, pm_grid_resolution)
temp_dir = tempfile.mkdtemp()
tmp_grid = os.path.join(temp_dir, 'tmp.grid')
groops_interface.make_specific_grid_in_polygon_file(
cg.current_groops_bin, tmp_grid, cg.current_polygon_file,
pm_grid_type, resolution)
loni, lati, _, _ = np.genfromtxt(tmp_grid, skip_header=2, unpack=True)
pm_lon, pm_lat = loni.tolist(), lati.tolist()
pm_count = np.size(pm_lon)
shutil.rmtree(temp_dir)
if not pm_count:
pm_count = cg.current_pm_count
if not pm_lon:
pm_lon = cg.current_pm_lon
if not pm_lat:
pm_lat = cg.current_pm_lat
if not pm_depth:
pm_depth = cg.current_pm_depths
# check if (re-)computation of normals is necessary
if (not cg.pm_count_is_parameter) and (
not cg.pm_positions_are_parameters) and (
not cg.pm_depths_are_parameters) and (
not cg.pm_grid_type_is_parameter) and (
not cg.pm_grid_resolution_is_parameter):
normals_path = normals_path.replace('genetic_algorithm',
'single_solution').replace(
'-' + ls_dict['ga_id'], '')
ls_dict['ga_id'] = None
LS = groops_files.read_groops_normals(normals_path)
else:
groops_interface.make_grid_file(cg.current_groops_bin, grid_file,
pm_lon, pm_lat,
(-np.array(pm_depth) * 1e3),
cg.current_pm_area,
**{'point_count': pm_count})
if not cg.picasso_on_leo:
groops_interface.build_pointmass_normals_ga(
cg.current_groops_bin,
mjd1,
mjd2,
grid_file,
normals_path,
love_enabled=cg.pm_magnitudes_are_parameters)
else:
ret = groops_interface.build_pointmass_normals_ga(
cg.current_groops_bin,
mjd1,
mjd2,
grid_file,
normals_path,
leo=True,
love_enabled=cg.pm_magnitudes_are_parameters)
#groops_command = 'module load mpich ; mpiexec -n %d %s ' % (cg.current_frontend_cores,ret)
#groops_command = 'module load mpich ; mpiexec -n %d %s ' % (40,ret.replace('bin/groops','bin/pgroops'))
groops_command = 'mpiexec -n %d %s ' % (
40, ret.replace('bin/groops', 'bin/pgroops'))
#print(groops_command)
temp_dir = tempfile.mkdtemp()
pbs_file = os.path.join(temp_dir, 'tmp.pbs')
#pbs_file = '/home/sreimond/data/scenario/projects/tmp.pbs'
f = open(pbs_file, "w")
f.write('#!/bin/bash\n')
f.write('#PBS -N picassoga\n')
f.write('#PBS -q normal\n')
f.write('#PBS -l walltime=23:59:00\n')
f.write('#PBS -d /scratch/sreimond/myrun\n')
f.write('#PBS -l nodes="1:ppn=40"\n')
f.write('module load mpich\n')
f.write(groops_command)
f.close()
os.system('/usr/bin/qsub %s' % pbs_file)
qstat = 'busy'
while qstat:
# qstat = subprocess.check_output("/usr/bin/qstat")
# print(qstat)
try:
#qstat = subprocess.check_output(["/usr/bin/pgrep","groops"])
qstat = subprocess.check_output("/usr/bin/qstat")
except subprocess.CalledProcessError as grepexc:
print("error code", grepexc.returncode, grepexc.output)
qstat = False
time.sleep(60)
shutil.rmtree(temp_dir)
groops_interface.eliminate_solve_pointmass_normals_ga(
cg.current_groops_bin, mjd1, mjd2, grid_file, normals_path)
LS = groops_files.read_groops_normals(normals_path)
# solve
R = matrices.IdentityMatrix(LS.col_count)
R.elements *= regularization
LS.define_regularization_matrix(R)
LS.regularized_normal_equation = matrices.MatrixEquation()
N_regularized = LS.normal_equation.A.add(LS.R)
LS.regularized_normal_equation.define_coefficient_matrix(N_regularized)
LS.regularized_normal_equation.define_right_hand_side(LS.normal_equation.b)
LS.solve()
if cg.current_regularization == -200:
LS.determine_vce_regularization(sig1=1.0, sigu=1e2)
elif cg.current_regularization == -300:
LS = rg.regularization_matlab(normals_path, method='lcurve')
elif cg.current_regularization == -400:
LS = rg.regularization_matlab(normals_path, method='gcv')
# update parameter vector if optimized
if pm_magnitude:
LS.x = matrices.return_matrix_from_numpy_array(np.array(pm_magnitude))
xNx = LS.x.return_transpose().multiply(LS.normal_equation.A).multiply(
LS.x).elements
nx2 = 2.0 * LS.normal_equation.b.return_transpose().multiply(
LS.x).elements
LS.rss = xNx - nx2 + LS.bb
xRx = LS.x.return_transpose().multiply(LS.x).elements
LS.xss = xRx
ls_dict['LS'] = LS
cg.ga_solutions += (ls_dict, )
def _compute_hydrology_grids(ui):
logging.info('Computing hydrological grids...')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
grace_path = os.path.join(project_dir, project_name, 'grace',
'single_solution')
grids_path = os.path.join(grace_path, 'grids')
corrections_path = os.path.join(project_dir, project_name, 'corrections')
# model names
models = ('gldas', 'lsdm', 'wghm')
if ui.rb_id109.isChecked():
models += ('goco', )
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1, '%Y-%m')[0].decode("utf-8")
grid_file = os.path.join(grids_path, '%s-grid.txt' % (ymstr))
for name in models:
hydro_grid = os.path.join(data_path, name.upper(),
'%s_TWS_%s.txt' % (name, ymstr))
if name is 'goco':
hydro_grid = os.path.join(data_path, 'GOCO', 'grids',
'GOCO-%s.txt' % (ymstr))
pointmass_file = os.path.join(
corrections_path, name, 'grids',
'%s_%s-pointmass.txt' % (name, ymstr))
# positions
if ui.rb_id050.isChecked():
try:
grid = np.array(np.genfromtxt(hydro_grid, skip_header=2),
ndmin=2)
grid = grid[pip_ix, :]
lon, lat, h, area_i, tws_i = grid[:,
0], grid[:,
1], grid[:,
2], grid[:,
3], grid[:,
4]
except:
continue
elif ui.rb_id051.isChecked():
try:
lon0, lat0, h0, area0, tws0 = np.genfromtxt(hydro_grid,
skip_header=2,
unpack=True)
lon, lat, h, area = np.genfromtxt(grid_file,
skip_header=2,
unpack=True)
tws_i = griddata((lon0, lat0), (tws0), (lon, lat),
method=ui.cob_id003.currentText())
area_i = griddata((lon0, lat0), (area0), (lon, lat),
method=ui.cob_id003.currentText())
except:
continue
point_count = np.size(lon)
# area
if ui.rb_id052.isChecked():
area = ui.dsb_id001.value() / point_count
elif ui.rb_id053.isChecked():
area = (area_i * defaults.EARTH_EQUATORIAL_RADIUS() *
defaults.EARTH_EQUATORIAL_RADIUS()) * 1e-6
elif ui.rb_id054.isChecked():
area = ui.dsb_id012.value() / point_count
# density
density = 1025.0
if ui.rb_id056.isChecked():
density = ui.dsb_id013.value()
# conversion to pointmass
kg_i = tws_i * (area * 1e6) * density
# write file
groops_interface.make_grid_file(groops_bin, pointmass_file, lon,
lat, h, area, *(kg_i, np.array(0)))
logging.info('Current file: %s', pointmass_file)
logging.info('Computing hydrological grids finished.')
def _compute_goco_grids_spline(ui):
logging.info('Computing GOCO grids (splines)...')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
grace_path = os.path.join(project_dir, project_name, 'grace',
'single_solution')
grids_path = os.path.join(grace_path, 'grids')
corrections_path = os.path.join(project_dir, project_name, 'corrections',
'goco')
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1, '%Y-%m')[0].decode("utf-8")
grid_file = os.path.join(grids_path, '%s-grid.txt' % (ymstr))
goco_grid = os.path.join(data_path, 'GOCO', 'grids',
'GOCO-%s.txt' % (ymstr))
pointmass_file = os.path.join(corrections_path, 'grids',
'goco_%s-pointmass.txt' % (ymstr))
# positions
try:
temp_dir = tempfile.mkdtemp()
tmp_input_grid_file = os.path.join(temp_dir, 'grid_in.txt')
tmp_output_grid_file = os.path.join(temp_dir, 'grid_out.txt')
lon, lat, h, area = np.genfromtxt(grid_file,
skip_header=2,
unpack=True)
groops_interface.make_grid_file(groops_bin, tmp_input_grid_file,
lon, lat, h, area)
groops_interface.compute_goco_grid(groops_bin, tmp_input_grid_file,
tmp_output_grid_file, mjd1)
lon, lat, h, area_i, tws_i = np.genfromtxt(tmp_output_grid_file,
skip_header=2,
unpack=True)
shutil.rmtree(temp_dir)
lon0, lat0, h0, area0, tws0 = np.genfromtxt(goco_grid,
skip_header=2,
unpack=True)
area_i = griddata((lon0, lat0), (area0), (lon, lat),
method=ui.cob_id003.currentText())
except:
continue
point_count = np.size(lon)
# area
if ui.rb_id052.isChecked():
area = ui.dsb_id001.value() / point_count
elif ui.rb_id053.isChecked():
area = (area_i * defaults.EARTH_EQUATORIAL_RADIUS() *
defaults.EARTH_EQUATORIAL_RADIUS()) * 1e-6
elif ui.rb_id054.isChecked():
area = ui.dsb_id012.value() / point_count
# density
density = 1025.0
if ui.rb_id056.isChecked():
density = ui.dsb_id013.value()
# conversion to pointmass
kg_i = tws_i * (area * 1e6) * density
# write file
groops_interface.make_grid_file(groops_bin, pointmass_file, lon, lat,
h, area, *(kg_i, np.array(0)))
logging.info('Current file: %s', pointmass_file)
logging.info('Computing GOCO grids (splines) done.')
def _compute_danubia_grid(ui):
logging.info('Computing PROMET/DANUBIA grids...')
# paths
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
grace_path = os.path.join(project_dir, project_name, 'grace',
'single_solution')
grids_path = os.path.join(grace_path, 'grids')
corrections_path = os.path.join(project_dir, project_name, 'corrections')
fh = Dataset(os.path.join(data_path, 'DANUBIA', 'Data-Danubia_update',
'Danubia.nc'),
mode='r')
fh.set_auto_mask(False)
lon0 = fh.variables['longitude'][:]
lat0 = fh.variables['latitude'][:]
LON0, LAT0 = np.meshgrid(lon0, lat0)
mjd0 = fh.variables['time'][:]
# loop over all dates
mjd_start, mjd_end = compute_grace._date_settings(ui)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
ymstr = date_functions.mjd2ymstring(mjd1, '%Y-%m')[0].decode("utf-8")
grid_file = os.path.join(grids_path, '%s-grid.txt' % (ymstr))
pointmass_file = os.path.join(corrections_path, 'promet', 'grids',
'promet_%s-pointmass.txt' % (ymstr))
ix = np.argwhere(mjd0 == mjd)
if not np.any(ix):
continue
ix = int(np.asscalar(ix))
cr = fh.variables['channel_runoff'][ix, :]
et = fh.variables['evapotranspiration'][ix, :]
gs = fh.variables['glacier_snow'][ix, :]
im = fh.variables['ice_melt'][ix, :]
pc = fh.variables['precipitation'][ix, :]
sm = fh.variables['soil_moisture'][ix, :]
tr = fh.variables['total_runoff'][ix, :]
ts = fh.variables['total_snow'][ix, :]
tws0 = eval(ui.le_id225.text()) * 1e-3
# mkae grid
grid = np.zeros((np.size(tws0), 5))
grid[:, 0] = LON0.flatten()
grid[:, 1] = LAT0.flatten()
grid[:, 4] = tws0.flatten()
# positions
if ui.rb_id050.isChecked():
try:
if (polygon is not None) and (pip is not None):
grid = grid[pip, :]
elif (polygon is not None) and (pip is None):
ix = list(range(grid.shape[0]))
p = Pool(None)
pip = p.map(partial(compute_grace._in_polygon_aux,
polygon=polygon,
grid=grid),
ix,
chunksize=25)
pip = np.nonzero(pip)[0]
grid = grid[pip, :]
lon, lat, h, area_i, tws_i = grid[:,
0], grid[:,
1], grid[:,
2], grid[:,
3], grid[:,
4]
except:
continue
elif ui.rb_id051.isChecked():
try:
lon0, lat0, h0, area0, tws0 = grid[:,
0], grid[:,
1], grid[:,
2], grid[:,
3], grid[:,
4]
lon, lat, h, area = np.genfromtxt(grid_file,
skip_header=2,
unpack=True)
tws_i = griddata((lon0, lat0), (tws0), (lon, lat),
method=ui.cob_id003.currentText())
area_i = griddata((lon0, lat0), (area0), (lon, lat),
method=ui.cob_id003.currentText())
except:
continue
point_count = np.size(lon)
# area
if ui.rb_id052.isChecked():
area = ui.dsb_id001.value() / point_count
elif ui.rb_id053.isChecked():
area = 1.0 # 1 km^2 per pixel
elif ui.rb_id054.isChecked():
area = ui.dsb_id012.value() / point_count
# density
density = 1025.0
if ui.rb_id056.isChecked():
density = ui.dsb_id013.value()
# conversion to pointmass
kg_i = tws_i * (area * 1e6) * density
# write file
groops_interface.make_grid_file(groops_bin, pointmass_file, lon, lat,
h, area, *(kg_i, np.array(0)))
logging.info('Current file: %s', pointmass_file)
fh.close()
logging.info('Computing PROMET/DANUBIA grids finished.')
def _evaluate_trend(ui):
logging.info('Trend evaluation started...')
# globals
global regression_data
mjd_ref = 54466.0
T = 365.25
mjd_eval = []
date_begin = ui.de_id003.date().toString("yyyy-MM-dd")
date_end = ui.de_id004.date().toString("yyyy-MM-dd")
mjd_start = date_functions.ymstring2mjd(date_begin)
tmp = date_functions.ymstring2mjd(date_end)
_, mjd_end = date_functions.mjd2mjdmrange(tmp)
mjd = mjd_start
while mjd <= mjd_end:
mjd1, mjd2 = date_functions.mjd2mjdmrange(mjd)
mjd = (mjd2 + 1)
mjd_eval.append(mjd1)
mjd = np.array(mjd_eval)
# regression
for key, value in regression_data.items():
logging.info('Current time series: %s', key)
regression_data[key]['mjd_regression'] = mjd
if regression_data[key]['LS'] is None:
regression_data[key]['kg_regression'] = None
continue
LS = regression_data[key]['LS']
# regression_data[key]['parameter_names'] = ()
# regression_data[key]['parameter_names'] += ('beta_0',)
# design matrix
A = np.ones(mjd.size)
t = (mjd - mjd_ref) / T
if ui.rb_id083.isChecked():
degree = 1
elif ui.rb_id084.isChecked():
degree = 2
elif ui.rb_id085.isChecked():
degree = 3
elif ui.rb_id086.isChecked():
degree = ui.sb_id015.value()
logging.info('Polynomial degree: %d', degree)
for p in np.arange(1, degree + 1, dtype=np.float_):
A = np.column_stack((A, t**p))
# sinusiodals
if ui.cb_id022.isChecked():
f = 1.0
A = np.column_stack((A, np.cos(2.0 * np.pi * f * t)))
A = np.column_stack((A, np.sin(2.0 * np.pi * f * t)))
logging.info('Parameters C_0 and S_0 added.')
if ui.cb_id023.isChecked():
f = 0.5
A = np.column_stack((A, np.cos(2.0 * np.pi * f * t)))
A = np.column_stack((A, np.sin(2.0 * np.pi * f * t)))
logging.info('Parameters C_1 and S_1 added.')
if ui.cb_id024.isChecked():
f = 161.0 / T
A = np.column_stack((A, np.cos(2.0 * np.pi * f * t)))
A = np.column_stack((A, np.sin(2.0 * np.pi * f * t)))
logging.info('Parameters C_2 and S_2 added.')
if ui.cb_id025.isChecked():
f = 3.73
A = np.column_stack((A, np.cos(2.0 * np.pi * f * t)))
A = np.column_stack((A, np.sin(2.0 * np.pi * f * t)))
logging.info('Parameters C_3 and S_3 added.')
if ui.cb_id026.isChecked():
periods = np.array(ui.le_id011.text().split(), dtype=np.float_)
for jx, period in enumerate(periods):
f = period / T
A = np.column_stack((A, np.cos(2.0 * np.pi * f * t)))
A = np.column_stack((A, np.sin(2.0 * np.pi * f * t)))
logging.info('Parameters C_%d and S_%d added.', jx + 4, jx + 4)
logging.info('Evaluationg LS system.')
# LS evaluation
A = matrices.return_matrix_from_numpy_array(A)
regression_data[key]['kg_regression'] = A.multiply(LS.x).elements
logging.info('LS system evaluated.')
logging.info('Trend evaluation finished...')
def _identify_outliers_remove_mean(ui):
logging.info('Identification of outliers and removal of mean started...')
# globals
global regression_data
# time data
date_begin = ui.de_id003.date().toString("yyyy-MM-dd")
date_end = ui.de_id004.date().toString("yyyy-MM-dd")
mjd_start = date_functions.ymstring2mjd(date_begin)
tmp = date_functions.ymstring2mjd(date_end)
_, mjd_end = date_functions.mjd2mjdmrange(tmp)
if ui.cb_id021.isChecked():
mjd_start = -np.inf
mjd_end = np.inf
# analyze data
for key, value in regression_data.items():
logging.info('Current time series: %s', key)
if np.all(np.isnan(regression_data[key]['mjd'])):
logging.info('Keeping all data.')
regression_data[key]['mjd_use'] = regression_data[key]['mjd']
regression_data[key]['kg_use'] = regression_data[key]['kg']
regression_data[key]['kg_sigma_use'] = regression_data[key][
'kg_sigma']
regression_data[key]['outliers'] = []
continue
ix1 = regression_data[key]['mjd'] >= mjd_start
ix2 = regression_data[key]['mjd'] <= mjd_end
tix = np.logical_and(ix1, ix2)
mjd = regression_data[key]['mjd'][tix]
kg = regression_data[key]['kg'][tix]
kg_sigma = regression_data[key]['kg_sigma'][tix]
if ui.rb_id074.isChecked():
oix = np.zeros(np.count_nonzero(tix), dtype=bool)
regression_data[key]['outliers'] = oix
elif not np.any(tix):
logging.info('No data in date range.')
oix = np.ones(np.count_nonzero(tix), dtype=bool)
regression_data[key]['outliers'] = oix
else:
ydt = signal.detrend(kg)
if ui.rb_id075.isChecked() or ui.rb_id076.isChecked():
m = np.nanmean(ydt)
if ui.rb_id077.isChecked() or ui.rb_id078.isChecked():
m = np.nanmedian(ydt)
if ui.rb_id075.isChecked() or ui.rb_id077.isChecked():
s = 3. * np.nanstd(ydt)
if ui.rb_id076.isChecked() or ui.rb_id078.isChecked():
s = 2. * np.nanstd(ydt)
ix1 = ydt > (m + s)
ix2 = ydt < (m - s)
oix = np.logical_or(ix1, ix2)
regression_data[key]['outliers'] = oix
kg_outliers = np.copy(kg)
mjd[oix] = np.nan
kg[oix] = np.nan
kg_sigma[oix] = np.nan
kg_outliers[~oix] = np.nan
regression_data[key]['kg_outliers'] = kg_outliers
regression_data[key]['mjd_use'] = mjd
regression_data[key]['kg_use'] = kg
regression_data[key]['kg_sigma_use'] = kg_sigma
logging.info('Number of outliers: %d', np.count_nonzero(oix))
if ui.cb_id077.isChecked():
regression_data[key]['kg_use'] -= np.nanmean(kg)
logging.info('Tempoaral mean removed')
logging.info('Identification of outliers and removal of mean finished.')
def make_plot(ui,skip_try=False):
if not skip_try:
logging.info('New plot...')
try:
make_plot(ui,skip_try=True)
except:
logging.info('Could not create plots. Check settings and result files.')
return
# paths
global project_name
global project_dir
project_name = str(ui.le_id001.text())
project_dir = str(ui.le_id002.text())
# upadte rc params
_update_rc_params(ui)
# initiate figure
fig,ax = plt.subplots()
# plot data
_plot_grace_goce(ui,ax)
_plot_grace(ui,ax)
_plot_goce(ui,ax)
_plot_hydrology(ui,ax)
_plot_gia(ui,ax)
_plot_internal_validation(ui,ax)
_plot_external_validation(ui,ax)
# ax
title = ui.le_id124.text()
if title:
ax.set_title(title)
xlabel = ui.le_id125.text()
if xlabel:
ax.set_xlabel(xlabel)
ylabel = ui.le_id126.text()
if ylabel:
ax.set_ylabel(ylabel)
if ui.rb_id097.isChecked():
xl = date_functions.mjd2datetimenumber(ui.dsb_id032.value())
xr = date_functions.mjd2datetimenumber(ui.dsb_id033.value())
ax.set_xlim(left=xl,right=xr)
else:
date_begin = ui.de_id003.date().toString("yyyy-MM-dd")
date_end = ui.de_id004.date().toString("yyyy-MM-dd")
mjd_start = date_functions.ymstring2mjd(date_begin)
tmp = date_functions.ymstring2mjd(date_end)
_, mjd_end = date_functions.mjd2mjdmrange(tmp)
xl = date_functions.mjd2datetimenumber(mjd_start)
xr = date_functions.mjd2datetimenumber(mjd_end)
ax.set_xlim(left=xl,right=xr)
if ui.rb_id099.isChecked():
ax.set_ylim(bottom=ui.dsb_id034.value(),top=ui.dsb_id035.value())
if ui.rb_id102.isChecked():
ax.set_yscale('linear')
elif ui.rb_id103.isChecked():
ax.set_yscale('log')
ax.legend()
fig.autofmt_xdate()
# export plot data
file_path = os.path.join(project_dir,project_name,'plot_data.txt')
f = open(file_path,'w')
w = csv.writer(f)
for line in ax.lines:
x = line.get_xdata()
x = date_functions.datetimenumber2mjd(x)
x = date_functions.mjd2ymstring(x,'%Y-%m-%d')
y = line.get_ydata()
l = line.get_label()
w.writerow(['%s_x' % l,x])
w.writerow(['%s_y' % l,y])
f.close()
# show figure
fig.show()
