def test_export(self):
try:
# input to inference model
dummy_input = torch.rand(size=(1, 3, *(self.img_size)), device='cpu')
self.model.eval()
torch.onnx.export(self.model, dummy_input, './mobilenetv3.onnx', verbose=False)
check_file_exist('./mobilenetv3.onnx')
except ExportError:
self.fail("Exception raised while exporting to ONNX")
def write_out(self, predicts, output_file, acc):
print("Write to file: {}".format(output_file))
print("Examples: {}, Predicts: {}".format(len(self.examples),
len(predicts)))
utils.check_file_exist(output_file)
with utils.create_write_file(output_file) as fw:
print('Dev: %.4f' % acc, file=fw)
for idx, example in enumerate(self.examples):
example_str = example.get_instance_string()
print("{}\t#\t{}".format(predicts[idx], example_str), file=fw)
def test_export(self):
# input to inference model
dummy_input = torch.rand(size=(1, 3, *(self.img_size)), device='cpu')
self.model.eval()
onnx_model_path = './mobilenetv3.onnx'
torch.onnx.export(self.model,
dummy_input,
onnx_model_path,
verbose=False)
check_file_exist(onnx_model_path)
def __init__(self, task_name='word2vec-word', init=False, expand=True):
tasks = {
'word2vec-lemma': {
'emb_file': config.word_embed_file,
'word_type': 'lemma'
},
'word2vec-word': {
'emb_file': config.word_embed_file,
'word_type': 'word'
},
'fasttext-lemma': {
'emb_file': config.fasttext_file,
'word_type': 'lemma'
},
'fasttext-word': {
'emb_file': config.fasttext_file,
'word_type': 'word'
},
'paragram-lemma': {
'emb_file': config.paragram_file,
'word_type': 'lemma'
},
'paragram-word': {
'emb_file': config.paragram_file,
'word_type': 'word'
},
}
task = tasks[task_name]
global WORD_TYPE
WORD_TYPE = task['word_type']
self.train_file = config.train_exp_file if expand else config.train_file
self.dev_file = config.dev_file
self.test_file = config.test_file
self.word_embed_file = task['emb_file']
self.word_dim = config.word_dim
self.max_len = config.max_sent_len
self.num_class = config.num_class
self.w2i_file, self.we_file = config.get_w2i_we_file(task_name)
utils.check_file_exist(self.w2i_file)
utils.check_file_exist(self.we_file)
if init:
word_vocab = self.build_vocab()
self.word_vocab, self.embed = data_utils.load_word_embedding(
word_vocab, self.word_embed_file, self.word_dim)
data_utils.save_params(self.word_vocab, self.w2i_file)
data_utils.save_params(self.embed, self.we_file)
else:
self.word_vocab = data_utils.load_params(self.w2i_file)
self.embed = data_utils.load_params(self.we_file)
print("vocab size: %d" % len(self.word_vocab), "we shape: ",
self.embed.shape)
self.train_data = Dataset(self.train_file, self.word_vocab,
self.max_len, self.num_class)
self.dev_data = Dataset(self.dev_file, self.word_vocab, self.max_len,
self.num_class)
if self.test_file:
self.test_data = Dataset(self.test_file, self.word_vocab,
self.max_len, self.num_class)
result_path = params.train_images
f = open(params.train_images_labels, "a+")
else:
result_path = params.test_images
f = open(params.test_images_labels, "a+")
image_path = '%s/%s' % (original_dir, file) # 图片路径
image = Image.open(image_path) # 打开图片文件
imgry = image.convert('L') # 转化为灰度图
recognizition = get_lable(file, original_dir)
line = file + " " + recognizition + "\n"
f.writelines(line)
# 保存图片
imgry.save(result_path + file)
print(file, recognizition)
i += 1
f.close()
return i
utils.check_file_exist(params.train_images_labels)
utils.check_file_exist(params.test_images_labels)
utils.check_floder_exist(params.train_images)
utils.check_floder_exist(params.test_images)
i = 0
i = main(params.original_dir, i, params.origin_train_num)
main(params.augmentation_dir, i, params.augmentation_train_num)
'''
Configuration file for model evaluation.
'''
import os
from utils import get_device, check_file_exist
from main_cfg import ARGS
CURR_FILE_PATH = '/'.join(os.path.realpath(__file__).split('/')
[:-1]) #the absolute path of this file.
DEVICE = get_device() #identify the device to be used for training/evaluation
MODEL_PATH = CURR_FILE_PATH + ARGS.model_path #trained model save path.
MODEL_NAME = ARGS.model_name
#checks if the torch model exists.
TRAINED_MODEL_PRESENCE = check_file_exist(file_path=MODEL_PATH,
file_name=MODEL_NAME)
RESIZED_IMAGE_SIZE = ARGS.image_size
CLASS_FILE = ARGS.class_file
CLASSES = []
#Get the classes name from the .txt file.
OPEN_CFILE = open(CLASS_FILE, 'r')
#Reads every line in the file and append the name into the list.
for line in OPEN_CFILE:
CLASSES.append(line.rstrip()) #strip the newline.
CLASSES.sort() #sort ascending order. IMPORTANT!
NUM_CLASSES = len(CLASSES)
def run(ini_file='TOPKAPI.ini'):
"""Run the model with the set-up defined by `ini_file`.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
print 'Read the file ', ini_file
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getfloat('numerical_options', 'solve_s')
solve_o = config.getfloat('numerical_options', 'solve_o')
solve_c = config.getfloat('numerical_options', 'solve_c')
only_channel_output = config.getboolean('numerical_options',
'only_channel_output')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
print 'Read the forcing data'
#~~~~Rainfall
h5file_in = h5.openFile(file_rain, mode='r')
group = '/' + group_name + '/'
node = h5file_in.getNode(group + 'rainfall')
ndar_rain = node.read()
h5file_in.close()
#~~~~ETr - Reference crop ET
h5file_in = h5.openFile(file_ET, mode='r')
group = '/' + group_name + '/'
node = h5file_in.getNode(group + 'ETr')
ndar_ETr = node.read()
h5file_in.close()
#~~~~ETo - Open water potential evap.
h5file_in = h5.openFile(file_ET, mode='r')
group = '/' + group_name + '/'
node = h5file_in.getNode(group + 'ETo')
ndar_ETo = node.read()
h5file_in.close()
#~~~~external_flow flows
if external_flow:
ar_Qexternal_flow = np.loadtxt(file_Qexternal_flow)[:, 5]
##============================##
## Pretreatment of input data ##
##============================##
print 'Pretreatment of input data'
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, ar_lambda, \
ar_Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L0, ar_Ks0, \
ar_theta_r, ar_theta_s, \
ar_n_o0, ar_n_c0, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
ar_kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L0 * fac_L
ar_Ks = ar_Ks0 * fac_Ks
ar_n_o = ar_n_o0 * fac_n_o
ar_n_c = ar_n_c0 * fac_n_c
print 'Max L=', max(ar_L)
print 'Max Ks=', max(ar_Ks)
print 'Max n_o=', max(ar_n_o)
print 'Max n_c=', max(ar_n_c)
#~~~~Computation of model parameters from physical parameters
ar_Vsm, ar_b_s, ar_b_o, \
ar_W, ar_b_c = pm.compute_cell_param(X, ar_Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
ar_lambda, ar_tan_beta,
ar_tan_beta_channel, ar_L,
ar_Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow, ar_coorx,
ar_coory, ar_lambda)
print 'external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')'
#~~~~Number of simulation time steps
nb_time_step = len(ndar_rain[:, 0])
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
print 'Initialize from file'
h5file_in = h5py.File(file_out)
ar_Vs0 = h5file_in['/Soil/V_s'][-1, :]
ar_Vc0 = h5file_in['/Channel/V_c'][-1, :]
ar_Vo0 = h5file_in['/Overland/V_o'][-1, :]
h5file_in.close()
else:
print 'Initialize from parms'
ar_Vs0 = fl.initial_volume_soil(ar_pVs_t0, ar_Vsm)
ar_Vo0 = ar_Vo_t0
ar_Vc0 = fl.initial_volume_channel(ar_Qc_t0, ar_W, X, ar_n_c)
## Computed variables
#Matrix of soil,overland and channel store at the end of the time step
ar_Vs1 = np.ones(nb_cell) * -99.9
ar_Vo1 = np.ones(nb_cell) * -99.9
ar_Vc1 = np.ones(nb_cell) * -99.9
#Matrix of outflows between two time steps
ar_Qs_out = np.ones(nb_cell) * -99.9
ar_Qo_out = np.ones(nb_cell) * -99.9
ar_Qc_out = np.zeros(nb_cell)
## Intermediate variables
ar_a_s = np.ones(nb_cell) * -99.9
ar_a_o = np.ones(nb_cell) * -99.9
ar_a_c = np.ones(nb_cell) * -99.9
ar_Q_to_next_cell = np.ones(nb_cell) * -99.9
ar_Q_to_channel = np.ones(nb_cell) * -99.9
ar_Q_to_channel_sub = np.zeros(nb_cell)
ar_Qc_cell_up = np.zeros(nb_cell)
ar_ETa = np.zeros(nb_cell)
ar_ET_channel = np.zeros(nb_cell)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file = h5.openFile(file_out, mode=fmode, title='TOPKAPI_out')
root = h5file.getNode('/')
root._v_attrs.pytopkapi_version = pytopkapi.__version__
root._v_attrs.pytopkapi_git_revision = pytopkapi.__git_revision__
atom = h5.Float32Atom()
h5filter = h5.Filters(9) # maximum compression
# create file structure as necessary
grp_name = '/Soil'
if grp_name not in h5file:
h5file.createGroup('/', 'Soil', 'Soil arrays')
if grp_name + '/Qs_out' not in h5file:
array_Qs_out = h5file.createEArray(grp_name,
'Qs_out',
atom,
shape=(0, nb_cell),
title='m3/s',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qs_out = h5file.getNode(grp_name + '/Qs_out')
if grp_name + '/V_s' not in h5file:
array_Vs = h5file.createEArray(grp_name,
'V_s',
atom,
shape=(0, nb_cell),
title='m3',
filters=h5filter,
expectedrows=nb_time_step + 1)
else:
array_Vs = h5file.getNode(grp_name + '/V_s')
grp_name = '/Overland'
if grp_name not in h5file:
h5file.createGroup('/', 'Overland', 'Overland arrays')
if grp_name + '/Qo_out' not in h5file:
array_Qo_out = h5file.createEArray(grp_name,
'Qo_out',
atom,
shape=(0, nb_cell),
title='m3/s',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qo_out = h5file.getNode(grp_name + '/Qo_out')
if grp_name + '/V_o' not in h5file:
array_Vo = h5file.createEArray(grp_name,
'V_o',
atom,
shape=(0, nb_cell),
title='m3',
filters=h5filter,
expectedrows=nb_time_step + 1)
else:
array_Vo = h5file.getNode(grp_name + '/V_o')
grp_name = '/Channel'
if grp_name not in h5file:
h5file.createGroup('/', 'Channel', 'Channel arrays')
if grp_name + '/Qc_out' not in h5file:
array_Qc_out = h5file.createEArray(grp_name,
'Qc_out',
atom,
shape=(0, nb_cell),
title='m3/s',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qc_out = h5file.getNode(grp_name + '/Qc_out')
if grp_name + '/V_c' not in h5file:
array_Vc = h5file.createEArray(grp_name,
'V_c',
atom,
shape=(0, nb_cell),
title='m3',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_Vc = h5file.getNode(grp_name + '/V_c')
if grp_name + '/Ec_out' not in h5file:
array_Ec_out = h5file.createEArray(grp_name,
'Ec_out',
atom,
shape=(0, nb_cell),
title='m3',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_Ec_out = h5file.getNode(grp_name + '/Ec_out')
if '/ET_out' not in h5file:
array_ET_out = h5file.createEArray('/',
'ET_out',
atom,
shape=(0, nb_cell),
title='mm',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_ET_out = h5file.getNode('/ET_out')
if '/Q_down' not in h5file:
array_Q_down = h5file.createEArray('/',
'Q_down',
atom,
shape=(0, nb_cell),
title='m3/s',
filters=h5filter,
expectedrows=nb_time_step)
else:
array_Q_down = h5file.getNode('/Q_down')
if append_output is False or first_run is True:
#Write the initial values into the output file
array_Vs.append(ar_Vs0.reshape((1, nb_cell)))
array_Vo.append(ar_Vo0.reshape((1, nb_cell)))
array_Vc.append(ar_Vc0.reshape((1, nb_cell)))
array_Qs_out.append(ar_Qs_out.reshape((1, nb_cell)))
array_Qo_out.append(ar_Qo_out.reshape((1, nb_cell)))
array_Qc_out.append(ar_Qc_out.reshape((1, nb_cell)))
array_Q_down.append(ar_Q_to_next_cell.reshape((1, nb_cell)))
array_ET_out.append(ar_ETa.reshape((1, nb_cell)))
E_vol = ar_ET_channel * 1e-3 * ar_W * ar_Xc
array_Ec_out.append(E_vol.reshape((1, nb_cell)))
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
print '** NB_CELL=', nb_cell
print '** NB_TIME_STEP=', nb_time_step
print '--> SIMULATIONS <--'
## Loop on time
for t in range(nb_time_step):
print t + 1, '/', nb_time_step
eff_sat = ar_Vs0 / ar_Vsm
# estimate soil suction head using Brookes and Corey (1964)
psi = psi_b / np.power(eff_sat, 1.0 / lamda)
## Loop on cells
n = -1
for cell1 in ar_label_sort:
cell = np.where(ar_cell_label == cell1)[0][0]
n = n + 1
## ======================== ##
## ===== INTERCEPTION ===== ##
## ======================== ##
## No interception for the moment
## ======================== ##
## ===== INFILTRATION ===== ##
## ======================== ##
rain_rate = ndar_rain[t, cell] / Dt
infiltration_depth = green_ampt_cum_infiltration(
rain_rate, psi[cell], eff_theta[cell], eff_sat[cell],
ar_Ks[cell], Dt)
## ====================== ##
## ===== SOIL STORE ===== ##
## ====================== ##
#~~~~ Computation of soil input
ar_a_s[cell] = fl.input_soil(infiltration_depth, Dt, X,
ar_Q_to_next_cell, li_cell_up[cell])
#~~~~ Resolution of the equation dV/dt=a_s-b_s*V^alpha_s
# Calculate the volume in the soil store at the end of the
# current time-step.
Vs_prim = om.solve_storage_eq(ar_a_s[cell], ar_b_s[cell], alpha_s,
ar_Vs0[cell], Dt, solve_s)
#~~~~ Computation of soil outflow and overland input
ar_Qs_out[cell], ar_Vs1[cell] = fl.output_soil(
ar_Vs0[cell], Vs_prim, ar_Vsm[cell], ar_a_s[cell],
ar_b_s[cell], alpha_s, Dt)
if ar_Qs_out[cell] < 0:
print 'Problem Soil:output greater than input....'
print 'n=', n, 'label=', cell
stop
## ========================== ##
## ===== OVERLAND STORE ===== ##
## ========================== ##
#~~~~ Computation of overland input
rain_excess = ndar_rain[t, cell] - infiltration_depth
# convert mm to m^3/s
rain_excess = max(0, (rain_excess * (10**-3) / Dt) * X**2)
ar_a_o[cell] = max(0,
ar_a_s[cell] \
- ((ar_Vs1[cell]-ar_Vs0[cell])/Dt \
+ ar_Qs_out[cell]) \
+ rain_excess)
#~~~~ Resolution of the equation dV/dt=a_o-b_o*V^alpha_o
ar_Vo1[cell] = om.solve_storage_eq(ar_a_o[cell], ar_b_o[cell],
alpha_o, ar_Vo0[cell], Dt,
solve_o)
#~~~~ Computation of overland outflows
ar_Qo_out[cell] = fl.Qout_computing(ar_Vo0[cell], ar_Vo1[cell],
ar_a_o[cell], Dt)
if ar_Qo_out[cell] < 0:
print 'Problem Overland:output greater than input....'
print 'n=', n, 'label=', cell
stop
## ============================= ##
## ===== FLOW PARTITIONING ===== ##
## ============================= ##
# ar_Q_to_channel_sub doesn't get used for anything?
ar_Q_to_next_cell[cell], \
ar_Q_to_channel[cell], \
ar_Q_to_channel_sub[cell] = fl.flow_partitioning(ar_lambda[cell],
ar_Qs_out[cell],
ar_Qo_out[cell],
ar_W[cell],
X, ar_Xc[cell])
## ======================== ##
## ===== CHANNEL STORE ==== ##
## ======================== ##
if ar_lambda[cell] == 1:
if ar_cell_down[cell] >= 0 \
and ar_lambda[ar_cell_down[cell]] == 0:
print 'Problem: the present cell has a channel but not the cell down...'
Stop
#~~~~ Computation of channel input
ar_a_c[cell], \
ar_Qc_cell_up[cell] = fl.input_channel(ar_Qc_out,
ar_Q_to_channel[cell],
li_cell_up[cell])
if external_flow \
and cell == np.where(ar_cell_label==cell_external_flow)[0][0]:
ar_a_c[cell] = ar_a_c[cell] + ar_Qexternal_flow[t]
#~~~~ Resolution of the equation dV/dt=a_c-b_c*V^alpha_c
ar_Vc1[cell] = om.solve_storage_eq(ar_a_c[cell], ar_b_c[cell],
alpha_c, ar_Vc0[cell], Dt,
solve_c)
#~~~~ Computation of channel outflows
ar_Qc_out[cell] = fl.Qout_computing(ar_Vc0[cell], ar_Vc1[cell],
ar_a_c[cell], Dt)
if ar_Qc_out[cell] < 0:
print 'Problem Channel: output greater than input....'
stop
if str(ar_Qc_out[cell]).count('N') > 0:
print ar_Qc_out[cell]
print 'Problem Channel: Non authorized operand....'
stop
else:
ar_a_c[cell] = 0.
ar_Vc1[cell] = 0.
ar_Qc_out[cell] = 0.
## ============================== ##
## ===== EVAPOTRANSPIRATION ===== ##
## ============================== ##
#~~~~~ From soil
ar_ETa[cell], \
ar_Vs1[cell], \
ar_Vo1[cell] = em.evapot_soil_overland(ar_Vo1[cell],
ar_Vs1[cell],
ar_Vsm[cell],
ar_kc[cell],
ndar_ETr[t, cell], X)
#~~~~~ Evaporation from channel
if ar_lambda[cell] == 1:
ar_ET_channel[cell], \
ar_Vc1[cell] = em.evapor_channel(ar_Vc1[cell],
ndar_ETo[t, cell],
ar_W[cell], ar_Xc[cell])
####===================================####
#### Affectation of new vector values ####
####===================================####
ar_Vs0 = np.array(ar_Vs1)
ar_Vo0 = np.array(ar_Vo1)
ar_Vc0 = np.array(ar_Vc1)
####===================================####
#### Results writing at each time step ####
####===================================####
array_Vs.append(ar_Vs1.reshape((1, nb_cell)))
array_Vo.append(ar_Vo1.reshape((1, nb_cell)))
array_Vc.append(ar_Vc1.reshape((1, nb_cell)))
array_Qs_out.append(ar_Qs_out.reshape((1, nb_cell)))
array_Qo_out.append(ar_Qo_out.reshape((1, nb_cell)))
array_Qc_out.append(ar_Qc_out.reshape((1, nb_cell)))
array_Q_down.append(ar_Q_to_next_cell.reshape((1, nb_cell)))
array_ET_out.append(ar_ETa.reshape((1, nb_cell)))
E_vol = ar_ET_channel * 1e-3 * ar_W * ar_Xc
array_Ec_out.append(E_vol.reshape((1, nb_cell)))
h5file.close()
print ' '
print '***** THE END *****'
def run(ini_file='TOPKAPI.ini'):
"""Run the model with the set-up defined by `ini_file`.
"""
##================================##
## Read the input file (*.ini) ##
##================================##
config = SafeConfigParser()
config.read(ini_file)
print 'Read the file ',ini_file
##~~~~~~ Numerical_options ~~~~~~##
solve_s = config.getfloat('numerical_options', 'solve_s')
solve_o = config.getfloat('numerical_options', 'solve_o')
solve_c = config.getfloat('numerical_options', 'solve_c')
only_channel_output = config.getboolean('numerical_options',
'only_channel_output')
##~~~~~~~~~~~ input files ~~~~~~~~~~~##
#Param
file_global_param = config.get('input_files', 'file_global_param')
file_cell_param = config.get('input_files', 'file_cell_param')
#Rain
file_rain = config.get('input_files', 'file_rain')
#ETP
file_ET = config.get('input_files', 'file_ET')
#~~~~~~~~~~~ Group (simulated event) ~~~~~~~~~~~##
group_name = config.get('groups', 'group_name')
##~~~~~~ Calibration ~~~~~~##
fac_L = config.getfloat('calib_params', 'fac_L')
fac_Ks = config.getfloat('calib_params', 'fac_Ks')
fac_n_o = config.getfloat('calib_params', 'fac_n_o')
fac_n_c = config.getfloat('calib_params', 'fac_n_c')
##~~~~~~ External flows ~~~~~~##
external_flow = config.getboolean('external_flow', 'external_flow')
if external_flow:
file_Qexternal_flow = config.get('external_flow',
'file_Qexternal_flow')
Xexternal_flow = config.getfloat('external_flow', 'Xexternal_flow')
Yexternal_flow = config.getfloat('external_flow', 'Yexternal_flow')
##~~~~~~~~~~~ output files ~~~~~~~~~~##
file_out = config.get('output_files', 'file_out')
ut.check_file_exist(file_out) #create path_out if it doesn't exist
if os.path.exists(file_out):
first_run = False
else:
first_run = True
append_output = config.getboolean('output_files', 'append_output')
if append_output is True:
fmode = 'a'
else:
fmode = 'w'
##============================##
## Read the forcing data ##
##============================##
print 'Read the forcing data'
#~~~~Rainfall
h5file_in = h5.openFile(file_rain,mode='r')
group = '/'+group_name+'/'
node = h5file_in.getNode(group+'rainfall')
ndar_rain = node.read()
h5file_in.close()
#~~~~ETr - Reference crop ET
h5file_in = h5.openFile(file_ET,mode='r')
group = '/'+group_name+'/'
node = h5file_in.getNode(group+'ETr')
ndar_ETr = node.read()
h5file_in.close()
#~~~~ETo - Open water potential evap.
h5file_in = h5.openFile(file_ET,mode='r')
group = '/'+group_name+'/'
node = h5file_in.getNode(group+'ETo')
ndar_ETo = node.read()
h5file_in.close()
#~~~~external_flow flows
if external_flow:
ar_Qexternal_flow = np.loadtxt(file_Qexternal_flow)[:, 5]
##============================##
## Pretreatment of input data ##
##============================##
print 'Pretreatment of input data'
#~~~~Read Global parameters file
X, Dt, alpha_s, \
alpha_o, alpha_c, \
A_thres, W_min, W_max = pm.read_global_parameters(file_global_param)
#~~~~Read Cell parameters file
ar_cell_label, ar_coorx, \
ar_coory, ar_lambda, \
ar_Xc, ar_dam, \
ar_tan_beta, ar_tan_beta_channel, \
ar_L0, ar_Ks0, \
ar_theta_r, ar_theta_s, \
ar_n_o0, ar_n_c0, \
ar_cell_down, ar_pVs_t0, \
ar_Vo_t0, ar_Qc_t0, \
ar_kc, psi_b, lamda = pm.read_cell_parameters(file_cell_param)
#~~~~Number of cell in the catchment
nb_cell = len(ar_cell_label)
#~~~~Computation of cell order
ar_label_sort = pm.sort_cell(ar_cell_label, ar_cell_down)
#~~~~Computation of upcells
li_cell_up = pm.direct_up_cell(ar_cell_label, ar_cell_down, ar_label_sort)
#~~~~Computation of drained area
ar_A_drained = pm.drained_area(ar_label_sort, li_cell_up, X)
#~~~~Apply calibration factors to the parameter values
ar_L = ar_L0*fac_L
ar_Ks = ar_Ks0*fac_Ks
ar_n_o = ar_n_o0*fac_n_o
ar_n_c = ar_n_c0*fac_n_c
print 'Max L=', max(ar_L)
print 'Max Ks=', max(ar_Ks)
print 'Max n_o=', max(ar_n_o)
print 'Max n_c=', max(ar_n_c)
#~~~~Computation of model parameters from physical parameters
ar_Vsm, ar_b_s, ar_b_o, \
ar_W, ar_b_c = pm.compute_cell_param(X, ar_Xc, Dt, alpha_s,
alpha_o, alpha_c, nb_cell,
A_thres, W_max, W_min,
ar_lambda, ar_tan_beta,
ar_tan_beta_channel, ar_L,
ar_Ks, ar_theta_r, ar_theta_s,
ar_n_o, ar_n_c, ar_A_drained)
#~~~~Look for the cell of external_flow tunnel
if external_flow:
cell_external_flow = ut.find_cell_coordinates(ar_cell_label,
Xexternal_flow,
Yexternal_flow,
ar_coorx,
ar_coory,
ar_lambda)
print 'external flows will be taken into account for cell no',\
cell_external_flow, ' coordinates ('\
,Xexternal_flow,',',Yexternal_flow,')'
#~~~~Number of simulation time steps
nb_time_step = len(ndar_rain[:,0])
##=============================##
## Variable array definition ##
##=============================##
## Initialisation of the reservoirs
#Matrix of soil,overland and channel store at the begining of the time step
if append_output and not first_run:
print 'Initialize from file'
# read from file
h5file_in = h5.openFile(file_out, mode='r')
node = h5file_in.getNode('/Soil/V_s')
ar_Vs0 = node.read()[-1, :]
node = h5file_in.getNode('/Overland/V_o')
ar_Vo0 = node.read()[-1, :]
node = h5file_in.getNode('/Channel/V_c')
ar_Vc0 = node.read()[-1, :]
h5file_in.close()
else:
print 'Initialize from parms'
ar_Vs0 = fl.initial_volume_soil(ar_pVs_t0, ar_Vsm)
ar_Vo0 = ar_Vo_t0
ar_Vc0 = fl.initial_volume_channel(ar_Qc_t0, ar_W, X, ar_n_c)
## Computed variables
#Matrix of soil,overland and channel store at the end of the time step
ar_Vs1 = np.ones(nb_cell)*-99.9
ar_Vo1 = np.ones(nb_cell)*-99.9
ar_Vc1 = np.ones(nb_cell)*-99.9
#Matrix of outflows between two time steps
ar_Qs_out = np.ones(nb_cell)*-99.9
ar_Qo_out = np.ones(nb_cell)*-99.9
ar_Qc_out = np.zeros(nb_cell)
## Intermediate variables
ar_a_s = np.ones(nb_cell)*-99.9
ar_a_o = np.ones(nb_cell)*-99.9
ar_a_c = np.ones(nb_cell)*-99.9
ar_Q_to_next_cell = np.ones(nb_cell)*-99.9
ar_Q_to_channel = np.ones(nb_cell)*-99.9
ar_Q_to_channel_sub = np.zeros(nb_cell)
ar_Qc_cell_up = np.zeros(nb_cell)
ar_ETa = np.zeros(nb_cell)
ar_ET_channel = np.zeros(nb_cell)
##=============================##
## HDF5 output file definition ##
##=============================##
h5file = h5.openFile(file_out, mode=fmode, title='TOPKAPI_out')
root = h5file.getNode('/')
root._v_attrs.pytopkapi_version = pytopkapi.__version__
root._v_attrs.pytopkapi_git_revision = pytopkapi.__git_revision__
atom = h5.Float32Atom()
h5filter = h5.Filters(9)# maximum compression
# create file structure as necessary
grp_name = '/Soil'
if grp_name not in h5file:
h5file.createGroup('/', 'Soil', 'Soil arrays')
if grp_name+'/Qs_out' not in h5file:
array_Qs_out = h5file.createEArray(grp_name, 'Qs_out',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qs_out = h5file.getNode(grp_name+'/Qs_out')
if grp_name+'/V_s' not in h5file:
array_Vs = h5file.createEArray(grp_name, 'V_s',
atom, shape=(0, nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step+1)
else:
array_Vs = h5file.getNode(grp_name+'/V_s')
grp_name = '/Overland'
if grp_name not in h5file:
h5file.createGroup('/', 'Overland', 'Overland arrays')
if grp_name+'/Qo_out' not in h5file:
array_Qo_out = h5file.createEArray(grp_name, 'Qo_out',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qo_out = h5file.getNode(grp_name+'/Qo_out')
if grp_name+'/V_o' not in h5file:
array_Vo = h5file.createEArray(grp_name, 'V_o',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step+1)
else:
array_Vo = h5file.getNode(grp_name+'/V_o')
grp_name = '/Channel'
if grp_name not in h5file:
h5file.createGroup('/', 'Channel', 'Channel arrays')
if grp_name+'/Qc_out' not in h5file:
array_Qc_out = h5file.createEArray(grp_name, 'Qc_out',
atom, shape=(0,nb_cell),
title='m3/s', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Qc_out = h5file.getNode(grp_name+'/Qc_out')
if grp_name+'/V_c' not in h5file:
array_Vc = h5file.createEArray(grp_name, 'V_c',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Vc = h5file.getNode(grp_name+'/V_c')
if grp_name+'/Ec_out' not in h5file:
array_Ec_out = h5file.createEArray(grp_name, 'Ec_out',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Ec_out = h5file.getNode(grp_name+'/Ec_out')
if '/ET_out' not in h5file:
array_ET_out = h5file.createEArray('/', 'ET_out',
atom, shape=(0,nb_cell),
title='mm', filters=h5filter,
expectedrows=nb_time_step)
else:
array_ET_out = h5file.getNode('/ET_out')
if '/Q_down' not in h5file:
array_Q_down = h5file.createEArray('/', 'Q_down',
atom, shape=(0,nb_cell),
title='m3', filters=h5filter,
expectedrows=nb_time_step)
else:
array_Q_down = h5file.getNode('/Q_down')
if append_output is False or first_run is True:
#Write the initial values into the output file
array_Vs.append(ar_Vs0.reshape((1,nb_cell)))
array_Vo.append(ar_Vo0.reshape((1,nb_cell)))
array_Vc.append(ar_Vc0.reshape((1,nb_cell)))
array_Qs_out.append(ar_Qs_out.reshape((1,nb_cell)))
array_Qo_out.append(ar_Qo_out.reshape((1,nb_cell)))
array_Qc_out.append(ar_Qc_out.reshape((1,nb_cell)))
array_Q_down.append(ar_Q_to_next_cell.reshape((1,nb_cell)))
array_ET_out.append(ar_ETa.reshape((1,nb_cell)))
E_vol = ar_ET_channel*1e-3 * ar_W * ar_Xc
array_Ec_out.append(E_vol.reshape((1,nb_cell)))
eff_theta = ar_theta_s - ar_theta_r
##===========================##
## Core of the Model ##
##===========================##
print '** NB_CELL=',nb_cell
print '** NB_TIME_STEP=',nb_time_step
print '--> SIMULATIONS <--'
## Loop on time
for t in range(nb_time_step):
print t+1, '/', nb_time_step
eff_sat = ar_Vs0/ar_Vsm
# estimate soil suction head using Brookes and Corey (1964)
psi = psi_b/np.power(eff_sat, 1.0/lamda)
## Loop on cells
n=-1
for cell1 in ar_label_sort:
cell=np.where(ar_cell_label==cell1)[0][0]
n=n+1
## ======================== ##
## ===== INTERCEPTION ===== ##
## ======================== ##
## No interception for the moment
## ======================== ##
## ===== INFILTRATION ===== ##
## ======================== ##
rain_rate = ndar_rain[t, cell]/Dt
infiltration_depth = green_ampt_cum_infiltration(rain_rate,
psi[cell],
eff_theta[cell],
eff_sat[cell],
ar_Ks[cell], Dt)
## ====================== ##
## ===== SOIL STORE ===== ##
## ====================== ##
#~~~~ Computation of soil input
ar_a_s[cell] = fl.input_soil(infiltration_depth,
Dt, X,
ar_Q_to_next_cell,
li_cell_up[cell])
#~~~~ Resolution of the equation dV/dt=a_s-b_s*V^alpha_s
# Calculate the volume in the soil store at the end of the
# current time-step.
Vs_prim = om.solve_storage_eq(ar_a_s[cell], ar_b_s[cell],
alpha_s, ar_Vs0[cell], Dt, solve_s)
#~~~~ Computation of soil outflow and overland input
ar_Qs_out[cell], ar_Vs1[cell] = fl.output_soil(ar_Vs0[cell],
Vs_prim,
ar_Vsm[cell],
ar_a_s[cell],
ar_b_s[cell],
alpha_s, Dt)
if ar_Qs_out[cell] < 0:
print 'Problem Soil:output greater than input....'
print 'n=', n, 'label=', cell
stop
## ========================== ##
## ===== OVERLAND STORE ===== ##
## ========================== ##
#~~~~ Computation of overland input
rain_excess = ndar_rain[t, cell] - infiltration_depth
# convert mm to m^3/s
rain_excess = max(0, (rain_excess*(10**-3)/Dt)*X**2)
ar_a_o[cell] = max(0,
ar_a_s[cell] \
- ((ar_Vs1[cell]-ar_Vs0[cell])/Dt \
+ ar_Qs_out[cell]) \
+ rain_excess)
#~~~~ Resolution of the equation dV/dt=a_o-b_o*V^alpha_o
ar_Vo1[cell] = om.solve_storage_eq(ar_a_o[cell],
ar_b_o[cell], alpha_o,
ar_Vo0[cell], Dt, solve_o)
#~~~~ Computation of overland outflows
ar_Qo_out[cell] = fl.Qout_computing(ar_Vo0[cell], ar_Vo1[cell],
ar_a_o[cell], Dt)
if ar_Qo_out[cell] < 0:
print 'Problem Overland:output greater than input....'
print 'n=', n, 'label=', cell
stop
## ============================= ##
## ===== FLOW PARTITIONING ===== ##
## ============================= ##
# ar_Q_to_channel_sub doesn't get used for anything?
ar_Q_to_next_cell[cell], \
ar_Q_to_channel[cell], \
ar_Q_to_channel_sub[cell] = fl.flow_partitioning(ar_lambda[cell],
ar_Qs_out[cell],
ar_Qo_out[cell],
ar_W[cell],
X, ar_Xc[cell])
## ======================== ##
## ===== CHANNEL STORE ==== ##
## ======================== ##
if ar_lambda[cell] == 1:
if ar_cell_down[cell] >= 0 \
and ar_lambda[ar_cell_down[cell]] == 0:
print 'Problem: the present cell has a channel but not the cell down...'
Stop
#~~~~ Computation of channel input
ar_a_c[cell], \
ar_Qc_cell_up[cell] = fl.input_channel(ar_Qc_out,
ar_Q_to_channel[cell],
li_cell_up[cell])
if external_flow \
and cell == np.where(ar_cell_label==cell_external_flow)[0][0]:
ar_a_c[cell] = ar_a_c[cell] + ar_Qexternal_flow[t]
#~~~~ Resolution of the equation dV/dt=a_c-b_c*V^alpha_c
ar_Vc1[cell] = om.solve_storage_eq(ar_a_c[cell],
ar_b_c[cell], alpha_c,
ar_Vc0[cell], Dt, solve_c)
#~~~~ Computation of channel outflows
ar_Qc_out[cell] = fl.Qout_computing(ar_Vc0[cell],
ar_Vc1[cell],
ar_a_c[cell], Dt)
if ar_Qc_out[cell] < 0:
print 'Problem Channel: output greater than input....'
stop
if str(ar_Qc_out[cell]).count('N') > 0:
print ar_Qc_out[cell]
print 'Problem Channel: Non authorized operand....'
stop
else:
ar_a_c[cell] = 0.
ar_Vc1[cell] = 0.
ar_Qc_out[cell] = 0.
## ============================== ##
## ===== EVAPOTRANSPIRATION ===== ##
## ============================== ##
#~~~~~ From soil
ar_ETa[cell], \
ar_Vs1[cell], \
ar_Vo1[cell] = em.evapot_soil_overland(ar_Vo1[cell],
ar_Vs1[cell],
ar_Vsm[cell],
ar_kc[cell],
ndar_ETr[t, cell], X)
#~~~~~ Evaporation from channel
if ar_lambda[cell] == 1:
ar_ET_channel[cell], \
ar_Vc1[cell] = em.evapor_channel(ar_Vc1[cell],
ndar_ETo[t, cell],
ar_W[cell], ar_Xc[cell])
####===================================####
#### Affectation of new vector values ####
####===================================####
ar_Vs0 = np.array(ar_Vs1)
ar_Vo0 = np.array(ar_Vo1)
ar_Vc0 = np.array(ar_Vc1)
####===================================####
#### Results writing at each time step ####
####===================================####
array_Vs.append(ar_Vs1.reshape((1,nb_cell)))
array_Vo.append(ar_Vo1.reshape((1,nb_cell)))
array_Vc.append(ar_Vc1.reshape((1,nb_cell)))
array_Qs_out.append(ar_Qs_out.reshape((1,nb_cell)))
array_Qo_out.append(ar_Qo_out.reshape((1,nb_cell)))
array_Qc_out.append(ar_Qc_out.reshape((1,nb_cell)))
array_Q_down.append(ar_Q_to_next_cell.reshape((1,nb_cell)))
array_ET_out.append(ar_ETa.reshape((1,nb_cell)))
E_vol = ar_ET_channel*1e-3 * ar_W * ar_Xc
array_Ec_out.append(E_vol.reshape((1,nb_cell)))
h5file.close()
print ' '
print '***** THE END *****'
asthma patients in the USA
(mentioned `asthma` or `inhaler` in one of the tweets)
created twitter account at 2013
who has geo information
"""
import re
from utils import check_file_exist, dump_pickle, load_pickle
##############################################################
# unique user ids in the USA, created account at 2013
path_uid_usa_2013 = '../data/twitter_asthma/unique_uid_usa_created_2013.txt'
# user_id | created_at
path_unique_user_ids_usa_2013 = '../data/twitter_asthma/unique_user_ids_usa_2013.pkl'
if not check_file_exist(path_unique_user_ids_usa_2013):
unique_user_ids_usa_2013 = set()
with open(path_uid_usa_2013, 'r') as f:
for line in f:
line = line.split('|')
user_id = line[0]
unique_user_ids_usa_2013.add(user_id)
dump_pickle(path_unique_user_ids_usa_2013, unique_user_ids_usa_2013)
else:
unique_user_ids_usa_2013 = load_pickle(path_unique_user_ids_usa_2013)
##############################################################
asthma = '.*asthma.*|.*inhaler.*'
asthma_pat = re.compile(asthma, flags=re.IGNORECASE)
line = line.split("|")
user_id = line[pos]
yield user_id
###################################################
'''
path_input = '../data/twitter_asthma/tweets_with_geo.txt'
# tweet_id | user_id | created_at | text | coordinates | location | time_zone
output = '../data/twitter_asthma/unique_user_ids.pkl'
if not check_file_exist(output):
user_ids = extract_user_id(path_input)
# dump unique user ids
unique_user_ids = set(user_ids)
# print(unique_user_ids)
dump_pickle(output, unique_user_ids)
'''
path_input = '../data/twitter_asthma/tweets_with_address_usa.txt'
# user_id | geo | location
output = '../data/twitter_asthma/unique_user_ids_usa.pkl'
if not check_file_exist(output):
user_ids = extract_user_id(path_input, pos=0)
# dump unique user ids
unique_user_ids = set(user_ids)
# print(unique_user_ids)
dump_pickle(output, unique_user_ids)
Configuration file for model evaluation.
'''
import os
from utils import get_device, check_file_exist
from main_cfg import ARGS
CURR_FILE_PATH = '/'.join(os.path.realpath(__file__).split('/')
[:-1]) #the absolute path of this file.
DEVICE = get_device() #identify the device to be used for training/evaluation
FEAT_MODEL_PATH = CURR_FILE_PATH + ARGS.model_path #trained model save path.
FEAT_MODEL_NAME = ARGS.model_name
CLASSIFIER_MODEL_PATH = FEAT_MODEL_PATH
CLASSIFIER_MODEL_NAME = ARGS.save_model_name
#checks if the torch model exists.
TRAINED_FEAT_MODEL_PRESENCE = check_file_exist(file_path=FEAT_MODEL_PATH,
file_name=FEAT_MODEL_NAME)
TRAINED_CLASSIFIER_MODEL_PRESENCE = check_file_exist(
file_path=CLASSIFIER_MODEL_PATH, file_name=CLASSIFIER_MODEL_NAME)
RESIZED_IMAGE_SIZE = ARGS.image_size
CLASS_FILE = ARGS.class_file
CLASSES = []
#Get the classes name from the .txt file.
OPEN_CFILE = open(CLASS_FILE, 'r')
#Reads every line in the file and append the name into the list.
for line in OPEN_CFILE:
CLASSES.append(line.rstrip()) #strip the newline.
# pass in the user_name or user_id of the account you want to download
# test
# if screen name
'''
screen_name = "MonaaTierra"
user_tweets_archive_2(screen_name)
path = f'..\\data\\twitter_asthma\\tweets\\{screen_name}.json'
with open(path) as f:
result = json.load(f, encoding='utf-8')
print(result)
'''
# if user_id
'''
user_id = '600070804'
user_tweets_archive(user_id)
'''
# input file: unique user ids just in the US
path_input = '../data/twitter_asthma/asthma_uid_usa_2013.txt'
with open(path_input, mode='r', encoding='utf-8',
errors='ignore') as infile:
for line in infile:
line = line.split('|')
user_id = line[0]
path = f'..\\data\\twitter_asthma\\tweets\\{user_id}.json'
if not check_file_exist(path):
user_tweets_archive(user_id, path)
