def main(_):
""" Just chooses the action according to the flag set. (code of the actions is in 'methods.py'. """
config = flags.FLAGS
print(config.mode)
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2 #+ 1500
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
test(config)
else:
print("Unknown mode")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == "prepro":
prepro(config)
else:
print("Unknown mode")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == "train":
train(config)
elif config.mode == "prepro":
copyfile(_[0], dir + 'config.py')
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
if config.use_cudnn:
print(
"Warning: Due to a known bug in Tensorlfow, the parameters of CudnnGRU may not be properly restored."
)
test(config)
elif config.mode == "predict":
if config.use_cudnn:
print(
"Warning: Due to a known bug in Tensorlfow, the parameters of CudnnGRU may not be properly restored."
)
predict(config)
elif config.mode == "test_sber":
if config.use_cudnn:
print(
"Warning: Due to a known bug in Tensorlfow, the parameters of CudnnGRU may not be properly restored."
)
test_sber(config)
else:
print("Unknown mode")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == "train":
while True:
try:
train(config)
except Exception:
print("exception...")
print('restart....')
pass
else:
break
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
test(config)
elif config.mode == "demo":
demo(config)
else:
print("Unknown mode")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == 'prepro':
prepro(config)
elif config.mode == 'train':
train(config)
elif config.mode == 'test':
test(config)
else:
raise ValueError('The mode must choose from [prepro/train/test]!')
def main(_):
config = flags.FLAGS
if config.mode == "train":
# train(config)
cv_train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "test":
cv_test(config)
elif config.mode == "demo":
demo(config)
def main(_):
config = flags.FLAGS
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = config.gpu_id
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "save_pb":
save_pb(config)
else:
print("Unknown mode")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
else:
print("Unknown mode")
exit(0)
def mysvr(qdata):
#data = np.loadtxt('airlines2.txt')
data = qdata
(x_train, y_train, x_val, y_val, x_test, y_test) = prepro(12, data["Open"])
print('initializing grid...')
best_model, best_predicts, best_error, best_param = grd(
x_train, y_train, x_val, y_val)
print('grid ready')
best_predicts = best_model.predict(x_test)
best_error = MSE(best_predicts, y_test)
print('initializing pso...')
g_best = pso(30, 50, x_train, y_train, x_val, y_val)
print('pso ready')
print('initializing svr')
model = SVR(C=g_best[0], gamma=g_best[1], epsilon=g_best[2])
print('svr ready')
model.fit(x_train, y_train)
predicts = model.predict(x_test)
return predicts, best_predicts
def main(_):
config = flags.FLAGS
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
test(config)
else:
print("Unknown mode, you must choose mode from [train/prepro/debug/test]")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == 'train':
train(config)
elif config.mode == 'prepro':
prepro(config)
elif config.mode == 'debug':
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == 'test':
test(config)
else:
print('Unknown mode')
exit(0)
def myar(qdata):
data = qdata
x_train, y_train, x_val, y_val, x_test, y_test = prepro(12, data["Open"])
in1 = np.linalg.pinv(x_train)
yt = y_train.T
coef = in1.dot(yt)
out = x_test.dot(coef.T)
return out, y_test
def main(_):
config = flags.FLAGS
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
if config.use_cudnn:
print("Warning: Due to a known bug in Tensorlfow, the parameters of CudnnGRU may not be properly restored.")
test(config)
else:
print("Unknown mode")
exit(0)
def main(_):
config = flags.FLAGS
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
test(config)
elif config.mode == "demo":
demo(config)
else:
print("Unknown mode")
exit(0)
def main(_):
jieba.re_han_default = re.compile("([\u4E00-\u9FD5a-zA-Z0-9+#&\._%\xd7]+)",
re.U)
config = flags.FLAGS
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
test(config)
elif config.mode == "demo":
demo(config)
else:
print("Unknown mode")
exit(0)
def mymlpr(qdata):
data = qdata
n_neuronios = 50
x_train, y_train, x_val, y_val, x_test, y_test = prepro(12, data["Open"])
model = MLPRegressor(hidden_layer_sizes=(n_neuronios),
activation='relu',
alpha=0.01)
model.fit(x_train, y_train)
pred = model.predict(x_test)
return pred
parser.add_argument('--data_split', type=str, default='train')
parser.add_argument('--fullwiki', action='store_true')
parser.add_argument('--prediction_file', type=str)
parser.add_argument('--sp_threshold', type=float, default=0.3)
config = parser.parse_args()
def _concat(filename):
if config.fullwiki:
return 'fullwiki.{}'.format(filename)
return filename
# config.train_record_file = _concat(config.train_record_file)
config.dev_record_file = _concat(config.dev_record_file)
config.test_record_file = _concat(config.test_record_file)
# config.train_eval_file = _concat(config.train_eval_file)
config.dev_eval_file = _concat(config.dev_eval_file)
config.test_eval_file = _concat(config.test_eval_file)
if config.mode == 'train':
train(config)
elif config.mode == 'prepro':
prepro(config)
elif config.mode == 'test':
test(config)
elif config.mode == 'count':
cnt_len(config)
def prepare(args):
prepro(args)
create_vocab(args)
add_selectbox = st.selectbox('Quelle entreprise vous intéresse?',
('NVDA', 'AMD', 'INTC'))
st.write("Voici les données récupérés par l'API")
df = generate_df(add_selectbox)
df
st.write("Les données préalablement récupérer nous décrit donc cette courbe")
st.set_option('deprecation.showPyplotGlobalUse', False)
plot_figs(df)
st.pyplot()
scaled_close_data, training_data_len, close_dataset, close_data, df = prepro(
df)
x_train, y_train, df = generate_x_train(scaled_close_data, training_data_len,
close_dataset, close_data, df)
st.write(
"En nous basant sur ces données et par le moyen d'une entraînement nous pouvons effectuer une prédiction par rapport à notre dataset"
)
valid, predictions, rmse, train, df = lstm(x_train, training_data_len,
close_dataset, close_data, df,
y_train)
valid
st.write(
"Ainsi il est aisé de constater que notre algorithme de prédiction respecte assez bien les tendances"
def main(_):
config = flags.FLAGS
#srun -p sugon --gres=gpu:1 python config-v1.py prepro Raw 2 128 50
mode = sys.argv[1]
model_name = sys.argv[2]
num_heads = sys.argv[3]
train_dir = 'train-v1'
if model_name in ['Soft_T5']:
fixed_c_maxlen = sys.argv[4]
learning_rate = sys.argv[5]
bucket_slop_min = float(sys.argv[6])
bucket_slop_max = float(sys.argv[7])
l1_width = int(sys.argv[8])
l2_width = int(sys.argv[9])
stddev = float(sys.argv[10])
soft_t5_activation = sys.argv[11]
trail = sys.argv[12]
dir_name = os.path.join(
train_dir, "_".join([
model_name,
str(num_heads),
str(fixed_c_maxlen),
str(learning_rate),
str(bucket_slop_min),
str(bucket_slop_max),
str(l1_width),
str(l2_width),
str(stddev), soft_t5_activation, trail
]))
elif model_name in ['Soft_T5_Nob']:
fixed_c_maxlen = sys.argv[4]
learning_rate = sys.argv[5]
soft_t5_activation = sys.argv[6]
trail = sys.argv[7]
dir_name = os.path.join(
train_dir, "_".join([
model_name,
str(num_heads),
str(fixed_c_maxlen),
str(learning_rate), soft_t5_activation, trail
]))
elif model_name in ['T5', 'T5_Nob']:
t5_num_buckets = int(sys.argv[4])
t5_max_distance = int(sys.argv[5])
trail = sys.argv[6]
dir_name = os.path.join(
train_dir, "_".join([
model_name,
str(num_heads),
str(t5_num_buckets),
str(t5_max_distance), trail
]))
else:
trail = sys.argv[4]
dir_name = os.path.join(train_dir,
"_".join([model_name,
str(num_heads), trail]))
if not os.path.exists(train_dir):
os.mkdir(train_dir)
if not os.path.exists(dir_name):
os.mkdir(dir_name)
event_log_dir = os.path.join(dir_name, "event")
save_dir = os.path.join(dir_name, "model")
answer_dir = os.path.join(dir_name, "answer")
answer_file = os.path.join(answer_dir, "answer.json")
if not os.path.exists(event_log_dir):
os.makedirs(event_log_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if not os.path.exists(answer_dir):
os.makedirs(answer_dir)
config.mode = mode
config.model = model_name
config.num_heads = int(num_heads)
config.trail = trail
if config.model in ['Soft_T5', 'Soft_T5_TPE']:
config.fixed_c_maxlen = int(fixed_c_maxlen)
config.learning_rate = float(learning_rate)
config.bucket_slop_min = bucket_slop_min
config.bucket_slop_max = bucket_slop_max
config.soft_t5_activation = soft_t5_activation
config.l1_width = l1_width
config.l2_width = l2_width
config.stddev = stddev
if config.model in ['Soft_T5_Nob']:
config.fixed_c_maxlen = int(fixed_c_maxlen)
config.learning_rate = float(learning_rate)
config.soft_t5_activation = soft_t5_activation
if config.model in ['T5', 'T5_TPE', 'T5_Nob']:
config.t5_num_buckets = t5_num_buckets
config.t5_max_distance = t5_max_distance
config.event_log_dir = event_log_dir
config.save_dir = save_dir
config.answer_file = answer_file
if config.mode == "train":
train(config)
elif config.mode == "prepro":
prepro(config)
elif config.mode == "debug":
config.num_steps = 2
config.val_num_batches = 1
config.checkpoint = 1
config.period = 1
train(config)
elif config.mode == "test":
test(config)
elif config.mode == "demo":
demo(config)
else:
print("Unknown mode")
exit(0)
"""
Script for running the QA system end to end
"""
from prepro import prepro
from evaluate import evaluate
import model
import baseline
data = prepro()
predictions = model.run()
results = evaluate(predictions, data['answers'])
print("Sentence score {} F1 score {}".format(results['sscore'], results['f1']))
baseline_predictions = baseline.run()
results = evaluate(baseline_predictions, data['answers'])
print("Baseline sentence score {}".format(data['sscore']))
