def initialize(self):
self.model_config = ModelConfig()
self.get_secure_cookie("session_id")
self.session = session.Session(self.application.session_manager, self)
self.logger = api_logger()
self.logger.info(
'session_id: %s, %s request body: %s' %
(self.get_secure_cookie("session_id"), self.request.path,
re.sub(r'(\\n|\\|\s+)', '', json.dumps(self.request.body))))
def initialize(self):
self.model_config = ModelConfig()
self.get_secure_cookie("session_id")
self.session = session.Session(self.application.session_manager, self)
self.logger = api_logger()
self.logger.info(
'%s: %s' % (self.__class__.__name__, re.sub(r'(\\n|\\|\s+)', '', json.dumps(self.request.body))))
class BaseHandler(tornado.web.RequestHandler):
def set_default_headers(self):
self.set_header('Access-Control-Allow-Origin', '*')
self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS')
self.set_header('Access-Control-Max-Age', 1000)
self.set_header('Access-Control-Allow-Headers', '*')
self.set_header('Content-type', 'application/json')
def initialize(self):
self.model_config = ModelConfig()
self.get_secure_cookie("session_id")
self.session = session.Session(self.application.session_manager, self)
self.logger = api_logger()
self.logger.info(
'session_id: %s, %s request body: %s' %
(self.get_secure_cookie("session_id"), self.request.path,
re.sub(r'(\\n|\\|\s+)', '', json.dumps(self.request.body))))
def on_finish(self):
self.model_config.close()
def get_need_args(self, args):
res = {}
for arg in args:
try:
param = self.get_argument(arg)
res[arg] = param
except:
raise ServerError(ServerError.ARGS_MISSING, args=arg)
return res
def get_inner_static_path(self):
return utils.config.get('global', 'inner_static')
def get_json_argument(self, name, default=None, allow_null=False):
r_body = self.request.body
if not r_body:
r_body = '{}'
request_body = json.loads(r_body)
if name not in request_body and not allow_null:
raise ServerError(ServerError.ARGS_MISSING, args=name)
return request_body.get(name, default)
class BaseHandler(tornado.web.RequestHandler):
def set_default_headers(self):
self.set_header('Access-Control-Allow-Origin', '*')
self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS')
self.set_header('Access-Control-Max-Age', 1000)
self.set_header('Access-Control-Allow-Headers', '*')
self.set_header('Content-type', 'application/json')
def initialize(self):
self.model_config = ModelConfig()
self.get_secure_cookie("session_id")
self.session = session.Session(self.application.session_manager, self)
self.logger = api_logger()
self.logger.info(
'%s: %s' % (self.__class__.__name__, re.sub(r'(\\n|\\|\s+)', '', json.dumps(self.request.body))))
def on_finish(self):
self.model_config.close()
def get_need_args(self, args):
res = {}
for arg in args:
try:
param = self.get_argument(arg)
res[arg] = param
except:
raise ServerError(ServerError.ARGS_MISSING, args=arg)
return res
def get_inner_static_path(self):
return utils.config.get('global', 'inner_static')
def get_json_argument(self, name, default=None, allow_null=False):
r_body = self.request.body
if not r_body:
r_body = '{}'
request_body = json.loads(r_body)
if name not in request_body and not allow_null:
raise ServerError(ServerError.ARGS_MISSING, args=name)
return request_body.get(name, default)
n_layer = 8
n_head = 8
elif args.arch == "m2":
n_embd = 240
n_layer = 12
n_head = 12
elif args.arch == "m1": # medium
n_embd = 200
n_layer = 10
n_head = 10
best_model_config = ModelConfig(
vocab_size=len(vocab) - 1,
n_positions=85 * 2,
n_ctx=85 * 2,
n_embd=n_embd,
n_layer=n_layer,
n_head=n_head,
loss_method="mse", # simple regression head
vocab_path=args.m2id,
model_path=args.best_model_path)
print(best_model_config)
# config for SelfPlay
selfplay_config = SelfPlayConfig(
depth=args.depth,
sims=args.sims,
buffer_size=args.buffer_size,
max_moves=args.max_moves,
n_data_collection_games=args.n_data_collection_games,
n_evaluation_games=args.n_evaluation_games,
)
def pred_fn(args, data_reader, figure_dir=None, prob_dir=None, log_dir=None):
current_time = time.strftime("%y%m%d-%H%M%S")
if log_dir is None:
log_dir = os.path.join(args.log_dir, "pred", current_time)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if (args.plot_figure == True) and (figure_dir is None):
figure_dir = os.path.join(log_dir, 'figures')
if not os.path.exists(figure_dir):
os.makedirs(figure_dir)
if (args.save_prob == True) and (prob_dir is None):
prob_dir = os.path.join(log_dir, 'probs')
if not os.path.exists(prob_dir):
os.makedirs(prob_dir)
if args.save_prob:
h5 = h5py.File(os.path.join(args.result_dir, "result.h5"),
"w",
libver='latest')
prob_h5 = h5.create_group("/prob")
logging.info("Pred log: %s" % log_dir)
logging.info("Dataset size: {}".format(data_reader.num_data))
with tf.compat.v1.name_scope('Input_Batch'):
if args.format == "mseed_array":
batch_size = 1
else:
batch_size = args.batch_size
dataset = data_reader.dataset(batch_size)
batch = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next()
config = ModelConfig(X_shape=data_reader.X_shape)
with open(os.path.join(log_dir, 'config.log'), 'w') as fp:
fp.write('\n'.join("%s: %s" % item for item in vars(config).items()))
model = UNet(config=config, input_batch=batch, mode="pred")
# model = UNet(config=config, mode="pred")
sess_config = tf.compat.v1.ConfigProto()
sess_config.gpu_options.allow_growth = True
# sess_config.log_device_placement = False
with tf.compat.v1.Session(config=sess_config) as sess:
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables(),
max_to_keep=5)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
latest_check_point = tf.train.latest_checkpoint(args.model_dir)
logging.info(f"restoring model {latest_check_point}")
saver.restore(sess, latest_check_point)
picks = []
amps = [] if args.amplitude else None
if args.plot_figure:
multiprocessing.set_start_method('spawn')
pool = multiprocessing.Pool(multiprocessing.cpu_count())
for _ in tqdm(range(0, data_reader.num_data, batch_size), desc="Pred"):
if args.amplitude:
pred_batch, X_batch, amp_batch, fname_batch, t0_batch, station_batch = sess.run(
[
model.preds, batch[0], batch[1], batch[2], batch[3],
batch[4]
],
feed_dict={
model.drop_rate: 0,
model.is_training: False
},
)
# X_batch, amp_batch, fname_batch, t0_batch = sess.run([batch[0], batch[1], batch[2], batch[3]])
else:
pred_batch, X_batch, fname_batch, t0_batch, station_batch = sess.run(
[model.preds, batch[0], batch[1], batch[2], batch[3]],
feed_dict={
model.drop_rate: 0,
model.is_training: False
},
)
# X_batch, fname_batch, t0_batch = sess.run([model.preds, batch[0], batch[1], batch[2]])
# pred_batch = []
# for i in range(0, len(X_batch), 1):
# pred_batch.append(sess.run(model.preds, feed_dict={model.X: X_batch[i:i+1], model.drop_rate: 0, model.is_training: False}))
# pred_batch = np.vstack(pred_batch)
picks_ = extract_picks(preds=pred_batch,
fnames=fname_batch,
station_ids=station_batch,
t0=t0_batch,
config=args)
picks.extend(picks_)
if args.amplitude:
amps_ = extract_amplitude(amp_batch, picks_)
amps.extend(amps_)
if args.plot_figure:
pool.starmap(
partial(
plot_waveform,
figure_dir=figure_dir,
),
zip(X_batch, pred_batch,
[x.decode() for x in fname_batch]),
)
if args.save_prob:
# save_prob(pred_batch, fname_batch, prob_dir=prob_dir)
save_prob_h5(pred_batch, [x.decode() for x in fname_batch],
prob_h5)
save_picks(picks,
args.result_dir,
amps=amps,
fname=args.result_fname + ".csv")
save_picks_json(picks,
args.result_dir,
dt=data_reader.dt,
amps=amps,
fname=args.result_fname + ".json")
print(
f"Done with {sum([len(x) for pick in picks for x in pick.p_idx])} P-picks and {sum([len(x) for pick in picks for x in pick.s_idx])} S-picks"
)
return 0
def test_fn(args, data_reader):
current_time = time.strftime("%y%m%d-%H%M%S")
logging.info("{} log: {}".format(args.mode, current_time))
if args.model_dir is None:
logging.error(f"model_dir = None!")
return -1
if not os.path.exists(args.result_dir):
os.makedirs(args.result_dir)
figure_dir = os.path.join(args.result_dir, "figures")
if not os.path.exists(figure_dir):
os.makedirs(figure_dir)
config = ModelConfig(X_shape=data_reader.X_shape,
Y_shape=data_reader.Y_shape)
config.update_args(args)
with open(os.path.join(args.result_dir, 'config.log'), 'w') as fp:
fp.write('\n'.join("%s: %s" % item for item in vars(config).items()))
with tf.compat.v1.name_scope('Input_Batch'):
dataset = data_reader.dataset(args.batch_size, shuffle=False)
batch = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next()
model = UNet(config, input_batch=batch, mode='test')
sess_config = tf.compat.v1.ConfigProto()
sess_config.gpu_options.allow_growth = True
# sess_config.log_device_placement = False
with tf.compat.v1.Session(config=sess_config) as sess:
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables())
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
logging.info("restoring models...")
latest_check_point = tf.train.latest_checkpoint(args.model_dir)
if latest_check_point is None:
logging.error(f"No models found in model_dir: {args.model_dir}")
return -1
saver.restore(sess, latest_check_point)
flog = open(os.path.join(args.result_dir, 'loss.log'), 'w')
test_loss = LMA()
progressbar = tqdm(range(0, data_reader.num_data, args.batch_size),
desc=args.mode)
picks = []
true_picks = []
for _ in progressbar:
loss_batch, preds_batch, X_batch, Y_batch, fname_batch, itp_batch, its_batch \
= sess.run([model.loss, model.preds, batch[0], batch[1], batch[2], batch[3], batch[4]],
feed_dict={model.drop_rate: 0, model.is_training: False})
test_loss(loss_batch)
progressbar.set_description(
"{}, loss={:.6f}, mean loss={:6f}".format(
args.mode, loss_batch, test_loss.value))
picks_ = extract_picks(preds_batch, fname_batch)
picks.extend(picks_)
true_picks.extend(
convert_true_picks(fname_batch, itp_batch, its_batch))
if args.plot_figure:
plot_waveform(data_reader.config,
X_batch,
preds_batch,
label=Y_batch,
fname=fname_batch,
itp=itp_batch,
its=its_batch,
figure_dir=figure_dir)
save_picks(picks, args.result_dir)
metrics = calc_performance(picks,
true_picks,
tol=3.0,
dt=data_reader.config.dt)
flog.write("mean loss: {}\n".format(test_loss))
flog.close()
return 0
dataConfig = DataConfig(
lm=args.lmtrain,
rf=args.res,
m2id=args.m2id,
maxlen=args.maxlen,
buffer= args.buffer,
)
if args.ds == "full":
dstrain = ChessDataInMemory(dataConfig)
else:
dstrain = ChessData(dataConfig)
modelConfig = ModelConfig(
vocab_size = len(dstrain.m2id),
n_positions=args.maxlen,
n_ctx=args.maxlen,
n_embd=args.n_embd,
n_layer=args.n_layer,
n_head=args.n_head
)
model = BaseHFGPT(modelConfig)
print(f"Model Size: {sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values())}")
trainerConf = TrainerConfig(
max_epochs = args.num_epochs,
batch_size = args.batch_size,
lr = args.lr,
betas = (args.beta1, args.beta2),
tb_path = model_folder,
save_every = args.save_every,
ckpt_path = model_path
)
def main(unused_argv):
# initialize
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.cuda_device
# config
config = ModelConfig()
# config > set the probability that a modality is invalidated
config.train_dropout_a_modality_prob = FLAGS.dropout_modality_prob
# save arguments
arguments_path = os.path.join(FLAGS.train_dir, 'arguments.json')
with open(arguments_path, 'w') as f:
f.write(json.dumps(FLAGS.flag_values_dict(), sort_keys=True, indent=2))
# data
data_loader = DataLoader(is_training=True, base_src=FLAGS.dataset_dir)
data_loader.load_data()
# graph
with tf.Graph().as_default():
model = Model(is_training=True, config=config)
model.build()
# training & session creation
global_step = tf.train.get_or_create_global_step()
logit_list = model.graph.output
label_list = tf.placeholder(tf.int64)
loss_list = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logit_list, labels=label_list)
total_loss = tf.reduce_mean(loss_list)
tf.summary.scalar('total_loss', total_loss)
opt = tf.train.AdamOptimizer(1e-3, epsilon=1e-2)
grads = opt.compute_gradients(total_loss)
train_op = opt.apply_gradients(grads, global_step=global_step)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.save_freq)
summary_op = tf.summary.merge_all()
init = tf.global_variables_initializer()
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True))
sess.run(init)
tf.train.start_queue_runners(sess=sess)
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
# print num trainable parameters
print('trainable parameters')
total_variable_parameters = 0
for variable in tf.trainable_variables():
shape = variable.get_shape()
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
print(' - %s: %d' % (variable.name, variable_parameters))
total_variable_parameters += variable_parameters
print('total: %d' % (total_variable_parameters))
# do iterations
for step in range(0, FLAGS.max_steps + 1):
start_time = time.time()
# data and label
r_left_data_list = []
r_right_data_list = []
r_label_list = []
for _ in range(FLAGS.batch_size):
original_data = data_loader.get_random_data()
r_left_data_list.append(original_data['left'])
r_right_data_list.append(original_data['right'])
r_label_list.append(original_data['label'])
# feed dict
feed_dict = {}
feed_dict[model.feeds.batch_size] = FLAGS.batch_size
feed_dict[model.feeds.input_left] = r_left_data_list
feed_dict[model.feeds.input_right] = r_right_data_list
feed_dict[label_list] = r_label_list
# run
if (step > 0) and (step % FLAGS.summary_freq == 0):
_, r_class_list, r_total_loss, summary_str = sess.run(
[train_op, model.graph.output_classes, total_loss, summary_op],
feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
else:
_, r_class_list, r_total_loss = sess.run(
[train_op, model.graph.output_classes, total_loss],
feed_dict=feed_dict)
duration = time.time() - start_time
if (FLAGS.sleep_ratio > 0 and duration > 0):
time.sleep(min(1.0, duration * FLAGS.sleep_ratio))
assert not np.isnan(r_total_loss), 'Model diverged with loss = NaN'
if (step % FLAGS.log_freq == 0):
sec_per_batch = float(duration)
accuracy = np.sum(
np.array(r_class_list).flatten() == np.array(r_label_list).
flatten()) / len(np.array(r_class_list).flatten())
print('step %d, accuracy = %.6f, loss = %.6f (%.3f sec/batch)' %
(step, accuracy, r_total_loss, sec_per_batch))
if (step > 0) and (step % FLAGS.save_freq == 0):
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
def train_fn(args, data_reader, data_reader_valid=None):
current_time = time.strftime("%y%m%d-%H%M%S")
log_dir = os.path.join(args.log_dir, current_time)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
logging.info("Training log: {}".format(log_dir))
model_dir = os.path.join(log_dir, 'models')
os.makedirs(model_dir)
figure_dir = os.path.join(log_dir, 'figures')
if not os.path.exists(figure_dir):
os.makedirs(figure_dir)
config = ModelConfig(X_shape=data_reader.X_shape,
Y_shape=data_reader.Y_shape)
if args.decay_step == -1:
args.decay_step = data_reader.num_data // args.batch_size
config.update_args(args)
with open(os.path.join(log_dir, 'config.log'), 'w') as fp:
fp.write('\n'.join("%s: %s" % item for item in vars(config).items()))
with tf.compat.v1.name_scope('Input_Batch'):
dataset = data_reader.dataset(args.batch_size, shuffle=True).repeat()
batch = tf.compat.v1.data.make_one_shot_iterator(dataset).get_next()
if data_reader_valid is not None:
dataset_valid = data_reader_valid.dataset(args.batch_size,
shuffle=False).repeat()
valid_batch = tf.compat.v1.data.make_one_shot_iterator(
dataset_valid).get_next()
model = UNet(config, input_batch=batch)
sess_config = tf.compat.v1.ConfigProto()
sess_config.gpu_options.allow_growth = True
# sess_config.log_device_placement = False
with tf.compat.v1.Session(config=sess_config) as sess:
summary_writer = tf.compat.v1.summary.FileWriter(log_dir, sess.graph)
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables(),
max_to_keep=5)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
if args.model_dir is not None:
logging.info("restoring models...")
latest_check_point = tf.train.latest_checkpoint(args.model_dir)
saver.restore(sess, latest_check_point)
if args.plot_figure:
multiprocessing.set_start_method('spawn')
pool = multiprocessing.Pool(multiprocessing.cpu_count())
flog = open(os.path.join(log_dir, 'loss.log'), 'w')
train_loss = EMA(0.9)
best_valid_loss = np.inf
for epoch in range(args.epochs):
progressbar = tqdm(range(0, data_reader.num_data, args.batch_size),
desc="{}: epoch {}".format(
log_dir.split("/")[-1], epoch))
for _ in progressbar:
loss_batch, _, _ = sess.run(
[model.loss, model.train_op, model.global_step],
feed_dict={
model.drop_rate: args.drop_rate,
model.is_training: True
})
train_loss(loss_batch)
progressbar.set_description(
"{}: epoch {}, loss={:.6f}, mean={:.6f}".format(
log_dir.split("/")[-1], epoch, loss_batch,
train_loss.value))
flog.write("epoch: {}, mean loss: {}\n".format(
epoch, train_loss.value))
if data_reader_valid is not None:
valid_loss = LMA()
progressbar = tqdm(range(0, data_reader_valid.num_data,
args.batch_size),
desc="Valid:")
for _ in progressbar:
loss_batch, preds_batch, X_batch, Y_batch, fname_batch = sess.run(
[
model.loss, model.preds, valid_batch[0],
valid_batch[1], valid_batch[2]
],
feed_dict={
model.drop_rate: 0,
model.is_training: False
})
valid_loss(loss_batch)
progressbar.set_description(
"valid, loss={:.6f}, mean={:.6f}".format(
loss_batch, valid_loss.value))
if valid_loss.value < best_valid_loss:
best_valid_loss = valid_loss.value
saver.save(
sess,
os.path.join(model_dir, "model_{}.ckpt".format(epoch)))
flog.write("Valid: mean loss: {}\n".format(valid_loss.value))
else:
loss_batch, preds_batch, X_batch, Y_batch, fname_batch = sess.run(
[model.loss, model.preds, batch[0], batch[1], batch[2]],
feed_dict={
model.drop_rate: 0,
model.is_training: False
})
saver.save(
sess, os.path.join(model_dir,
"model_{}.ckpt".format(epoch)))
if args.plot_figure:
pool.starmap(
partial(
plot_waveform,
figure_dir=figure_dir,
),
zip(X_batch, preds_batch,
[x.decode() for x in fname_batch], Y_batch),
)
# plot_waveform(X_batch, preds_batch, fname_batch, label=Y_batch, figure_dir=figure_dir)
flog.flush()
flog.close()
return 0
from game import Player, RandomPlayer, GameEngine, Move
from model import BaseHFGPT, ModelConfig, BetaChess
# make the app and run the server
app = Flask(__name__)
game = GameEngine()
# player = RandomPlayer()
# define the NeuraPlayer
with open("assets/moves.json") as f:
vocab_size = len(json.load(f))
config = ModelConfig(vocab_size=vocab_size,
n_positions=170,
n_ctx=170,
n_embd=128,
n_layer=8,
n_head=8,
loss_method="mse")
player = Player(config,
"models/useful/splendid-donkey-197/cgpt_38401.pt",
"assets/moves.json",
search="sample",
depth=2,
model_class=BetaChess)
@app.route('/move')
def make_move():
# let the player
move_dict = request.args
def main(args):
config_args = {
arg: val
for arg, val in vars(args).items() if val is not None
}
configs = ModelConfig(**config_args)
is_training = tf.placeholder(tf.bool)
x_train, x_test, y_train, y_test = load_data(args.datadir)
in_tensors = tf.placeholder(tf.float32, shape=(None, 64, 64, 3))
in_classes = tf.placeholder(tf.float32, shape=(None, args.num_classes))
model = build_model(in_tensors, configs, is_training)
with tf.name_scope('predictions'):
out_y_pred = tf.nn.softmax(model)
with tf.name_scope('loss_score'):
loss_score = tf.nn.softmax_cross_entropy_with_logits(logits=model,
labels=in_classes)
with tf.name_scope('loss'):
loss = tf.reduce_mean(loss_score)
tf.summary.scalar('loss', loss)
with tf.name_scope('train'):
optimizer = tf.train.AdamOptimizer(args.learn_rate).minimize(loss)
output_dir = args.outputdir
tb_log_dir = os.path.join(output_dir, 'tensorboard_logs')
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if not os.path.exists(tb_log_dir):
os.mkdir(tb_log_dir)
merged = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(tb_log_dir)
with tf.Session() as session:
summary_writer.add_graph(session.graph)
session.run(tf.global_variables_initializer())
with Timer("Training"):
for epoch in range(args.epochs):
with Timer("Epoch #{}".format(epoch)):
print("Epoch=", epoch)
tf_score = []
for mb in minibatcher(x_train,
y_train,
args.batch_size,
shuffle=True):
tf_output = session.run(
[optimizer, loss, merged],
feed_dict={
in_tensors: mb[0],
in_classes: mb[1],
is_training: True
})
summary_writer.add_summary(tf_output[2],
global_step=epoch)
tf_score.append(tf_output[1])
print(" train_loss_score=", np.mean(tf_score))
# after the training is done, time to test it on the test set
print("TEST SET PERFORMANCE")
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
y_test_pred, test_loss = session.run([out_y_pred, loss],
feed_dict={
in_tensors: x_test,
in_classes: y_test,
is_training: False
},
options=run_options,
run_metadata=run_metadata)
summary_writer.add_run_metadata(run_metadata=run_metadata,
tag='predict')
print(" test_loss_score=", test_loss)
y_test_pred_classified = np.argmax(y_test_pred,
axis=1).astype(np.int32)
y_test_true_classified = np.argmax(y_test, axis=1).astype(np.int32)
print(
classification_report(y_test_true_classified,
y_test_pred_classified))
cm = confusion_matrix(y_test_true_classified, y_test_pred_classified)
plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)
plt.colorbar()
plt.tight_layout()
plt.savefig(os.path.join(output_dir, 'confusion.png'))
# And the log2 version, to emphasize the misclassifications
plt.imshow(np.log2(cm + 1),
interpolation='nearest',
cmap=plt.get_cmap("tab20"))
plt.colorbar()
plt.tight_layout()
plt.savefig(os.path.join(output_dir, 'confusion_log.png'))
tokenizer = BertTokenizer.from_pretrained(bert_model_name)
# use pretrained wiki_vector model
wiki_model_file = os.path.join(BASE_DIR, 'wiki_vector.model')
wiki2vec = Wikipedia2Vec.load(wiki_model_file)
en_embd_dim = 100 # entity embedding dim
en_pad_size = 12 # max entity number of one data
device = torch.device("cpu") # use CPU or GPU
app.logger.info('use device: {}'.format(device))
processor = DataProcess()
entity_vector = processor.load_entity_vector(
entity_vector_root) # get pretrained entity_vector
mconf = ModelConfig(bert_model_name,
entity_vector,
en_embd_dim,
en_hidden_size1=128,
en_hidden_size2=128,
use_en_encoder=True)
bert = Model(mconf).to(device)
bert.load_state_dict(
torch.load(bert_model_file,
map_location=torch.device('cpu'))) #load bert model
xgboost_model = joblib.load(xgboost_model_file) #load xgboost model
test_batch = 32
idf_dict, unk_idf = processor.load_idf(idf_file)
entity_score_dict = processor.load_entity_score_dict(entity_frep_file)
keyword_entropy_dict = processor.load_keyword_entropy_dict(
keyword_entropy_file)
domain_score_dict = processor.load_domain_score_dict(domain_frep_file)
def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
cond_logging('args = %s', args)
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE'])
args.device = 'cdua:0'
args.world_size = 1
args.rank = 0
if args.distributed:
args.num_gpu = 1
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
args.world_size = int(os.environ['WORLD_SIZE'])
args.rank = int(os.environ['RANK'])
torch.distributed.init_process_group(backend='nccl',
init_method='env://',
rank=args.rank,
world_size=args.world_size)
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
assert args.rank >= 0
cond_logging("new args = %s", args)
train_queue, valid_queue, test_queue = create_dataset_loader(
args.dataset, args.dataset_dir, args.train_portion, args.batch_size,
args.workers, args.distributed)
model_config = ModelConfig(arch=args.arch,
depth=args.depth,
width_multiplier=args.width_multiplier,
init_channels=args.init_channels,
channel_padding=args.channel_padding,
classes=args.classes)
model_config.set_fur_criterion(get_criterion(args.classes, 0.))
if args.init_channel_config:
init_channel_numbers = eval('init_channel_config.%s' %
args.init_channel_config)
model_config.set_channel_numbers(init_channel_numbers)
for i in range(args.iters):
cond_logging('******* search iter:{} *********'.format(i))
model = model_config.build_model()
if args.distributed:
if args.distributed:
model = DDP(model, device_ids=[args.local_rank])
cond_logging("before adjustment:")
model_config.logging_config(args.local_rank)
model = train_new_model(model, train_queue, valid_queue, test_queue)
if args.eval:
break
model_config.computing_fur(valid_queue, args.base_drop_rate,
args.times, args.world_size,
args.distributed, args.local_rank)
update_num = max(round(args.update_num - i * args.update_num_decay), 0)
model_config.update_chanel_with_fur(update_num,
args.arch_learning_rate,
args.arch_learning_rate_decay,
args.local_rank)
model_config.scale_to_ori_flops()
cond_logging("After adjustment:")
model_config.list_channel_config(args.local_rank)
del model
torch.cuda.synchronize()
def main(unused_argv):
# initialize
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.cuda_device
# config
config = ModelConfig()
config.test_modality_availabilities = [(not FLAGS.dropout_left),
(not FLAGS.dropout_right)]
# data
data_loader = DataLoader(is_training=False, base_src=FLAGS.dataset_dir)
data_loader.load_data()
test_data_list = data_loader.get_all_data()
# graph
with tf.Graph().as_default():
model = Model(is_training=False, config=config)
model.build()
# test & session creation
saver = tf.train.Saver()
init = tf.global_variables_initializer()
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True))
sess.run(init)
saver.restore(sess, FLAGS.restore_path)
tf.train.start_queue_runners(sess=sess)
# run session for each data
total_data_count = 0
correct_data_count = 0
for (data_index, original_data) in enumerate(test_data_list):
if (data_index % 1000 == 0):
print('- %d / %d' % (data_index, len(test_data_list)))
r_left_data_list = []
r_right_data_list = []
r_label_list = []
r_left_data_list.append(original_data['left'])
r_right_data_list.append(original_data['right'])
r_label_list.append(original_data['label'])
feed_dict = {}
feed_dict[model.feeds.batch_size] = 1
feed_dict[model.feeds.input_left] = r_left_data_list
feed_dict[model.feeds.input_right] = r_right_data_list
_, r_output_classes = sess.run(
[model.graph.output_probs, model.graph.output_classes],
feed_dict=feed_dict)
r_class_list = r_output_classes.flatten()
r_label_list = np.array(r_label_list).flatten()
total_data_count += len(r_class_list)
correct_data_count += np.sum((r_class_list == r_label_list))
accuracy = correct_data_count * 1.0 / total_data_count
# print results
print('checkpoint path: %s' % (FLAGS.restore_path))
print('accuracy: %f' % (accuracy))
print('error rate: %f' % (1.0 - accuracy))
law_path = 'data/criminal_law.txt'
vec_path = 'data/words.vec'
emb_path = 'data/few_shot_emb.npy'
w2id_path = 'data/w2id.pkl'
# vec_path='E:\\iCloudDrive\\Projects\\GitHub\\attribute_charge\\attribute_charge\\data\\words.vec'
# emb_path='E:\\iCloudDrive\\Projects\\GitHub\\attribute_charge\\attribute_charge\\few_shot_emb.npy'
# w2id_path='E:\\iCloudDrive\\Projects\\GitHub\\attribute_charge\\attribute_charge\\w2id.pkl'
data_path = 'data/legal_data.pkl'
checkpoint_dir = 'model_save'
restore_path = None
save_path = 'model_save/legal_basis-time-{}-{}-{}'.format(
time.localtime(time.time()).tm_mon,
time.localtime(time.time()).tm_mday,
time.localtime(time.time()).tm_hour)
# NO SPACE in path !!!!!!!!!!!!!!!!!!!
model_config = ModelConfig()
lr = model_config.learning_rate
if restore and restore_path is None:
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt:
restore_path = ckpt.model_checkpoint_path
print('restore_path:', restore_path)
with open('data/accu.txt', 'r', encoding='utf-8') as f:
accu_class = [i for i in f.read().split('\n')[:-1]]
with open('data/law.txt', 'r') as f:
law_class = [int(i) for i in f.read().split('\n')[:-1]]
file_order = {law_class[i]: i + 1 for i in range(len(law_class))}
law_class = sorted(law_class)
return None
move = random.choice(legal_moves)
return move
def __repr__(self):
return f"<Random Player {self.id}>"
# test script
if __name__ == "__main__":
with open("assets/moves.json") as f:
vocab_size = len(json.load(f))
config = ModelConfig(vocab_size=vocab_size,
n_positions=60,
n_ctx=60,
n_embd=128,
n_layer=30,
n_head=8)
game = GameEngine()
pgn_writer = chess.pgn.Game()
pgn_writer_node = pgn_writer
pgn_writer.headers["Event"] = "Test"
player1 = Player(config, ".model_sample/z4_0.pt",
"assets/moves.json") # assume to be white
player2 = Player(config, ".model_sample/z4_0.pt",
"assets/moves.json") # assume to be black
# play
mv = 0