def main():
train_graph = tf.Graph()
config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
# please do not use the totality of the GPU memory
config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
config.gpu_options.allocator_type = 'BFC'
with tf.Session(graph=train_graph, config=config) as sess:
args = load_args()
tf.set_random_seed(args.seed)
np.random.seed(seed=args.seed)
train_docs_all, train_queries_all, train_summaries_all, doc_count, query_count, summary_count = load_train_dataid_pickle()
val_docs_all, val_queries_all, val_summaries_all, doc_count, query_count, summary_count = load_val_dataid_pickle()
if(args.toy):
train_docs_all, train_queries_all, train_summaries_all = train_docs_all[:args.toy], train_queries_all[:args.toy], train_summaries_all[:args.toy]
val_docs_all, val_queries_all, val_summaries_all = val_docs_all[:args.toy], val_queries_all[:args.toy], val_summaries_all[:args.toy]
print ("LENGTH OF TRAIN DOC ALL: ", len(train_docs_all))
train_set = create_bucket(list(zip(train_docs_all, train_queries_all, train_summaries_all)))
val_set = create_bucket(list(zip(val_docs_all, val_queries_all, val_summaries_all)))
batched_train_set = batchify(train_set, _buckets, args.train_batch_size, args.max_word_size)
batched_val_set = batchify(val_set, _buckets, args.train_batch_size, args.max_word_size)
data_all = batched_train_set, batched_val_set
model = create_model(sess, args.train_batch_size, args.train_load_checkpoint, args, True)
train(args, sess, data_all, model)
def main():
args, labels = load_args()
interpreter = Interpreter(args.model)
interpreter.allocate_tensors()
_, height, width, _ = interpreter.get_input_details()[0]['shape']
#with picamera.PiCamera(resolution=(640, 480), framerate=30) as camera:
#camera.start_preview()
cap = set_cap()
times=1
old_labels = ""
frame_count = 0
diff_frame_count = 0
while True:
image = read_cap(cap)
start_time = time.time()
results = classify_image(interpreter, image)
elapsed_ms = (time.time() - start_time) * 1000
label_id, prob = results[0]
if frame_count == 0:
old_labels = labels[label_id]
if old_labels != labels[label_id]:
diff_frame_count = diff_frame_count+1
else:
diff_frame_count = 0
if diff_frame_count>=THRESHOLD:
line_bot("this is "+ labels[label_id])
old_labels = labels[label_id]
diff_frame_count = 0
frame_count = frame_count + 1
print("diff_frame_count:"+str(diff_frame_count) +" tensor id:" + labels[label_id] + " and old id: " + str(old_labels))
close_cap()
def run_inference_decoding():
print ("In Decode")
# Load vocabularies.
args = load_args()
doc_dict = load_dict(args.doc_dict_path)
sum_dict = load_dict(args.sum_dict_path)
if doc_dict is None or sum_dict is None:
logging.warning("Dict not found.")
val_docid, val_queryid, val_sumid = load_test_dataid_pickle()
dev_set = create_bucket(val_docid, val_queryid, val_sumid)
batched_test_data = batchify (dev_set, _buckets, args.test_batch_size)
tf.reset_default_graph()
with tf.Session() as sess:
# Create model and load parameters.
print("Creating %d layers of %d units." %
(args.num_layers, args.size))
model = create_model(sess, doc_dict, sum_dict, args.test_batch_size, args.test_load_checkpoint, args.test_forward_only, args)
result = []
idx = 0
for batch_test in batched_test_data:
idx += 1
encoder_doc_inputs, encoder_query_inputs, decoder_inputs, encoder_doc_len, encoder_query_len, decoder_len = batch_test
#encoder_doc_inputs = list(map(lambda x: x if x!=0, encoder_doc_inputs))
print ("OUTSIDE STEP: ", encoder_query_len)
outputs = model.step_beam(sess, encoder_doc_inputs, encoder_query_inputs, encoder_doc_len, encoder_query_len, args.beam_width)
outputs = np.array(outputs).flatten()
#print ("OUTPUT FROM BEAM SEARCH DECODER", outputs)
# If there is an EOS symbol in outputs, cut them at that point.
if ID_EOS in outputs:
outputs = outputs[:outputs.index(ID_EOS)]
gen_sum = " ".join(sen_map2tok(outputs, sum_dict[1]))
result.append(gen_sum)
print("Finish {} samples. :: {}".format(idx, gen_sum))
with open(args.test_output, "w") as f:
for item in result:
print(item, file=f)
from video_entities import video_entities
from args import Args, load_args
from log import configure_log
from cfg import load_cfg
import asyncio
async def run(args: Args):
'''loads video named entities from a list of video id's in a jsonl.gz file'''
cfg = await load_cfg()
log = configure_log(cfg)
if (cfg.state.videoPaths is None and args.videos is None):
raise Exception('Need either videoPaths or videos')
log.info('video_entities - {machine} started: {state}',
machine=cfg.machine,
state=cfg.state.to_json())
try:
video_entities(cfg, args, log)
except Exception as e:
log.error("error running video_entities", exc_info=True)
if __name__ == "__main__":
asyncio.run(run(load_args()))
def run_training():
print("In Train")
try:
os.makedirs(args.train_dir)
except:
pass
print("Preparing summarization data.")
args = load_args()
if (args.reload_data or args.reload_all):
docid, queryid, sumid, doc_dict, sum_dict = load_data(
args, args.full_data_dir)
train_docid, val_docid, train_queryid, val_queryid, train_sumid, val_sumid = train_test_split(
docid,
queryid,
sumid,
test_size=args.train_test_split,
shuffle=False,
random_state=42)
store_train_dataid_pickle(train_docid, train_queryid, train_sumid)
store_test_dataid_pickle(val_docid, val_queryid, val_sumid)
else:
train_docid, train_queryid, train_sumid = load_train_dataid_pickle()
val_docid, val_queryid, val_sumid = load_test_dataid_pickle()
doc_dict = load_dict(args.doc_dict_path, args.doc_vocab_size)
sum_dict = load_dict(args.sum_dict_path, args.sum_vocab_size)
tf.reset_default_graph()
config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)
# please do not use the totality of the GPU memory
config.gpu_options.per_process_gpu_memory_fraction = 0.90
config.gpu_options.allow_growth = True
config.gpu_options.allocator_type = 'BFC'
with tf.Graph().as_default(), tf.Session(config=config) as sess:
# tensorflow seed must be inside graph
tf.set_random_seed(args.seed)
np.random.seed(seed=args.seed)
# Create model.
print("Creating %d layers of %d units." % (args.num_layers, args.size))
train_writer = tf.summary.FileWriter(args.tfboard + '/train',
sess.graph)
model = create_model(sess, doc_dict, sum_dict, args.train_batch_size,
args.train_load_checkpoint,
args.train_forward_only, args)
# Read data into buckets and compute their sizes.
print("Create buckets.")
dev_set = create_bucket(val_docid, val_queryid, val_sumid)
train_set = create_bucket(train_docid, train_queryid, train_sumid)
train_bucket_sizes = [len(train_set[b]) for b in range(len(_buckets))]
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in range(len(train_bucket_sizes))
]
for (s_size, q_size, t_size), nsample in zip(_buckets,
train_bucket_sizes):
print("Train set bucket ({}, {}, {}) has {} samples.".format(
s_size, q_size, t_size, nsample))
batched_train_set = batchify(train_set, _buckets,
args.train_batch_size)
batched_dev_set = batchify(dev_set, _buckets, args.train_batch_size)
# This is the training loop.
step_time, train_acc, train_loss = 0.0, 0.0, 0.0
step_start_time = 0
num_epoch = 0
step_time = 0
while num_epoch <= args.max_epochs:
epoch_train_loss = 0.0
epoch_train_acc = 0.0
current_train_step = 0
epoch_start_time = time.time()
for batch_train in batched_train_set:
step_start_time = time.time()
encoder_doc_inputs, encoder_query_inputs, decoder_inputs, encoder_doc_len, encoder_query_len, decoder_len = batch_train
print("ENCODER QUERY LEN: ", encoder_query_len)
step_train_loss, step_train_acc, _ = model.step(
sess, encoder_doc_inputs, encoder_query_inputs,
decoder_inputs, encoder_doc_len, encoder_query_len,
decoder_len, False, train_writer)
step_time = time.time() - step_start_time
#print ("CURRENT STEP: ", current_train_step, " STEP TIME: ", step_time)
step_train_loss = (step_train_loss *
args.train_batch_size) / np.sum(decoder_len)
epoch_train_loss += step_train_loss
epoch_train_acc += step_train_acc
current_train_step += 1
# Once in a while, we save checkpoint.
if current_train_step % args.steps_per_checkpoint == 0:
# Save checkpoint and zero timer and loss.
save_time_start = time.time()
checkpoint_path = os.path.join(args.train_dir,
"model.ckpt")
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
time_taken_to_save = time.time() - save_time_start
print("Time taken to save checkpoint: ",
time_taken_to_save)
# Once in a while, we print statistics and run evals.
if current_train_step % args.steps_per_print == 0:
# Print statistics for the previous epoch.
print(
"Epoch: %d, GlobalStep: %d, step-time %.2f, Acc: %.4f, Loss: %.4f, Perpxty: %.2f"
% (num_epoch, model.global_step.eval(), step_time,
step_train_acc, step_train_loss,
np.exp(float(step_train_loss))))
step_time, train_acc, train_loss = 0.0, 0.0, 0.0
#epoch_train_loss, epoch_train_acc, current_train_step = 1., 2., 15
epoch_eval_loss, epoch_eval_acc = 0.0, 0.0
current_eval_step = 0
for batch_dev in batched_dev_set:
encoder_doc_inputs, encoder_query_inputs, decoder_inputs, encoder_doc_len, encoder_query_len, decoder_len = batch_dev
step_eval_loss, step_eval_acc, _ = model.step(
sess, encoder_doc_inputs, encoder_query_inputs,
decoder_inputs, encoder_doc_len, encoder_query_len,
decoder_len, True)
step_eval_loss = (step_eval_loss *
args.train_batch_size) / np.sum(decoder_len)
epoch_eval_loss += step_eval_loss
epoch_eval_acc += step_eval_acc
current_eval_step += 1
print("at the end of epoch:", num_epoch)
print("Average train loss = %6.8f, Average perplexity = %6.8f" %
(epoch_train_loss / current_train_step,
np.exp(epoch_train_loss / current_train_step)))
print("Average train acc = %6.8f" %
(epoch_train_acc / current_train_step))
print("validation loss = %6.8f, perplexity = %6.8f" %
(epoch_eval_loss / current_eval_step,
np.exp(epoch_eval_loss / current_eval_step)))
print("Average Validation acc = %6.8f" %
(epoch_eval_acc / current_eval_step))
# Save checkpoint and zero timer and loss.
save_time_start = time.time()
checkpoint_path = os.path.join(args.train_dir, "model.ckpt")
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
time_taken_to_save = time.time() - save_time_start
print("Time taken to save checkpoint: ", time_taken_to_save)
num_epoch += 1
sys.stdout.flush()
else:
data.clean(folder, args.dataset_type, out_path)
print("\nNothing else to do! Closing")
if __name__ == "__main__":
"""
Run main with argument
"""
parser = argparse.ArgumentParser(
description="Run PARIS entity matching and compute metrics"
)
parser.add_argument(
"--json_args"
)
parser.add_argument(
"--dataset_ea",
default = ""
)
parser.add_argument(
"--fold_number",
default = None
)
args_main = parser.parse_args()
args = load_args(args_main.json_args)
args.dataset_ea = args_main.dataset_ea
args.fold_number = args_main.fold_number
main(args)
docid = np.array(docid)
queryid = np.array(queryid)
queryid = np.array(queryid)
num_batches_per_epoch = (len(docid) - 1) // batch_size + 1
for epoch in range(num_epochs):
for batch_num in range(num_batches_per_epoch):
start_index = batch_num * batch_size
end_index = min((batch_num + 1) * batch_size, len(docid))
yield docid[start_index:end_index], queryid[
start_index:end_index], sumid[start_index:end_index]
if __name__ == '__main__':
args = load_args()
dicts = load_dictionaries()
train_docs_all, train_queries_all, train_summaries_all, doc_count, query_count, summary_count = get_train_val_data(
args, 'train', args.train_data_dir, dicts, 0, 0, 0)
print("train: ", train_docs_all[0][2][0])
print("COUNT: ", doc_count)
print("LENGTH OF TRAIN: ", len(train_docs_all))
val_docs_all, val_queries_all, val_summaries_all, doc_count, query_count, summary_count = get_train_val_data(
args, 'val', args.val_data_dir, dicts, doc_count, query_count,
summary_count)
print("train: ", val_docs_all[0][2][0])
print("COUNT: ", doc_count)
print("LENGTH OF TRAIN: ", len(val_docs_all))
test_docs_all, test_queries_all, test_summaries_all, doc_count, query_count = get_test_data(
'test', args.test_data_dir, dicts, doc_count, query_count)
print("train: ", test_docs_all[0][2][0])
actions = torch.cat(actions, dim=1)
logps = torch.stack(logps, dim=1)
rewards = np.transpose(np.asarray(rewards))
p_loss, v_loss, e_loss = cost_func(args, values, logps, actions,
rewards)
loss = p_loss + 0.5 * v_loss + 0.01 * e_loss
eploss += loss.item()
optim = H.batch_zero_optim_hn(optim)
optim = H.update_hn(args, loss, optim)
values, logps, actions, rewards = [], [], [], []
if __name__ == "__main__":
args = args.load_args()
args.save_dir = '{}/'.format(args.env.lower())
if args.render:
args.processes = 1
args.test = True
if args.test:
args.lr = 0
if args.scratch:
print('training on server; saving to /scratch/eecs-share')
args.n_actions = gym.make(args.env).action_space.n
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
# print ('=> Multienvironment settings')
envs1 = MultiEnvironment(args.env1, args.batch_size, args.frame_skip)
envs2 = MultiEnvironment(args.env2, args.batch_size, args.frame_skip)
torch.manual_seed(args.seed)