def main():
# Load config
load_config()
for monitor_info in settings.MONITOR_INFOS:
try:
# Get topic list
topic_list = get_topic_list(monitor_info['board_url'])
# Get new topic list
new_topic_list = get_new_topic_list(topic_list, monitor_info['board_url'])
# Get spam topics
topic_to_delete = []
for topic in new_topic_list:
if is_title_match(topic['title']) or is_topic_content_match(topic['url']):
topic_to_delete.append(topic)
log('Spam <%s> detected.' % topic['title'])
# Delete spam topics
if len(topic_to_delete) > 0:
admin = Admin.login(
monitor_info['login_username'],
monitor_info['login_password'],
monitor_info['board_url']
)
if admin is not None:
for topic in topic_to_delete:
admin.delete_topic(topic['url'])
admin.clean()
except Exception, e:
log(unicode(traceback.format_exc()))
continue
def load():
config = utility.load_config()
if config == None:
tkMessageBox.showerror('Message', 'Configuration file not existed, please input manually')
else:
e1.delete(0, END)
e1.insert(0, config[0].strip())
e2.delete(0, END)
e2.insert(0, config[1].strip())
e3.delete(0, END)
e3.insert(0, config[2].strip())
e4.delete(0, END)
e4.insert(0, config[3].strip())
e6.delete(0, END)
e6.insert(0, config[4].strip())
e7.delete(0, END)
e7.insert(0, config[5].strip())
e8.delete(0, END)
e8.insert(0, config[6].strip())
e9.delete(0, END)
e9.insert(0, config[7].strip())
e10.delete(0, END)
e10.insert(0, config[8].strip())
e11.delete(0, END)
e11.insert(0, config[9].strip())
tkMessageBox.showinfo('Message', 'import finish')
def main(_):
utility.set_up_logging()
if not FLAGS.config:
raise KeyError('You must specify a configuration.')
if FLAGS.load_from:
logdir = FLAGS.logdir = FLAGS.load_from
else:
if FLAGS.logdir and os.path.exists(FLAGS.logdir):
run_number = [
int(f.split("-")[0]) for f in os.listdir(FLAGS.logdir)
if os.path.isdir(os.path.join(FLAGS.logdir, f))
and FLAGS.config in f
]
run_number = max(run_number) + 1 if len(run_number) > 0 else 0
else:
run_number = 0
logdir = FLAGS.logdir and os.path.expanduser(
os.path.join(FLAGS.logdir, '{}-{}'.format(run_number,
FLAGS.config)))
try:
config = utility.load_config(logdir)
except IOError:
config = tools.AttrDict(getattr(configs, FLAGS.config)())
config = utility.save_config(config, logdir)
train(config, logdir)
def batch_vardev():
conf = load_config("VarDevConf")
try:
main(conf)
except:
print traceback.format_exc()
def main(argv):
del argv # Unused.
print('loading and building expert policy')
checkpoint_file = os.path.join(FLAGS.logdir, FLAGS.checkpoint)
lin_policy = np.load(checkpoint_file, encoding='bytes')
lin_policy = lin_policy.items()[0][1]
M = lin_policy[0]
# mean and std of state vectors estimated online by ARS.
mean = lin_policy[1]
std = lin_policy[2]
config = utility.load_config(FLAGS.logdir)
print("config=",config)
env = config['env'](hard_reset=True, render=True)
ob_dim = env.observation_space.shape[0]
ac_dim = env.action_space.shape[0]
# set policy parameters. Possible filters: 'MeanStdFilter' for v2, 'NoFilter' for v1.
policy_params = {
'type': 'linear',
'ob_filter': config['filter'],
'ob_dim': ob_dim,
'ac_dim': ac_dim,
"weights": M,
"mean": mean,
"std": std,
}
policy = policies.LinearPolicy(policy_params, update_filter=False)
returns = []
observations = []
actions = []
for i in range(FLAGS.num_rollouts):
print('iter', i)
obs = env.reset()
done = False
totalr = 0.
steps = 0
while not done:
action = policy.act(obs)
observations.append(obs)
actions.append(action)
obs, r, done, _ = env.step(action)
time.sleep(1./100.)
totalr += r
steps += 1
if steps % 100 == 0:
print('%i/%i' % (steps, config['rollout_length']))
if steps >= config['rollout_length']:
break
returns.append(totalr)
print('returns', returns)
print('mean return', np.mean(returns))
print('std of return', np.std(returns))
def main(argv):
del argv # Unused.
print('loading and building expert policy')
checkpoint_file = os.path.join(FLAGS.logdir, FLAGS.checkpoint)
lin_policy = np.load(checkpoint_file, encoding='bytes')
lin_policy = lin_policy.items()[0][1]
M = lin_policy[0]
# mean and std of state vectors estimated online by ARS.
mean = lin_policy[1]
std = lin_policy[2]
config = utility.load_config(FLAGS.logdir)
print("config=", config)
env = config['env'](hard_reset=True, render=True)
ob_dim = env.observation_space.shape[0]
ac_dim = env.action_space.shape[0]
# set policy parameters. Possible filters: 'MeanStdFilter' for v2, 'NoFilter' for v1.
policy_params = {
'type': 'linear',
'ob_filter': config['filter'],
'ob_dim': ob_dim,
'ac_dim': ac_dim,
"weights": M,
"mean": mean,
"std": std,
}
policy = policies.LinearPolicy(policy_params, update_filter=False)
returns = []
observations = []
actions = []
for i in range(FLAGS.num_rollouts):
print('iter', i)
obs = env.reset()
done = False
totalr = 0.
steps = 0
while not done:
action = policy.act(obs)
observations.append(obs)
actions.append(action)
obs, r, done, _ = env.step(action)
time.sleep(1. / 100.)
totalr += r
steps += 1
if steps % 100 == 0:
print('%i/%i' % (steps, config['rollout_length']))
if steps >= config['rollout_length']:
break
returns.append(totalr)
print('returns', returns)
print('mean return', np.mean(returns))
print('std of return', np.std(returns))
def batch_countdev():
c = 1
for num_steps in xrange(3+c,7):
conf = load_config("CountDevConf")
conf.NUM_STEPS = num_steps
conf.reinit()
try:
main(conf)
except:
print traceback.format_exc()
def __init__(self):
config = utility.load_config(FLAGS.logdir)
policy_layers = config.policy_layers
value_layers = config.value_layers
network = config.network
self.sess = tf.Session()
self.agent = ppo_agent.SimplePPOPolicy(self.sess,
network,
policy_layers=policy_layers,
value_layers=value_layers,
checkpoint=os.path.join(
FLAGS.logdir,
FLAGS.checkpoint))
def main(_):
"""Create or load configuration and launch the trainer."""
utility.set_up_logging()
if not FLAGS.config:
raise KeyError('You must specify a configuration.')
logdir = FLAGS.logdir and os.path.expanduser(
os.path.join(FLAGS.logdir, '{}-{}'.format(FLAGS.timestamp,
FLAGS.config)))
try:
config = utility.load_config(logdir)
except IOError:
config = tools.AttrDict(getattr(configs, FLAGS.config)())
config = utility.save_config(config, logdir)
for score in train(config, FLAGS.env_processes):
tf.logging.info('Score {}.'.format(score))
def run():
try:
config = utility.load_config()
vk_session = VkApi(token=config["token"])
vk_api = vk_session.get_api()
longpoll = VkBotLongPoll(vk_session, config["group_id"])
for event in longpoll.listen():
try:
if event.type == VkBotEventType.MESSAGE_NEW:
#Слушаем longpoll, если пришло сообщение то:
if (event.raw["object"]["text"].startswith("!")):
text = re.match(r"(\W\w*)(.*)",
event.raw["object"]["text"],
flags=re.DOTALL)
command = text.group(1).strip()
message = text.group(2).strip()
handler = None
if event.from_user and str(
event.raw["object"]
["from_id"]) in config["admin_ids"]:
handler = utility.admin_commands.get(
command.lower())
if not handler:
handler = utility.public_commands.get(
command.lower())
if handler:
attachment = utility.parse_attachment(event)
result = handler.exec(message, attachment)
for msg in result:
utility.send_msg(vk_api, event, msg[0], msg[1])
else:
utility.send_msg(
vk_api, event, "Я не смог найти команду %s. \n\
Воспользуйтесь !помощь для получения списка команд"
% command)
except exceptions.NikolayException as e:
e.resolve(vk_api, event)
except RequestException as e:
try:
with open("./log/request_errors.log", mode="a") as log:
log.write(str(datetime.now()) + '\n')
log.write(traceback.format_exc())
log.write(str(e) + '\n')
except OSError:
print("Error occured while opening log file")
time.sleep(60)
run()
def batch_stepdev():
c = 0
for num_uavs in xrange(3,9):
for num_steps in xrange(3,5):
if c < 7:
print "Skipping (%s,%s)" % (num_uavs,num_steps)
c += 1
continue
conf = load_config("StepDevConf")
conf.NUM_UAVS = num_uavs
conf.NUM_STEPS = num_steps
conf.reinit()
try:
main(conf)
except:
print traceback.format_exc()
def visualize(logdir,
outdir,
num_agents,
num_episodes,
checkpoint=None,
env_processes=True):
"""Recover checkpoint and render videos from it.
Args:
logdir: Logging directory of the trained algorithm.
outdir: Directory to store rendered videos in.
num_agents: Number of environments to simulate in parallel.
num_episodes: Total number of episodes to simulate.
checkpoint: Checkpoint name to load; defaults to most recent.
env_processes: Whether to step environments in separate processes.
"""
config = utility.load_config(logdir)
with tf.device('/cpu:0'):
batch_env = utility.define_batch_env(
lambda: _create_environment(config, outdir),
num_agents,
env_processes=False)
graph = utility.define_simulation_graph(batch_env, config.algorithm,
config)
total_steps = num_episodes * config.max_length
loop = _define_loop(graph, total_steps)
saver = utility.define_saver(exclude=(r'.*_temporary/.*', r'global_step'))
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
#utility.initialize_variables(sess, saver, config.logdir, checkpoint, resume=True)
utility.initialize_variables(sess,
saver,
logdir,
checkpoint,
resume=True)
for unused_score in loop.run(sess, saver, total_steps):
pass
batch_env.close()
def main(unused_args):
length_normalization_factor = FLAGS.length_normalization_factor
# Load model configuration
config_path = os.path.join(os.path.dirname(__file__), 'model_conf', FLAGS.model_name + '.py')
config = utility.load_config(config_path)
config.trainCollection = FLAGS.train_collection
config.word_cnt_thr = FLAGS.word_cnt_thr
config.rootpath = FLAGS.rootpath
train_collection = FLAGS.train_collection
test_collection = FLAGS.test_collection
overwrite = FLAGS.overwrite
feature = FLAGS.vf_name
img_set_file = os.path.join(rootpath, test_collection, 'VideoSets', '%s.txt' % test_collection)
if not os.path.exists(img_set_file):
img_set_file = os.path.join(rootpath, test_collection, 'ImageSets', '%s.txt' % test_collection)
img_list = map(str.strip, open(img_set_file).readlines())
# have visual feature ready
vf_dir = utility.get_feat_dir(test_collection, feature, rootpath)
vf_reader = BigFile( vf_dir )
textbank = TextBank(utility.get_train_vocab_file(FLAGS))
config.vocab_size = len(textbank.vocab)
config.vf_size = int(open(os.path.join(vf_dir, 'shape.txt')).read().split()[1])
model_dir = utility.get_model_dir(FLAGS)
output_dir = utility.get_pred_dir(FLAGS)
checkpoint_style = FLAGS.checkpoint_style
if checkpoint_style == 'file':
#output_per_filename = 'model_perf_in_topk_%d_%s' % (FLAGS.top_k, FLAGS.eval_model_list_file)
# read validated top models
validation_output_dir = utility.get_sim_dir(FLAGS)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
eval_model_list_file = os.path.join(validation_output_dir, 'loss_info.txt') #FLAGS.eval_model_list_file)
shutil.copy(eval_model_list_file, output_dir)
test_iter_list = []
for line in open(eval_model_list_file).readlines()[:FLAGS.top_k]:
iter_current = int(line.strip().split()[0])
test_iter_list.append(iter_current)
elif checkpoint_style == 'iter_interval':
#output_per_filename = 'model_perf_in_%s' % FLAGS.eval_stat
test_iter_list = range(*[int(x) for x in FLAGS.eval_stat.split("-")])
elif checkpoint_style == 'iter_num':
#output_per_filename = 'model_perf_in_iter_%d' % FLAGS.iter_num
test_iter_list = [FLAGS.iter_num]
with_image_embedding = True if FLAGS.with_image_embedding != 0 else False
g = tf.Graph()
with g.as_default():
model = InferenceWrapper(config=config,model_dir=model_dir,
gpu_memory_fraction=FLAGS.gpu_memory_fraction,
gpu=FLAGS.gpu,
with_image_embedding=with_image_embedding)
model.build_model()
for k, iter_n in enumerate(test_iter_list):
model_path = os.path.join(model_dir, 'variables', 'model_%d.ckpt' % iter_n)
while not os.path.exists(model_path+'.meta'):
logger.error('Model path: %s', model_path)
logger.error('Cannot load model file and exit')
sys.exit(0)
top_one_pred_sent_file = os.path.join(output_dir, 'top%d' % k, 'top_one_pred_sent.txt')
top_n_pred_sent_file = os.path.join(output_dir, 'top%d' % k, 'top_n_pred_sent.txt')
# perf_file = os.path.join(output_dir, 'model_%d.ckpt' % iter_n, 'perf.txt')
if os.path.exists(top_one_pred_sent_file) and not overwrite:
# write existing perf file and print out
logger.info('%s exists. skip', top_one_pred_sent_file)
continue
if not os.path.exists(os.path.split(top_one_pred_sent_file)[0]):
os.makedirs(os.path.split(top_one_pred_sent_file)[0])
logger.info('save results to %s', top_one_pred_sent_file)
# load the trained model
generator = CaptionGenerator(config, model, length_normalization_factor = length_normalization_factor)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads, gpu_options=gpu_options, allow_soft_placement=True)
#with tf.Session(config=config_proto) as session:
#model.build_model(session, model_path)
model.load_model(model_path)
fout_one_sent = codecs.open(top_one_pred_sent_file, 'w','utf-8')
fout_n_sent = codecs.open(top_n_pred_sent_file, 'w','utf-8')
for progress,img in enumerate(img_list):
# predict sentences given a visual feature
visual_feature = np.array(vf_reader.read_one(img))
sentences = generator.beam_search( visual_feature, FLAGS.beam_size)
# output top one sentence info
sent_score = sentences[0].score
sent = ' '.join(sentences[0].words)
fout_one_sent.write(img + ' ' + '%.3f' % sent_score + ' ' + sent + '\n')
logger.debug(img + ' ' + '%.3f' % sent_score + ' ' + sent)
# output top n sentences info
fout_n_sent.write(img)
for sentence in sentences:
sent_score = sentence.score
sent = ' '.join(sentence.words)
fout_n_sent.write('\t' + '%.3f' % sent_score + '\t' + sent)
fout_n_sent.write('\n')
if progress % 100 == 0:
logger.info('%d images decoded' % (progress+1))
logger.info('%d images decoded' % (progress+1))
fout_one_sent.close()
fout_n_sent.close()
def main(unused_args):
length_normalization_factor = FLAGS.length_normalization_factor
# Load model configuration
config_path = os.path.join(os.path.dirname(__file__), 'model_conf', FLAGS.model_name + '.py')
config = utility.load_config(config_path)
config.trainCollection = FLAGS.train_collection
config.word_cnt_thr = FLAGS.word_cnt_thr
config.rootpath = FLAGS.rootpath
train_collection = FLAGS.train_collection
test_collection = FLAGS.test_collection
overwrite = FLAGS.overwrite
feature = FLAGS.vf_name
img_set_file = os.path.join(rootpath, test_collection, 'VideoSets', '%s.txt' % test_collection)
if not os.path.exists(img_set_file):
img_set_file = os.path.join(rootpath, test_collection, 'ImageSets', '%s.txt' % test_collection)
img_list = map(str.strip, open(img_set_file).readlines())
# have visual feature ready
FLAGS.vf_dir = os.path.join(rootpath, test_collection, 'FeatureData', feature)
vf_reader = BigFile(FLAGS.vf_dir)
textbank = TextBank(utility.get_train_vocab_file(FLAGS))
config.vocab_size = len(textbank.vocab)
config.vf_size = int(open(os.path.join(FLAGS.vf_dir, 'shape.txt')).read().split()[1])
model_dir = utility.get_model_dir(FLAGS)
output_dir = utility.get_pred_dir(FLAGS)
checkpoint_style = FLAGS.checkpoint_style
if checkpoint_style == 'file':
#output_per_filename = 'model_perf_in_topk_%d_%s' % (FLAGS.top_k, FLAGS.eval_model_list_file)
# read validated top models
validation_output_dir = utility.get_sim_dir(FLAGS)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
eval_model_list_file = os.path.join(validation_output_dir, 'loss_info.txt') #FLAGS.eval_model_list_file)
shutil.copy(eval_model_list_file, output_dir)
test_iter_list = []
for line in open(eval_model_list_file).readlines()[:FLAGS.top_k]:
iter_current = int(line.strip().split()[0])
test_iter_list.append(iter_current)
elif checkpoint_style == 'iter_interval':
#output_per_filename = 'model_perf_in_%s' % FLAGS.eval_stat
test_iter_list = range(*[int(x) for x in FLAGS.eval_stat.split("-")])
elif checkpoint_style == 'iter_num':
#output_per_filename = 'model_perf_in_iter_%d' % FLAGS.iter_num
test_iter_list = [FLAGS.iter_num]
with_image_embedding = True if FLAGS.with_image_embedding != 0 else False
g = tf.Graph()
with g.as_default():
model = InferenceWrapper(config=config,model_dir=model_dir,
gpu_memory_fraction=FLAGS.gpu_memory_fraction,
gpu=FLAGS.gpu,
with_image_embedding=with_image_embedding)
model.build_model()
for k, iter_n in enumerate(test_iter_list):
model_path = os.path.join(model_dir, 'variables', 'model_%d.ckpt' % iter_n)
while not os.path.exists(model_path+'.meta'):
logger.error('Model path: %s', model_path)
logger.error('Cannot load model file and exit')
sys.exit(0)
top_one_pred_sent_file = os.path.join(output_dir, 'top%d' % k, 'top_one_pred_sent.txt')
top_n_pred_sent_file = os.path.join(output_dir, 'top%d' % k, 'top_n_pred_sent.txt')
# perf_file = os.path.join(output_dir, 'model_%d.ckpt' % iter_n, 'perf.txt')
if os.path.exists(top_one_pred_sent_file) and not overwrite:
# write existing perf file and print out
logger.info('%s exists. skip', top_one_pred_sent_file)
continue
if not os.path.exists(os.path.split(top_one_pred_sent_file)[0]):
os.makedirs(os.path.split(top_one_pred_sent_file)[0])
logger.info('save results to %s', top_one_pred_sent_file)
# load the trained model
generator = CaptionGenerator(config, model, length_normalization_factor = length_normalization_factor)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads, gpu_options=gpu_options, allow_soft_placement=True)
#with tf.Session(config=config_proto) as session:
#model.build_model(session, model_path)
model.load_model(model_path)
fout_one_sent = codecs.open(top_one_pred_sent_file, 'w','utf-8')
fout_n_sent = codecs.open(top_n_pred_sent_file, 'w','utf-8')
for progress,img in enumerate(img_list):
print(img)
# predict sentences given a visual feature
visual_feature = np.array(vf_reader.read_one(img))
sentences = generator.beam_search( visual_feature, FLAGS.beam_size)
# output top one sentence info
sent_score = sentences[0].score
sent = ' '.join(sentences[0].words)
fout_one_sent.write(img + ' ' + '%.3f' % sent_score + ' ' + sent + '\n')
logger.debug(img + ' ' + '%.3f' % sent_score + ' ' + sent)
# output top n sentences info
fout_n_sent.write(img)
for sentence in sentences:
sent_score = sentence.score
sent = ' '.join(sentence.words)
fout_n_sent.write('\t' + '%.3f' % sent_score + '\t' + sent)
fout_n_sent.write('\n')
if progress % 100 == 0:
logger.info('%d images decoded' % (progress+1))
logger.info('%d images decoded' % (progress+1))
fout_one_sent.close()
fout_n_sent.close()
import requests
import utility as util
# read params from config.json
config = util.load_config()
# general session that can be used for future API calls
session = requests.session()
session.headers['AUTHORIZATION'] = 'apikey %s' % config['APIKey']
# print api result
def print_result(response):
print("---------returns----------")
print(response.status_code)
print(response.text)
# api request caller
def api_resp_check(response):
if response.status_code != 200:
print("----------ERROR-------------")
print_result(response)
util.script_exit()
# api endpoint constructor
def api_url_ctor(data_type, name=None, id=None):
if name:
return '/adminapi/v1/' + data_type.value + '?name=' + name
elif id:
def main(unused_args):
train_collection = FLAGS.train_collection
val_collection = FLAGS.val_collection
overwrite = FLAGS.overwrite
output_dir = utility.get_sim_dir(FLAGS)
loss_info_file = os.path.join(output_dir, 'loss_info.txt')
if os.path.exists(loss_info_file) and not overwrite:
logger.info('%s exists. quit', loss_info_file)
sys.exit(0)
model_dir = utility.get_model_dir(FLAGS)
config_path = os.path.join(os.path.dirname(__file__), 'model_conf',
FLAGS.model_name + '.py')
config = utility.load_config(config_path)
if FLAGS.fluency_method == 'None':
FLAGS.fluency_method = None
config.fluency_method = FLAGS.fluency_method
if config.fluency_method == 'weighted':
config.use_weighted_loss = True
else:
config.use_weighted_loss = False
textbank = TextBank(utility.get_train_vocab_file(FLAGS))
config.vocab_size = len(textbank.vocab)
config.vf_size = int(
open(os.path.join(utility.get_val_feat_dir(FLAGS),
'shape.txt')).read().split()[1])
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads,
gpu_options=gpu_options,
allow_soft_placement=True)
with_image_embedding = True if FLAGS.with_image_embedding > 0 else False
with tf.Graph().as_default(), tf.Session(config=config_proto) as session:
assert len(config.buckets) >= 1
assert config.buckets[-1] == config.max_num_steps
with tf.device('/gpu:%d' % FLAGS.gpu):
with tf.variable_scope("LSTMModel", reuse=None):
if with_image_embedding:
model = LSTMModel(
mode='eval',
num_steps=config.buckets[-1],
config=config,
model_dir=model_dir, #model_name=FLAGS.model_name,
flag_with_saver=True)
#model_root=FLAGS.model_root)
else:
# deprecating this option
print('Plz use image_embedding')
sys.exit(-1)
model.build()
model_path_list = []
_dir = os.path.join(model_dir, 'variables')
for _file in os.listdir(_dir):
if _file.startswith('model_') and _file.endswith('.ckpt.meta'):
iter_n = int(_file[6:-10])
model_path = os.path.join(_dir, 'model_%d.ckpt' % iter_n)
model_path_list.append((iter_n, model_path))
data_provider = BucketDataProvider(val_collection,
utility.get_train_vocab_file(FLAGS),
feature=FLAGS.vf_name,
language=FLAGS.language,
flag_shuffle=False,
method=config.fluency_method,
rootpath=FLAGS.rootpath)
iter2loss = {}
for iter_n, model_path in model_path_list:
loss_file = os.path.join(output_dir, 'model_%d.ckpt' % iter_n,
'loss.txt')
if os.path.exists(loss_file) and not overwrite:
logger.info('load loss from %s', loss_file)
loss = float(open(loss_file).readline().strip())
iter2loss[iter_n] = loss
continue
if not os.path.exists(os.path.split(loss_file)[0]):
os.makedirs(os.path.split(loss_file)[0])
model.saver.restore(session, model_path)
# print([v.name for v in tf.trainable_variables()])
logger.info('Continue to train from %s', model_path)
val_cost = run_epoch(session, config.batch_size,
config.buckets[-1], config, model,
data_provider)
logger.info(
"Validation cost for checkpoint model_%d.ckpt is %.3f" %
(iter_n, val_cost))
iter2loss[iter_n] = val_cost
with open(loss_file, "w") as fw:
fw.write('%g' % val_cost)
fw.close()
sorted_iter2loss = sorted(iter2loss.iteritems(), key=lambda x: x[1])
with open(loss_info_file, 'w') as fw:
fw.write('\n'.join(
['%d %g' % (iter_n, loss) for (iter_n, loss) in sorted_iter2loss]))
fw.close()
def test():
rootpath = FLAGS.rootpath
overwrite = FLAGS.overwrite
train_collection = FLAGS.train_collection
aux_train_collection = FLAGS.aux_train_collection
val_collection = FLAGS.val_collection
test_collection = FLAGS.test_collection
vf_name = FLAGS.vf_name
annotation_name = FLAGS.annotation_name
aux_annotation_name = FLAGS.aux_annotation_name
top_k = FLAGS.top_k
config_path = os.path.join(os.path.dirname(__file__), 'model_conf',
FLAGS.model_name + '.py')
config = utility.load_config(config_path)
model_dir = utility.get_model_dir(FLAGS)
val_perf_file = os.path.join(model_dir, '%s.txt' % val_collection)
epoch, val_perf = open(val_perf_file).readline().strip().split()
epoch = int(epoch.rsplit('_', 1)[1])
model_path = os.path.join(model_dir, 'epoch_%d.ckpt' % epoch)
test_data_iter = TestDataBatchIterator(test_collection,
vf_name,
FLAGS.batch_size,
rootpath=rootpath)
res_dir = utility.get_pred_dir(FLAGS)
res_file = os.path.join(res_dir, 'id.tagvotes.txt')
if os.path.exists(res_file) and not overwrite:
logger.info('%s exists. quit', res_file)
return
concept_file = os.path.join(rootpath, train_collection, 'Annotations',
annotation_name)
concepts = map(str.strip, open(concept_file).readlines())
if FLAGS.multi_task:
aux_concept_file = os.path.join(rootpath, aux_train_collection,
'Annotations', aux_annotation_name)
aux_concepts = map(str.strip, open(aux_concept_file).readlines())
else:
aux_concepts = []
mlp_predictor = MLPPredictor(
n_layers=[test_data_iter.feat_dim, config.hidden_size,
len(concepts)],
last_nl_fun=config.output,
loss_fun=config.loss,
top_k=min(len(concepts), FLAGS.top_k),
ses_threads=FLAGS.ses_threads,
aux_classes=len(aux_concepts),
gpu_memory_fraction=FLAGS.gpu_memory_fraction,
gpu=FLAGS.gpu)
mlp_predictor.load_model(model_path)
progbar = utils.Progbar(test_data_iter.num_batches)
if not os.path.exists(res_dir):
os.makedirs(res_dir)
fout = open(res_file, "w")
for minibatch_index, (ids_test, X_test, _, _) in enumerate(test_data_iter):
pred_top_values_batch, pred_top_indices_batch = mlp_predictor.predict(
X_test)
for idx, img_id in enumerate(ids_test):
tagvotes = zip(pred_top_indices_batch[idx],
pred_top_values_batch[idx])
res = ' '.join(
['%s %g' % (concepts[x[0]], x[1]) for x in tagvotes])
fout.write('%s %s\n' % (img_id, res))
progbar.add(1)
fout.close()
beam = Caption(sentence, state, logprob, score, metadata_list)
partial_captions.push(beam)
if partial_captions.size() == 0:
# We have run out of partial candidates; happens when beam_size = 1.
break
# If we have no complete captions then fall back to the partial captions.
# But never output a mixture of complete and partial captions because a
# partial caption could have a higher score than all the complete captions.
if not complete_captions.size():
complete_captions = partial_captions
captions = complete_captions.extract(sort=True)
for i, caption in enumerate(captions):
caption.words = self.textbank.decode_tokens(caption.sentence[1:])
return captions
if __name__ == '__main__':
from utility import load_config
from constant import ROOT_PATH, DEFAULT_WORD_COUNT
config = load_config('model_conf/8k_neuraltalk.py')
config.trainCollection = 'weibotrain'
config.word_cnt_thr = DEFAULT_WORD_COUNT
config.rootpath = ROOT_PATH
model = None
generator = CaptionGenerator(config, model)
#print(generator.beam_search())
beam = Caption(sentence, state, logprob, score,
metadata_list)
partial_captions.push(beam)
if partial_captions.size() == 0:
# We have run out of partial candidates; happens when beam_size = 1.
break
# If we have no complete captions then fall back to the partial captions.
# But never output a mixture of complete and partial captions because a
# partial caption could have a higher score than all the complete captions.
if not complete_captions.size():
complete_captions = partial_captions
captions = complete_captions.extract(sort=True)
for i, caption in enumerate(captions):
caption.words = self.textbank.decode_tokens(caption.sentence[1:])
return captions
if __name__ == '__main__':
from utility import load_config
from constant import ROOT_PATH, DEFAULT_WORD_COUNT
config = load_config('model_conf/8k_neuraltalk.py')
config.trainCollection = 'weibotrain'
config.word_cnt_thr = DEFAULT_WORD_COUNT
config.rootpath = ROOT_PATH
model = None
generator = CaptionGenerator(config, model)
#print(generator.beam_search())
def main(unused_args):
model_dir = utility.get_model_dir(FLAGS)
if os.path.exists(model_dir) and not FLAGS.overwrite:
logger.info('%s exists. quit', model_dir)
sys.exit(0)
# Load model configuration
config_path = os.path.join(os.path.dirname(__file__), 'model_conf',
FLAGS.model_name + '.py')
config = utility.load_config(config_path)
FLAGS.vf_dir = os.path.join(FLAGS.rootpath, FLAGS.train_collection,
'FeatureData', FLAGS.vf_name)
vocab_file = utility.get_vocab_file(FLAGS.train_collection,
FLAGS.word_cnt_thr, FLAGS.rootpath)
textbank = TextBank(vocab_file)
config.vocab_size = len(textbank.vocab)
config.vf_size = int(
open(os.path.join(FLAGS.vf_dir, 'shape.txt')).read().split()[1])
if hasattr(config, 'num_epoch_save'):
num_epoch_save = config.num_epoch_save
else:
num_epoch_save = 1
if FLAGS.fluency_method == 'None':
FLAGS.fluency_method = None
config.fluency_method = FLAGS.fluency_method
if config.fluency_method == 'weighted':
config.use_weighted_loss = True
else:
config.use_weighted_loss = False
train_image_embedding = True
try:
if config.train_image_embedding == False:
assert ('freeze' in FLAGS.model_name)
train_image_embedding = False
logger.info('Not training image embedding')
except:
pass
with_image_embedding = True if FLAGS.with_image_embedding != 0 else False
# Start model training
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads,
gpu_options=gpu_options,
allow_soft_placement=True)
with tf.Graph().as_default(), tf.Session(config=config_proto) as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
assert len(config.buckets) >= 1
assert config.buckets[-1] == config.max_num_steps
models = []
with tf.device('gpu:%s' % FLAGS.gpu):
with tf.variable_scope("LSTMModel",
reuse=None,
initializer=initializer):
if with_image_embedding:
m = LSTMModel(mode='train',
num_steps=config.buckets[0],
config=config,
model_dir=model_dir,
flag_with_saver=True,
train_image_embedding=train_image_embedding)
#model_root=FLAGS.model_root)
else:
# deprecating this function
logger.info('Plz use with_image_embedding=1')
sys.exit(-1)
'''m = PCALSTMModel(mode='train',
num_steps=config.buckets[0],
config=config,
model_name=FLAGS.model_name,
flag_with_saver=True,
train_image_embedding=train_image_embedding,
model_root=FLAGS.model_root)
'''
m.build()
models.append(m)
pre_trained_iter = 0
if FLAGS.pre_trained_model_path:
pre_trained_iter = int(
FLAGS.pre_trained_model_path.split('model_')[1].split('.')[0])
hdlr = logging.FileHandler(
os.path.join(m.model_dir, 'log%d.txt' % pre_trained_iter))
hdlr.setLevel(logging.INFO)
hdlr.setFormatter(logging.Formatter(formatter_log))
logger.addHandler(hdlr)
if FLAGS.pre_trained_model_path:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
models[0].saver.restore(session, FLAGS.pre_trained_model_path)
logger.info('Continue to train from %s',
FLAGS.pre_trained_model_path)
elif FLAGS.pre_trained_imembedding_path:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
models[0].imemb_saver.restore(session,
FLAGS.pre_trained_imembedding_path)
logger.info('Init image-embedding from %s',
FLAGS.pre_trained_imembedding_path)
elif FLAGS.pre_trained_lm_path:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
models[0].lm_saver.restore(session, FLAGS.pre_trained_lm_path)
logger.info('Init language from %s', FLAGS.pre_trained_lm_path)
else:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
# print([v.name for v in tf.trainable_variables()])
iters_done = 0
data_provider = BucketDataProvider(FLAGS.train_collection,
vocab_file,
FLAGS.vf_name,
language=FLAGS.language,
method=config.fluency_method,
rootpath=FLAGS.rootpath)
for i in range(config.num_epoch):
logger.info('epoch %d', i)
data_provider.shuffle_data_queue()
train_cost, iters_done = run_epoch(session,
iters_done,
config,
models,
data_provider,
verbose=True)
logger.info("Train cost for epoch %d is %.3f" % (i, train_cost))
# save the current model if necessary
if (i + 1) % num_epoch_save == 0:
models[0].saver.save(
session,
os.path.join(
m.variable_dir,
'model_%d.ckpt' % (iters_done + pre_trained_iter)))
if with_image_embedding:
models[0].imemb_saver.save(session, os.path.join(m.variable_dir, \
'imembedding_model_%d.ckpt' % (iters_done)))
logger.info("Model saved at iteration %d", iters_done)
# copy the configure file in to checkpoint direction
os.system("cp %s %s" % (config_path, model_dir))
if FLAGS.pre_trained_model_path:
os.system("echo %s > %s" %
(FLAGS.pre_trained_model_path,
os.path.join(model_dir, 'pre_trained_model_path.txt')))
if FLAGS.pre_trained_imembedding_path:
os.system(
"echo %s > %s" %
(FLAGS.pre_trained_imembedding_path,
os.path.join(model_dir, 'pre_trained_imembedding_path.txt')))
import numpy
import traceback,sys
from deap import algorithms
from bootstrap import bootstrap
from progress import Progress
from utility import load_config
def main(conf):
pop, toolbox, hof, stats = bootstrap(conf)
prog = Progress(conf)
pop, logbook = algorithms.eaMuPlusLambda(pop, toolbox, mu=conf.MU, lambda_=conf.LAMBDA,
cxpb=conf.CXPB, mutpb=conf.MUTPB, ngen=0, stats=stats, halloffame=hof)
prog.update(0,pop,hof)
for gen in range(conf.NGEN/5):
pop, logbook = algorithms.eaMuPlusLambda(pop, toolbox, mu=conf.MU, lambda_=conf.LAMBDA,
cxpb=conf.CXPB, mutpb=conf.MUTPB, ngen=5, stats=stats, halloffame=hof)
prog.update((gen+1)*5,pop,hof)
#prog.hold()
return pop, stats, hof
if __name__ == "__main__":
conf = load_config("StepDevConf")
main(conf)
db = metadata_db if db is None else db
RE.subscribe(db.insert)
RE.subscribe(BestEffortCallback())
return RE
RE = get_runengine()
keywords_vars['RE'] = 'Default RunEngine instance'
# --- import utility functions
from utility import load_config
keywords_func['load_config'] = load_config.__doc__
# --- load devices info from yaml file
_devices = load_config('seisidd/config/tomo_devices.yml')
# --- exp safeguard suspender
from bluesky.suspenders import SuspendFloor
# --- define shutter
keywords_func['get_shutter'] = 'Return a connection to a sim/real shutter'
def get_shutter(mode='debug'):
"""
return
simulated shutter <-- dryrun, debug
acutal shutter <-- production
"""
import apstools.devices as APS_devices
redirect, session, jsonify, send_from_directory
import os
import gc # flask and garbage collection is not a good combo, clear session should invite a gc.collect()
from functools import wraps
from datetime import datetime
import time
import sigma
# for pickles sake...
from sigma import KnowledgeMap, Topic
import auth
from utility import load_config
app = Flask(__name__) # - Initialize the Flask-app object
load_config(app) # - Loads config
# - Create a secret key to be used by session = {}
# to encrypt sensitive data
# ----------- LOGIN WRAP -----------
def login_required(f):
@wraps(f)
def wrap(*args, **kwargs):
if 'innlogget' in session:
return f(*args, **kwargs)
else:
return redirect(url_for('index'))
return wrap
def main(unused_args):
model_dir=utility.get_model_dir(FLAGS)
if os.path.exists(model_dir) and not FLAGS.overwrite:
logger.info('%s exists. quit', model_dir)
sys.exit(0)
# Load model configuration
config_path = os.path.join(os.path.dirname(__file__), 'model_conf', FLAGS.model_name + '.py')
config = utility.load_config(config_path)
# pdb.set_trace()
FLAGS.vf_dir = os.path.join(FLAGS.rootpath, FLAGS.train_collection, 'FeatureData', FLAGS.vf_name)
vocab_file = utility.get_vocab_file(FLAGS.train_collection, FLAGS.word_cnt_thr, FLAGS.rootpath)
textbank = TextBank(vocab_file)
config.vocab_size = len(textbank.vocab)
config.vf_size = int(open(os.path.join(FLAGS.vf_dir, 'shape.txt')).read().split()[1])
if hasattr(config,'num_epoch_save'):
num_epoch_save = config.num_epoch_save
else:
num_epoch_save = 1
# if FLAGS.fluency_method == 'None':
# FLAGS.fluency_method = None
# config.fluency_method = FLAGS.fluency_method
# if config.fluency_method == 'weighted':
# config.use_weighted_loss = True
# else:
# config.use_weighted_loss = False
train_image_embedding = True
try:
if config.train_image_embedding == False:
assert('freeze' in FLAGS.model_name)
train_image_embedding = False
logger.info('Not training image embedding')
except:
pass
with_image_embedding = True if FLAGS.with_image_embedding != 0 else False
# Start model training
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads, gpu_options=gpu_options, allow_soft_placement=True)
with tf.Graph().as_default(), tf.Session(config=config_proto) as session:
writer = tf.train.SummaryWriter("logs/", session.graph)
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
assert len(config.buckets) >= 1
assert config.buckets[-1] == config.max_num_steps
models = []
with tf.device('gpu:%s'%FLAGS.gpu):
with tf.variable_scope("LSTMModel", reuse=None, initializer=initializer):
if with_image_embedding:
m = LSTMModel(mode='train',
num_steps=config.buckets[0],
config=config,
model_dir=model_dir,
flag_with_saver=True,
train_image_embedding=train_image_embedding)
#model_root=FLAGS.model_root)
else:
# deprecating this function
logger.info('Plz use with_image_embedding=1')
sys.exit(-1)
m.build()
models.append(m)
pre_trained_iter=0
if FLAGS.pre_trained_model_path:
pre_trained_iter = int(FLAGS.pre_trained_model_path.split('model_')[1].split('.')[0])
hdlr = logging.FileHandler(os.path.join(m.model_dir, 'log%d.txt'%pre_trained_iter))
hdlr.setLevel(logging.INFO)
hdlr.setFormatter(logging.Formatter(formatter_log))
logger.addHandler(hdlr)
if FLAGS.pre_trained_model_path:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
models[0].saver.restore(session, FLAGS.pre_trained_model_path)
logger.info('Continue to train from %s', FLAGS.pre_trained_model_path)
elif FLAGS.pre_trained_imembedding_path:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
models[0].imemb_saver.restore(session, FLAGS.pre_trained_imembedding_path)
logger.info('Init image-embedding from %s', FLAGS.pre_trained_imembedding_path)
elif FLAGS.pre_trained_lm_path:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
models[0].lm_saver.restore(session, FLAGS.pre_trained_lm_path)
logger.info('Init language from %s', FLAGS.pre_trained_lm_path)
else:
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
# print([v.name for v in tf.trainable_variables()])
iters_done = 0
data_provider = BucketDataProvider(FLAGS.train_collection, vocab_file, FLAGS.vf_name,
language=FLAGS.language, # method=config.fluency_method,
rootpath=FLAGS.rootpath)
for i in range(config.num_epoch):
logger.info('epoch %d', i)
data_provider.shuffle_data_queue() ####################################
train_cost, iters_done, result = run_epoch(session, iters_done, config, models, data_provider , verbose=True)
logger.info("Train cost for epoch %d is %.3f" % (i, train_cost))
writer.add_summary(result, i)
# save the current model if necessary
if (i+1)% num_epoch_save == 0:
models[0].saver.save(session, os.path.join(m.variable_dir,
'model_%d.ckpt' % (iters_done+pre_trained_iter)))
if with_image_embedding:
models[0].imemb_saver.save(session, os.path.join(m.variable_dir, \
'imembedding_model_%d.ckpt' % (iters_done)))
logger.info("Model saved at iteration %d", iters_done)
# copy the configure file in to checkpoint direction
os.system("cp %s %s" % (config_path, model_dir))
if FLAGS.pre_trained_model_path:
os.system("echo %s > %s" % (FLAGS.pre_trained_model_path, os.path.join(model_dir, 'pre_trained_model_path.txt')))
if FLAGS.pre_trained_imembedding_path:
os.system("echo %s > %s" % (FLAGS.pre_trained_imembedding_path, os.path.join(model_dir, 'pre_trained_imembedding_path.txt')))
pop, logbook = algorithms.eaMuPlusLambda(pop,
toolbox,
mu=conf.MU,
lambda_=conf.LAMBDA,
cxpb=conf.CXPB,
mutpb=conf.MUTPB,
ngen=0,
stats=stats,
halloffame=hof)
prog.update(0, pop, hof)
for gen in range(conf.NGEN / 5):
pop, logbook = algorithms.eaMuPlusLambda(pop,
toolbox,
mu=conf.MU,
lambda_=conf.LAMBDA,
cxpb=conf.CXPB,
mutpb=conf.MUTPB,
ngen=5,
stats=stats,
halloffame=hof)
prog.update((gen + 1) * 5, pop, hof)
#prog.hold()
return pop, stats, hof
if __name__ == "__main__":
conf = load_config("StepDevConf")
main(conf)
def main(unused_args):
train_collection = FLAGS.train_collection
val_collection = FLAGS.val_collection
overwrite = FLAGS.overwrite
output_dir = utility.get_sim_dir(FLAGS)
loss_info_file = os.path.join(output_dir, 'loss_info.txt')
if os.path.exists(loss_info_file) and not overwrite:
logger.info('%s exists. quit', loss_info_file)
sys.exit(0)
model_dir=utility.get_model_dir(FLAGS)
config_path = os.path.join(os.path.dirname(__file__), 'model_conf', FLAGS.model_name + '.py')
config = utility.load_config(config_path)
# if FLAGS.fluency_method == 'None':
# FLAGS.fluency_method = None
# config.fluency_method = FLAGS.fluency_method
# if config.fluency_method == 'weighted':
# config.use_weighted_loss = True
# else:
# config.use_weighted_loss = False
textbank = TextBank(utility.get_train_vocab_file(FLAGS))
config.vocab_size = len(textbank.vocab)
config.vf_size = int(open(os.path.join(utility.get_val_feat_dir(FLAGS), 'shape.txt')).read().split()[1])
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads, gpu_options=gpu_options, allow_soft_placement=True)
with_image_embedding = True if FLAGS.with_image_embedding > 0 else False
with tf.Graph().as_default(), tf.Session(config=config_proto) as session:
assert len(config.buckets) >= 1
assert config.buckets[-1] == config.max_num_steps
with tf.device('/gpu:%d'%FLAGS.gpu):
with tf.variable_scope("LSTMModel", reuse=None):
if with_image_embedding:
model = LSTMModel(mode='eval',
num_steps=config.buckets[-1],
config=config,
model_dir=model_dir, #model_name=FLAGS.model_name,
flag_with_saver=True)
#model_root=FLAGS.model_root)
else:
# deprecating this option
print('Plz use image_embedding')
sys.exit(-1)
model.build()
model_path_list = []
_dir = os.path.join(model_dir,'variables')
for _file in os.listdir(_dir):
if _file.startswith('model_') and _file.endswith('.ckpt.meta'):
iter_n = int(_file[6:-10])
model_path = os.path.join(_dir, 'model_%d.ckpt'%iter_n)
model_path_list.append((iter_n, model_path))
data_provider = BucketDataProvider(val_collection, utility.get_train_vocab_file(FLAGS),
feature=FLAGS.vf_name,
language=FLAGS.language,
flag_shuffle=False,
# method=config.fluency_method,
rootpath=FLAGS.rootpath)
iter2loss = {}
for iter_n, model_path in model_path_list:
loss_file = os.path.join(output_dir, 'model_%d.ckpt' % iter_n, 'loss.txt')
if os.path.exists(loss_file) and not overwrite:
logger.info('load loss from %s', loss_file)
loss = float(open(loss_file).readline().strip())
iter2loss[iter_n] = loss
continue
if not os.path.exists(os.path.split(loss_file)[0]):
os.makedirs(os.path.split(loss_file)[0])
model.saver.restore(session, model_path)
# print([v.name for v in tf.trainable_variables()])
logger.info('Continue to train from %s', model_path)
val_cost = run_epoch(session, config.batch_size, config.buckets[-1], config,model, data_provider)
logger.info("Validation cost for checkpoint model_%d.ckpt is %.3f" % (iter_n, val_cost))
iter2loss[iter_n] = val_cost
with open(loss_file, "w") as fw:
fw.write('%g' % val_cost)
fw.close()
sorted_iter2loss = sorted(iter2loss.iteritems(), key=lambda x: x[1])
with open(loss_info_file, 'w') as fw:
fw.write('\n'.join(['%d %g' % (iter_n, loss) for (iter_n, loss) in sorted_iter2loss]))
fw.close()
def main(unused_args):
model_dir = utility.get_model_dir(FLAGS)
if os.path.exists(model_dir) and not FLAGS.overwrite:
logger.info('%s exists. quit', model_dir)
sys.exit(0)
rootpath = FLAGS.rootpath
overwrite = FLAGS.overwrite
train_collection = FLAGS.train_collection
aux_train_collection = FLAGS.aux_train_collection
val_collection = FLAGS.val_collection
vf_name = FLAGS.vf_name
annotation_name = FLAGS.annotation_name
aux_annotation_name = FLAGS.aux_annotation_name
# Load model configuration
config_path = os.path.join(os.path.dirname(__file__), 'model_conf',
FLAGS.model_name + '.py')
config = utility.load_config(config_path)
train_loss_hist_file = os.path.join(model_dir, 'train_loss_hist.txt')
val_per_hist_file = os.path.join(model_dir, '%s.txt' % val_collection)
if os.path.exists(train_loss_hist_file) and not overwrite:
logger.info('%s exists. quit', train_loss_hist_file)
sys.exit(0)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
else:
os.system('rm -rf %s/*' % model_dir)
collections = [train_collection, aux_train_collection
] if FLAGS.multi_task else [train_collection]
annotation_names = [annotation_name, aux_annotation_name
] if FLAGS.multi_task else [annotation_name]
concept_files = [
os.path.join(rootpath, col, 'Annotations', anno)
for (col, anno) in zip(collections, annotation_names)
]
train_dataset = DataBatchIterator(collections, concept_files, vf_name,
config.batch_size, rootpath)
val_dataset = DataBatchIterator([val_collection], concept_files[:1],
vf_name, config.batch_size, rootpath)
aux_classes = train_dataset.aux_num_labels
# Start model training
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config_proto = tf.ConfigProto(
intra_op_parallelism_threads=FLAGS.ses_threads,
gpu_options=gpu_options,
allow_soft_placement=True)
with tf.Graph().as_default(), tf.Session(config=config_proto) as session:
with tf.variable_scope("MLPModel"):
model = MLP(is_training=True,
n_layers=[
train_dataset.feat_dim, config.hidden_size,
train_dataset.num_labels
],
last_nl_fun=config.output,
loss_fun=config.loss,
keep_prob=config.keep_prob,
top_k=config.top_k,
aux_classes=aux_classes,
flag_with_saver=True,
gpu=FLAGS.gpu)
if tf.__version__ < '1.0':
tf.initialize_all_variables().run()
else:
tf.global_variables_initializer().run()
model.assign_lr(session, config.learning_rate)
best_val_perf = 0
count_down = config.tolerate
train_loss_hist = []
val_per_hist = []
learning_rate = config.learning_rate
for epoch in range(config.max_epochs):
logger.info('epoch %d', epoch)
train_dataset.shuffle()
train_loss, val_perf = run_epoch(session, model, train_dataset,
val_dataset, FLAGS.multi_task)
logger.info('train loss %.4f, val perf %.4f', train_loss, val_perf)
train_loss_hist.append(train_loss)
val_per_hist.append(val_perf)
if val_perf > best_val_perf:
best_val_perf = val_perf
best_ckpt_name = 'epoch_%d.ckpt' % epoch
model_file = os.path.join(model_dir, best_ckpt_name)
model.saver.save(session, model_file)
logger.info('*** better model -> val_perf=%.4f', val_perf)
else:
count_down -= 1
if 0 == count_down:
if learning_rate > 1e-8:
count_down = config.tolerate
learning_rate /= 2.0
model.assign_lr(session, learning_rate)
logger.info('adjust lr to %g', learning_rate)
else:
logger.info("early-stop happend at epoch %d", epoch)
break
with open(train_loss_hist_file, 'w') as fout:
for i, loss in enumerate(train_loss_hist):
fout.write("epoch_" + str(i) + " " + str(loss) + "\n")
fout.close()
sorted_epoch_perf = sorted(zip(range(len(val_per_hist)), val_per_hist),
key=lambda x: x[1],
reverse=True)
with open(val_per_hist_file, 'w') as fout:
for i, perf in sorted_epoch_perf:
fout.write("epoch_" + str(i) + " " + str(perf) + "\n")
fout.close()
