def cmd_autoupdate_main_wrapper(args):
logger.info('Auto Update')
settings.DRY_RUN = args.dry_run if args.dry_run else False
settings.FORCE_RERUN = args.force if args.force else False
settings.ENABLE_JUNIT = args.junit if args.junit else False
# always use browserstack
settings.TESTENV = 'bs'
settings.SAVE_DB = True
logger.setLevel(logging.DEBUG) if args.verbose else logger.setLevel(
logging.INFO)
get_config()
cmd_autoupdate_main()
def cdn_install():
"""Installs sam from cdn
Note:
Following samtools.properties config values are must to continue:
- rh_portal_username
- rh_portal_password
"""
# Register and subscribe the host to portal
register_subscribe()
# Disable unwanted repos
run('yum-config-manager --disable "*"', quiet=True)
# Enable rhel6 repos and sam repos
run('yum-config-manager --enable rhel-6-server-rpms')
run('yum-config-manager --enable rhel-6-server-sam-rpms')
custom_url = get_config('samtools', 'custom_url')
if custom_url is not None:
create_repo(custom_url)
# Install sam
run('yum install -y katello-headpin-all')
# Run katello-configure
run('katello-configure --deployment=sam --user-pass=admin')
def __create_phase_lines(self, session):
"""
Create lines for the different phases in the session. Green ones for the start of a phase and red ones for the
end.
:param session: The session this phase lines should represent.
:type session: Session
"""
config = helper.get_config(session.session_num)
if config is None:
return
# Get all the times of all phases and creates a line for each one.
for start, end in config.phase_timestamps.itervalues():
start = helper.csvtime_to_float(session.date,
start) - session.start_time
end = helper.csvtime_to_float(session.date,
end) - session.start_time
for time, color in [(start, global_values.COLOR_PHASE_GREEN),
(end, global_values.COLOR_PHASE_RED)]:
pos_x = (time / self.__duration) * self.__time_slider.width
line_node = avg.LineNode(color=color,
pos1=(pos_x, 25),
pos2=(pos_x, 48),
strokewidth=2,
opacity=0.8,
active=True,
parent=self.__time_bar)
self.__phase_lines[time] = line_node
def thread_decode(test_path, vocab, FLAGS):
sess = tf.Session(config=get_config())
if FLAGS.beam == True:
FLAGS.batch_size = FLAGS.beam_size
FLAGS.max_dec_steps = 1
print('batch size ', FLAGS.batch_size)
summarizationModel = PointerNet(FLAGS, vocab)
summarizationModel.build_graph()
saver = tf.train.Saver()
COORD = tf.train.Coordinator()
best_model = load_best_model(FLAGS.restore_path)
print('best model : {0}'.format(best_model))
saver.restore(sess, save_path=best_model)
batcher = Batcher(test_path,
vocab,
FLAGS,
single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
batches = batcher.cpu_fill_batch_queue() # 1 example repeated across batch
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
#切分数据
split_datas, count_array = split_batches(batches, FLAGS.work_num)
print("len batches : {0}".format(len(batches)))
assert len(split_datas) == FLAGS.work_num
work_threads = []
for i in range(FLAGS.work_num):
job = lambda: do(split_datas[i], summarizationModel, vocab, sess,
FLAGS, count_array[i], i)
t = threading.Thread(target=job)
t.start()
work_threads.append(t)
print('work : {0}'.format(i))
COORD.join(work_threads)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
def cmd_run_main_wrapper(args):
settings.TESTENV = args.env if args.env else 'bs'
settings.SAVE_DB = True
if args.testcases: settings.TESTCASES = args.testcases
elif args.json:
settings.TESTCASES = None
with open(
os.path.abspath(os.path.dirname(__file__)) + '/' + args.json,
"r") as read_it:
settings.TESTOBJECTS = json.load(read_it)
else:
settings.TESTCASES, settings.TESTOBJECTS = None, None
logger.setLevel(logging.DEBUG) if args.verbose else logger.setLevel(
logging.INFO)
settings.DRY_RUN = args.dry_run if args.dry_run else False
settings.FORCE_RERUN = args.force if args.force else False
get_config()
cmd_run_bs_main() if settings.TESTENV == 'bs' else cmd_run_local_main()
def cmd_runlocal_main_wrapper(args):
settings.TESTENV = 'local'
settings.DRY_RUN = args.dry_run if args.dry_run else False
settings.ENABLE_JUNIT = args.junit if args.junit else False
settings.CATCH_FAIL = args.catch_fail if args.catch_fail else False
logger.setLevel(logging.DEBUG)
get_config()
if args.testcases: settings.TESTCASES = args.testcases
elif args.all:
settings.TESTCASES = []
for key, value in settings.dataJson.items():
settings.TESTCASES.append(key)
elif args.all_live:
settings.TESTCASES = []
for key, value in settings.dataJson.items():
if settings.dataJson[key]['isLive']:
settings.TESTCASES.append(key)
cmd_runlocal_main()
def get(self, tenant_id):
if not tenant_id:
error = "Tenant not provided"
logging.error(error)
return {"error": error}, 400
response = {'config': get_config('configuration.json', tenant_id)}
return jsonify(response)
def create_plugin(name, version, clazz, **kwargs):
config = helper.get_config()
return Thread_Plugin(client=clazz(),
name=name,
version=version,
bg_host=config.bg_host,
bg_port=config.bg_port,
ssl_enabled=config.ssl_enabled,
**kwargs)
def run_training():
print('batch size', FLAGS.batch_size)
summarizationModel = PointerNet(FLAGS, vocab)
summarizationModel.build_graph()
batcher = Batcher(FLAGS.data_path,
vocab,
FLAGS,
single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
val_batcher = Batcher(FLAGS.val_data_path,
vocab,
FLAGS,
single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
sess = tf.Session(config=get_config())
sess.run(tf.global_variables_initializer())
eval_max_reward = -float('inf')
saver = tf.train.Saver(max_to_keep=10)
if FLAGS.restore_path:
print('loading params...')
saver.restore(sess, FLAGS.restore_path)
epoch = FLAGS.epoch
step = 0
patient = FLAGS.patient
while epoch > 0:
batches = batcher.fill_batch_queue()
print('load batch...')
for batch in batches:
print('start training...')
step += 1
feed_dict = make_feed_dict(summarizationModel, batch)
loss, _ = sess.run(
[summarizationModel.loss, summarizationModel.train_op],
feed_dict)
print("epoch : {0}, step : {1}, loss : {2}".format(
abs(epoch - FLAGS.epoch), step, loss))
if step % FLAGS.eval_step == 0:
eval_reward = run_eval(summarizationModel, val_batcher, sess)
print('eval reward ', eval_reward)
if eval_max_reward < eval_reward:
if not os.path.exists(FLAGS.checkpoint):
os.mkdir(FLAGS.checkpoint)
saver.save(sess,
save_path=os.path.join(
FLAGS.checkpoint,
'model_{0}_{1}.ckpt'.format(
step, eval_reward)))
eval_max_reward = eval_reward
patient = FLAGS.patient
print('eval max reward : {0}'.format(eval_max_reward))
if patient < 0:
break
if eval_max_reward - eval_reward > FLAGS.threshold:
patient -= 1
def cmd_runbs_main_wrapper(args):
settings.IS_RUNBS = True
settings.TESTENV = 'bs'
settings.ENABLE_JUNIT = args.junit if args.junit else False
logger.setLevel(logging.INFO) if args.save_db else logger.setLevel(
logging.INFO)
if args.testcases: settings.TESTCASES = args.testcases
elif args.json:
settings.TESTCASES = None
with open(
os.path.abspath(os.path.dirname(__file__)) + '/' + args.json,
"r") as read_it:
settings.TESTOBJECTS = json.load(read_it)
else:
settings.TESTCASES, settings.TESTOBJECTS = None, None
settings.DRY_RUN = args.dry_run if args.dry_run else False
settings.SAVE_DB = args.save_db if args.save_db else False
logger.setLevel(logging.INFO) if settings.SAVE_DB else logger.setLevel(
logging.DEBUG)
get_config()
cmd_runbs_main()
def register_subscribe():
"""Registers and subscribes to portal
Note:
Following samtools.properties config values are must to continue:
- rh_portal_username
- rh_portal_password
"""
rh_portal_username = get_config('samtools', 'rh_portal_username')
rh_portal_password = get_config('samtools', 'rh_portal_password')
if rh_portal_username is None or rh_portal_password is None:
print ('Missing Parameters: rh_portal_username AND rh_portal_password '
'must be defined in samtools.properties to continue '
'installation')
sys.exit(1)
# Subscribe to RH portal
run('subscription-manager register --username="******" --password="******" '
'--autosubscribe --force'.format(rh_portal_username,
rh_portal_password))
def _initialise(self, args):
logging.debug('Starting client mode checks on config file')
config = get_config(args.config_file)
server_url = get_server_url(config)
self._proxy = xmlrpclib.ServerProxy(server_url)
self._hostname = gethostname()
self._artifact = args.artifact
self._version = args.version
logging.debug('Client checks passed')
def _initialise(self, args):
logging.debug('Starting client mode checks on config file')
config = get_config(args.config_file)
server_url = get_server_url(config)
self._proxy = xmlrpclib.ServerProxy(server_url)
self._hostname = gethostname()
self._running = threading.Event()
self._running.clear()
logging.debug('Client checks passed')
def send_mail(subject, message, attachment_path=None):
config = helper.get_config()
print(config)
attachment = None
if attachment_path is not None:
attachment = [{"attachment", open(attachment_path, "rb")}]
return requests.post(config['mailgun_api_url'],
auth=("api", config['mailgun_key']),
data={
"from": config['mailgun_sender'],
"to": [config['mailgun_sample_receiver']],
"subject": subject,
"text": message
},
files=attachment)
def create_plugin(name, version, clazz, **kwargs):
config = helper.get_config()
# Brewtils 3.2.0ish has a bug that breaks when reusing the same client class
# It's been fixed in brewtils, but fix it here so we can run tests on that version
# See https://github.com/beer-garden/beer-garden/issues/1014
if hasattr(clazz, "current_request"):
delattr(clazz, "current_request")
if hasattr(clazz, "_current_request"):
delattr(clazz, "_current_request")
return Thread_Plugin(client=clazz(),
name=name,
version=version,
bg_host=config.bg_host,
bg_port=config.bg_port,
ssl_enabled=config.ssl_enabled,
**kwargs)
def test_merged_config(self):
config_dir = normpath(join(self.current_dir, "config"))
filename = 'unit_test_secrets.ini'
config = get_config(config_dir, filename)
# Test options that are specified only in 'unit_test_secrets.ini'
self.assertEqual(config.get('secrets', 'SECRET_KEY'), 'k#%(x)s$y')
self.assertEqual(config.get('database', 'DATABASE_PASSWORD'), 'bbb')
# Test options that are specified only in the included file.
self.assertEqual(config.get('database', 'DATABASE_USER'), 'tribe')
self.assertEqual(config.get('database', 'DATABASE_PORT'), '5432')
# Test options that are specified in both unit_test_secrets.ini
# and its included file. (unit_test_secrets.ini should always
# take precedence.)
self.assertEqual(config.get('database', 'DATABASE_NAME'), '')
self.assertEqual(config.get('database', 'DATABASE_HOST'), 'aaa')
self.assertEqual(config.get('debug', 'ALLOWED_HOSTS'),
'localhost 127.0.0.1')
def install_from_repo():
"""Task to install SAM from repo URL
Note:
Following samtools.properties config values are must to continue:
- rh_portal_username
- rh_portal_password
- base_url
"""
base_url = get_config('samtools', 'base_url')
if base_url is None:
print ('Missing Parameters: base_url must be defined in '
'samtools.properties to continue installation')
sys.exit(1)
# Register and subscribe
register_subscribe()
# Disable unwanted repos
print 'Disabling all repos...'
run('yum-config-manager --disable "*"', quiet=True)
# Enable rhel6 repos only
run('yum-config-manager --enable rhel-6-server-rpms')
create_repo(base_url)
# Install sam
run('yum install -y katello-headpin-all')
# Run katello-configure
run('katello-configure --deployment=sam --user-pass=admin')
# Run ping test
run_ping_command()
def decode(test_path, rl):
sess = tf.Session(config=get_config())
if FLAGS.beam == True:
FLAGS.batch_size = FLAGS.beam_size
FLAGS.max_dec_steps = 1
print('batch size ', FLAGS.batch_size)
#if rl == False:
summarizationModel = PointerNet(FLAGS, vocab)
#elif rl==True:
# if FLAGS.gamma > 0:
# import rl_model_gamma
# summarizationModel = rl_model_gamma.RLNet(FLAGS, vocab)
# else:
# import rl_model
# summarizationModel = rl_model.RLNet(FLAGS, vocab)
summarizationModel.build_graph()
saver = tf.train.Saver()
best_model = load_best_model(FLAGS.restore_path)
print('best model : {0}'.format(best_model))
saver.restore(sess, save_path=best_model)
counter = 0
batcher = Batcher(test_path,
vocab,
FLAGS,
single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
batches = batcher.fill_batch_queue(
is_training=False) # 1 example repeated across batch
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
if FLAGS.beam == False:
for batch in batches:
article = batch.original_articles[0]
original_abstract_sents = batch.original_abstracts_sents # list of strings
#print('*****************start**************')
best_hyps = beam_search.run_greedy_search(sess, summarizationModel,
vocab, batch)
output_ids = [[int(t) for t in best_hyp.tokens[1:]]
for best_hyp in best_hyps]
decoded_words = data.outputids2words_greedy(
output_ids, vocab,
(batch.art_oovs[0] if FLAGS.pointer_gen else None))
decoded_words = remove_stop_index(decoded_words, data)
write_for_rouge_greedy(
original_abstract_sents, decoded_words, article, counter,
FLAGS.dec_path, FLAGS.ref_path, FLAGS.all_path
) # write ref summary and decoded summary to file, to eval with pyrouge later
counter += FLAGS.batch_size # this is how many examples we've decoded
print('counter ... ', counter)
if counter % (5 * 64) == 0:
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
else:
for batch in batches:
article = batch.original_articles[0]
original_abstract_sents = batch.original_abstracts_sents[
0] # list of strings
#print('*****************start**************')
best_hyps = beam_search.run_beam_search(sess, summarizationModel,
vocab, batch)
#print('best hyp : {0}'.format(best_hyp))
output_ids = [int(t) for t in best_hyps.tokens[1:]]
decoded_words = data.outputids2words_beam(
output_ids, vocab,
(batch.art_oovs[0] if FLAGS.pointer_gen else None))
try:
fst_stop_idx = decoded_words.index(
data.STOP_DECODING) # index of the (first) [STOP] symbol
decoded_words = decoded_words[:fst_stop_idx]
except ValueError:
decoded_words = decoded_words
#decoded_words = ' '.join(decoded_words)
write_for_rouge_beam(
original_abstract_sents, decoded_words, article, counter,
FLAGS.dec_path, FLAGS.ref_path, FLAGS.all_path
) # write ref summary and decoded summary to file, to eval with pyrouge later
counter += 1 # this is how many examples we've decoded
print('counter ... ', counter)
if counter % 100 == 0:
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
def run_rl_training():
summarizationModel = RLNet(FLAGS, vocab)
summarizationModel.build_graph()
batcher = Batcher(FLAGS.data_path,
vocab,
FLAGS,
single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
val_batcher = Batcher(FLAGS.val_data_path,
vocab,
FLAGS,
single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
sess = tf.Session(config=get_config())
saver = tf.train.Saver(max_to_keep=100)
if FLAGS.restore_rl_path:
saver.restore(sess, FLAGS.restore_rl_path)
else:
sess.run(tf.global_variables_initializer())
sess.run(
loading_variable([v for v in tf.trainable_variables()],
reader_params(load_best_model(
FLAGS.restore_path))))
print('loading params...')
epoch = FLAGS.epoch
step = 0
eval_loss = float('inf')
while epoch > 0:
batches = batcher.fill_batch_queue()
for batch in batches:
step += 1
sampled_sentence_r_values = []
greedy_sentence_r_values = []
feed_dict = make_feed_dict(summarizationModel, batch)
to_return = {
'sampled_sentences':
summarizationModel.sampled_sentences,
'greedy_search_sentences':
summarizationModel.greedy_search_sentences
}
ret_dict = sess.run(to_return, feed_dict)
# calculate reward
for sampled_sentence, greedy_search_sentence, target_sentence in zip(
ret_dict['sampled_sentences'],
ret_dict['greedy_search_sentences'], batch.target_batch):
assert len(sampled_sentence[0]) == len(target_sentence) == len(
greedy_search_sentence[0])
reference_sent = ' '.join([str(k) for k in target_sentence])
sampled_sent = ' '.join([str(k) for k in sampled_sentence[0]])
sampled_sentence_r_values.append(
reward_function(reference_sent, sampled_sent))
greedy_sent = ' '.join(
[str(k) for k in greedy_search_sentence[0]])
greedy_sentence_r_values.append(
reward_function(reference_sent, greedy_sent))
to_return = {
'train_op': summarizationModel.train_op,
'pgen_loss': summarizationModel._pgen_loss,
'rl_loss': summarizationModel._rl_loss,
'loss': summarizationModel.loss
}
to_return['s_r'] = summarizationModel._sampled_sentence_r_values
to_return['g_r'] = summarizationModel._greedy_sentence_r_values
feed_dict[summarizationModel.
_sampled_sentence_r_values] = sampled_sentence_r_values
feed_dict[summarizationModel.
_greedy_sentence_r_values] = greedy_sentence_r_values
feed_dict[summarizationModel._eta] = 0.5
res = sess.run(to_return, feed_dict)
print(
'step : {0},pgen_loss : {1}, rl_loss : {2}, loss : {3}, reward : {4}'
.format(step, res['pgen_loss'], res['rl_loss'], res['loss'],
np.sum(res['s_r'] - res['g_r'])))
if step % FLAGS.eval_step == 0:
eval_ = run_rl_eval(summarizationModel, val_batcher, sess, 0.5)
if eval_ < eval_loss:
if not os.path.exists(FLAGS.checkpoint):
os.mkdir(FLAGS.checkpoint)
saver.save(sess,
save_path=os.path.join(
FLAGS.checkpoint,
'model_{0}_{1}.ckpt'.format(step, eval_)))
eval_loss = eval_
patient = FLAGS.patient
print('eval loss : ', eval_loss)
if patient < 0:
break
if eval_ - eval_loss > FLAGS.threshold:
patient -= 1
def create_session(session_number):
config = get_config(session_number)
video_timestamps = {} if config is None else config.video_timestamps
tracking_filename = input_filename_prefix + str(
session_number) + "_users.csv"
touch_filename = input_filename_prefix + str(session_number) + "_touch.csv"
device_touch_filename = input_filename_prefix + str(
session_number) + "_device_touch.csv"
device_filename = input_filename_prefix + str(
session_number) + "_device.csv"
number_of_users = get_number_of_users(tracking_filename, touch_filename)
print "number of users in session: ", number_of_users
user_pitch_offsets = [0] * number_of_users
session_time_offset = 0.0 if config is None else config.session_time_offset
video_time_offset = 0.0 if config is None else config.video_offsets
current_dir = os.getcwd()
os.chdir("{}{}".format(input_video_dir, session_number))
filenames = glob.glob(input_video_prefix + str(session_number) +
"_video*.mp4")
if len(filenames) < 1 and len(video_timestamps) < 1:
print "ERROR: cannot find video files or the config variables for the session " + str(
session_number) + "."
assert False
if len(filenames) > 0:
print "video file for session " + str(
session_number) + ":", filenames[0]
video_filenames = [
"{}{}/{}".format(input_video_dir, session_number, fn)
for fn in filenames
]
os.chdir(current_dir)
video_start_times = []
date = None
video_filenames = video_filenames if len(
video_filenames) > 0 else video_timestamps.keys()
for video_filename in video_filenames:
possible_timestamps = [
ts for fn, ts in video_timestamps.iteritems()
if fn in video_filename
]
if len(possible_timestamps) > 0:
video_start_time_unix = possible_timestamps[0]
else:
video_start_time_unix = float(video_filename.split("_")[-1][:-4])
print "video start time unix: {0: .6f}".format(video_start_time_unix)
date_time = datetime.datetime.fromtimestamp(
video_start_time_unix).strftime('%Y-%m-%d %H:%M:%S.%f')
print "video start time:", date_time
date_split = date_time.split(' ')
if date is None:
date = date_split[0]
video_start_times.append(date_split[1])
return Session(session_num=session_number,
level_num=1,
data_dir=current_dir,
optitrack_filename=tracking_filename,
touch_filename=touch_filename,
device_touch_filename=device_touch_filename,
device_filename=device_filename,
video_filenames=video_filenames,
date=date,
video_start_times=video_start_times,
video_time_offset=video_time_offset,
session_time_offset=session_time_offset,
num_users=number_of_users,
user_pitch_offsets=user_pitch_offsets)
def main():
config = helper.get_config()
init(config)
logging.info('Tesco Scraper has starterd')
tesco.scrape_tesco()
from pathlib import Path
import torch
from onnxruntime import InferenceSession, SessionOptions
from transformers import pipeline, AutoTokenizer
import numpy as np
from helper import get_config, group_sub_entities, group_entities, render_ner_html_custom
from urllib import request
from quant_flair_model import QuantizableLanguageModel
from transformers.models.bert.tokenization_bert import BertTokenizer
flair.models.LanguageModel = QuantizableLanguageModel
flair.models.language_model.LanguageModel = QuantizableLanguageModel
IGNORE_LABELS = set(["O"])
config = get_config()
colors = {
"PRS": "#F7FF53", # YELLOW
"PER": "#F7FF53", # YELLOW
"ORG": "#E8902E", # ORANGE (darker)
"LOC": "#FF40A3", # PINK
"MISC": "#4647EB", # PURPLE
"EVN": "#06b300", # GREEN
"MSR": "#FFEDD5", # ORANGE (lighter)
"TME": "#ff7398", # PINK (pig)
"WRK": "#c5ff73", # YELLOW (REFLEX)
"OBJ": "#4ed4b0", # TURQUOISE
"O": "#ddd", # GRAY
}
return True
if __name__ == '__main__':
if getpass.getuser() == "root":
print "You're root, Please run as normal user"
sys.exit()
#Register to socket to prevent run many instance at one time
helper.get_lock('bing_wallpaper')
freeze_support()
temp_config = helper.get_config()
if temp_config is not None:
print "Found previous config"
if 'curr_mode' in temp_config and temp_config['curr_mode'] in all_modes:
config['curr_mode'] = temp_config['curr_mode']
if 'timer_milliseconds' in temp_config:
config['timer_milliseconds'] = temp_config['timer_milliseconds']
if 'auto_download' in temp_config:
config['auto_download'] = temp_config['auto_download']
date_ranges = helper.get_range_dates(config['curr_mode'])
def run_rl_training_gamma(FLAGS, vocab):
summarizationModel = RLNet(FLAGS, vocab)
summarizationModel.build_graph()
batcher = Batcher(FLAGS.data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after)
val_batcher = Batcher(FLAGS.val_data_path, vocab, FLAGS, single_pass=FLAGS.single_pass,
decode_after=FLAGS.decode_after)
sess = tf.Session(config=get_config())
saver = tf.train.Saver(max_to_keep=10)
if FLAGS.restore_rl_path:
print('restore rl model...')
saver.restore(sess, FLAGS.restore_rl_path)
else:
sess.run(tf.global_variables_initializer())
sess.run(
loading_variable([v for v in tf.trainable_variables()], reader_params(load_best_model(FLAGS.restore_path))))
print('loading params...')
epoch = FLAGS.epoch
step = 0
patient = FLAGS.patient
eval_max_reward = -float('inf')
while epoch > 0:
batches = batcher.fill_batch_queue()
for batch in batches:
step += 1
sampled_sentence_r_values = []
greedy_sentence_r_values = []
feed_dict = make_feed_dict(summarizationModel, batch)
to_return = {
'sampled_sentences': summarizationModel.sampled_sentences,
}
ret_dict = sess.run(to_return, feed_dict)
Rs = []
# calculate reward
for sampled_sentence, target_sentence in zip(ret_dict['sampled_sentences'],batch.target_batch):
# print('sampled : ',sampled_sentence)
# print('target : ', target_sentence)
reward = compute_reward(sampled_sentence[0], target_sentence)
R = 0
R_l = []
for r in reward[::-1]:
R = r + FLAGS.gamma * R
R_l.insert(0,R)
#avg = np.mean(R_l)
#R_l = list(map(lambda a:a-avg, R_l))
Rs.append(R_l)
to_return = {
'train_op': summarizationModel.train_op,
'pgen_loss': summarizationModel._pgen_loss,
'rl_loss': summarizationModel._rl_loss,
'loss': summarizationModel.loss
}
to_return['reward'] = summarizationModel._reward
feed_dict[summarizationModel._reward] = Rs
feed_dict[summarizationModel._eta] = 0.1
res = sess.run(to_return, feed_dict)
print('step : {0}, pgen_loss : {1}, rl_loss : {2}, loss : {3}, reward : {4}'.format(step,res['pgen_loss'], res['rl_loss'],
res['loss'],
np.mean(res['reward'],axis=0)[0]
))
if step % FLAGS.eval_step == 0:
#eval_ = run_rl_eval_gramma(summarizationModel, val_batcher, sess, 0.5)
eval_reward = run_eval(summarizationModel, val_batcher, sess)
print('eval reward ', eval_reward)
if eval_reward > eval_max_reward:
if not os.path.exists(FLAGS.checkpoint): os.mkdir(FLAGS.checkpoint)
saver.save(sess, save_path=os.path.join(FLAGS.checkpoint, 'model_{0}_{1}.ckpt'.format(step, eval_reward)))
eval_max_reward = eval_reward
patient = FLAGS.patient
print('eval max ward ', eval_max_reward)
if patient < 0:
break
if eval_max_reward - eval_reward > FLAGS.threshold:
patient -= 1
def main(layer_sizes= [2, 1, 1], L2_reg= 0.0001, param_scale = 0.1, batch_size = 20, num_epochs = 10, step_size = 1.0, hyper_iter =450, hyper_step_size = 2.5, hyper_decay = 0.333, hyper_decay_after = 300, hyper_decay_every = 300, hyper_L2_reg = 0.000002, rank_loss_scaling_factor = 20.0, mse_tolerance = 0.01, outputFname = '/tmp/results.txt', sorted_c_r_list = None, init_params_list=[], combined_init_params = None, mask_params = None, random_state = None, hyper_train = True, fake_train_images = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]), fake_train_labels = np.array([[1.0],[0.0],[0.0],[1.0]]), fake_valid_images = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]), fake_valid_labels = np.array([[1.0],[0.0],[0.0],[1.0]])):
#import pickle
#with open("mnist19_try.pkl", 'rb') as f_rd:
# sorted_c_r_list = pickle.load(f_rd)
if False: #layer_sizes is None:
import helper
args = helper.get_config()
# Model parameters
layer_sizes = args.layer_sizes
L2_reg = args.L2_reg
# Training parameters
param_scale = args.param_scale #Very important param. Should be low
batch_size = args.batch_size #inner loop batch size. Ignore if number of training examples < batch size
num_epochs = args.num_epochs #inner loop epochs
step_size = args.step_size #Very important param.
#This is the not hyper_epochs. If hyper_iter > num_batches, then batches repeat.
#Ideally, we should have hyper_iter = num_batches * hyper_epochs.
hyper_iter = args.hyper_iter
hyper_step_size = args.hyper_step_size
hyper_L2_reg = args.hyper_L2_reg
rank_loss_scaling_factor = args.rank_loss_scaling_factor
#if DEBUG: print("Loading training data...")
#N, actual_train_images, actual_train_labels, actual_test_images, actual_test_labels = load_mnist()
#Following should be float values. Required by autograd. Cannot take gradient w.r.t integers
fake_train_images = np.array(args.fake_train_images)
fake_train_labels = np.array(args.fake_train_labels)
#fake_test_images = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]])
#fake_test_labels = np.array([[1.0], [0.0], [0.0], [1.0]]) #np.zeros((10,1))
#fake_test_images = np.array([[1.0, 0.0], [0.0, 1.0]])
#fake_test_labels = np.array([[0.0], [0.0]]) #np.zeros((10,1))
test_batch_size = batch_size
# Define training objective
def objective(train_images,train_labels,params, mask_param, c_list, iter):
num_batches = int(np.ceil(len(train_images) / batch_size))
def batch_indices(iter):
idx = iter % num_batches
return slice(idx * batch_size, (idx+1) * batch_size)
idx = batch_indices(iter)
#if DEBUG: print ('In train objective')
return -log_posterior_binary(params, mask_param, train_images[idx], train_labels[idx], L2_reg, c_list)
# Define test objective
def test_objective(params, mask_param, test_images, test_labels, c_list, iter):
num_batches = int(np.ceil(len(test_images) / test_batch_size))
def batch_indices(iter):
idx = iter % num_batches
return slice(idx * batch_size, (idx+1) * test_batch_size)
idx = batch_indices(iter)
if DEBUG: print ('Test BCE loss')
return [-x for x in log_posterior_binary_test(params, mask_param, test_images[idx], test_labels[idx], L2_reg, c_list)]
def inner_loop(train_images,train_labels, init_params, mask_param, c_list):
def objective_wrapper(params,iter): #sgd in line 108 magically supplies the iter parameter
return objective(train_images,train_labels,params, mask_param, c_list, iter)
objective_grad = grad(objective_wrapper)
num_batches = int(np.ceil(len(train_images) / batch_size))
##if DEBUG: print(" Epoch | Train accuracy | Test accuracy ")
#def print_perf(params, iter, gradient):
# if iter % num_batches == 0:
# train_acc = accuracy(params, train_images, train_labels, c_list)
# test_acc = accuracy(params, test_images, test_labels, c_list)
#if DEBUG: print("{:15}|{:20}|{:20}".format(iter//num_batches, train_acc, test_acc))
# The optimizers provided can optimize lists, tuples, or dicts of parameters.
#if DEBUG: print("DOING INNER LOOP SGD")
optimized_params = adam(objective_grad, init_params, step_size=step_size,
num_iters=num_epochs * num_batches,callback=None)
#test_loss = test_objective(optimized_params, train_images, train_labels, c, 0)
return optimized_params
trained_weights_dict = dict()
test_loss_dict = dict()
#sorted_c_r_list = [(0.32, 0.95), (1.2, 0.91), (1.6, 0.87)]
#sorted_c_r_list = [(0.45444444444444443, 0.9613), (0.12, 0.957), (1.1211111111111112, 0.9369), (1.5655555555555554, 0.9167), (2.01, 0.9076), (0.01, 0.8999)]#, (0.2322, 0.9727)]
#c_1 = 0.2322
#c_2 = 0.32
def check_success(scores):
sorted_scores = []
for idx, val in enumerate(scores):
sorted_scores.append((idx, val))
sorted_scores.sort(key=lambda x: x[1], reverse=True)
for idx, val in enumerate(sorted_scores):
if (idx != val[0]):
if DEBUG: print ('Failed')
return False
if DEBUG: print ('Success in rank')
return True
#init_params_list = generate_init_params(sorted_c_r_list, param_scale, layer_sizes)
def test_loss(train_images, init_params, mask_param, c_list, t):
[train_images, valid_images] = np.split(train_images, 2) #Extracting train and valid images
trained_weights = inner_loop(train_images,fake_train_labels, init_params, mask_param, c_list)
return (test_objective(trained_weights, mask_param, valid_images, fake_valid_labels, c_list, t), trained_weights)
def pairwise_rankloss(score_1, score_2, target):
scaling_factor = rank_loss_scaling_factor
if (score_1 > score_2):
scaling_factor = 0.0
diff = (score_1 - score_2)*scaling_factor
rank_loss = -target*diff + np.log(1 + np.exp(diff))
#print ('Score1, Score2, Rank Loss', score_1, score_2, rank_loss)
return rank_loss
def rankloss(scores):
total_rank_loss = 0.0
rank_loss_count = 0
for i in range(len(scores)-1):
for j in range(i, len(scores)):
if i !=j:
score_1 = scores[i]
score_2 = scores[j]
target = 1.0 #This is assuming scores are already placed in their actual/target index (based on sorted_c_r_list ordering)
#1.0 means score_1 > score_2 and 0.0 means score_1 < score_2
rank_loss = pairwise_rankloss(score_1, score_2, target)
total_rank_loss += rank_loss
rank_loss_count += 1 #Total number of rank comparisons
if DEBUG: print ('Pairs', i, j, 'Score', score_1, score_2, rank_loss)
total_rank_loss = total_rank_loss/rank_loss_count
if DEBUG: print ('Total Rank Loss', total_rank_loss)
return total_rank_loss
def mse_loss(scores):
mean, std = compute_mean_std(scores)
normalized_pred_scores = [ (x-mean)/std for x in scores] #Whiten the data
for index, (i, j) in enumerate(zip(scores, normalized_pred_scores)):
if DEBUG: print (sorted_c_r_list[index][0], 'Predicted scores', i, 'Normalized Pred scores', j)
true_scores = [ r for (c,r) in sorted_c_r_list ] #Convert accuracy to error
mean, std = compute_mean_std(true_scores)
normalized_true_scores = [ (x-mean)/std for x in true_scores]
for index, (i, j) in enumerate(zip(true_scores, normalized_true_scores)):
if DEBUG: print (sorted_c_r_list[index][0], 'True scores', i, 'Normalized True scores', j)
mse = np.mean(np.array([(i-j)**2 for i, j in zip(normalized_true_scores, normalized_pred_scores)]))
if DEBUG: print ('Mean Squared Error', mse)
#print ('True Scores and Normalized scores', true_scores, normalized_true_scores)
#print ('Pred Scores and Normalized scores', scores, normalized_pred_scores)
return mse
def get_scores(train_images, t):
score_list = [] #Scores will be appended in the same order as the sorted_c_r_list
trained_weights_list = []
#for idx, (c, r) in enumerate(sorted_c_r_list):
init_idx = get_init_idx(combined_init_params, mask_params, random_state)
c_list = [c for (c, r) in sorted_c_r_list]
test_loss_list, trained_weights_list = test_loss(train_images, combined_init_params[init_idx], mask_params[init_idx], c_list, t) #Reuse init_params
score_list = [-loss for loss in test_loss_list] #Validation loss needs to be inverted to get rank score
#score_list.append(score)
#trained_weights_list.append(trained_weights)
return score_list, trained_weights_list
#Train the network and save the weights
def hyper_objective(train_images, t): #t is test iteration
if DEBUG: print ('Hyper iteration', t)
score_list, trained_weights_list = get_scores(train_images, t)
success_flag = check_success(score_list)
if success_flag == True:
if DEBUG: print ('Successful Learnt Images', train_images)
#with open (outputFname, 'a') as fwr:
# fwr.writelines("Hyperepoch "+str(t)+" Rank order correct "+str(train_images)+"\n")
if (t==0 or t==hyper_iter-1 or success_flag == True):
if DEBUG: print ('Hyper Iteration', t, 'scores')
if DEBUG: [print (score) for score in score_list]
if (t==hyper_iter-1 or success_flag == True):
if DEBUG==True:
for idx, trained_weights in enumerate(trained_weights_list):
print ('Final Weights', idx, trained_weights)
#print ('Final trained_images', train_images)
#Write to output file from inside the petridish. This is to ensure that we capture the result before we lose in the next iter
mse = mse_loss(score_list)
#if mse < mse_tolerance:
# with open (outputFname, 'a') as fwr:
# fwr.writelines("Hyperepoch "+str(t)+" MSE Loss converged "+str(mse)+str(train_images)+"\n")
return mse+l2_norm(train_images)*hyper_L2_reg
def get_results(learnt_images, train, iteration=0):
score_list, trained_weights_list = get_scores(learnt_images, t=iteration) #t is not useful unless number of images > batch_size
success_flag = check_success(score_list)
return success_flag, score_list, trained_weights_list
if hyper_train == True:
hyper_objective_grad = grad(hyper_objective)
h_i = 0 #Initializing hyper-iteration
optimized_params = np.concatenate((fake_train_images, fake_valid_images), axis=0) #Valid images and labels have the same dimension as the train images and labels
#Following while loop runs Hyper-training with hyper step size decay
while h_i < hyper_iter:
hyper_lr, num_iters = compute_lr(init_lr=hyper_step_size, current_iter=h_i, decay_after=hyper_decay_after, decay_factor=hyper_decay, decay_every=hyper_decay_every)
print ('Hyper Learning Rate', hyper_lr, num_iters)
optimized_params = adam(hyper_objective_grad, optimized_params, step_size=hyper_lr,num_iters=num_iters)
h_i = h_i + num_iters
#optimized_params = adam(hyper_objective_grad, optimized_params, step_size=hyper_step_size/3,num_iters=300)
if DEBUG: print ('Learned training images', optimized_params)
success_flag, score_list, trained_weights_list = get_results(optimized_params, train=hyper_train, iteration=hyper_iter)
mse = mse_loss(score_list)
#Since the train_images and train_labels have the same dimensionality,
#we can split the concatenated optimized_params into two equal parts
#Added validation images and labels that have the same dimensionality as the training images and labels
[optimized_train_images, optimized_valid_images] = np.split(optimized_params, 2)
return mse, success_flag, score_list, optimized_train_images, fake_train_labels, optimized_valid_images, fake_valid_labels
#Test. Only return scores
else:
optimized_params = np.concatenate((fake_train_images, fake_valid_images), axis=0) #Valid images and labels have the same dimension as the train images and labels
success_flag, score_list, trained_weights_list = get_results(optimized_params, train=hyper_train, iteration=0)
mse = mse_loss(score_list)
#Since the train_images and train_labels have the same dimensionality,
#we can split the concatenated optimized_params into two equal parts
#Added validation images and labels that have the same dimensionality as the training images and labels
[optimized_train_images, optimized_valid_images] = np.split(optimized_params, 2)
return mse, success_flag, score_list, optimized_train_images, fake_train_labels, optimized_valid_images, fake_valid_labels
def _initialise(self, args):
logging.debug('Starting server mode checks on config file')
config = get_config(args.config_file)
self._clients = {}
self._backup_location = ''
self._port = 9001
if config.has_option('server', 'backup_location'):
self._backup_location = config.get('server', 'backup_location')
if not os.path.isdir(self._backup_location):
logging.warn("Backup location '%s' does not exist, attempting to create it" % self._backup_location)
try:
os.makedirs(self._backup_location)
except:
raise RuntimeError('Could not create the requested backup location')
else:
raise RuntimeError('Backup location not specified in config file')
if not config.has_option('server', 'port'):
logging.warn('No port specified, using 9001')
else:
try:
self._port = int(config.get('server', 'port'))
except:
raise RuntimeError('Server port must be an integer')
for section in config.sections():
if not section == 'server':
logging.debug('Found a client: %s' % section)
if not config.has_option(section, 'artifacts'):
raise RuntimeError('Client sections require an artifacts option')
artifacts_string = config.get(section, 'artifacts')
artifacts = {}
if artifacts_string == '':
raise RuntimeError('Artifacts list cannot be empty')
for artifact in artifacts_string.split(','):
logging.debug('Found an artifact: %s' % artifact)
file_based = True
filename = ''
backup_command = ''
restore_command = ''
cleanup = False
versions = 1
interval = '1h'
if config.has_option(section, artifact + '_filename'):
filename = config.get(section, artifact + '_filename')
else:
raise RuntimeError("Artifacts must have at least a file specified. Error in client '%s'" % section)
if config.has_option(section, artifact + '_backup_command'):
file_based = False
backup_command = config.get(section, artifact + '_backup_command')
if config.has_option(section, artifact + '_restore_command'):
restore_command = config.get(section, artifact + '_restore_command')
else:
raise RuntimeError("A backup command was specified without a restore command. A restore command is required in client '%s', artifact '%s'" % (section, artifact))
if config.has_option(section, artifact + '_cleanup'):
tmp = config.get(section, artifact + '_cleanup')
if tmp.lower() == 'true':
cleanup = True
elif tmp.lower() == 'false':
cleanup = False
else:
raise RuntimeError("Invalid option for cleanup in client '%s', artifact '%s'" % (section, artifact))
if config.has_option(section, artifact + '_versions'):
try:
versions = int(config.get(section, artifact + '_versions'))
except:
raise RuntimeError("Version option must be an integer in client '%s', artifact '%s'" % (section, artifact))
if config.has_option(section, artifact + '_interval'):
interval = config.get(section, artifact + '_interval')
regex = "^(\d+w ?)?(\d+d ?)?(\d+h ?)?(\d+m ?)?(\d+s ?)?$"
if not re.search(regex, interval):
raise RuntimeError("Interval option must in valid timedelta format. e.g. 1w2d3h4m. In client '%s', artifact '%s'" % (section, artifact))
artifacts[artifact] = {
'file_based': file_based,
'filename': filename,
'backup_command': backup_command,
'restore_command': restore_command,
'cleanup': cleanup,
'versions': versions,
'interval': interval
}
self._clients[section] = artifacts
if not len(self._clients) > 0:
raise RuntimeError('No clients specified')
self._server = None
def _initialise(self, args):
logging.debug('Starting server mode checks on config file')
config = get_config(args.config_file)
self._clients = {}
self._backup_location = ''
self._port = 9001
if config.has_option('server', 'backup_location'):
self._backup_location = config.get('server', 'backup_location')
if not os.path.isdir(self._backup_location):
logging.warn(
"Backup location '%s' does not exist, attempting to create it"
% self._backup_location)
try:
os.makedirs(self._backup_location)
except:
raise RuntimeError(
'Could not create the requested backup location')
else:
raise RuntimeError('Backup location not specified in config file')
if not config.has_option('server', 'port'):
logging.warn('No port specified, using 9001')
else:
try:
self._port = int(config.get('server', 'port'))
except:
raise RuntimeError('Server port must be an integer')
for section in config.sections():
if not section == 'server':
logging.debug('Found a client: %s' % section)
if not config.has_option(section, 'artifacts'):
raise RuntimeError(
'Client sections require an artifacts option')
artifacts_string = config.get(section, 'artifacts')
artifacts = {}
if artifacts_string == '':
raise RuntimeError('Artifacts list cannot be empty')
for artifact in artifacts_string.split(','):
logging.debug('Found an artifact: %s' % artifact)
file_based = True
filename = ''
backup_command = ''
restore_command = ''
cleanup = False
versions = 1
interval = '1h'
if config.has_option(section, artifact + '_filename'):
filename = config.get(section, artifact + '_filename')
else:
raise RuntimeError(
"Artifacts must have at least a file specified. Error in client '%s'"
% section)
if config.has_option(section,
artifact + '_backup_command'):
file_based = False
backup_command = config.get(
section, artifact + '_backup_command')
if config.has_option(section,
artifact + '_restore_command'):
restore_command = config.get(
section, artifact + '_restore_command')
else:
raise RuntimeError(
"A backup command was specified without a restore command. A restore command is required in client '%s', artifact '%s'"
% (section, artifact))
if config.has_option(section, artifact + '_cleanup'):
tmp = config.get(section, artifact + '_cleanup')
if tmp.lower() == 'true':
cleanup = True
elif tmp.lower() == 'false':
cleanup = False
else:
raise RuntimeError(
"Invalid option for cleanup in client '%s', artifact '%s'"
% (section, artifact))
if config.has_option(section, artifact + '_versions'):
try:
versions = int(
config.get(section, artifact + '_versions'))
except:
raise RuntimeError(
"Version option must be an integer in client '%s', artifact '%s'"
% (section, artifact))
if config.has_option(section, artifact + '_interval'):
interval = config.get(section, artifact + '_interval')
regex = "^(\d+w ?)?(\d+d ?)?(\d+h ?)?(\d+m ?)?(\d+s ?)?$"
if not re.search(regex, interval):
raise RuntimeError(
"Interval option must in valid timedelta format. e.g. 1w2d3h4m. In client '%s', artifact '%s'"
% (section, artifact))
artifacts[artifact] = {
'file_based': file_based,
'filename': filename,
'backup_command': backup_command,
'restore_command': restore_command,
'cleanup': cleanup,
'versions': versions,
'interval': interval
}
self._clients[section] = artifacts
if not len(self._clients) > 0:
raise RuntimeError('No clients specified')
self._server = None
import math
from datetime import datetime, timedelta
from csv_data_reader import get_data
from cStringIO import StringIO
from helper import map_kinect_to_device_coordinates, map_device_coordinates_to_wall_pixels, \
mapping_kinect_to_giant_coordinates, map_wall_pixels_to_device_coordinates, get_config
input_folder_prefix = '../data_logs/session_'
input_filename_prefix = 'session_'
output_filename_prefix = 'giant_session_'
# session_ids = ['0', '1', '2', '3', '4', '5', '6', '13', '14', '16']
session_ids = ['6']
for session_id in session_ids:
config = get_config(session_id)
joint_type_body_to_device = "OKS_SPINE_MID" if config is None else config.joint_type_body_to_device
joint_type_body_to_touch = "OKS_SPINE_SHOULDER" if config is None else config.joint_type_body_to_touch
device_id_count_threshold = 5 if config is None else config.device_id_count_threshold
max_time_delta_body_to_device = timedelta(
milliseconds=150
) if config is None else config.max_time_delta_body_to_device
max_dist_delta_body_to_device = 0.6 if config is None else config.max_dist_delta_body_to_device
max_time_delta_touch_to_inj = timedelta(
milliseconds=150
) if config is None else config.max_time_delta_touch_to_inj
max_time_delta_inj_to_de = timedelta(
milliseconds=2) if config is None else config.max_time_delta_inj_to_de
max_time_delta_body_to_touch = timedelta(
milliseconds=75
# Site name:
SITE_NAME = 'tribe'
# Settings files
settings_root = dirname(abspath(__file__))
config_dir = normpath(join(settings_root, 'config'))
# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
sys.path.append(PROJECT_ROOT)
########## END PATH CONFIGURATION
########## INI CONFIGURATION
config = get_config(config_dir, 'secrets.ini')
########## END INI CONFIGURATION
########## SECRET CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
# Note: This key only used for development and testing.
SECRET_KEY = config.get('secrets', 'SECRET_KEY')
########## END SECRET CONFIGURATION
########## DEBUG CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = config.getboolean('debug', 'DEBUG')
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = config.getboolean('debug', 'TEMPLATE_DEBUG')
