def cmd_autoupdate_main_wrapper(args): logger.info('Auto Update') settings.DRY_RUN = args.dry_run if args.dry_run else False settings.FORCE_RERUN = args.force if args.force else False settings.ENABLE_JUNIT = args.junit if args.junit else False # always use browserstack settings.TESTENV = 'bs' settings.SAVE_DB = True logger.setLevel(logging.DEBUG) if args.verbose else logger.setLevel( logging.INFO) get_config() cmd_autoupdate_main()
def cdn_install(): """Installs sam from cdn Note: Following samtools.properties config values are must to continue: - rh_portal_username - rh_portal_password """ # Register and subscribe the host to portal register_subscribe() # Disable unwanted repos run('yum-config-manager --disable "*"', quiet=True) # Enable rhel6 repos and sam repos run('yum-config-manager --enable rhel-6-server-rpms') run('yum-config-manager --enable rhel-6-server-sam-rpms') custom_url = get_config('samtools', 'custom_url') if custom_url is not None: create_repo(custom_url) # Install sam run('yum install -y katello-headpin-all') # Run katello-configure run('katello-configure --deployment=sam --user-pass=admin')
def __create_phase_lines(self, session): """ Create lines for the different phases in the session. Green ones for the start of a phase and red ones for the end. :param session: The session this phase lines should represent. :type session: Session """ config = helper.get_config(session.session_num) if config is None: return # Get all the times of all phases and creates a line for each one. for start, end in config.phase_timestamps.itervalues(): start = helper.csvtime_to_float(session.date, start) - session.start_time end = helper.csvtime_to_float(session.date, end) - session.start_time for time, color in [(start, global_values.COLOR_PHASE_GREEN), (end, global_values.COLOR_PHASE_RED)]: pos_x = (time / self.__duration) * self.__time_slider.width line_node = avg.LineNode(color=color, pos1=(pos_x, 25), pos2=(pos_x, 48), strokewidth=2, opacity=0.8, active=True, parent=self.__time_bar) self.__phase_lines[time] = line_node
def thread_decode(test_path, vocab, FLAGS): sess = tf.Session(config=get_config()) if FLAGS.beam == True: FLAGS.batch_size = FLAGS.beam_size FLAGS.max_dec_steps = 1 print('batch size ', FLAGS.batch_size) summarizationModel = PointerNet(FLAGS, vocab) summarizationModel.build_graph() saver = tf.train.Saver() COORD = tf.train.Coordinator() best_model = load_best_model(FLAGS.restore_path) print('best model : {0}'.format(best_model)) saver.restore(sess, save_path=best_model) batcher = Batcher(test_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) batches = batcher.cpu_fill_batch_queue() # 1 example repeated across batch print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) #切分数据 split_datas, count_array = split_batches(batches, FLAGS.work_num) print("len batches : {0}".format(len(batches))) assert len(split_datas) == FLAGS.work_num work_threads = [] for i in range(FLAGS.work_num): job = lambda: do(split_datas[i], summarizationModel, vocab, sess, FLAGS, count_array[i], i) t = threading.Thread(target=job) t.start() work_threads.append(t) print('work : {0}'.format(i)) COORD.join(work_threads) print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
def cmd_run_main_wrapper(args): settings.TESTENV = args.env if args.env else 'bs' settings.SAVE_DB = True if args.testcases: settings.TESTCASES = args.testcases elif args.json: settings.TESTCASES = None with open( os.path.abspath(os.path.dirname(__file__)) + '/' + args.json, "r") as read_it: settings.TESTOBJECTS = json.load(read_it) else: settings.TESTCASES, settings.TESTOBJECTS = None, None logger.setLevel(logging.DEBUG) if args.verbose else logger.setLevel( logging.INFO) settings.DRY_RUN = args.dry_run if args.dry_run else False settings.FORCE_RERUN = args.force if args.force else False get_config() cmd_run_bs_main() if settings.TESTENV == 'bs' else cmd_run_local_main()
def cmd_runlocal_main_wrapper(args): settings.TESTENV = 'local' settings.DRY_RUN = args.dry_run if args.dry_run else False settings.ENABLE_JUNIT = args.junit if args.junit else False settings.CATCH_FAIL = args.catch_fail if args.catch_fail else False logger.setLevel(logging.DEBUG) get_config() if args.testcases: settings.TESTCASES = args.testcases elif args.all: settings.TESTCASES = [] for key, value in settings.dataJson.items(): settings.TESTCASES.append(key) elif args.all_live: settings.TESTCASES = [] for key, value in settings.dataJson.items(): if settings.dataJson[key]['isLive']: settings.TESTCASES.append(key) cmd_runlocal_main()
def get(self, tenant_id): if not tenant_id: error = "Tenant not provided" logging.error(error) return {"error": error}, 400 response = {'config': get_config('configuration.json', tenant_id)} return jsonify(response)
def create_plugin(name, version, clazz, **kwargs): config = helper.get_config() return Thread_Plugin(client=clazz(), name=name, version=version, bg_host=config.bg_host, bg_port=config.bg_port, ssl_enabled=config.ssl_enabled, **kwargs)
def run_training(): print('batch size', FLAGS.batch_size) summarizationModel = PointerNet(FLAGS, vocab) summarizationModel.build_graph() batcher = Batcher(FLAGS.data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) val_batcher = Batcher(FLAGS.val_data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) sess = tf.Session(config=get_config()) sess.run(tf.global_variables_initializer()) eval_max_reward = -float('inf') saver = tf.train.Saver(max_to_keep=10) if FLAGS.restore_path: print('loading params...') saver.restore(sess, FLAGS.restore_path) epoch = FLAGS.epoch step = 0 patient = FLAGS.patient while epoch > 0: batches = batcher.fill_batch_queue() print('load batch...') for batch in batches: print('start training...') step += 1 feed_dict = make_feed_dict(summarizationModel, batch) loss, _ = sess.run( [summarizationModel.loss, summarizationModel.train_op], feed_dict) print("epoch : {0}, step : {1}, loss : {2}".format( abs(epoch - FLAGS.epoch), step, loss)) if step % FLAGS.eval_step == 0: eval_reward = run_eval(summarizationModel, val_batcher, sess) print('eval reward ', eval_reward) if eval_max_reward < eval_reward: if not os.path.exists(FLAGS.checkpoint): os.mkdir(FLAGS.checkpoint) saver.save(sess, save_path=os.path.join( FLAGS.checkpoint, 'model_{0}_{1}.ckpt'.format( step, eval_reward))) eval_max_reward = eval_reward patient = FLAGS.patient print('eval max reward : {0}'.format(eval_max_reward)) if patient < 0: break if eval_max_reward - eval_reward > FLAGS.threshold: patient -= 1
def cmd_runbs_main_wrapper(args): settings.IS_RUNBS = True settings.TESTENV = 'bs' settings.ENABLE_JUNIT = args.junit if args.junit else False logger.setLevel(logging.INFO) if args.save_db else logger.setLevel( logging.INFO) if args.testcases: settings.TESTCASES = args.testcases elif args.json: settings.TESTCASES = None with open( os.path.abspath(os.path.dirname(__file__)) + '/' + args.json, "r") as read_it: settings.TESTOBJECTS = json.load(read_it) else: settings.TESTCASES, settings.TESTOBJECTS = None, None settings.DRY_RUN = args.dry_run if args.dry_run else False settings.SAVE_DB = args.save_db if args.save_db else False logger.setLevel(logging.INFO) if settings.SAVE_DB else logger.setLevel( logging.DEBUG) get_config() cmd_runbs_main()
def register_subscribe(): """Registers and subscribes to portal Note: Following samtools.properties config values are must to continue: - rh_portal_username - rh_portal_password """ rh_portal_username = get_config('samtools', 'rh_portal_username') rh_portal_password = get_config('samtools', 'rh_portal_password') if rh_portal_username is None or rh_portal_password is None: print ('Missing Parameters: rh_portal_username AND rh_portal_password ' 'must be defined in samtools.properties to continue ' 'installation') sys.exit(1) # Subscribe to RH portal run('subscription-manager register --username="******" --password="******" ' '--autosubscribe --force'.format(rh_portal_username, rh_portal_password))
def _initialise(self, args): logging.debug('Starting client mode checks on config file') config = get_config(args.config_file) server_url = get_server_url(config) self._proxy = xmlrpclib.ServerProxy(server_url) self._hostname = gethostname() self._artifact = args.artifact self._version = args.version logging.debug('Client checks passed')
def _initialise(self, args): logging.debug('Starting client mode checks on config file') config = get_config(args.config_file) server_url = get_server_url(config) self._proxy = xmlrpclib.ServerProxy(server_url) self._hostname = gethostname() self._running = threading.Event() self._running.clear() logging.debug('Client checks passed')
def send_mail(subject, message, attachment_path=None): config = helper.get_config() print(config) attachment = None if attachment_path is not None: attachment = [{"attachment", open(attachment_path, "rb")}] return requests.post(config['mailgun_api_url'], auth=("api", config['mailgun_key']), data={ "from": config['mailgun_sender'], "to": [config['mailgun_sample_receiver']], "subject": subject, "text": message }, files=attachment)
def create_plugin(name, version, clazz, **kwargs): config = helper.get_config() # Brewtils 3.2.0ish has a bug that breaks when reusing the same client class # It's been fixed in brewtils, but fix it here so we can run tests on that version # See https://github.com/beer-garden/beer-garden/issues/1014 if hasattr(clazz, "current_request"): delattr(clazz, "current_request") if hasattr(clazz, "_current_request"): delattr(clazz, "_current_request") return Thread_Plugin(client=clazz(), name=name, version=version, bg_host=config.bg_host, bg_port=config.bg_port, ssl_enabled=config.ssl_enabled, **kwargs)
def test_merged_config(self): config_dir = normpath(join(self.current_dir, "config")) filename = 'unit_test_secrets.ini' config = get_config(config_dir, filename) # Test options that are specified only in 'unit_test_secrets.ini' self.assertEqual(config.get('secrets', 'SECRET_KEY'), 'k#%(x)s$y') self.assertEqual(config.get('database', 'DATABASE_PASSWORD'), 'bbb') # Test options that are specified only in the included file. self.assertEqual(config.get('database', 'DATABASE_USER'), 'tribe') self.assertEqual(config.get('database', 'DATABASE_PORT'), '5432') # Test options that are specified in both unit_test_secrets.ini # and its included file. (unit_test_secrets.ini should always # take precedence.) self.assertEqual(config.get('database', 'DATABASE_NAME'), '') self.assertEqual(config.get('database', 'DATABASE_HOST'), 'aaa') self.assertEqual(config.get('debug', 'ALLOWED_HOSTS'), 'localhost 127.0.0.1')
def install_from_repo(): """Task to install SAM from repo URL Note: Following samtools.properties config values are must to continue: - rh_portal_username - rh_portal_password - base_url """ base_url = get_config('samtools', 'base_url') if base_url is None: print ('Missing Parameters: base_url must be defined in ' 'samtools.properties to continue installation') sys.exit(1) # Register and subscribe register_subscribe() # Disable unwanted repos print 'Disabling all repos...' run('yum-config-manager --disable "*"', quiet=True) # Enable rhel6 repos only run('yum-config-manager --enable rhel-6-server-rpms') create_repo(base_url) # Install sam run('yum install -y katello-headpin-all') # Run katello-configure run('katello-configure --deployment=sam --user-pass=admin') # Run ping test run_ping_command()
def decode(test_path, rl): sess = tf.Session(config=get_config()) if FLAGS.beam == True: FLAGS.batch_size = FLAGS.beam_size FLAGS.max_dec_steps = 1 print('batch size ', FLAGS.batch_size) #if rl == False: summarizationModel = PointerNet(FLAGS, vocab) #elif rl==True: # if FLAGS.gamma > 0: # import rl_model_gamma # summarizationModel = rl_model_gamma.RLNet(FLAGS, vocab) # else: # import rl_model # summarizationModel = rl_model.RLNet(FLAGS, vocab) summarizationModel.build_graph() saver = tf.train.Saver() best_model = load_best_model(FLAGS.restore_path) print('best model : {0}'.format(best_model)) saver.restore(sess, save_path=best_model) counter = 0 batcher = Batcher(test_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) batches = batcher.fill_batch_queue( is_training=False) # 1 example repeated across batch print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) if FLAGS.beam == False: for batch in batches: article = batch.original_articles[0] original_abstract_sents = batch.original_abstracts_sents # list of strings #print('*****************start**************') best_hyps = beam_search.run_greedy_search(sess, summarizationModel, vocab, batch) output_ids = [[int(t) for t in best_hyp.tokens[1:]] for best_hyp in best_hyps] decoded_words = data.outputids2words_greedy( output_ids, vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) decoded_words = remove_stop_index(decoded_words, data) write_for_rouge_greedy( original_abstract_sents, decoded_words, article, counter, FLAGS.dec_path, FLAGS.ref_path, FLAGS.all_path ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += FLAGS.batch_size # this is how many examples we've decoded print('counter ... ', counter) if counter % (5 * 64) == 0: print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) else: for batch in batches: article = batch.original_articles[0] original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings #print('*****************start**************') best_hyps = beam_search.run_beam_search(sess, summarizationModel, vocab, batch) #print('best hyp : {0}'.format(best_hyp)) output_ids = [int(t) for t in best_hyps.tokens[1:]] decoded_words = data.outputids2words_beam( output_ids, vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words #decoded_words = ' '.join(decoded_words) write_for_rouge_beam( original_abstract_sents, decoded_words, article, counter, FLAGS.dec_path, FLAGS.ref_path, FLAGS.all_path ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded print('counter ... ', counter) if counter % 100 == 0: print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
def run_rl_training(): summarizationModel = RLNet(FLAGS, vocab) summarizationModel.build_graph() batcher = Batcher(FLAGS.data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) val_batcher = Batcher(FLAGS.val_data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) sess = tf.Session(config=get_config()) saver = tf.train.Saver(max_to_keep=100) if FLAGS.restore_rl_path: saver.restore(sess, FLAGS.restore_rl_path) else: sess.run(tf.global_variables_initializer()) sess.run( loading_variable([v for v in tf.trainable_variables()], reader_params(load_best_model( FLAGS.restore_path)))) print('loading params...') epoch = FLAGS.epoch step = 0 eval_loss = float('inf') while epoch > 0: batches = batcher.fill_batch_queue() for batch in batches: step += 1 sampled_sentence_r_values = [] greedy_sentence_r_values = [] feed_dict = make_feed_dict(summarizationModel, batch) to_return = { 'sampled_sentences': summarizationModel.sampled_sentences, 'greedy_search_sentences': summarizationModel.greedy_search_sentences } ret_dict = sess.run(to_return, feed_dict) # calculate reward for sampled_sentence, greedy_search_sentence, target_sentence in zip( ret_dict['sampled_sentences'], ret_dict['greedy_search_sentences'], batch.target_batch): assert len(sampled_sentence[0]) == len(target_sentence) == len( greedy_search_sentence[0]) reference_sent = ' '.join([str(k) for k in target_sentence]) sampled_sent = ' '.join([str(k) for k in sampled_sentence[0]]) sampled_sentence_r_values.append( reward_function(reference_sent, sampled_sent)) greedy_sent = ' '.join( [str(k) for k in greedy_search_sentence[0]]) greedy_sentence_r_values.append( reward_function(reference_sent, greedy_sent)) to_return = { 'train_op': summarizationModel.train_op, 'pgen_loss': summarizationModel._pgen_loss, 'rl_loss': summarizationModel._rl_loss, 'loss': summarizationModel.loss } to_return['s_r'] = summarizationModel._sampled_sentence_r_values to_return['g_r'] = summarizationModel._greedy_sentence_r_values feed_dict[summarizationModel. _sampled_sentence_r_values] = sampled_sentence_r_values feed_dict[summarizationModel. _greedy_sentence_r_values] = greedy_sentence_r_values feed_dict[summarizationModel._eta] = 0.5 res = sess.run(to_return, feed_dict) print( 'step : {0},pgen_loss : {1}, rl_loss : {2}, loss : {3}, reward : {4}' .format(step, res['pgen_loss'], res['rl_loss'], res['loss'], np.sum(res['s_r'] - res['g_r']))) if step % FLAGS.eval_step == 0: eval_ = run_rl_eval(summarizationModel, val_batcher, sess, 0.5) if eval_ < eval_loss: if not os.path.exists(FLAGS.checkpoint): os.mkdir(FLAGS.checkpoint) saver.save(sess, save_path=os.path.join( FLAGS.checkpoint, 'model_{0}_{1}.ckpt'.format(step, eval_))) eval_loss = eval_ patient = FLAGS.patient print('eval loss : ', eval_loss) if patient < 0: break if eval_ - eval_loss > FLAGS.threshold: patient -= 1
def create_session(session_number): config = get_config(session_number) video_timestamps = {} if config is None else config.video_timestamps tracking_filename = input_filename_prefix + str( session_number) + "_users.csv" touch_filename = input_filename_prefix + str(session_number) + "_touch.csv" device_touch_filename = input_filename_prefix + str( session_number) + "_device_touch.csv" device_filename = input_filename_prefix + str( session_number) + "_device.csv" number_of_users = get_number_of_users(tracking_filename, touch_filename) print "number of users in session: ", number_of_users user_pitch_offsets = [0] * number_of_users session_time_offset = 0.0 if config is None else config.session_time_offset video_time_offset = 0.0 if config is None else config.video_offsets current_dir = os.getcwd() os.chdir("{}{}".format(input_video_dir, session_number)) filenames = glob.glob(input_video_prefix + str(session_number) + "_video*.mp4") if len(filenames) < 1 and len(video_timestamps) < 1: print "ERROR: cannot find video files or the config variables for the session " + str( session_number) + "." assert False if len(filenames) > 0: print "video file for session " + str( session_number) + ":", filenames[0] video_filenames = [ "{}{}/{}".format(input_video_dir, session_number, fn) for fn in filenames ] os.chdir(current_dir) video_start_times = [] date = None video_filenames = video_filenames if len( video_filenames) > 0 else video_timestamps.keys() for video_filename in video_filenames: possible_timestamps = [ ts for fn, ts in video_timestamps.iteritems() if fn in video_filename ] if len(possible_timestamps) > 0: video_start_time_unix = possible_timestamps[0] else: video_start_time_unix = float(video_filename.split("_")[-1][:-4]) print "video start time unix: {0: .6f}".format(video_start_time_unix) date_time = datetime.datetime.fromtimestamp( video_start_time_unix).strftime('%Y-%m-%d %H:%M:%S.%f') print "video start time:", date_time date_split = date_time.split(' ') if date is None: date = date_split[0] video_start_times.append(date_split[1]) return Session(session_num=session_number, level_num=1, data_dir=current_dir, optitrack_filename=tracking_filename, touch_filename=touch_filename, device_touch_filename=device_touch_filename, device_filename=device_filename, video_filenames=video_filenames, date=date, video_start_times=video_start_times, video_time_offset=video_time_offset, session_time_offset=session_time_offset, num_users=number_of_users, user_pitch_offsets=user_pitch_offsets)
def main(): config = helper.get_config() init(config) logging.info('Tesco Scraper has starterd') tesco.scrape_tesco()
from pathlib import Path import torch from onnxruntime import InferenceSession, SessionOptions from transformers import pipeline, AutoTokenizer import numpy as np from helper import get_config, group_sub_entities, group_entities, render_ner_html_custom from urllib import request from quant_flair_model import QuantizableLanguageModel from transformers.models.bert.tokenization_bert import BertTokenizer flair.models.LanguageModel = QuantizableLanguageModel flair.models.language_model.LanguageModel = QuantizableLanguageModel IGNORE_LABELS = set(["O"]) config = get_config() colors = { "PRS": "#F7FF53", # YELLOW "PER": "#F7FF53", # YELLOW "ORG": "#E8902E", # ORANGE (darker) "LOC": "#FF40A3", # PINK "MISC": "#4647EB", # PURPLE "EVN": "#06b300", # GREEN "MSR": "#FFEDD5", # ORANGE (lighter) "TME": "#ff7398", # PINK (pig) "WRK": "#c5ff73", # YELLOW (REFLEX) "OBJ": "#4ed4b0", # TURQUOISE "O": "#ddd", # GRAY }
return True if __name__ == '__main__': if getpass.getuser() == "root": print "You're root, Please run as normal user" sys.exit() #Register to socket to prevent run many instance at one time helper.get_lock('bing_wallpaper') freeze_support() temp_config = helper.get_config() if temp_config is not None: print "Found previous config" if 'curr_mode' in temp_config and temp_config['curr_mode'] in all_modes: config['curr_mode'] = temp_config['curr_mode'] if 'timer_milliseconds' in temp_config: config['timer_milliseconds'] = temp_config['timer_milliseconds'] if 'auto_download' in temp_config: config['auto_download'] = temp_config['auto_download'] date_ranges = helper.get_range_dates(config['curr_mode'])
def run_rl_training_gamma(FLAGS, vocab): summarizationModel = RLNet(FLAGS, vocab) summarizationModel.build_graph() batcher = Batcher(FLAGS.data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) val_batcher = Batcher(FLAGS.val_data_path, vocab, FLAGS, single_pass=FLAGS.single_pass, decode_after=FLAGS.decode_after) sess = tf.Session(config=get_config()) saver = tf.train.Saver(max_to_keep=10) if FLAGS.restore_rl_path: print('restore rl model...') saver.restore(sess, FLAGS.restore_rl_path) else: sess.run(tf.global_variables_initializer()) sess.run( loading_variable([v for v in tf.trainable_variables()], reader_params(load_best_model(FLAGS.restore_path)))) print('loading params...') epoch = FLAGS.epoch step = 0 patient = FLAGS.patient eval_max_reward = -float('inf') while epoch > 0: batches = batcher.fill_batch_queue() for batch in batches: step += 1 sampled_sentence_r_values = [] greedy_sentence_r_values = [] feed_dict = make_feed_dict(summarizationModel, batch) to_return = { 'sampled_sentences': summarizationModel.sampled_sentences, } ret_dict = sess.run(to_return, feed_dict) Rs = [] # calculate reward for sampled_sentence, target_sentence in zip(ret_dict['sampled_sentences'],batch.target_batch): # print('sampled : ',sampled_sentence) # print('target : ', target_sentence) reward = compute_reward(sampled_sentence[0], target_sentence) R = 0 R_l = [] for r in reward[::-1]: R = r + FLAGS.gamma * R R_l.insert(0,R) #avg = np.mean(R_l) #R_l = list(map(lambda a:a-avg, R_l)) Rs.append(R_l) to_return = { 'train_op': summarizationModel.train_op, 'pgen_loss': summarizationModel._pgen_loss, 'rl_loss': summarizationModel._rl_loss, 'loss': summarizationModel.loss } to_return['reward'] = summarizationModel._reward feed_dict[summarizationModel._reward] = Rs feed_dict[summarizationModel._eta] = 0.1 res = sess.run(to_return, feed_dict) print('step : {0}, pgen_loss : {1}, rl_loss : {2}, loss : {3}, reward : {4}'.format(step,res['pgen_loss'], res['rl_loss'], res['loss'], np.mean(res['reward'],axis=0)[0] )) if step % FLAGS.eval_step == 0: #eval_ = run_rl_eval_gramma(summarizationModel, val_batcher, sess, 0.5) eval_reward = run_eval(summarizationModel, val_batcher, sess) print('eval reward ', eval_reward) if eval_reward > eval_max_reward: if not os.path.exists(FLAGS.checkpoint): os.mkdir(FLAGS.checkpoint) saver.save(sess, save_path=os.path.join(FLAGS.checkpoint, 'model_{0}_{1}.ckpt'.format(step, eval_reward))) eval_max_reward = eval_reward patient = FLAGS.patient print('eval max ward ', eval_max_reward) if patient < 0: break if eval_max_reward - eval_reward > FLAGS.threshold: patient -= 1
def main(layer_sizes= [2, 1, 1], L2_reg= 0.0001, param_scale = 0.1, batch_size = 20, num_epochs = 10, step_size = 1.0, hyper_iter =450, hyper_step_size = 2.5, hyper_decay = 0.333, hyper_decay_after = 300, hyper_decay_every = 300, hyper_L2_reg = 0.000002, rank_loss_scaling_factor = 20.0, mse_tolerance = 0.01, outputFname = '/tmp/results.txt', sorted_c_r_list = None, init_params_list=[], combined_init_params = None, mask_params = None, random_state = None, hyper_train = True, fake_train_images = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]), fake_train_labels = np.array([[1.0],[0.0],[0.0],[1.0]]), fake_valid_images = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]), fake_valid_labels = np.array([[1.0],[0.0],[0.0],[1.0]])): #import pickle #with open("mnist19_try.pkl", 'rb') as f_rd: # sorted_c_r_list = pickle.load(f_rd) if False: #layer_sizes is None: import helper args = helper.get_config() # Model parameters layer_sizes = args.layer_sizes L2_reg = args.L2_reg # Training parameters param_scale = args.param_scale #Very important param. Should be low batch_size = args.batch_size #inner loop batch size. Ignore if number of training examples < batch size num_epochs = args.num_epochs #inner loop epochs step_size = args.step_size #Very important param. #This is the not hyper_epochs. If hyper_iter > num_batches, then batches repeat. #Ideally, we should have hyper_iter = num_batches * hyper_epochs. hyper_iter = args.hyper_iter hyper_step_size = args.hyper_step_size hyper_L2_reg = args.hyper_L2_reg rank_loss_scaling_factor = args.rank_loss_scaling_factor #if DEBUG: print("Loading training data...") #N, actual_train_images, actual_train_labels, actual_test_images, actual_test_labels = load_mnist() #Following should be float values. Required by autograd. Cannot take gradient w.r.t integers fake_train_images = np.array(args.fake_train_images) fake_train_labels = np.array(args.fake_train_labels) #fake_test_images = np.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]]) #fake_test_labels = np.array([[1.0], [0.0], [0.0], [1.0]]) #np.zeros((10,1)) #fake_test_images = np.array([[1.0, 0.0], [0.0, 1.0]]) #fake_test_labels = np.array([[0.0], [0.0]]) #np.zeros((10,1)) test_batch_size = batch_size # Define training objective def objective(train_images,train_labels,params, mask_param, c_list, iter): num_batches = int(np.ceil(len(train_images) / batch_size)) def batch_indices(iter): idx = iter % num_batches return slice(idx * batch_size, (idx+1) * batch_size) idx = batch_indices(iter) #if DEBUG: print ('In train objective') return -log_posterior_binary(params, mask_param, train_images[idx], train_labels[idx], L2_reg, c_list) # Define test objective def test_objective(params, mask_param, test_images, test_labels, c_list, iter): num_batches = int(np.ceil(len(test_images) / test_batch_size)) def batch_indices(iter): idx = iter % num_batches return slice(idx * batch_size, (idx+1) * test_batch_size) idx = batch_indices(iter) if DEBUG: print ('Test BCE loss') return [-x for x in log_posterior_binary_test(params, mask_param, test_images[idx], test_labels[idx], L2_reg, c_list)] def inner_loop(train_images,train_labels, init_params, mask_param, c_list): def objective_wrapper(params,iter): #sgd in line 108 magically supplies the iter parameter return objective(train_images,train_labels,params, mask_param, c_list, iter) objective_grad = grad(objective_wrapper) num_batches = int(np.ceil(len(train_images) / batch_size)) ##if DEBUG: print(" Epoch | Train accuracy | Test accuracy ") #def print_perf(params, iter, gradient): # if iter % num_batches == 0: # train_acc = accuracy(params, train_images, train_labels, c_list) # test_acc = accuracy(params, test_images, test_labels, c_list) #if DEBUG: print("{:15}|{:20}|{:20}".format(iter//num_batches, train_acc, test_acc)) # The optimizers provided can optimize lists, tuples, or dicts of parameters. #if DEBUG: print("DOING INNER LOOP SGD") optimized_params = adam(objective_grad, init_params, step_size=step_size, num_iters=num_epochs * num_batches,callback=None) #test_loss = test_objective(optimized_params, train_images, train_labels, c, 0) return optimized_params trained_weights_dict = dict() test_loss_dict = dict() #sorted_c_r_list = [(0.32, 0.95), (1.2, 0.91), (1.6, 0.87)] #sorted_c_r_list = [(0.45444444444444443, 0.9613), (0.12, 0.957), (1.1211111111111112, 0.9369), (1.5655555555555554, 0.9167), (2.01, 0.9076), (0.01, 0.8999)]#, (0.2322, 0.9727)] #c_1 = 0.2322 #c_2 = 0.32 def check_success(scores): sorted_scores = [] for idx, val in enumerate(scores): sorted_scores.append((idx, val)) sorted_scores.sort(key=lambda x: x[1], reverse=True) for idx, val in enumerate(sorted_scores): if (idx != val[0]): if DEBUG: print ('Failed') return False if DEBUG: print ('Success in rank') return True #init_params_list = generate_init_params(sorted_c_r_list, param_scale, layer_sizes) def test_loss(train_images, init_params, mask_param, c_list, t): [train_images, valid_images] = np.split(train_images, 2) #Extracting train and valid images trained_weights = inner_loop(train_images,fake_train_labels, init_params, mask_param, c_list) return (test_objective(trained_weights, mask_param, valid_images, fake_valid_labels, c_list, t), trained_weights) def pairwise_rankloss(score_1, score_2, target): scaling_factor = rank_loss_scaling_factor if (score_1 > score_2): scaling_factor = 0.0 diff = (score_1 - score_2)*scaling_factor rank_loss = -target*diff + np.log(1 + np.exp(diff)) #print ('Score1, Score2, Rank Loss', score_1, score_2, rank_loss) return rank_loss def rankloss(scores): total_rank_loss = 0.0 rank_loss_count = 0 for i in range(len(scores)-1): for j in range(i, len(scores)): if i !=j: score_1 = scores[i] score_2 = scores[j] target = 1.0 #This is assuming scores are already placed in their actual/target index (based on sorted_c_r_list ordering) #1.0 means score_1 > score_2 and 0.0 means score_1 < score_2 rank_loss = pairwise_rankloss(score_1, score_2, target) total_rank_loss += rank_loss rank_loss_count += 1 #Total number of rank comparisons if DEBUG: print ('Pairs', i, j, 'Score', score_1, score_2, rank_loss) total_rank_loss = total_rank_loss/rank_loss_count if DEBUG: print ('Total Rank Loss', total_rank_loss) return total_rank_loss def mse_loss(scores): mean, std = compute_mean_std(scores) normalized_pred_scores = [ (x-mean)/std for x in scores] #Whiten the data for index, (i, j) in enumerate(zip(scores, normalized_pred_scores)): if DEBUG: print (sorted_c_r_list[index][0], 'Predicted scores', i, 'Normalized Pred scores', j) true_scores = [ r for (c,r) in sorted_c_r_list ] #Convert accuracy to error mean, std = compute_mean_std(true_scores) normalized_true_scores = [ (x-mean)/std for x in true_scores] for index, (i, j) in enumerate(zip(true_scores, normalized_true_scores)): if DEBUG: print (sorted_c_r_list[index][0], 'True scores', i, 'Normalized True scores', j) mse = np.mean(np.array([(i-j)**2 for i, j in zip(normalized_true_scores, normalized_pred_scores)])) if DEBUG: print ('Mean Squared Error', mse) #print ('True Scores and Normalized scores', true_scores, normalized_true_scores) #print ('Pred Scores and Normalized scores', scores, normalized_pred_scores) return mse def get_scores(train_images, t): score_list = [] #Scores will be appended in the same order as the sorted_c_r_list trained_weights_list = [] #for idx, (c, r) in enumerate(sorted_c_r_list): init_idx = get_init_idx(combined_init_params, mask_params, random_state) c_list = [c for (c, r) in sorted_c_r_list] test_loss_list, trained_weights_list = test_loss(train_images, combined_init_params[init_idx], mask_params[init_idx], c_list, t) #Reuse init_params score_list = [-loss for loss in test_loss_list] #Validation loss needs to be inverted to get rank score #score_list.append(score) #trained_weights_list.append(trained_weights) return score_list, trained_weights_list #Train the network and save the weights def hyper_objective(train_images, t): #t is test iteration if DEBUG: print ('Hyper iteration', t) score_list, trained_weights_list = get_scores(train_images, t) success_flag = check_success(score_list) if success_flag == True: if DEBUG: print ('Successful Learnt Images', train_images) #with open (outputFname, 'a') as fwr: # fwr.writelines("Hyperepoch "+str(t)+" Rank order correct "+str(train_images)+"\n") if (t==0 or t==hyper_iter-1 or success_flag == True): if DEBUG: print ('Hyper Iteration', t, 'scores') if DEBUG: [print (score) for score in score_list] if (t==hyper_iter-1 or success_flag == True): if DEBUG==True: for idx, trained_weights in enumerate(trained_weights_list): print ('Final Weights', idx, trained_weights) #print ('Final trained_images', train_images) #Write to output file from inside the petridish. This is to ensure that we capture the result before we lose in the next iter mse = mse_loss(score_list) #if mse < mse_tolerance: # with open (outputFname, 'a') as fwr: # fwr.writelines("Hyperepoch "+str(t)+" MSE Loss converged "+str(mse)+str(train_images)+"\n") return mse+l2_norm(train_images)*hyper_L2_reg def get_results(learnt_images, train, iteration=0): score_list, trained_weights_list = get_scores(learnt_images, t=iteration) #t is not useful unless number of images > batch_size success_flag = check_success(score_list) return success_flag, score_list, trained_weights_list if hyper_train == True: hyper_objective_grad = grad(hyper_objective) h_i = 0 #Initializing hyper-iteration optimized_params = np.concatenate((fake_train_images, fake_valid_images), axis=0) #Valid images and labels have the same dimension as the train images and labels #Following while loop runs Hyper-training with hyper step size decay while h_i < hyper_iter: hyper_lr, num_iters = compute_lr(init_lr=hyper_step_size, current_iter=h_i, decay_after=hyper_decay_after, decay_factor=hyper_decay, decay_every=hyper_decay_every) print ('Hyper Learning Rate', hyper_lr, num_iters) optimized_params = adam(hyper_objective_grad, optimized_params, step_size=hyper_lr,num_iters=num_iters) h_i = h_i + num_iters #optimized_params = adam(hyper_objective_grad, optimized_params, step_size=hyper_step_size/3,num_iters=300) if DEBUG: print ('Learned training images', optimized_params) success_flag, score_list, trained_weights_list = get_results(optimized_params, train=hyper_train, iteration=hyper_iter) mse = mse_loss(score_list) #Since the train_images and train_labels have the same dimensionality, #we can split the concatenated optimized_params into two equal parts #Added validation images and labels that have the same dimensionality as the training images and labels [optimized_train_images, optimized_valid_images] = np.split(optimized_params, 2) return mse, success_flag, score_list, optimized_train_images, fake_train_labels, optimized_valid_images, fake_valid_labels #Test. Only return scores else: optimized_params = np.concatenate((fake_train_images, fake_valid_images), axis=0) #Valid images and labels have the same dimension as the train images and labels success_flag, score_list, trained_weights_list = get_results(optimized_params, train=hyper_train, iteration=0) mse = mse_loss(score_list) #Since the train_images and train_labels have the same dimensionality, #we can split the concatenated optimized_params into two equal parts #Added validation images and labels that have the same dimensionality as the training images and labels [optimized_train_images, optimized_valid_images] = np.split(optimized_params, 2) return mse, success_flag, score_list, optimized_train_images, fake_train_labels, optimized_valid_images, fake_valid_labels
def _initialise(self, args): logging.debug('Starting server mode checks on config file') config = get_config(args.config_file) self._clients = {} self._backup_location = '' self._port = 9001 if config.has_option('server', 'backup_location'): self._backup_location = config.get('server', 'backup_location') if not os.path.isdir(self._backup_location): logging.warn("Backup location '%s' does not exist, attempting to create it" % self._backup_location) try: os.makedirs(self._backup_location) except: raise RuntimeError('Could not create the requested backup location') else: raise RuntimeError('Backup location not specified in config file') if not config.has_option('server', 'port'): logging.warn('No port specified, using 9001') else: try: self._port = int(config.get('server', 'port')) except: raise RuntimeError('Server port must be an integer') for section in config.sections(): if not section == 'server': logging.debug('Found a client: %s' % section) if not config.has_option(section, 'artifacts'): raise RuntimeError('Client sections require an artifacts option') artifacts_string = config.get(section, 'artifacts') artifacts = {} if artifacts_string == '': raise RuntimeError('Artifacts list cannot be empty') for artifact in artifacts_string.split(','): logging.debug('Found an artifact: %s' % artifact) file_based = True filename = '' backup_command = '' restore_command = '' cleanup = False versions = 1 interval = '1h' if config.has_option(section, artifact + '_filename'): filename = config.get(section, artifact + '_filename') else: raise RuntimeError("Artifacts must have at least a file specified. Error in client '%s'" % section) if config.has_option(section, artifact + '_backup_command'): file_based = False backup_command = config.get(section, artifact + '_backup_command') if config.has_option(section, artifact + '_restore_command'): restore_command = config.get(section, artifact + '_restore_command') else: raise RuntimeError("A backup command was specified without a restore command. A restore command is required in client '%s', artifact '%s'" % (section, artifact)) if config.has_option(section, artifact + '_cleanup'): tmp = config.get(section, artifact + '_cleanup') if tmp.lower() == 'true': cleanup = True elif tmp.lower() == 'false': cleanup = False else: raise RuntimeError("Invalid option for cleanup in client '%s', artifact '%s'" % (section, artifact)) if config.has_option(section, artifact + '_versions'): try: versions = int(config.get(section, artifact + '_versions')) except: raise RuntimeError("Version option must be an integer in client '%s', artifact '%s'" % (section, artifact)) if config.has_option(section, artifact + '_interval'): interval = config.get(section, artifact + '_interval') regex = "^(\d+w ?)?(\d+d ?)?(\d+h ?)?(\d+m ?)?(\d+s ?)?$" if not re.search(regex, interval): raise RuntimeError("Interval option must in valid timedelta format. e.g. 1w2d3h4m. In client '%s', artifact '%s'" % (section, artifact)) artifacts[artifact] = { 'file_based': file_based, 'filename': filename, 'backup_command': backup_command, 'restore_command': restore_command, 'cleanup': cleanup, 'versions': versions, 'interval': interval } self._clients[section] = artifacts if not len(self._clients) > 0: raise RuntimeError('No clients specified') self._server = None
def _initialise(self, args): logging.debug('Starting server mode checks on config file') config = get_config(args.config_file) self._clients = {} self._backup_location = '' self._port = 9001 if config.has_option('server', 'backup_location'): self._backup_location = config.get('server', 'backup_location') if not os.path.isdir(self._backup_location): logging.warn( "Backup location '%s' does not exist, attempting to create it" % self._backup_location) try: os.makedirs(self._backup_location) except: raise RuntimeError( 'Could not create the requested backup location') else: raise RuntimeError('Backup location not specified in config file') if not config.has_option('server', 'port'): logging.warn('No port specified, using 9001') else: try: self._port = int(config.get('server', 'port')) except: raise RuntimeError('Server port must be an integer') for section in config.sections(): if not section == 'server': logging.debug('Found a client: %s' % section) if not config.has_option(section, 'artifacts'): raise RuntimeError( 'Client sections require an artifacts option') artifacts_string = config.get(section, 'artifacts') artifacts = {} if artifacts_string == '': raise RuntimeError('Artifacts list cannot be empty') for artifact in artifacts_string.split(','): logging.debug('Found an artifact: %s' % artifact) file_based = True filename = '' backup_command = '' restore_command = '' cleanup = False versions = 1 interval = '1h' if config.has_option(section, artifact + '_filename'): filename = config.get(section, artifact + '_filename') else: raise RuntimeError( "Artifacts must have at least a file specified. Error in client '%s'" % section) if config.has_option(section, artifact + '_backup_command'): file_based = False backup_command = config.get( section, artifact + '_backup_command') if config.has_option(section, artifact + '_restore_command'): restore_command = config.get( section, artifact + '_restore_command') else: raise RuntimeError( "A backup command was specified without a restore command. A restore command is required in client '%s', artifact '%s'" % (section, artifact)) if config.has_option(section, artifact + '_cleanup'): tmp = config.get(section, artifact + '_cleanup') if tmp.lower() == 'true': cleanup = True elif tmp.lower() == 'false': cleanup = False else: raise RuntimeError( "Invalid option for cleanup in client '%s', artifact '%s'" % (section, artifact)) if config.has_option(section, artifact + '_versions'): try: versions = int( config.get(section, artifact + '_versions')) except: raise RuntimeError( "Version option must be an integer in client '%s', artifact '%s'" % (section, artifact)) if config.has_option(section, artifact + '_interval'): interval = config.get(section, artifact + '_interval') regex = "^(\d+w ?)?(\d+d ?)?(\d+h ?)?(\d+m ?)?(\d+s ?)?$" if not re.search(regex, interval): raise RuntimeError( "Interval option must in valid timedelta format. e.g. 1w2d3h4m. In client '%s', artifact '%s'" % (section, artifact)) artifacts[artifact] = { 'file_based': file_based, 'filename': filename, 'backup_command': backup_command, 'restore_command': restore_command, 'cleanup': cleanup, 'versions': versions, 'interval': interval } self._clients[section] = artifacts if not len(self._clients) > 0: raise RuntimeError('No clients specified') self._server = None
import math from datetime import datetime, timedelta from csv_data_reader import get_data from cStringIO import StringIO from helper import map_kinect_to_device_coordinates, map_device_coordinates_to_wall_pixels, \ mapping_kinect_to_giant_coordinates, map_wall_pixels_to_device_coordinates, get_config input_folder_prefix = '../data_logs/session_' input_filename_prefix = 'session_' output_filename_prefix = 'giant_session_' # session_ids = ['0', '1', '2', '3', '4', '5', '6', '13', '14', '16'] session_ids = ['6'] for session_id in session_ids: config = get_config(session_id) joint_type_body_to_device = "OKS_SPINE_MID" if config is None else config.joint_type_body_to_device joint_type_body_to_touch = "OKS_SPINE_SHOULDER" if config is None else config.joint_type_body_to_touch device_id_count_threshold = 5 if config is None else config.device_id_count_threshold max_time_delta_body_to_device = timedelta( milliseconds=150 ) if config is None else config.max_time_delta_body_to_device max_dist_delta_body_to_device = 0.6 if config is None else config.max_dist_delta_body_to_device max_time_delta_touch_to_inj = timedelta( milliseconds=150 ) if config is None else config.max_time_delta_touch_to_inj max_time_delta_inj_to_de = timedelta( milliseconds=2) if config is None else config.max_time_delta_inj_to_de max_time_delta_body_to_touch = timedelta( milliseconds=75
# Site name: SITE_NAME = 'tribe' # Settings files settings_root = dirname(abspath(__file__)) config_dir = normpath(join(settings_root, 'config')) # Add our project to our pythonpath, this way we don't need to type our project # name in our dotted import paths: sys.path.append(PROJECT_ROOT) ########## END PATH CONFIGURATION ########## INI CONFIGURATION config = get_config(config_dir, 'secrets.ini') ########## END INI CONFIGURATION ########## SECRET CONFIGURATION # See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key # Note: This key only used for development and testing. SECRET_KEY = config.get('secrets', 'SECRET_KEY') ########## END SECRET CONFIGURATION ########## DEBUG CONFIGURATION # See: https://docs.djangoproject.com/en/dev/ref/settings/#debug DEBUG = config.getboolean('debug', 'DEBUG') # See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug TEMPLATE_DEBUG = config.getboolean('debug', 'TEMPLATE_DEBUG')