def test_target_language(self): """Test the target language (-t) flag.""" args = ["arg0", "-ttl", "-wTest.foo", "file.xml"] sys.argv = args c = util.get_config() self.assertTrue(c.command_line.xml_input_file == "file.xml") self.assertTrue(c.command_line.target_language == "tl") args[1] = "-tandroid" c = util.get_config() self.assertTrue(c.command_line.target_language == "android") args[1] = "-tddcpp" c = util.get_config() self.assertTrue(c.command_line.target_language == "ddcpp") args[1] = "-tFoo" try: c.parse() self.assertTrue(False, "Invalid target language should result in a SystemExit.") except SystemExit: self.assertTrue(sys.exc_info() is not None) return
def get_config(self,module_id,option,default,boolean=False): """ Pass-through function for convenience - module_id - - option - - default - - boolean - """ util.get_config(self.cfg,module_id,option,default,boolean)
def test_object_path(self): """Test the object path (-b) flag.""" args = ["arg0", "-b/TestFoo", "-ttl", "-wTest.Foo", "file.xml"] sys.argv = args c = util.get_config() self.assertTrue(c.command_line.xml_input_file == "file.xml") self.assertTrue(c.command_line.target_language == "tl") self.assertTrue(c.command_line.object_path == "/TestFoo") args = ["arg0", "-b", "/TestFoo", "-wTest.Foo", "-ttl", "file.xml"] sys.argv = args c = util.get_config() self.assertTrue(c.command_line.object_path == "/TestFoo") return
def test_well_known_name(self): """Test the well known name flag.""" args = ["arg0", "-ttl", "-wTest.My.Foo", "file.xml"] sys.argv = args c = util.get_config() self.assertTrue(c.command_line.well_known_name == "Test.My.Foo") return
def main(): hooks = [CheckTabs(), CheckIndentation()] should_check_pep8 = get_config("check-pep8", as_bool=True, default=True) if should_check_pep8: hooks += [CheckPep8()] # create temp directory for getting copies of files from staging. # TODO: Make all the hooks operate on strings instead of files, and get rid of this. temp_dir = tempfile.mkdtemp() temp_dir_with_slash = temp_dir + os.sep failure_encountered = False try: for filename in changed_files(): if not is_python_file(filename): continue relevant_hooks = [hook for hook in hooks if hook.should_process_file(filename)] if relevant_hooks: temp_filename = make_temp_copy(temp_dir_with_slash, filename) for relevant_hook in relevant_hooks: passes, error_message = relevant_hook.file_passes(temp_filename, original_filename=filename) if not passes: failure_encountered = True print (error_message) break finally: shutil.rmtree(temp_dir, True) return int(failure_encountered)
def setup_training(model, batcher): """Does setup before starting training (run_training)""" train_dir = os.path.join(FLAGS.log_root, "train") if not os.path.exists(train_dir): os.makedirs(train_dir) model.build_graph() # build the graph if FLAGS.convert_to_coverage_model: assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True" convert_to_coverage_model() if FLAGS.restore_best_model: restore_best_model() saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time sv = tf.train.Supervisor(logdir=train_dir, is_chief=True, saver=saver, summary_op=None, save_summaries_secs=60, # save summaries for tensorboard every 60 secs save_model_secs=60, # checkpoint every 60 secs global_step=model.global_step) summary_writer = sv.summary_writer tf.logging.info("Preparing or waiting for session...") sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config()) tf.logging.info("Created session.") try: run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted except KeyboardInterrupt: tf.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...") sv.stop()
def initUI(self): self.setGeometry(200, 200, 600, 200) self.setWindowTitle(__name_app__ + " Config") self.config = get_config() self.src_path = self.config.get('main', 'source_dir') self.dst_path = self.config.get('main', 'dest_dir') QtGui.QLabel("Parameter", self).setGeometry(QtCore.QRect(10, 10, 140, 20)) QtGui.QLabel("Source folder: ", self).setGeometry(QtCore.QRect(10, 30, 140, 20)) QtGui.QLabel("Destination folder: ", self).setGeometry(QtCore.QRect(10, 50, 140, 20)) QtGui.QLabel("Value", self).setGeometry(QtCore.QRect(150, 10, 300, 20)) self.src = QtGui.QLabel(self.src_path, self) self.src.setGeometry(QtCore.QRect(150, 30, 300, 20)) self.src.setFrameStyle(QtGui.QFrame.Panel | QtGui.QFrame.Sunken) self.dst = QtGui.QLabel(self.dst_path, self) self.dst.setGeometry(QtCore.QRect(150, 50, 300, 20)) self.dst.setFrameStyle(QtGui.QFrame.Panel | QtGui.QFrame.Sunken) self.src_btn = QtGui.QPushButton("...", self) self.src_btn.setGeometry(QtCore.QRect(450, 30, 30, 20)) self.src_btn.clicked.connect(lambda: self.ChgDirBtnClk(self.src)) self.dst_btn = QtGui.QPushButton("...", self) self.dst_btn.setGeometry(QtCore.QRect(450, 50, 30, 20)) self.dst_btn.clicked.connect(lambda: self.ChgDirBtnClk(self.dst)) self.save_btn = QtGui.QPushButton("Save", self) self.save_btn.setGeometry(QtCore.QRect((450 - 80), 80, 80, 30)) self.save_btn.clicked.connect(self.SaveBtnClk) self.cancel_btn = QtGui.QPushButton("Cancel", self) self.cancel_btn.setGeometry(QtCore.QRect(450, 80, 80, 30)) self.cancel_btn.clicked.connect(self.close)
def __init__(self, model, batcher, vocab, dqn = None): """Initialize decoder. Args: model: a Seq2SeqAttentionModel object. batcher: a Batcher object. vocab: Vocabulary object """ self._model = model self._model.build_graph() self._batcher = batcher self._vocab = vocab self._saver = tf.train.Saver() # we use this to load checkpoints for decoding self._sess = tf.Session(config=util.get_config()) if FLAGS.ac_training: self._dqn = dqn self._dqn_graph = tf.Graph() with self._dqn_graph.as_default(): self._dqn.build_graph() self._dqn_saver = tf.train.Saver() # we use this to load checkpoints for decoding self._dqn_sess = tf.Session(config=util.get_config()) _ = util.load_dqn_ckpt(self._dqn_saver, self._dqn_sess) # Load an initial checkpoint to use for decoding ckpt_path = util.load_ckpt(self._saver, self._sess, FLAGS.decode_from) if FLAGS.single_pass: # Make a descriptive decode directory name ckpt_name = "{}-ckpt-".format(FLAGS.decode_from) + ckpt_path.split('-')[ -1] # this is something of the form "ckpt-123456" self._decode_dir = os.path.join(FLAGS.log_root, get_decode_dir_name(ckpt_name)) else: # Generic decode dir name self._decode_dir = os.path.join(FLAGS.log_root, "decode") # Make the decode dir if necessary if not os.path.exists(self._decode_dir): os.mkdir(self._decode_dir) if FLAGS.single_pass: # Make the dirs to contain output written in the correct format for pyrouge self._rouge_ref_dir = os.path.join(self._decode_dir, "reference") if not os.path.exists(self._rouge_ref_dir): os.mkdir(self._rouge_ref_dir) self._rouge_dec_dir = os.path.join(self._decode_dir, "decoded") if not os.path.exists(self._rouge_dec_dir): os.mkdir(self._rouge_dec_dir)
def test_no_args(self): """Test what happens when no arguments are given.""" sys.argv = ["DummyArg0"] with self.assertRaises(SystemExit) as cm: c = util.get_config() self.assertTrue(sys.exc_info() is not None) return
def test_well_known_name_optional(self): """Test the well known name flag.""" args = ["arg0", "-ttl", "file.xml"] sys.argv = args # non-optional for 'tl' self.assertRaises(config.ConfigException, util.get_config) # optional for 'ddcpp' args[1] = "-tddcpp" c = util.get_config() self.assertTrue(c.command_line.well_known_name == None) return
def post(self): try: title = self.request.get('title').strip() if title: config = util.get_config() del config.config[title] config.put() memcache.delete(title, namespace='config') # else, just redirect to this dir self.redirect('.') except: self.redirect('.')
def search_track(): lines = open('data/SongswithChords.csv').read().split('\r') api_key = util.get_config([musicovery_api])[musicovery_api] f = open('data/songswithchords_musicoveryid.csv','w') http = urllib3.PoolManager() for line in lines[1:: if line.strip() == '': continue data = re.compile(',').split(line.strip()) artist = data[1].strip().replace(' ','%20').replace('&', '%27') title = data[2].strip().replace(' ', '%20').replace('&', '%27') url = musicovery_search.format(**{'title':title}, 'artist':artist, 'api_key':api_key})
def main(debug=False): try: util.load_logging_system() logger = logging.getLogger(__name__) config = util.get_config() pidfile = config.get("PATH","pid") pid = os.getpid() open(pidfile,'w').write("%d"%pid) manager.Manager(debug).run() except Exception, ex: logger.exception("Houston, we have a problem!")
def get_echonest_music_qualities(file_name = 'data/echo_music_songids.txt'): ''' Function used to retrieve the information in EchoNest for all the songs in MSD subset ''' api_key = util.get_config([echonest_api])[echonest_api] outfile = open('data/echo_musicovery_qualities.csv','w') track_count, echo_count = 0, 0 with open(file_name) as f: http = urllib3.PoolManager() for line in f: print line track_count += 1 song_id = line.strip() params = {} params['key'] = api_key params['id'] = song_id url = song_profile_url.format(**params) print url try_again = True count = 0 while try_again: count += 1 r = http.request('GET', url) jsondata = json.loads(r.data) if jsondata['response']['status']['code'] == 3: time.sleep(3) else: try_again = False if count > 15: print song_id + ' information not found' + r.data jsondata['response']['status']['code'] = 5 try_again = False if jsondata['response']['status']['code'] == 5: print song_id + ' information not found' + r.data if len(jsondata['response']['songs']) != 0: echo_count += 1 interested_keys = ['energy', 'liveness', 'tempo', 'speechiness', 'acousticness', 'danceability','instrumentalness', 'loudness', 'valence','key','mode', 'time_signature'] outline = song_id for ikey in interested_keys: outline = outline + ',' + \ str(jsondata['response']['songs'][0]['audio_summary'][ikey]) outfile.write(outline+ '\n') outfile.flush() outfile.close() print 'Outof : ' + str(track_count) + ' EchoNest had information for:' + str(echo_count)
def get(self): try: title = self.request.get('title').strip() if title: config = util.get_config() vals = { 'item' : { 'title' : title, 'value' : config.config[title], } } self.template( 'config-del.html', vals, 'admin' ); else: self.redirect('.') except: self.redirect('.')
def get(self): config = util.get_config() title = None value = None if self.request.get('title'): title = self.request.get('title') if title in config.config: value = config.config[title] vals = { 'item' : { 'title' : title, 'value' : value, } } self.template( 'config-form.html', vals, 'admin' );
def setup_training_discriminator(model): """Does setup before starting training (run_training)""" train_dir = os.path.join(FLAGS.log_root, "train-discriminator") if not os.path.exists(train_dir): os.makedirs(train_dir) model.build_graph() # build the graph saver = tf.train.Saver(max_to_keep=10) # we use this to load checkpoints for decoding sess = tf.Session(config=util.get_config()) init = tf.global_variables_initializer() sess.run(init) #util.load_ckpt(saver, sess, ckpt_dir="train-discriminator") return sess, saver,train_dir
def run_eval(model, batcher, vocab): """Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far.""" model.build_graph() # build the graph saver = tf.train.Saver(max_to_keep=3) # we will keep 3 best checkpoints at a time sess = tf.Session(config=util.get_config()) eval_dir = os.path.join(FLAGS.log_root, "eval") # make a subdir of the root dir for eval data bestmodel_save_path = os.path.join(eval_dir, 'bestmodel') # this is where checkpoints of best models are saved summary_writer = tf.summary.FileWriter(eval_dir) running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to implement early stopping best_loss = None # will hold the best loss achieved so far while True: _ = util.load_ckpt(saver, sess) # load a new checkpoint batch = batcher.next_batch() # get the next batch # run eval on the batch t0=time.time() results = model.run_eval_step(sess, batch) t1=time.time() tf.logging.info('seconds for batch: %.2f', t1-t0) # print the loss and coverage loss to screen loss = results['loss'] tf.logging.info('loss: %f', loss) if FLAGS.coverage: coverage_loss = results['coverage_loss'] tf.logging.info("coverage_loss: %f", coverage_loss) # add summaries summaries = results['summaries'] train_step = results['global_step'] summary_writer.add_summary(summaries, train_step) # calculate running avg loss running_avg_loss = calc_running_avg_loss(np.asscalar(loss), running_avg_loss, summary_writer, train_step) # If running_avg_loss is best so far, save this checkpoint (early stopping). # These checkpoints will appear as bestmodel-<iteration_number> in the eval dir if best_loss is None or running_avg_loss < best_loss: tf.logging.info('Found new best model with %.3f running_avg_loss. Saving to %s', running_avg_loss, bestmodel_save_path) saver.save(sess, bestmodel_save_path, global_step=train_step, latest_filename='checkpoint_best') best_loss = running_avg_loss # flush the summary writer every so often if train_step % 100 == 0: summary_writer.flush()
def post(self): title = None value = None config = util.get_config() try: # get all the incoming values title = self.request.get('title').strip() value = self.request.get('value').strip() config.config[title] = value config.put() memcache.delete(title, namespace='config') self.redirect('.') except Exception, err: vals['item'] = self.request.POST vals['err'] = err self.template( 'config-form.html', vals, 'admin' );
def get_echonest_id(file_name = 'data/musicovery_unique.csv'): api_key = util.get_config([echonest_api])[echonest_api] lines = open(file_name).read().split('\r') # weird mac issue http = urllib3.PoolManager() outfile = open('data/EchoMusic_musicovery.csv','w') for line in lines: sparams = {} song_id, title, artist, mbid, valence, arousal = re.compile(',').split(line.strip()) print line generic = title + ' ' + artist generic = generic.replace('\'','%27') generic = generic.replace('&', ' ') splitAtSpace = re.compile('\s+').split(generic) generic = '%20'.join(splitAtSpace) sparams['key'] = api_key sparams['combined'] = generic url = song_search_url.format(**sparams) try_again = True count = 0 while try_again: count += 1 r = http.request('GET', url) jsondata = json.loads(r.data) if jsondata['response']['status']['code'] == 3: time.sleep(3) else: try_again = False if count > 10: print song_id + ' information not found' + r.data jsondata['response']['status']['code'] = 5 try_again = False if jsondata['response']['status']['code'] == 5: print song_id + ' information not found' + r.data if len(jsondata['response']['songs']) != 0: dataline = unicode(line,errors='replace') + ',' dataline = dataline + jsondata['response']['songs'][0]['artist_name'].replace(',',' ') + \ ',' + jsondata['response']['songs'][0]['artist_id'] + \ ',' + jsondata['response']['songs'][0]['title'].replace(',',' ') + \ ',' + jsondata['response']['songs'][0]['id'] + \ ','+ song_id outfile.write(dataline + '\n') outfile.flush() outfile.close()
def file_passes(self, temp_filename, original_filename=None): if original_filename is None: original_filename = temp_filename pep8_path = os.path.join(os.path.dirname(__file__), "pep8", "pep8.py") pep8_ignore = get_config("pep8-ignore") pep8_command = "{pep8_path} --ignore={ignore} -r {filename}".format(pep8_path=pep8_path, ignore=pep8_ignore, filename=temp_filename) pep8_out, pep8_err, pep8_rc = run_command(pep8_command) if len(pep8_err) > 0: return False, "# Internal error checking pep8:\n{pep8_err}\n".format(pep8_err=pep8_err) if len(pep8_out) > 0: assert temp_filename.endswith(original_filename) temp_dir = temp_filename[:-len(original_filename)] error_message = "# pep8 problems with {f}:".format(f=original_filename) pep8_formatted = "\n".join(["# " + line.replace(temp_dir, "") for line in pep8_out.splitlines()]) return False, error_message + "\n" + pep8_formatted return True, None
def convert_to_coverage_model(): """Load non-coverage checkpoint, add initialized extra variables for coverage, and save as new checkpoint""" tf.logging.info("converting non-coverage model to coverage model..") # initialize an entire coverage model from scratch sess = tf.Session(config=util.get_config()) print "initializing everything..." sess.run(tf.global_variables_initializer()) # load all non-coverage weights from checkpoint saver = tf.train.Saver([v for v in tf.global_variables() if "coverage" not in v.name and "Adagrad" not in v.name]) print "restoring non-coverage variables..." curr_ckpt = util.load_ckpt(saver, sess) print "restored." # save this model and quit new_fname = curr_ckpt + '_cov_init' print "saving model to %s..." % (new_fname) new_saver = tf.train.Saver() # this one will save all variables that now exist new_saver.save(sess, new_fname) print "saved." exit()
def setup_training(model, batcher): """Does setup before starting training (run_training)""" train_dir = os.path.join(FLAGS.log_root, "train") if FLAGS.finetune: if not os.path.exists(train_dir): print (util.bcolors.OKGREEN + 'Copying See et al. pre-trained model (%s) to (%s) to be fine-tuned' % (os.path.join(FLAGS.pretrained_path, 'train'), train_dir) + util.bcolors.ENDC) os.makedirs(train_dir) files = glob.glob(os.path.join(os.path.join(FLAGS.pretrained_path, 'train'), "*model*")) files.extend(glob.glob(os.path.join(os.path.join(FLAGS.pretrained_path, 'train'), "*checkpoint*"))) for file in files: if os.path.isfile(file): shutil.copy2(file, train_dir) if not os.path.exists(train_dir): os.makedirs(train_dir) model.build_graph() # build the graph if FLAGS.convert_to_coverage_model: assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True" convert_to_coverage_model() if FLAGS.restore_best_model: restore_best_model() saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time sv = tf.train.Supervisor(logdir=train_dir, is_chief=True, saver=saver, summary_op=None, save_summaries_secs=60, # save summaries for tensorboard every 60 secs save_model_secs=60, # checkpoint every 60 secs global_step=model.global_step) summary_writer = sv.summary_writer logging.info("Preparing or waiting for session...") sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config()) logging.info("Created session.") try: run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted except KeyboardInterrupt: logging.info("Caught keyboard interrupt on worker. Stopping supervisor...") sv.stop()
def setup_training(model, batcher, config): train_directory = config['train_folder'] if not os.path.exists(train_directory): os.makedirs(train_directory) saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time sv = tf.train.Supervisor( logdir=train_directory, is_chief=True, saver=saver, summary_op=None, save_summaries_secs=60, # save summaries for tensorboard every 60 secs save_model_secs=60, # checkpoint every 60 secs #global_step=model.global_step ) #sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config()) sess_context_manager = sv.prepare_or_wait_for_session( config=util.get_config()) ### managed_session with sess_context_manager as sess: loss_lst = [] loss_long, loss_all = 0, 0 for it in range(config['train_iter']): t1 = time() batch = batcher.next_batch() to_return = model.model_train(batch, sess) loss = to_return['loss'] loss_all += loss loss_long += loss t2 = time() #print(t2 - t1) if it > 0 and it % 100 == 0: print('iter: {}, loss: {}'.format(it, str(loss_all)[:6])) loss_all = 0 if it % 1000 == 0 and it > 0: loss_lst.append(loss_long) print(loss_lst) loss_long = 0
def setup_training(mode, generator, discriminator, generator_batcher, discriminator_batcher): train_dir = os.path.join(FLAGS.log_root, "train") if not os.path.exists(train_dir): os.makedirs(train_dir) if FLAGS.restore_best_model: restore_best_model() return saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time supervisor = tf.train.Supervisor( logdir=train_dir, is_chief=True, saver=saver, summary_op=None, save_summaries_secs=60, # save summaries for tensorboard every 60 secs save_model_secs=60, # checkpoint every 60 secs global_step=generator.global_step) summary_writer = supervisor.summary_writer sess_context_manager = supervisor.prepare_or_wait_for_session( config=util.get_config()) try: if mode == "pretrain": trainer.pretrain_generator(generator, generator_batcher, summary_writer, sess_context_manager) elif mode == "train": trainer.pretrain_discriminator(discriminator, sess_context_manager) trainer.adversarial_train(generator, discriminator, generator_batcher, discriminator_batcher, summary_writer, sess_context_manager) else: raise ValueError("Caught invalid value of mode!") except KeyboardInterrupt: tf.logging.info( "Caught keyboard interrupt on worker. Stopping supervisor...") supervisor.stop()
def __init__(self, model, batcher, vocab): """Initialize decoder. Args: model: a Seq2SeqAttentionModel object. batcher: a Batcher object. vocab: Vocabulary object """ self._model = model self._model.build_graph() self._batcher = batcher self._vocab = vocab self._saver = tf.train.Saver() # we use this to load checkpoints for decoding self._sess = tf.Session(config=util.get_config()) # Load an initial checkpoint to use for decoding ckpt_path = util.load_ckpt(self._saver, self._sess) if FLAGS.single_pass: # Make a descriptive decode directory name ckpt_name = "ckpt-" + ckpt_path.split('-')[-1] # this is something of the form "ckpt-123456" self._decode_dir = os.path.join(FLAGS.log_root, get_decode_dir_name(ckpt_name)) if os.path.exists(self._decode_dir): raise Exception("single_pass decode directory %s should not already exist" % self._decode_dir) else: # Generic decode dir name self._decode_dir = os.path.join(FLAGS.log_root, "decode") # Make the decode dir if necessary if not os.path.exists(self._decode_dir): os.mkdir(self._decode_dir) #if FLAGS.single_pass: if FLAGS.single_pass: # Make the dirs to contain output written in the correct format for pyrouge self._rouge_ref_dir = os.path.join(self._decode_dir, "reference") if not os.path.exists(self._rouge_ref_dir): os.mkdir(self._rouge_ref_dir) self._rouge_dec_dir = os.path.join(self._decode_dir, "decoded") if not os.path.exists(self._rouge_dec_dir): os.mkdir(self._rouge_dec_dir)
def setup_training(model, batcher): """Does setup before starting training (run_training)""" train_dir = os.path.join(FLAGS.log_root, "train") if not os.path.exists(train_dir): os.makedirs(train_dir) model.build_graph() # build the graph if FLAGS.convert_to_coverage_model: if not FLAGS.coverage: raise ValueError( "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True" "and coverage=True") convert_to_coverage_model() if FLAGS.restore_best_model: restore_best_model() saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time sv = tf.train.Supervisor( logdir=train_dir, is_chief=True, saver=saver, summary_op=None, save_summaries_secs=60, # save summaries for tensorboard every 60 secs save_model_secs=60, # checkpoint every 60 secs global_step=model.global_step) summary_writer = sv.summary_writer tf.logging.info("Preparing or waiting for session...") sess_context_manager = sv.prepare_or_wait_for_session( config=util.get_config()) tf.logging.info("Created session.") try: run_training( model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted except KeyboardInterrupt: tf.logging.info( "Caught keyboard interrupt on worker. Stopping supervisor...") sv.stop()
def restore_best_model(): """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory""" tf.logging.info("Restoring bestmodel for training...") # Initialize all vars in the model sess = tf.Session(config=util.get_config()) print "Initializing all variables..." sess.run(tf.initialize_all_variables()) # Restore the best model from eval dir saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name]) print "Restoring all non-adagrad variables from best model in eval dir..." curr_ckpt = util.load_ckpt(saver, sess, "eval") print "Restored %s." % curr_ckpt # Save this model to train dir and quit new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model") new_fname = os.path.join(FLAGS.log_root, "train", new_model_name) print "Saving model to %s..." % (new_fname) new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables new_saver.save(sess, new_fname) print "Saved." exit()
def restore_best_model(): """Load bestmodel file from eval directory, add variables for adagrad, and save to train directory""" tf.logging.info("Restoring bestmodel for training...") # Initialize all vars in the model sess = tf.Session(config=util.get_config()) print("Initializing all variables...") sess.run(tf.initialize_all_variables()) # Restore the best model from eval dir saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name]) print("Restoring all non-adagrad variables from best model in eval dir...") curr_ckpt = util.load_ckpt(saver, sess, "eval") print ("Restored %s." % curr_ckpt) # Save this model to train dir and quit new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model") new_fname = os.path.join(FLAGS.log_root, "train", new_model_name) print ("Saving model to %s..." % (new_fname)) new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables new_saver.save(sess, new_fname) print ("Saved.") exit()
def restore_best_model(): print("Restoring bestmodel for training...") # Initialize all vars in the model sess = tf.Session(config=util.get_config()) print("Initializing all variables...") sess.run(tf.initialize_all_variables()) # Restore the best model from eval dir saver = tf.train.Saver( [v for v in tf.all_variables() if "Adagrad" not in v.name]) print("Restoring all non-adagrad variables from best model in eval dir...") curr_ckpt = util.load_ckpt(saver, sess, "val") print("Restored %s." % curr_ckpt) new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model") new_fname = os.path.join(FLAGS.exp_name, "train", new_model_name) print("Saving model to %s..." % (new_fname)) new_saver = tf.train.Saver() new_saver.save(sess, new_fname) print("Saved.") exit()
def convert_to_coverage_model(): """Load non-coverage checkpoint, add initialized extra variables for coverage, and save as new checkpoint""" tf.logging.info("converting non-coverage model to coverage model..") # initialize an entire coverage model from scratch sess = tf.Session(config=util.get_config()) print("initializing everything...") sess.run(tf.global_variables_initializer()) # load all non-coverage weights from checkpoint saver = tf.train.Saver([v for v in tf.global_variables( ) if "coverage" not in v.name and "Adagrad" not in v.name]) print("restoring non-coverage variables...") curr_ckpt = util.load_ckpt(saver, sess) print("restored.") # save this model and quit new_fname = curr_ckpt + '_cov_init' print("saving model to %s..." % (new_fname)) new_saver = tf.train.Saver() # this one will save all variables that now exist new_saver.save(sess, new_fname) print("saved.") exit()
def __init__(self, model, batcher, vocab, ckpt_path): self._model = model self._model.build_graph() self._batcher = batcher self._vocab = vocab self.ckpt_path = ckpt_path self._saver = tf.train.Saver() self._sess = tf.Session(config=util.get_config()) self._saver.restore(self._sess, self.ckpt_path) print("load mode from %s" % self.ckpt_path) self.model_num = self.ckpt_path.split('-')[-1] ckpt_name = "ckpt-" + self.model_num # this is something of the form "ckpt-123456" self._decode_dir = os.path.join(FLAGS.log_root, get_infer_dir(ckpt_name)) # Make the decode dir if necessary if not os.path.exists(self._decode_dir): os.mkdir(self._decode_dir) else: raise Exception("infer directory %s should not already exist")
def infer(self): def build_infer(): embedding = Nmt._build_embedding(self.src_vocab_size, self.opts.embedding_size, name="source_embedding") input_, encoder_outputs, encoder_state = self._build_encoder( embedding) embedding = Nmt._build_embedding(self.tgt_vocab_size, self.opts.embedding_size, name="tgt_embedding") samples, logits = self._build_decoder(encoder_outputs, encoder_state, embedding, train=False) return samples, logits with ipu_scope('/device:IPU:0'): data, vocab = self._build_inputs() batch = ipu_compiler.compile(build_infer, []) # Create a restoring object saver = tf.train.Saver() ipu_options = util.get_config(report_n=0) utils.configure_ipu_system(ipu_options) session = tf.Session() checkpoint = CHECKPOINT_FILE + 'ckpt' saver.restore(session, checkpoint) # Run a dummy value to force the graph compilation session.run(batch, feed_dict=next(data)) while True: feed_dict = next(data) predictions, _ = session.run(batch, feed_dict=feed_dict) print_data(feed_dict[self.placeholders['source']], vocab[0], predictions, vocab[1]) if not self.opts.interact: break
def setup_training(model, batcher): """Does setup before starting training (run_training)""" train_dir = os.path.join(FLAGS.log_root, "train") if not os.path.exists(train_dir): os.makedirs(train_dir) model.build_graph() # build the graph if FLAGS.convert_to_coverage_model: assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True" convert_to_coverage_model() if FLAGS.restore_best_model: restore_best_model() saver = tf.train.Saver(max_to_keep=30) # keep 3 checkpoints at a time saver_hook = tf.train.CheckpointSaverHook( train_dir, save_steps=5000, saver=saver) #Saves checkpoints every N steps or seconds. summary_writer = tf.summary.FileWriter(train_dir) #将汇总的写入文件 #Saves summaries every N steps. summary_hook = tf.train.SummarySaverHook(save_steps=20, summary_op=model._summaries, summary_writer=summary_writer) tf.logging.info("Created session.") session = tf.train.MonitoredTrainingSession( hooks=[saver_hook, summary_hook], config=util.get_config(), checkpoint_dir=train_dir) summary_writer.add_graph(session.graph) try: run_training( model, batcher, session, summary_writer) # this is an infinite loop until interrupted except KeyboardInterrupt: tf.logging.info( "Caught keyboard interrupt on worker. Stopping supervisor...") session.close()
def get_songs(): api_key = util.get_config([musicovery_api])[musicovery_api] valences = range(50000, 1000000, 100000) arousals = range(50000, 1000000, 100000) f = open('data/musicovery_60.csv','w') http = urllib3.PoolManager() for valence in valences: for arousal in arousals: print 'getting songs for valence ' + str(valence) + ' arousal: ' + str(arousal) url = musicovery_url.format(**{'valence':valence, 'arousal':arousal, 'api_key':api_key}) r = http.request('GET', url) jsondata = json.loads(r.data) print url if jsondata['root']['response']['code'] != '100': print 'ERROR out ' + jsondata['root']['response']['code'] time.sleep(2) r = http.request('GET', url) jsondata = json.loads(r.data) for track in jsondata['root']['tracks']['track']: try: title = defaultMe(track['title']) id = defaultMe(track['id']) artist = defaultMe(track['artist']['name']) artist_mbid = defaultMe(track['artist']['mbid']) music_valence = defaultMe(track['valence']) music_arousal = defaultMe(track['arousal']) releasedate = defaultMe(track['releasedate']) genre = defaultMe(track['genre']) line = str(id) + ',' + title + ',' + artist + ',' + \ artist_mbid + ',' + str(music_valence) + ',' + \ str(music_arousal) f.write(line + '\n') f.flush() except: print 'some error for one track!!' continue f.close()
def setup_training(generator, discriminator, generator_batcher, discriminator_batcher): """Does setup before starting training (run_training)""" train_dir = os.path.join(FLAGS.log_root, "train") if not os.path.exists(train_dir): os.makedirs(train_dir) if FLAGS.restore_best_model: restore_best_model() saver = tf.train.Saver(max_to_keep=5) # keep 3 checkpoints at a time sv = tf.train.Supervisor( logdir=train_dir, is_chief=True, saver=saver, summary_op=None, save_summaries_secs=60, # save summaries for tensorboard every 60 secs save_model_secs=60, # checkpoint every 60 secs global_step=generator.global_step) summary_writer = sv.summary_writer tf.logging.info("Preparing or waiting for session...") sess_context_manager = sv.prepare_or_wait_for_session( config=util.get_config()) tf.logging.info("Created session.") try: if FLAGS.pretrain_discriminator: pre_train_discriminator(discriminator, sess_context_manager) run_training(generator, discriminator, generator_batcher, discriminator_batcher, summary_writer, sess_context_manager) #model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted except KeyboardInterrupt: tf.logging.info( "Caught keyboard interrupt on worker. Stopping supervisor...") sv.stop()
def get_echonest_analysis(file_name = 'data/echo_music_songids.txt'): api_key = util.get_config([echonest_api])[echonest_api] with open(file_name) as f: for line in f: http = urllib3.PoolManager() song_id = line.strip() params = {} params['key'] = api_key params['id'] = song_id url = song_profile_url.format(**params) print url try_again = True count = 0 while try_again: count += 1 print count r = http.request('GET', url) jsondata = json.loads(r.data) if jsondata['response']['status']['code'] == 3: time.sleep(3) else: try_again = False if count > 15: print song_id + ' information not found' + r.data jsondata['response']['status']['code'] = 5 try_again = False if jsondata['response']['status']['code'] == 5: print song_id + ' information not found' + r.data if len(jsondata['response']['songs']) != 0: amazon_url = jsondata['response']['songs'][0]['audio_summary']['analysis_url'] r_amazon = http.request('GET', amazon_url) try: json_amazon = json.loads(r_amazon.data) json.dump(json_amazon,open('data/echo_analysis/'+song_id+'.json','w')) except: print 'song_id: ' + song_id + ' has no analysis data'
def test_encoder_layer_build(self, batch_size, prefix): os.chdir(os.path.dirname(os.path.realpath(__file__))) os.chdir('../') from model import AMConfig from model import ConformerAM from util import get_config cfg = get_config(1) cfg.configure_ipu_system() config_file = 'configs/train_fp32_kl_loss.yaml' config = AMConfig.from_yaml(config_file) config['use_ipu_dropout'] = False model = ConformerAM(config) x = tf.random_normal(shape=(batch_size, model.config['maxlen_in'], model.config['adim'])) mask_adder = tf.random_normal(shape=(batch_size, 1, 1, model.config['maxlen_in'])) pos_emb = tf.random_normal(shape=(1, model.config['maxlen_in'], model.config['adim'])) layer = model._build_encoder_layer(x, mask_adder, pos_emb, prefix) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) output = sess.run(layer) assert output.shape == (batch_size, model.config['maxlen_in'], model.config['adim'])
def helper_infection_rate_param_sweep(filename, expected_avg_list, infection_rate_list=None): ''' Purpose: helper function for task 6 Inputs: filename: (str) json file to open expected_avg_list: (list) expected value infection_rate_list: (list) optional input parameter ''' starting_state, random_seed, max_num_days, _, num_trials = \ util.get_config(filename) if infection_rate_list is None: infection_rate_list = [0, 0.25, 0.5, 0.75, 1.0] # helper function for testing actual_avg_list = sir.infection_rate_param_sweep(starting_state, random_seed, max_num_days, infection_rate_list, num_trials) if actual_avg_list != pytest.approx(expected_avg_list): s = "Actual ({:}) and expected ({:}) final states do not match" pytest.fail(s.format(actual_avg_list, expected_avg_list))
def run_decoding(model, batcher): model.build_graph() # build the graph saver = tf.train.Saver( max_to_keep=3) # we will keep 3 best checkpoints at a time sess = tf.Session(config=util.get_config()) decode_dir = os.path.join( FLAGS.exp_name, "decode") # make a subdir of the root dir for eval data batches = batcher.getBatches() _ = util.load_ckpt(saver, sess) # load a new checkpoint epoch_avg_loss = 0. epoch_decode_steps = 0 for batch in batches: results = model.run_decode_step(sess, batch) loss = results['loss'] epoch_decode_steps += 1 epoch_avg_loss = (epoch_avg_loss * (epoch_decode_steps - 1.) + loss) / epoch_decode_steps print("Average loss %f" % (epoch_avg_loss))
def __init__(self, model, vocab,single_pass,hps,pointer_gen,log_root): """Initialize decoder. Args: model: a Seq2SeqAttentionModel object. batcher: a Batcher object. vocab: Vocabulary object """ self._model = model self._model.build_graph() # self._batcher = batcher self._vocab = vocab self._saver = tf.train.Saver() # we use this to load checkpoints for decoding self._sess = tf.Session(config=util.get_config()) self.single_pass=single_pass self. max_dec_steps=hps.max_dec_steps self.min_dec_steps=hps.min_dec_steps self.max_dec_steps=hps.max_dec_steps self.beam_size=hps.beam_size self.pointer_gen=pointer_gen # Load an initial checkpoint to use for decoding ckpt_path = util.load_ckpt(self._saver, self._sess,log_root) print ckpt_path
def get_cluster() -> Cluster: """ Get the cluster to connect to. Because we only ever want to connect to a single cassandra cluster, the output of this function is cached. Effectively, the cluster is a singleton. """ config = get_config() ips = config['db_addresses'] port = int(config['db_port']) keyspace = config['db_keyspace'] username = config['db_username'] password = config['db_password'] auth: PlainTextAuthProvider = PlainTextAuthProvider(username, password) cluster: Cluster = Cluster(ips, port=port, auth_provider=auth) cluster.register_user_type(keyspace, 'class_session', ClassSession) cluster.register_user_type(keyspace, 'class_stream', ClassStream) cluster.register_user_type(keyspace, 'class', Class) return cluster
def main(): """ Main function, it will start the timer, instantiate de logger and inititate de build/get process :return: """ try: start = time.time() config = util.get_config() set_globals(config) util.change_cwd(BASE_PATH) except Exception as err: print(err) try: create_logger() except Exception as err: print(err) return logger.info('Process starded....') print('####################') is_proc_ok = start_process(config) minutes = (time.time() - start) / 60 if is_proc_ok: util.print_and_log(logger.info, 'Complete process took {} minutes'.format(minutes)) else: util.print_and_log( logger.info, 'Complete process took {} minutes, with ERRORS. Check log.'.format( minutes))
def config_collection(shutit): """Collect core config from config files for all seen modules. """ cfg = shutit.cfg shutit_map = shutit.shutit_map for mid in module_ids(shutit): # Default to None so we can interpret as ifneeded util.get_config(cfg,mid,'build',None,boolean=True) util.get_config(cfg,mid,'remove',False,boolean=True) util.get_config(cfg,mid,'do_repository_work',False,boolean=True) # ifneeded will (by default) only take effect if 'build' is not specified # It can, however, be forced to a value, but this should be unusual if cfg[mid]['build'] is None: shutit.get_config(mid,'build_ifneeded',True,boolean=True) cfg[mid]['build'] = False else: shutit.get_config(mid,'build_ifneeded',False,boolean=True)
def copy_checkpoint(source, target): for ext in (".index", ".data-00000-of-00001"): shutil.copyfile(source + ext, target + ext) if __name__ == "__main__": # util.set_gpus() if len(sys.argv) > 1: name = sys.argv[1] print("Running experiment: {} (from command-line argument).".format(name)) else: name = os.environ["EXP"] print("Running experiment: {} (from environment variable).".format(name)) config = util.get_config("experiments.conf")[name] config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name)) # Dynamic batch size. config["batch_size"] = -1 config["max_tokens_per_batch"] = -1 # Use dev lm, if provided. if config["lm_path"] and "lm_path_dev" in config and config["lm_path_dev"]: config["lm_path"] = config["lm_path_dev"] util.print_config(config) data = LSGNData(config) model = SRLModel(data, config) evaluator = LSGNEvaluator(config)
# -*- coding: utf-8 -*- import wrap # 用于输出运行日志 import logging logging.basicConfig( level=logging.INFO, format= '127.0.0.1 - - [%(asctime)s - %(name)s - %(levelname)s - %(message)s]') logger = logging.getLogger(__name__) # 数据库连接信息 import mysql.connector import util config = util.get_config() USER = config["user"] PASSWORD = config["password"] DATABASE = config["database"] # 连接mysql数据库 def get_connect(): conn = mysql.connector.connect(user=USER, password=PASSWORD, database=DATABASE) return conn # ==============TODO留作参考注册函数======================= # def save_user(username, password):
#!/bin/env python # -*- encoding: utf-8 -*- # 导入web目录__init__ 创建flask实例 from web import app import os, sys, logging, logging.config import db, util work_dir = os.path.dirname(os.path.realpath(__file__)) app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT' service_conf = os.path.join(work_dir, 'conf/service.conf') config = util.get_config(service_conf, 'web') #print config #将参数追加到app.config字典中,就可以随意调用了 app.config.update(config) if __name__ == '__main__': app.run(host=config.get('bind', '0.0.0.0'), port=int(config.get('port')), debug=True)
pred_answers.append(best_ans) f1s.append(f1) hits.append(hit) print('evaluation.......') print('how many eval samples......', len(f1s)) print('avg_f1', np.mean(f1s)) print('avg_hits', np.mean(hits)) model.train() return np.mean(f1s), np.mean(hits) if __name__ == "__main__": # config_file = sys.argv[2] cfg = get_config() random.seed(cfg['seed']) np.random.seed(cfg['seed']) torch.manual_seed(cfg['seed']) torch.cuda.manual_seed_all(cfg['seed']) if cfg['mode'] == 'train': train(cfg) elif cfg['mode'] == 'test': documents = load_documents(cfg['data_folder'] + cfg['{}_documents'.format(cfg['mode'])]) test_data = DataLoader(cfg, documents, mode='test') model = KAReader(cfg) model = model.to(torch.device('cuda')) model_save_path = 'model/{}/{}_best.pt'.format(cfg['name'], cfg['model_id']) model.load_state_dict(torch.load(model_save_path))
def __init__(self, model, batcher, vocab): """Initialize decoder. Args: model: a SentSelector object. batcher: a Batcher object. vocab: Vocabulary object """ # get the data split set if "train" in FLAGS.data_path: self._dataset = "train" elif "val" in FLAGS.data_path: self._dataset = "val" elif "test" in FLAGS.data_path: self._dataset = "test" else: raise ValueError( "FLAGS.data_path %s should contain one of train, val or test" % (FLAGS.data_path)) # create the data loader self._batcher = batcher if FLAGS.eval_gt_rouge: # no need to load model, default is fasle # Make a descriptive decode directory name self._decode_dir = os.path.join(FLAGS.log_root, 'select_gt' + self._dataset) tf.logging.info('Save evaluation results to ' + self._decode_dir) if os.path.exists(self._decode_dir): raise Exception( "single_pass decode directory %s should not already exist" % self._decode_dir) # Make the decode dir os.makedirs(self._decode_dir) # Make the dirs to contain output written in the correct format for pyrouge self._rouge_ref_dir = os.path.join(self._decode_dir, "reference") if not os.path.exists(self._rouge_ref_dir): os.mkdir(self._rouge_ref_dir) self._rouge_gt_dir = os.path.join(self._decode_dir, "gt_selected") if not os.path.exists(self._rouge_gt_dir): os.mkdir(self._rouge_gt_dir) else: # FALSE self._model = model self._model.build_graph() self._vocab = vocab self._saver = tf.train.Saver( ) # we use this to load checkpoints for decoding self._sess = tf.Session(config=util.get_config()) # Load an initial checkpoint to use for decoding print(" eval_ckpt_path ", FLAGS.eval_ckpt_path) if FLAGS.load_best_eval_model: tf.logging.info('Loading best eval checkpoint') ckpt_path = util.load_ckpt(self._saver, self._sess, ckpt_dir='eval') elif FLAGS.eval_ckpt_path: ckpt_path = util.load_ckpt(self._saver, self._sess, ckpt_path=FLAGS.eval_ckpt_path, ckpt_dir='eval') else: tf.logging.info('Loading best train checkpoint') ckpt_path = util.load_ckpt(self._saver, self._sess) if FLAGS.single_pass: # Make a descriptive decode directory name ckpt_name = "ckpt-" + ckpt_path.split('-')[ -1] # this is something of the form "ckpt-123456" decode_root_dir, decode_dir = get_decode_dir_name( ckpt_name, self._dataset) self._decode_root_dir = os.path.join(FLAGS.log_root, decode_root_dir) self._decode_dir = os.path.join(FLAGS.log_root, decode_root_dir, decode_dir) tf.logging.info('Save evaluation results to ' + self._decode_dir) if os.path.exists(self._decode_dir): raise Exception( "single_pass decode directory %s should not already exist" % self._decode_dir) else: # Generic decode dir name self._decode_dir = os.path.join(FLAGS.log_root, "select") # Make the decode dir if necessary if not os.path.exists(self._decode_dir): os.makedirs(self._decode_dir) if FLAGS.single_pass: # Make the dirs to contain output written in the correct format for pyrouge self._rouge_ref_dir = os.path.join(self._decode_dir, "reference") if not os.path.exists(self._rouge_ref_dir): os.mkdir(self._rouge_ref_dir) self._rouge_dec_dir = os.path.join(self._decode_dir, "selected") if not os.path.exists(self._rouge_dec_dir): os.mkdir(self._rouge_dec_dir) if FLAGS.save_pkl: self._result_dir = os.path.join(self._decode_dir, "select_result") if not os.path.exists(self._result_dir): os.mkdir(self._result_dir) self._probs_pkl_path = os.path.join(self._decode_root_dir, "probs.pkl") if not os.path.exists(self._probs_pkl_path): self._make_probs_pkl = True else: self._make_probs_pkl = False self._precision = [] self._recall = [] self._accuracy = [] self._ratio = [] self._select_sent_num = []
from api import app, zabbix_api, zabbix_Graph_api import os,sys,logging,logging.config import db,util from celery import Celery,platforms #session使用需要设置secret_key app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT' #导入自定义的各种配置参数,最终参数以字典形式返回 work_dir = os.path.dirname(os.path.realpath(__file__)) service_conf = os.path.join(work_dir, 'conf/service.conf') img = os.path.join(work_dir,'web/static/zabbix') img_url = {"zabbix_img_url":img} config = util.get_config(service_conf, 'api') cobbler_config = util.get_config(service_conf, 'cobbler') zabbix_config = util.get_config(service_conf, 'zabbix') #将参数追加到app.config字典中,就可以随意调用了 app.config.update(config) app.config.update(cobbler_config) app.config.update(img_url) app.config.update(zabbix_config) #print app.config #实例化数据库类,并将实例化的对象导入配置 app.config['cursor'] = db.Cursor(config)
# エキスパートPythonプログラミング 改訂2版 13章 # マルチスレッドその2 # ワーカースレッド内で発生した例外をメインスレッドに返す import time from gmaps import Geocoding from threading import Thread, current_thread from queue import Queue, Empty from util import PLACES, get_config api = Geocoding(api_key=get_config()["geocode_api_key"]) def fetch_place(place): geocoded = api.geocode(place)[0] return geocoded, current_thread().name def present_result(geocoded, thread_name): print( f'{geocoded["formatted_address"]:>60s}, {geocoded["geometry"]["location"]["lat"]:6.2f}, {geocoded["geometry"]["location"]["lng"]:6.2f} [{thread_name}]' ) def worker(work_queue, results_queue): # 各々のスレッドは、work_queue から取り出して処理する # どの都市をどのスレッドが処理するかは、その時次第 while not work_queue.empty(): try: item = work_queue.get(block=False) except Empty:
my_model = use_cuda( GraftNet(word_emb_file, entity_emb_file, entity_kge_file, relation_emb_file, relation_kge_file, cfg['num_layer'], num_kb_relation, num_entities, num_vocab, cfg['entity_dim'], cfg['word_dim'], cfg['kge_dim'], cfg['pagerank_lambda'], cfg['fact_scale'], cfg['lstm_dropout'], cfg['linear_dropout'], cfg['use_kb'], cfg['use_doc'])) if cfg['load_model_file'] is not None: print('loading model from', cfg['load_model_file']) pretrained_model_states = torch.load(cfg['load_model_file']) if word_emb_file is not None: del pretrained_model_states['word_embedding.weight'] if entity_emb_file is not None: del pretrained_model_states['entity_embedding.weight'] my_model.load_state_dict(pretrained_model_states, strict=False) return my_model if __name__ == "__main__": config_file = sys.argv[2] CFG = get_config(config_file) if '--train' == sys.argv[1]: train(CFG) elif '--test' == sys.argv[1]: test(CFG) else: assert False, "--train or --test?"
def run_eval(model, batcher, re_vocab, embed): """Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far.""" model.build_graph(embed) # build the graph saver = tf.train.Saver( max_to_keep=10) # we will keep 3 best checkpoints at a time sess = tf.Session(config=util.get_config()) eval_dir = os.path.join( FLAGS.log_root, "eval") # make a subdir of the root dir for eval data bestmodel_save_path = os.path.join( eval_dir, 'bestmodel') # this is where checkpoints of best models are saved summary_writer = tf.summary.FileWriter(eval_dir) running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to implement early stopping best_loss = None # will hold the best loss achieved so far count = 0 while True: _ = util.load_ckpt(saver, sess) # load a new checkpoint batch = batcher.next_batch() # get the next batch if not os.path.exists(FLAGS.model_dir): os.makedirs(FLAGS.model_dir) os.makedirs(FLAGS.system_dir) # run eval on the batch t0 = time.time() # results = model.run_eval_step(sess, batch) step_output = model.run_eval_step(sess, batch) t1 = time.time() #tf.logging.info('seconds for batch: %.2f', t1-t0) (summaries, loss, train_step) = step_output[0] (out_decoder_outputs, out_sent_decoder_outputs, final_dists) = step_output[1] (step_loss, word_loss, sent_loss, word_loss_null, sent_loss_null, switch_loss) = step_output[2] coverage_loss = 0.0 running_avg_loss = calc_running_avg_loss(np.asscalar(loss), running_avg_loss, summary_writer, train_step) if best_loss is None or running_avg_loss < (best_loss): if best_loss is None: best_loss = 0.0 tf.logging.info( 'Found new best model with %.3f running_avg_loss. Saving to %s', running_avg_loss, bestmodel_save_path) saver.save(sess, bestmodel_save_path, global_step=train_step, latest_filename='checkpoint_best') best_loss = running_avg_loss last_step = train_step tf.logging.info('loss: %f rloss: %f', loss, rloss) # print the loss to screen tf.logging.info( 'step_loss: %f word_loss: %f ,sent_loss: %f ,word_loss_null: %f,sent_loss_null: %f ,switch_loss: %f,cover_loss: %f', step_loss, word_loss, sent_loss, word_loss_null, sent_loss_null, switch_loss, coverage_loss) os.system("rm -rf " + FLAGS.model_dir + ' ' + FLAGS.system_dir) count = count + 1 if train_step % 100 == 0: summary_writer.flush()
def main(): hooks = [CheckTabs(), CheckIndentation()] debug = get_config("debug", as_bool=True, default=False) should_check_pep8 = get_config("check-pep8", as_bool=True, default=True) if should_check_pep8: hooks += [CheckPep8()] incremental = get_config("incremental", as_bool=True, default=False) incremental_verbose = get_config("incremental.verbose", as_bool=True, default=False) if debug: print "Starting hooks, with pep8 %s, incremental %s, hooks [%s]" % (should_check_pep8, incremental, ", ".join(map(str, hooks))) # create temp directory for getting copies of files from staging. # TODO: Make all the hooks operate on strings instead of files, and get rid of this. temp_dir = tempfile.mkdtemp() temp_dir_with_slash = temp_dir + os.sep failure_encountered = False try: for filename in changed_files(): if debug: print "Examining %s" % filename if not is_python_file(filename): if debug: print "Skipping %s, not a python file" % filename continue relevant_hooks = [hook for hook in hooks if hook.should_process_file(filename)] if not relevant_hooks: if debug: print "Skipping %s, no relevant hooks" % filename continue if incremental: head_temp_filename = make_temp_copy(temp_dir_with_slash, filename, head=True) incremental_hooks = copy.copy(relevant_hooks) if os.path.isfile(head_temp_filename): # This is not a newly added file, so check whether it used to fail the hooks. for relevant_hook in relevant_hooks: head_passes, unused_error_message = relevant_hook.file_passes(head_temp_filename, original_filename=filename) if not head_passes: # Incremental checking was requested, and current HEAD doesn't pass, # so don't bother checking this file with this hook. incremental_hooks.remove(relevant_hook) if incremental_verbose: print "Hook %s failed on current HEAD for file %s"%(relevant_hook, filename) relevant_hooks = incremental_hooks if not relevant_hooks: if debug: print "Skipping %s, no relevant hooks (after incremental check)" % filename continue temp_filename = make_temp_copy(temp_dir_with_slash, filename) for relevant_hook in relevant_hooks: passes, error_message = relevant_hook.file_passes(temp_filename, original_filename=filename) if not passes: failure_encountered = True print(error_message) break finally: shutil.rmtree(temp_dir, True) return int(failure_encountered)
import os import sys import subprocess import logging import archive from time import sleep import util config = util.get_config() bash_path = config.get('treasurecolumn', 'bash_path') filter_shell_script = config.get('treasurecolumn', 'filter_shell_script') tmp_mp3 = config.get('treasurecolumn', 'tmp_mp3') tmp_processed_mp3 = config.get('treasurecolumn', 'tmp_processed_mp3') audio_filter_schell_script = config.get('treasurecolumn', 'audio_filter_schell_script') ffmpeg_location = config.get('treasurecolumn', 'ffmpeg_location') def make_movie(input_dir, output_file, frame_count, rate=5): files = map(lambda f: input_dir + '/' + f, filter(lambda f: f.endswith('.jpg'), os.listdir(input_dir))) queue = [] done = 0 for i, jpg in enumerate(files): parts = os.path.split(jpg) num = parts[1].split('.')[0] newfilename = os.path.join(parts[0], 'processed_%s.jpg' % num.zfill(5)) phase = 'diag4' if i % 2 else 'diag42' p = subprocess.Popen([bash_path, filter_shell_script, jpg, newfilename, phase, '&']) queue.append(p) p.communicate()
import os from datetime import datetime from enum import Enum from fabric.api import run, env, task from fabric.colors import yellow from fabric.contrib.console import confirm from fabric.utils import puts, error from definitions import ROOT_DIR, CONFIG_PATH from util import adjust_expanduser, get_config config = get_config(CONFIG_PATH) env.user = config.get('ssh', 'user') env.hosts = config.get('hosts', env.dest).split(',') if 'dest' in env else None env.gateway = config.get('hosts', 'step', fallback=None) env.ssh_config_path = adjust_expanduser(config.get('ssh', 'config_path')) env.use_ssh_config = True MYSQL_EXEC = config.get('mysql', 'exec') class Style(Enum): txt = { 'message': 'result as text file', 'format': '.txt', 'encode': 'utf-8' } csv = {'message': 'result as csv', 'format': '.csv', 'encode': 'utf-8'} excel = {
def main(_): # Configuration. num_unrolls = FLAGS.num_steps if FLAGS.seed: tf.set_random_seed(FLAGS.seed) # Problem. # problem, net_config, net_assignments = util.get_config(FLAGS.problem, # FLAGS.path) param_dict = {} param_dict['bs'] = FLAGS.bs param_dict['m'] = FLAGS.m param_dict['n'] = FLAGS.n print(param_dict) problem, net_config, net_assignments = util.get_config( FLAGS.problem, net_name="RNNprop", mode=FLAGS.mode, #加入mode num_linear_heads=1, init=FLAGS.init, path=FLAGS.path, param=param_dict) # Optimizer setup. if FLAGS.optimizer == "Adam": cost_op = problem() problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) problem_reset = tf.variables_initializer(problem_vars) optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate) optimizer_reset = tf.variables_initializer(optimizer.get_slot_names()) update = optimizer.minimize(cost_op) reset = [problem_reset, optimizer_reset] elif FLAGS.optimizer == "L2L": if FLAGS.path is None: logging.warning("Evaluating untrained L2L optimizer") optimizer = meta.MetaOptimizer(FLAGS.num_mt, FLAGS.beta1, FLAGS.beta2, **net_config) # meta_loss = optimizer.meta_loss(problem, 1, net_assignments=net_assignments) meta_loss, _, _, _, _, seq_step, \ _, _, _, _, _, _ = optimizer.meta_loss(problem, 1, net_assignments=net_assignments) #这里原来是各种名字的变量的,但是似乎object never used就是指这些,那我就全部用下划线变量代替了 _, update, reset, cost_op, _ = meta_loss else: raise ValueError("{} is not a valid optimizer".format(FLAGS.optimizer)) config = tf.ConfigProto() config.gpu_options.allow_growth = True with ms.MonitoredSession() as sess: # with tf.Session(config=config) as sess: sess.run(reset) # Prevent accidental changes to the graph. tf.get_default_graph().finalize() total_time = 0 total_cost = 0 loss_record = [] for ep in xrange(FLAGS.num_epochs): # Training. time, cost = util.run_eval_epoch(sess, cost_op, [update], num_unrolls, step=seq_step) total_time += time total_cost += sum(cost) / num_unrolls loss_record += cost print(ep, cost[-1]) # Results. util.print_stats("Epoch {}".format(FLAGS.num_epochs), total_cost, total_time, FLAGS.num_epochs) with open( '{}/{}_eval_loss_record.pickle'.format(FLAGS.path, FLAGS.optimizer), 'wb') as l_record: pickle.dump(loss_record, l_record) print("Saving evaluate loss record {}".format(FLAGS.path))
def main(_): # Configuration. # Problem. with tf.variable_scope("problem", partitioner=tf.min_max_variable_partitioner( max_partitions=2, min_slice_size=10 << 10)): problem, net_config, net_assignments = util.get_config(FLAGS.problem) loss = problem() global_step = tf.Variable(0, dtype=tf.int64, trainable=False) # Optimizer setup. var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='problem') print(var_list) #adam_opt = tf.train.GradientDescentOptimizer(FLAGS.learning_rate) adam_opt = tf.train.AdamOptimizer(FLAGS.learning_rate) opt = adam_opt.minimize(loss, global_step) if FLAGS.mode != 1: optimizer = l2l_optimizer.L2LOptimizer( internal_optimizer=adam_opt, loss_func=problem, opt_last=FLAGS.opt_last, preprocessor=LogAndSign(10), co_opt=FLAGS.co_opt, rnn_layer_cnt=FLAGS.layer, delta_ratio=FLAGS.delta_ratio, update_ratio=FLAGS.update_ratio, dynamic_unroll=FLAGS.dynamic_unroll) opt = optimizer.minimize(loss, global_step=global_step, unroll_len=FLAGS.unroll_len) if FLAGS.mode == 1: print('use adam opt') else: print('use l2l opt') slot_reset = tf.no_op() if FLAGS.mode != 1: slot_reset = tf.variables_initializer(optimizer._slot_vars + optimizer._opt_vars) init = tf.group( *[tf.global_variables_initializer(), tf.local_variables_initializer()]) var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) print(var_list) #saver = tf.train.Saver(var_list = var_list) with ms.MonitoredSession() as sess: #with tf.Session() as sess: # Prevent accidental changes to the graph. tf.get_default_graph().finalize() sess.run(init) print('trainable variables') trainable_vars = tf.trainable_variables() for v in trainable_vars: print("parameter:", v.name, "device:", v.device, "shape:", v.get_shape()) best_evaluation = float("inf") total_time = 0 accum_loss = 0.0 total_cost = 0 for e in xrange(FLAGS.num_epochs): # Training. step, curr_loss, _ = sess.run([global_step, loss, opt]) accum_loss += curr_loss if step % 100 == 0: print('step:%d,loss:%f' % (step, accum_loss / 100)) accum_loss = 0 if step % FLAGS.reset_interval == 0: #print('reset') sess.run(slot_reset)
def main(): #hooks = [CheckTabs(), CheckIndentation()] hooks = [CheckComment()] #debug = get_config("debug", as_bool=True, default=False) debug = False # @TODO: hard code for now, use get_config later on incremental = get_config("incremental", as_bool=True, default=False) incremental_verbose = get_config("incremental.verbose", as_bool=True, default=False) if debug: print "Starting hooks, incremental %s, hooks [%s]" % (incremental, ", ".join(map(str, hooks))) # create temp directory for getting copies of files from staging. # TODO: Make all the hooks operate on strings instead of files, and get rid of this. temp_dir = tempfile.mkdtemp() temp_dir_with_slash = temp_dir + os.sep atexit.register(shutil.rmtree, temp_dir, True) # clean up after ourselves failure_encountered = False if incremental: # Incremental checking requires checking the previous version of a # file for errors, but if the file was added, we can't do that. # Get a list of non-added changed files here to check against. modified_files = frozenset(changed_files(include_added_files=False)) for filename in changed_files(): if debug: print "Examining %s" % filename if not is_file_type(filename, 'scala'): if debug: print "Skipping %s, not a scala file" % filename continue relevant_hooks = [hook for hook in hooks if hook.should_process_file(filename)] if debug: print "relevant hooks are: ", relevant_hooks if not relevant_hooks: if debug: print "Skipping %s, no relevant hooks" % filename continue if incremental and filename in modified_files: head_temp_filename = make_temp_copy(temp_dir_with_slash, filename) incremental_hooks = copy.copy(relevant_hooks) if os.path.isfile(head_temp_filename): # This is not a newly added file, so check whether it used to fail the hooks. for relevant_hook in relevant_hooks: head_passes, unused_error_message = relevant_hook.file_passes(head_temp_filename, original_filename=filename) if not head_passes: # Incremental checking was requested, and current HEAD doesn't pass, # so don't bother checking this file with this hook. incremental_hooks.remove(relevant_hook) if incremental_verbose: print "Hook %s failed on current HEAD for file %s" % (relevant_hook, filename) relevant_hooks = incremental_hooks if not relevant_hooks: if debug: print "Skipping %s, no relevant hooks (after incremental check)" % filename continue temp_filename = make_temp_copy(temp_dir_with_slash, filename) for relevant_hook in relevant_hooks: passes, error_message = relevant_hook.file_passes(temp_filename, original_filename=filename) print(filename + " --> " + error_message) if not passes: failure_encountered = True break return int(failure_encountered)