def _main(args: CommandLineArgs) -> None: print('COX pairs generator.') print('CONFIGURATION:\t{}'.format(args.config)) with open(args.config) as json_config_file: config = utils.AttrDict(json.load(json_config_file)) people_folds = _split_people_into_folds(config.dataset.coxs2v.subject_list, config.dataset.coxs2v.num_fold) for video in config.dataset.coxs2v.video_list: print('Generating pairs for {}.'.format(video)) image_dir = os.path.join(config.dataset.coxs2v.video_dir, video) pairs_file_name = os.path.join(config.dataset.coxs2v.video_pairs, video + '_pairs.txt') matches = [] mismatches = [] for i, fold in enumerate(people_folds): print('Fold {}/ {} subjects'.format(i, len(fold))) matches.append( _make_matches(image_dir, fold, config.num_matches_mismatches)) mismatches.append( _make_mismatches(image_dir, fold, config.num_matches_mismatches)) write_pairs(pairs_file_name, matches, mismatches, config.dataset.coxs2v.num_fold, config.num_matches_mismatches)
def load_api_params(tf_config, graph, api_config="../config/api2nl.yaml"): params = {} # read config file and default config with open('../config/default.yaml') as f: default_config = utils.AttrDict(yaml.safe_load(f)) with open(api_config) as f: api_config = utils.AttrDict(yaml.safe_load(f)) # set default values for parameters that are not defined for k, v in default_config.items(): api_config.setdefault(k, v) api_config.checkpoint_dir = os.path.join(api_config.model_dir, 'checkpoints') #tasks = [api_config] #for task in tasks: # for parameter, value in api_config.items(): # task.setdefault(parameter, value) api_config.encoders = [ utils.AttrDict(encoder) for encoder in api_config.encoders ] api_config.decoders = [ utils.AttrDict(decoder) for decoder in api_config.decoders ] for encoder_or_decoder in api_config.encoders + api_config.decoders: for parameter, value in api_config.items(): encoder_or_decoder.setdefault(parameter, value) with tf.Session(config=tf_config, graph=graph) as sess: api_model = TranslationModel(**api_config) ckpt = tf.train.get_checkpoint_state(api_config.checkpoint_dir) saver = tf.train.Saver(tf.global_variables()) print("Reading api model parameters from %s" % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) for v in tf.trainable_variables(): params[v.name] = sess.run(v.value()) return params
def get_data_and_meta(self, subset=None): if subset is None: return self.embeddings, self.meta, self.idx_to_str else: (found, indx) = utils.ismember(subset, list(self.ids)) if np.sum(found) != len(subset): print( "Some of the files (%d) in the subset are missing. Ignoring them" % (len(subset) - np.sum(found))) indx = indx[found] # Subset the embeddings and the metadata to the required samples embeddings = self.embeddings[indx] meta = dict() for k, v in self.meta.items(): meta[k] = v[indx] meta = utils.AttrDict(meta) return embeddings, meta, self.idx_to_str
def segment(self, text, lower=False): sentences = self.ss.segment(text) words_no_filter = self.ws.segment_sentences( sentences=sentences, lower=lower, use_stop_words=False, use_speech_tags_filter=False) words_no_stop_words = self.ws.segment_sentences( sentences=sentences, lower=lower, use_stop_words=True, use_speech_tags_filter=False) words_all_filters = self.ws.segment_sentences( sentences=sentences, lower=lower, use_stop_words=True, use_speech_tags_filter=True) return utils.AttrDict(sentences=sentences, words_no_filter=words_no_filter, words_no_stop_words=words_no_stop_words, words_all_filters=words_all_filters)
def compute_stats(x, class_ids, class_weights): # Zeroth, first and second order stats per speaker, considering that # each speaker is weighted by the provided weights stats = utils.AttrDict(dict()) nsamples = class_ids.shape[0] classmat = coo_matrix((np.ones(nsamples), (class_ids, np.arange(nsamples)))).tocsr() stats.weights = np.atleast_2d(class_weights).T # Global mean, considering that each speaker's data should be # multiplied by the corresponding weight for that speaker sample_weights = stats.weights[class_ids] stats.mu = np.array(sample_weights.T @ x) / np.sum(sample_weights) # N and F are stats per speaker, weights are not involved in the computation of those. # S, on the other hand, is already a sum over speakers so the sample from each speaker # has to be weighted by the corresponding weight xcent = x - stats.mu xcentw = xcent * sample_weights stats.N = np.array(classmat.sum(1)) stats.F = np.array(classmat @ xcent) stats.S = xcent.T @ xcentw return stats
def main(args): print('Feature extractor training.') print('CONFIGURATION:\t{}'.format(args.config)) with open(args.config) as json_config_file: config = utils.AttrDict(json.load(json_config_file)) # Set up output directory experiment_name = generate_experiment_name(config) model_dir = os.path.join(os.path.expanduser(config.output.output_dir), experiment_name) if not os.path.exists(model_dir): os.makedirs(model_dir) print('Model saved at {}'.format(model_dir)) config_filename = utils.path_leaf(args.config) copyfile(args.config, os.path.join(model_dir, config_filename)) # CUDA for PyTorch use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") # device = torch.device("cpu") # Get dataloaders train_loader = dataloaders.get_traindataloaders(config.train_dataset, config) evaluators_list = dataloaders.get_evaluators(config.evaluation_datasets, config) # Set up training model print('Building training model') if config.model.checkpoint: checkpoint_path = config.model.checkpoint_path else: checkpoint_path = None model = models.load_model(config.model.model_arch, device, checkpoint_path=checkpoint_path, embedding_size=config.model.embedding_size, imgnet_pretrained=config.model.pretrained_imagenet) optimizer = optim.SGD(model.parameters(), lr=config.hyperparameters.learning_rate, momentum=0.9, nesterov=True, weight_decay=2e-4) # scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.1) # scheduler = lr_scheduler.ExponentialLR(optimizer, config.hyperparameters.learning_rate_decay_factor) scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=config.hyperparameters.n_epochs, eta_min=1e-6) plotter = utils.VisdomPlotter(config.visdom.server, env_name=experiment_name, port=config.visdom.port) miner = miners.FunctionSemihardTripletSelector(config.hyperparameters.margin, plotter) loss = nn.TripletMarginLoss(config.hyperparameters.margin, swap=config.hyperparameters.triplet_swap) my_trainer = trainer.Triplet_Trainer(model, miner, loss, optimizer, scheduler, device, plotter, config.hyperparameters.margin, config.model.embedding_size, evaluation.pair_evaluate, batch_size=config.hyperparameters.batch_size) # Loop over epochs epoch = 0 print('Training Launched.') while epoch < config.hyperparameters.n_epochs: # Validation for evaluator in evaluators_list: print('\nEvaluation on {}'.format(evaluator.test_name)) evaluator.evaluate(model, device, plotter=plotter, epoch=epoch) # Training print('\nTrain Epoch {}'.format(epoch)) my_trainer.Train_Epoch(train_loader, epoch) # Save model if not (epoch + 1) % config.output.save_interval: model_file_path = os.path.join(model_dir, 'model_{}.pth'.format(epoch)) print('\nSave model at {}'.format(model_file_path)) torch.save({'epoch': epoch, 'model_state_dict': utils.state_dict_to_cpu(model.state_dict()), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'embedding_size': config.model.embedding_size, }, model_file_path) epoch += 1 # Final save. model_file_path = os.path.join(model_dir, 'model_{}.pth'.format(epoch)) print('\nSave model at {}'.format(model_file_path)) torch.save({'epoch': epoch, 'model_state_dict': utils.state_dict_to_cpu(model.state_dict()), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'embedding_size': config.model.embedding_size, }, model_file_path) print('Finish.') return model
Usage: train.py <config>""") parser.add_argument("config", help="path to config file") parser.add_argument('--continue-training', type=utils.str2bool, default=False, help='Continue training from last_model.pt.') args = parser.parse_args() return args if __name__ == "__main__": timer = utils.Timer() args = get_args() timer.tic() config = utils.AttrDict(yaml.load(open(args.config))) dataconfig = config["data"] trainingconfig = config["training"] modelconfig = config["model"] feat_range = [int(i) for i in dataconfig['feat_range'].split(',')] label_range = [int(i) for i in dataconfig['label_range'].split(',')] ngpu = 1 if "multi_gpu" in trainingconfig and trainingconfig["multi_gpu"] == True: ngpu = torch.cuda.device_count() tokenizer = data_utils.CharTokenizer(dataconfig["vocab_path"], add_blk=modelconfig['add_blk']) modelconfig["decoder"]["vocab_size"] = tokenizer.unit_num() if modelconfig['signal']["feature_type"] == 'offline':
def respfunc_viewer(path): app = QtGui.QApplication([]) pyqtgraph.setConfigOption("background", "w") pyqtgraph.setConfigOption("foreground", "k") win = QtGui.QMainWindow() win.setWindowTitle("MT response function data viewer") darea = dockarea.DockArea() w = QtGui.QWidget() win.setCentralWidget(darea) taglist = QtGui.QListWidget(win) taglist.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection) taglist_dock = dockarea.Dock("Tags") taglist_dock.addWidget(taglist) darea.addDock(taglist_dock) sitelist = QtGui.QListWidget() sitelist.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection) sitelist_dock = dockarea.Dock("Tree...") sitelist_dock.addWidget(sitelist) darea.addDock(sitelist_dock, "left", taglist_dock) resplot = pyqtgraph.PlotWidget() resplot_dock = dockarea.Dock("APPARENT RESISTIVITY") resplot_dock.addWidget(resplot) darea.addDock(resplot_dock, "left", sitelist_dock) phaseplot = pyqtgraph.PlotWidget() phaseplot_dock = dockarea.Dock("PHASE") phaseplot_dock.addWidget(phaseplot) darea.addDock(phaseplot_dock, "bottom", resplot_dock) default_pen = [[(255, 255, 255, 90)], dict(width=1)] select_pen = [["r"], dict(width=1.5)] skipflag_pen = [[(255, 255, 255, 30)], dict(width=0.5)] resplotitem = resplot.getPlotItem() phaseplotitem = phaseplot.getPlotItem() resplotitem.invertX(True) phaseplotitem.invertX(True) resplotitem.setLogMode(x=True, y=True) phaseplotitem.setLogMode(x=True, y=False) phaseplotitem.vb.setXLink(resplotitem.vb) resplotitem.setYRange(np.log10(0.1), np.log10(1000)) phaseplotitem.setYRange(0, 90) resvb = resplotitem.vb phasevb = phaseplotitem.vb data = utils.AttrDict() tagfns = glob.glob(op.join(path, "*-cal.json")) tag2fn = {} fn2tag = {} sites = set() tagfns.sort() data = utils.AttrDict() with open(op.join(path, "maskedfreqs.json"), mode="r") as f: maskedfreqs = utils.read_json(f) maskedlines = utils.AttrDict() datasymbols = utils.AttrDict() psymbols = utils.AttrDict({ "xy": dict(pen=None, symbol="o", symbolBrush="b"), "yx": dict(pen=None, symbol="s", symbolBrush="r") }) plines = utils.AttrDict({"xy": dict(pen="b"), "yx": dict(pen="r")}) plotpens = utils.AttrDict({ "xy": "b", "yx": "r", }) plotsymbols = utils.AttrDict({"xy": "o", "yx": "s"}) def plot(tag): if not hasattr(datasymbols[tag], "res_xy"): datasymbols[tag].res_xy = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].res_xy, **psymbols.xy) datasymbols[tag].res_yx = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].res_yx, **psymbols.yx) datasymbols[tag].phase_xy = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].phase_xy, **psymbols.xy) datasymbols[tag].phase_yx = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].phase_yx, **psymbols.yx) maskedlines[tag].res_xy = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].res_xy, **plines.xy) maskedlines[tag].res_yx = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].res_yx, **plines.yx) maskedlines[tag].phase_xy = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].phase_xy, **plines.xy) maskedlines[tag].phase_yx = pyqtgraph.PlotDataItem( data[tag].freqs, data[tag].phase_yx, **plines.yx) resplotitem.addItem(datasymbols[tag].res_xy) resplotitem.addItem(datasymbols[tag].res_yx) resplotitem.addItem(maskedlines[tag].res_xy) resplotitem.addItem(maskedlines[tag].res_yx) phaseplotitem.addItem(datasymbols[tag].phase_xy) phaseplotitem.addItem(datasymbols[tag].phase_yx) phaseplotitem.addItem(maskedlines[tag].phase_xy) phaseplotitem.addItem(maskedlines[tag].phase_yx) for i, freq in enumerate(data[tag].freqs): if maskedfreqs[tag]["masks"][i] == 0: data[tag].freqs[i] = float(maskedfreqs[tag]["freqs"][i]) else: data[tag].freqs[i] = np.nan maskedlines[tag].res_xy.setData(data[tag].freqs, data[tag].res_xy) maskedlines[tag].res_yx.setData(data[tag].freqs, data[tag].res_yx) maskedlines[tag].phase_xy.setData(data[tag].freqs, data[tag].phase_xy) maskedlines[tag].phase_yx.setData(data[tag].freqs, data[tag].phase_yx) progress = QtGui.QProgressDialog("Loading data...", "Abort", 0, len(tagfns), win) progress.setWindowModality(QtCore.Qt.WindowModal) for i, tagfn in enumerate(tagfns): progress.setValue(i) tag = op.basename(tagfn).replace("-cal.json", "") tag2fn[tag] = tagfn fn2tag[tagfn] = tag site = tag.split("-")[0] sites.add(site) data[tag] = utils.read_json(tagfn) if not tag in maskedfreqs: maskedfreqs[tag] = utils.AttrDict({ "freqs": data[tag].freqs.copy(), "masks": np.empty_like(data[tag].freqs) * 0 }) if not tag in maskedlines: maskedlines[tag] = utils.AttrDict() datasymbols[tag] = utils.AttrDict() plot(tag) if progress.wasCanceled(): break progress.setValue(len(tagfns)) resfreqselect = pyqtgraph.LinearRegionItem([0, -1]) phasefreqselect = pyqtgraph.LinearRegionItem([0, -1]) resplotitem.addItem(resfreqselect) phaseplotitem.addItem(phasefreqselect) def res_region_moved(): phasefreqselect.setRegion(resfreqselect.getRegion()) def phase_region_moved(): resfreqselect.setRegion(phasefreqselect.getRegion()) resfreqselect.sigRegionChanged.connect(res_region_moved) phasefreqselect.sigRegionChanged.connect(phase_region_moved) def populate_tag_list(filter_sites=None): if filter_sites: tags = [ t for t in tag2fn.keys() if t.split("-")[0] in filter_sites ] else: tags = sorted(tag2fn.keys()) tags.sort() taglist.clear() for tag in tags: # print tag tagitem = QtGui.QListWidgetItem(taglist) tagitem.setText(tag) plot_per_tag_list() print def plot_per_tag_list(): tags = [t.text() for t in taglist.selectedItems()] if not tags: tags = [ t.text() for t in [taglist.item(i) for i in xrange(taglist.count())] ] for plotitemtag, tagitems in datasymbols.items(): if plotitemtag in tags: for item_name, item in tagitems.items(): item.setSymbol(plotsymbols[item_name[-2:]]) # item.setPen(None)#plotpens[item_name[-2:]]) else: for item in tagitems.values(): item.setSymbol(None) # item.setPen(None) for plotitemtag, tagitems in maskedlines.items(): if plotitemtag in tags: for item_name, item in tagitems.items(): item.setPen(plotpens[item_name[-2:]]) else: for item in tagitems.values(): item.setPen(None) def selected_site_names(): return [s.text() for s in sitelist.selectedItems()] def pick_site(): newsites = selected_site_names() populate_tag_list(newsites) # plot_per_tag_list() def toggle_selected_mask(value): tags = [str(t.text()) for t in taglist.selectedItems()] log_mask_range = resfreqselect.getRegion() fmin = 10**log_mask_range[0] fmax = 10**log_mask_range[1] for tag in tags: for i, freq in enumerate(maskedfreqs[tag]["freqs"]): if freq >= fmin and freq <= fmax: maskedfreqs[tag]["masks"][i] = value plot(tag) print log_mask_range, tags, "\n" disable = QtGui.QPushButton("&Delete selected frequencies") enable = QtGui.QPushButton("&Enable selected frequencies") sitelist_dock.addWidget(disable) sitelist_dock.addWidget(enable) disable.clicked.connect(lambda: toggle_selected_mask(1)) enable.clicked.connect(lambda: toggle_selected_mask(0)) # def generate_key_press_event_handler(self, vb, event): # vb.keyPressEvent(self, event) # if event.key() is Qt.Key_X: # toggle_selected_mask(mode="xy") # elif event.key() is Qt.Key_Y: # toggle_selected_mask(mode="yx") # resplotitem.vb.keyPressEvent = lambda populate_tag_list() sites = sorted(list(sites)) for site in sites: siteitem = QtGui.QListWidgetItem(sitelist) siteitem.setText(site) sitelist.itemSelectionChanged.connect(pick_site) taglist.itemSelectionChanged.connect(plot_per_tag_list) def cleanup(): with open(op.join(path, "maskedfreqs.json"), mode="w") as f: utils.write_json(maskedfreqs, f) win.showMaximized() app.aboutToQuit.connect(cleanup) app.exec_()
def __init__(self, encoders, decoders, checkpoint_dir, learning_rate, learning_rate_decay_factor, batch_size, keep_best=1, dev_prefix=None, score_function='corpus_scores', name=None, ref_ext=None, pred_edits=False, dual_output=False, binary=None, truncate_lines=True, ensemble=False, checkpoints=None, beam_size=1, len_normalization=1, early_stopping=True, **kwargs): self.batch_size = batch_size self.character_level = {} self.binary = [] for encoder_or_decoder in encoders + decoders: encoder_or_decoder.ext = encoder_or_decoder.ext or encoder_or_decoder.name self.character_level[ encoder_or_decoder.ext] = encoder_or_decoder.character_level self.binary.append(encoder_or_decoder.get('binary', False)) self.char_output = decoders[0].character_level self.src_ext = [encoder.ext for encoder in encoders] self.trg_ext = [decoder.ext for decoder in decoders] self.extensions = self.src_ext + self.trg_ext self.ref_ext = ref_ext if self.ref_ext is not None: self.binary.append(False) self.pred_edits = pred_edits self.dual_output = dual_output self.dev_prefix = dev_prefix self.name = name self.max_input_len = [encoder.max_len for encoder in encoders] self.max_output_len = [decoder.max_len for decoder in decoders] if truncate_lines: self.max_len = None # we let seq2seq.get_batch handle long lines (by truncating them) else: # the line reader will drop lines that are too long self.max_len = dict( zip(self.extensions, self.max_input_len + self.max_output_len)) self.learning_rate = tf.Variable(learning_rate, trainable=False, name='learning_rate', dtype=tf.float32) self.learning_rate_decay_op = self.learning_rate.assign( self.learning_rate * learning_rate_decay_factor) with tf.device('/cpu:0'): self.global_step = tf.Variable(0, trainable=False, name='global_step') self.baseline_step = tf.Variable(0, trainable=False, name='baseline_step') self.filenames = utils.get_filenames(extensions=self.extensions, dev_prefix=dev_prefix, name=name, ref_ext=ref_ext, binary=self.binary, **kwargs) utils.debug('reading vocabularies') self.vocabs = None self.src_vocab, self.trg_vocab = None, None self.read_vocab() for encoder_or_decoder, vocab in zip(encoders + decoders, self.vocabs): if vocab: encoder_or_decoder.vocab_size = len(vocab.reverse) utils.debug('creating model') self.models = [] if ensemble and checkpoints is not None: for i, _ in enumerate(checkpoints, 1): with tf.variable_scope('model_{}'.format(i)): model = Seq2SeqModel(encoders, decoders, self.learning_rate, self.global_step, name=name, pred_edits=pred_edits, dual_output=dual_output, baseline_step=self.baseline_step, **kwargs) self.models.append(model) self.seq2seq_model = self.models[0] else: self.seq2seq_model = Seq2SeqModel(encoders, decoders, self.learning_rate, self.global_step, name=name, pred_edits=pred_edits, dual_output=dual_output, baseline_step=self.baseline_step, **kwargs) self.models.append(self.seq2seq_model) self.seq2seq_model.create_beam_op(self.models, beam_size, len_normalization, early_stopping) self.batch_iterator = None self.dev_batches = None self.train_size = None self.saver = None self.keep_best = keep_best self.checkpoint_dir = checkpoint_dir self.epoch = None self.training = utils.AttrDict() # used to keep track of training try: self.reversed_scores = getattr( evaluation, score_function).reversed # the lower the better except AttributeError: self.reversed_scores = False # the higher the better
all_text = list(brown.words()) use_words_ = z.valfilter(lambda x: x >= 5, Counter(all_text)) use_words = [w for w in all_text if w in use_words_] le = LabelEncoder() toka = le.fit_transform(use_words) tokl = list(toka) vc = le.classes_ # In[ ]: cnf = ut.AttrDict( eta=.1, min_eta=.0001, accumsec=0, N=100, C=4, K=6, iter=0, thresh=15, epoch=0, pad=0, term=dict(iters=None, secs=None ), dir='cache', ) cnf = Conf(cnf) cnf_ = cnf del cnf cfp = update(cnf_, pad=2, term={}) W = wut.init_w(len(vc), cnf_.N, seed=1) We = W.copy() # ### Sgd # One final speedup I got was from putting the entire inner loop routine into a numba JIT'd function, instead of calling each JIT'd function separately. Numba seems to reduce some of python's function call overhead, as well as the numpy indexing. For some reason, merely indexing into a numpy array (to calculate the gradient and then update the original matrix) still ended up being a significant bottleneck, though numba seemed to reduce it.
def main(args=None): args = parser.parse_args(args) # read config file and default config with open('../model/default.yaml') as f: default_config = utils.AttrDict(yaml.safe_load(f)) with open(args.config) as f: config = utils.AttrDict(yaml.safe_load(f)) if args.learning_rate is not None: args.reset_learning_rate = True # command-line parameters have higher precedence than config file for k, v in vars(args).items(): if v is not None: config[k] = v # set default values for parameters that are not defined for k, v in default_config.items(): config.setdefault(k, v) # if config.score_function: # config.score_functions = evaluation.name_mapping[config.score_function] if args.crash_test: config.max_train_size = 0 if not config.debug: os.environ[ 'TF_CPP_MIN_LOG_LEVEL'] = '3' # disable TensorFlow's debugging logs decoding_mode = any(arg is not None for arg in (args.decode, args.eval, args.align)) # enforce parameter constraints assert config.steps_per_eval % config.steps_per_checkpoint == 0, ( 'steps-per-eval should be a multiple of steps-per-checkpoint') assert decoding_mode or args.train or args.save, ( 'you need to specify at least one action (decode, eval, align, or train)' ) assert not (args.average and args.ensemble) if args.train and args.purge: utils.log('deleting previous model') shutil.rmtree(config.model_dir, ignore_errors=True) os.makedirs(config.model_dir, exist_ok=True) # copy config file to model directory config_path = os.path.join(config.model_dir, 'config.yaml') if args.train and not os.path.exists(config_path): with open(args.config) as config_file, open(config_path, 'w') as dest_file: content = config_file.read() content = re.sub(r'model_dir:.*?\n', 'model_dir: {}\n'.format(config.model_dir), content, flags=re.MULTILINE) dest_file.write(content) # also copy default config config_path = os.path.join(config.model_dir, 'default.yaml') if args.train and not os.path.exists(config_path): shutil.copy('../config/default.yaml', config_path) logging_level = logging.DEBUG if args.verbose else logging.INFO # always log to stdout in decoding and eval modes (to avoid overwriting precious train logs) log_path = os.path.join(config.model_dir, config.log_file) logger = utils.create_logger(log_path if args.train else None) logger.setLevel(logging_level) utils.log('label: {}'.format(config.label)) utils.log('description:\n {}'.format('\n '.join( config.description.strip().split('\n')))) utils.log(' '.join(sys.argv)) # print command line try: # print git hash commit_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip() utils.log('commit hash {}'.format(commit_hash)) except: pass utils.log('tensorflow version: {}'.format(tf.__version__)) # log parameters utils.debug('program arguments') for k, v in sorted(config.items(), key=itemgetter(0)): utils.debug(' {:<20} {}'.format(k, pformat(v))) if isinstance(config.dev_prefix, str): config.dev_prefix = [config.dev_prefix] config.encoders = [utils.AttrDict(encoder) for encoder in config.encoders] config.decoders = [utils.AttrDict(decoder) for decoder in config.decoders] for encoder_or_decoder in config.encoders + config.decoders: for parameter, value in config.items(): encoder_or_decoder.setdefault(parameter, value) if args.max_output_len is not None: # override decoder's max len config.decoders[0].max_len = args.max_output_len config.checkpoint_dir = os.path.join(config.model_dir, 'checkpoints') # setting random seeds if config.seed is None: config.seed = random.randrange(sys.maxsize) if config.tf_seed is None: config.tf_seed = random.randrange(sys.maxsize) utils.log('python random seed: {}'.format(config.seed)) utils.log('tf random seed: {}'.format(config.tf_seed)) random.seed(config.seed) tf.set_random_seed(config.tf_seed) device = None if config.no_gpu: device = '/cpu:0' device_id = None elif config.gpu_id is not None: device = '/gpu:{}'.format(config.gpu_id) device_id = config.gpu_id else: device_id = 0 # hide other GPUs so that TensorFlow won't use memory on them os.environ['CUDA_VISIBLE_DEVICES'] = '' if device_id is None else str( device_id) tf_config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) tf_config.gpu_options.allow_growth = config.allow_growth tf_config.gpu_options.per_process_gpu_memory_fraction = config.mem_fraction config.api_params = None api_graph = tf.Graph() transfer_graph = tf.Graph() if config.use_transfer: # utils.log("loading api params") ckpt = tf.train.get_checkpoint_state(config.checkpoint_dir) if not ckpt or not ckpt.model_checkpoint_path: utils.log("loading api params") config.api_params = load_api_params(tf_config=tf_config, graph=api_graph) def average_checkpoints(main_sess, sessions): for var in tf.global_variables(): avg_value = sum(sess.run(var) for sess in sessions) / len(sessions) main_sess.run(var.assign(avg_value)) with tf.Session(config=tf_config, graph=transfer_graph) as sess: global global_tf_config, global_transfer_graph global_tf_config = tf_config global_transfer_graph = transfer_graph utils.log('creating model') utils.log('using device: {}'.format(device)) with tf.device(device): if config.weight_scale: if config.initializer == 'uniform': initializer = tf.random_uniform_initializer( minval=-config.weight_scale, maxval=config.weight_scale) else: initializer = tf.random_normal_initializer( stddev=config.weight_scale) else: initializer = None tf.get_variable_scope().set_initializer(initializer) # exempt from creating gradient ops config.decode_only = decoding_mode model = TranslationModel(**config) # count parameters # not counting parameters created by training algorithm (e.g. Adam) variables = [ var for var in tf.global_variables() if not var.name.startswith('gradients') ] utils.log('model parameters ({})'.format(len(variables))) parameter_count = 0 for var in sorted(variables, key=lambda var: var.name): utils.log(' {} {}'.format(var.name, var.get_shape())) v = 1 for d in var.get_shape(): v *= d.value parameter_count += v utils.log('number of parameters: {:.2f}M'.format(parameter_count / 1e6)) best_checkpoint = os.path.join(config.checkpoint_dir, 'best') params = { 'variable_mapping': config.variable_mapping, 'reverse_mapping': config.reverse_mapping } if config.ensemble and len(config.checkpoints) > 1: model.initialize(config.checkpoints, **params) elif config.average and len(config.checkpoints) > 1: model.initialize(reset=True) sessions = [ tf.Session(config=tf_config) for _ in config.checkpoints ] for sess_, checkpoint in zip(sessions, config.checkpoints): model.initialize(sess=sess_, checkpoints=[checkpoint], **params) average_checkpoints(sess, sessions) elif (not config.checkpoints and decoding_mode and (os.path.isfile(best_checkpoint + '.index') or os.path.isfile(best_checkpoint + '.index'))): # in decoding and evaluation mode, unless specified otherwise (by `checkpoints`), # try to load the best checkpoint global global_sess global_sess = sess model.initialize(config.checkpoints, **params) else: # loads last checkpoint, unless `reset` is true model.initialize(sess=sess, **config) if args.save: model.save() elif args.decode is not None: global global_config, global_model global_config = config global_model = model utils.log('starting decoding') # model.decode(code_string, **config) app.run(host='0.0.0.0') elif args.eval is not None: model.evaluate(on_dev=False, **config) elif args.align is not None: model.align(**config) elif args.train: try: model.train(**config) except KeyboardInterrupt: sys.exit()
def __init__(self, encoders, decoders, learning_rate, global_step, max_gradient_norm, use_dropout=False, freeze_variables=None, feed_previous=0.0, optimizer='sgd', decode_only=False, len_normalization=1.0, name=None, chained_encoders=False, pred_edits=False, baseline_step=None, use_baseline=True, reverse_input=False, moving_average=None, **kwargs): self.encoders = encoders self.decoders = decoders self.temperature = self.decoders[0].temperature self.name = name self.learning_rate = learning_rate self.global_step = global_step self.baseline_step = baseline_step self.use_baseline = use_baseline self.max_output_len = [decoder.max_len for decoder in decoders] self.max_input_len = [encoder.max_len for encoder in encoders] self.len_normalization = len_normalization self.reverse_input = reverse_input dropout_on = [] dropout_off = [] if use_dropout: for encoder_or_decoder in encoders + decoders: names = [ 'rnn_input', 'rnn_output', 'rnn_state', 'initial_state', 'word', 'input_layer', 'output', 'attn', 'deep_layer', 'inter_layer', 'embedding' ] for name in names: value = encoder_or_decoder.get(name + '_dropout') var_name = name + '_keep_prob' if not value: encoder_or_decoder[var_name] = 1.0 continue var = tf.Variable(1 - value, trainable=False, name=var_name) encoder_or_decoder[var_name] = var dropout_on.append(var.assign(1.0 - value)) dropout_off.append(var.assign(1.0)) self.dropout_on = tf.group(*dropout_on) self.dropout_off = tf.group(*dropout_off) self.feed_previous = tf.constant(feed_previous, dtype=tf.float32) self.feed_argmax = tf.constant( True, dtype=tf.bool) # feed with argmax or sample from softmax self.encoder_inputs = [] self.encoder_input_length = [] for encoder in encoders: shape = [None, None, encoder.embedding_size ] if encoder.binary else [None, None] dtype = tf.float32 if encoder.binary else tf.int32 encoder_input = tf.placeholder(dtype=dtype, shape=shape, name='encoder_{}'.format( encoder.name)) encoder_input_length = tf.placeholder( dtype=tf.int32, shape=[None], name='encoder_input_length_{}'.format(encoder.name)) self.encoder_inputs.append(encoder_input) self.encoder_input_length.append(encoder_input_length) # starts with BOS, and ends with EOS self.targets = tuple([ tf.placeholder(tf.int32, shape=[None, None], name='target_{}'.format(decoder.name)) for decoder in decoders ]) if chained_encoders and pred_edits: architecture = models.chained_encoder_decoder # no REINFORCE for now else: architecture = models.encoder_decoder tensors = architecture(encoders, decoders, self.encoder_inputs, self.targets, self.feed_previous, encoder_input_length=self.encoder_input_length, feed_argmax=self.feed_argmax, **kwargs) (self.losses, self.outputs, self.encoder_state, self.attention_states, self.attention_weights, self.samples, self.beam_fun, self.initial_data) = tensors self.xent_loss = self.losses self.loss = self.xent_loss # main loss optimizers = self.get_optimizers(optimizer, learning_rate) if not decode_only: get_update_ops = functools.partial( self.get_update_op, opts=optimizers, max_gradient_norm=max_gradient_norm, freeze_variables=freeze_variables) self.update_ops = utils.AttrDict({ 'xent': get_update_ops(self.xent_loss, global_step=self.global_step), }) self.models = [self] self.beam_outputs = tf.expand_dims(tf.argmax(self.outputs[0], axis=2), axis=1) self.beam_scores = tf.zeros(shape=[tf.shape(self.beam_outputs)[0], 1]) self.beam_size = 1
def __init__(self, emb_file, meta_file=None, meta_is_dur_only=False, device=None): """ Args: emb_file (string): File with embeddings and sample ids in npz format meta_file (string): File with metadata for each id we want to store from the file above. Should contain: sample_id speaker_id session_id domain_id * The speaker id is a unique string identifying the speaker * The session id is a unique string identifying the recording session from which the audio sample was extracted (ie, if a waveform is split into chunks as a pre- processing step, the chunks all belong to the same session, or if several mics were used to record a person speaking all these recordings belong to the same session). This information is used by the loader to avoid creating same-session trials which would mess up calibration. * The domain id is a unique string identifying the domain. Domains should correspond to disjoint speaker sets. This information is also used by the loader. Only same- domain trials are created since cross-domain trials would never include target trials and would likely result in very easy impostor trials. """ if meta_file is not None: print("Loading data from %s\n with metadata file %s" % (emb_file, meta_file)) else: print("Loading data from %s without metadata" % emb_file) if emb_file.endswith(".npz"): data_dict = np.load(emb_file) elif emb_file.endswith(".h5"): with h5py.File(emb_file, 'r') as f: data_dict = {'ids': f['ids'][()], 'data': f['data'][()]} else: raise Exception("Unrecognized format for embeddings file %s" % emb_file) embeddings_all = data_dict['data'] if type(data_dict['ids'][0]) == np.bytes_: ids_all = [i.decode('UTF-8') for i in data_dict['ids']] elif type(data_dict['ids'][0]) == np.str_: ids_all = data_dict['ids'] else: raise Exception( "Bad format for ids in embeddings file %s (should be strings)" % emb_file) self.idx_to_str = dict() self.meta = dict() if meta_file is None: fields = ('sample_id', ) formats = ('O', ) self.meta_raw = np.array( ids_all, np.dtype({ 'names': fields, 'formats': ('O', ) })) else: if meta_is_dur_only: fields, formats = zip(*[('sample_id', 'O'), ('duration', 'float32')]) else: fields, formats = zip( *[('sample_id', 'O'), ('speaker_id', 'O'), ('session_id', 'O'), ('domain_id', 'O'), ('duration', 'float32')]) self.meta_raw = np.loadtxt( meta_file, np.dtype({ 'names': fields, 'formats': formats })) # Convert the metadata strings into indices print(" Converting metadata strings into indices") for field, fmt in zip(fields, formats): if fmt == 'O': # Convert all the string fields into indices names, nmap = np.unique(self.meta_raw[field], return_inverse=True) if field == 'sample_id' and len(names) != len(self.meta_raw): raise Exception( "Metadata file %s has repeated sample ids" % meta_file) # Index to string and string to index maps self.idx_to_str[field] = dict(zip(np.arange(len(names)), names)) self.idx_to_str[field + "_inv"] = dict( zip(names, np.arange(len(names)))) self.meta[field] = np.array(nmap, dtype=np.int32) else: self.meta[field] = self.meta_raw[field] self.meta = utils.AttrDict(self.meta) # Subset the embeddings to only those in the metadata file name_to_idx = dict(zip(ids_all, np.arange(len(ids_all)))) keep_idxs = np.array( [name_to_idx.get(n, -1) for n in self.meta_raw['sample_id']]) if np.any(keep_idxs == -1): raise Exception( "There are %d sample ids (out of %d in the metadata file %s) that are missing from the embeddings file %s.\nPlease, remove those files from the metadata file and try again" % (np.sum(keep_idxs == -1), len( self.meta_raw), meta_file, emb_file)) self.embeddings = embeddings_all[keep_idxs] self.ids = np.array(ids_all)[keep_idxs] if device is not None: # Move the embeddings and the durations to the device self.embeddings = utils.np_to_torch(self.embeddings, device) if 'duration' in self.meta: self.meta['duration'] = utils.np_to_torch( self.meta['duration'], device) print("Done. Loaded %d embeddings from %s" % (len(self.embeddings), emb_file), flush=True)
def main(args): print('Feature extractor training.') print('CONFIGURATION:\t{}'.format(args.config)) with open(args.config) as json_config_file: config = utils.AttrDict(json.load(json_config_file)) # Set up output directory experiment_name = generate_experiment_name(config) model_dir = os.path.join(os.path.expanduser(config.output.output_dir), experiment_name) if not os.path.exists(model_dir): os.makedirs(model_dir) print('Model saved at {}'.format(model_dir)) config_filename = path_leaf(args.config) copyfile(args.config, os.path.join(model_dir, config_filename)) # CUDA for PyTorch use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") source_loader = dataloaders.get_traindataloaders(config.source_dataset, config) target_loader = dataloaders.get_traindataloaders(config.target_dataset, config) evaluators_list = dataloaders.get_evaluators(config.evaluation_datasets, config) # Set up training model print('Building training model') if config.model.checkpoint: checkpoint_path = config.model.checkpoint_path else: checkpoint_path = None model = models.load_model(config.model.model_arch, device, checkpoint_path=checkpoint_path, embedding_size=config.model.embedding_size, imgnet_pretrained=config.model.pretrained_imagenet) optimizer = optim.SGD(model.parameters(), lr=config.hyperparameters.learning_rate, momentum=0.9, nesterov=True, weight_decay=2e-4) scheduler = lr_scheduler.ExponentialLR(optimizer, config.hyperparameters.learning_rate_decay_factor) model = model.to(device) plotter = utils.VisdomPlotter(config.visdom.server ,env_name=experiment_name, port=config.visdom.port) print('Fitting source dataset.') gmixture = clustering.distance_supervised_gaussian_mixture(source_loader, model, device, _plotter=plotter, name='Source Gaussians') print('Fitting target dataset.') clustering.update_gaussian_mixture(gmixture, target_loader, model, device, _plotter=plotter, name='Target Gaussians') print('DualTriplet loss training mode.') miner = miners.get_miner(config.miner, config.hyperparameters.margin, config.hyperparameters.people_per_batch, plotter, deadzone_ratio=config.hyperparameters.deadzone_ratio) miner.gmixture = gmixture loss = losses.DualtripletLoss(config.hyperparameters.margin, config.hyperparameters.lamda, plotter) model_trainer = trainer.Dualtriplet_Trainer(model, miner, loss, optimizer, scheduler, device, plotter, config.hyperparameters.margin, config.model.embedding_size, batch_size=config.hyperparameters.batch_size) if not os.path.isdir(model_dir): # Create the model directory if it doesn't exist os.makedirs(model_dir) # Loop over epochs epoch = 0 print('Training Launched.') while epoch < config.hyperparameters.n_epochs: # Validation for evaluator in evaluators_list: print('\nEvaluation on {}'.format(evaluator.test_name)) evaluator.evaluate(model, device, plotter=plotter, epoch=epoch) # Training print('\nExperimentation {}'.format(config.experiment)) print('Train Epoch {}'.format(epoch)) model_trainer.Train_Epoch(source_loader, target_loader, epoch) # Save model # if not (epoch + 1) % config.output.save_interval: # # model_file_path = os.path.join(model_dir, 'model_{}.pth'.format(epoch)) # print('\nSave model at {}'.format(model_file_path)) # torch.save({'epoch': epoch, # 'model_state_dict': utils.state_dict_to_cpu(model.state_dict()), # 'optimizer_state_dict': optimizer.state_dict(), # 'scheduler_state_dict': scheduler.state_dict(), # 'embedding_size': config.model.embedding_size # }, model_file_path) epoch += 1 model_file_path = os.path.join(model_dir, 'model_{}.pth'.format(epoch)) print('\nSave model at {}'.format(model_file_path)) torch.save({'epoch': epoch, 'model_state_dict': utils.state_dict_to_cpu(model.state_dict()), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'embedding_size': config.model.embedding_size }, model_file_path) print('Finish.')
parser.add_argument('--experiment', type=str, help='Dataset to experiment', default='lfw') return parser.parse_args(argv) if __name__ == '__main__': args = parse_arguments(sys.argv[1:]) print('Feature extractor evaluation.') print('CONFIGURATION:\t{}'.format(args.config)) with open(args.config) as json_config_file: config = utils.AttrDict(json.load(json_config_file)) # CUDA for PyTorch use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") # Load model if config.model.checkpoint: checkpoint_path = config.model.checkpoint_path else: checkpoint_path = None model = models.load_model( config.model.model_arch, device, checkpoint_path=checkpoint_path, embedding_size=config.model.embedding_size,
def main(args): print('CONFIGURATION:\t{}'.format(args.config)) with open(args.config) as json_config_file: config = utils.AttrDict(json.load(json_config_file)) # Set up output directory # subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S') subdir = config.visdom.environment_name model_dir = os.path.join(os.path.expanduser(config.output.output_dir), subdir) if not os.path.isdir(model_dir): # Create the model directory if it doesn't exist os.makedirs(model_dir) else: raise Exception('Environment name {} already taken.'.format(subdir)) config_filename = path_leaf(args.config) copyfile(args.config, os.path.join(model_dir, config_filename)) # CUDA for PyTorch use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") data_transform = transforms.Compose([ transforms.Resize((config.hyperparameters.image_size, config.hyperparameters.image_size), interpolation=1), transforms.ToTensor() ]) test_batch_size = (config.hyperparameters.people_per_batch * config.hyperparameters.images_per_person) // 2 nrof_folds = config.dataset.cross_validation.num_fold fold_tool = utils.FoldGenerator(nrof_folds, config.dataset.cross_validation.num_train_folds, config.dataset.cross_validation.num_val_folds) train_folds, val_folds, test_folds = fold_tool.get_fold() ########################### # SET UP DATALOADERS HERE # ########################### source_loader = dataset_utils.get_coxs2v_trainset(config.dataset.coxs2v.still_dir, config.dataset.coxs2v.video2_dir, config.dataset.coxs2v.video2_pairs, train_folds, nrof_folds, data_transform, config.hyperparameters.people_per_batch, config.hyperparameters.images_per_person) target_loader = dataset_utils.get_coxs2v_trainset(config.dataset.coxs2v.still_dir, config.dataset.coxs2v.video4_dir, config.dataset.coxs2v.video4_pairs, val_folds, nrof_folds, data_transform, config.hyperparameters.people_per_batch, config.hyperparameters.images_per_person) test_loaders_list = dataloaders.get_testevaluators(config, data_transform, test_batch_size, test_folds, nrof_folds, is_vggface2=False, is_lfw=True, is_cox_video1=False, is_cox_video2=True, is_cox_video3=False, is_cox_video4=True) ################### # DATALOADERS END # ################### #Set up training model print('Building training model') if config.model.checkpoint: print('Loading from checkpoint {}'.format(config.model.checkpoint_path)) checkpoint = torch.load(config.model.checkpoint_path) embedding_size = checkpoint['embedding_size'] # start_epoch = checkpoint['epoch'] start_epoch = 0 else: embedding_size = config.model.embedding_size start_epoch = 0 model = models.load_model(config.model.model_arch, embedding_size=embedding_size, imgnet_pretrained=config.model.pretrained_imagenet) optimizer = optim.SGD(model.parameters(), lr=config.hyperparameters.learning_rate, momentum=0.9, nesterov=True, weight_decay=2e-4) scheduler = lr_scheduler.ExponentialLR(optimizer, config.hyperparameters.learning_rate_decay_factor) if config.model.checkpoint: model.load_state_dict(checkpoint['model_state_dict']) # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # scheduler.load_state_dict(checkpoint['scheduler_state_dict']) model = model.to(device) plotter = utils.VisdomPlotter(env_name=config.visdom.environment_name, port=config.visdom.port) print('Quadruplet loss training mode.') miner = miners.SemihardNegativeQuadrupletSelector(config.hyperparameters.margin) loss = losses.QuadrupletLoss2(config.hyperparameters.margin, config.hyperparameters.margin2, lamda=config.hyperparameters.lamda) trainer = Quadruplet_Trainer(model, miner, loss, optimizer, scheduler, device, plotter, config.hyperparameters.margin, config.model.embedding_size, config.visdom.log_interval) # Loop over epochs print('Training Launched.') for epoch in range(start_epoch, config.hyperparameters.n_epochs): # Validation for test_name, test_loader in test_loaders_list: print('\nEvaluation on {}'.format(test_name)) trainer.Evaluate(test_loader, name=test_name, nrof_folds=nrof_folds, val_far=config.hyperparameters.val_far) # Training print('\nTrain Epoch {}'.format(epoch)) trainer.Train_Epoch(source_loader, target_loader) # Save model if not (epoch + 1) % config.output.save_interval: if not os.path.isdir(model_dir): # Create the model directory if it doesn't exist os.makedirs(model_dir) model_file_path = os.path.join(model_dir, 'model_{}.pth'.format(epoch)) print('\nSave model at {}'.format(model_file_path)) torch.save({'epoch': epoch, 'model_state_dict': utils.state_dict_to_cpu(model.state_dict()), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'embedding_size': config.model.embedding_size }, model_file_path) print('Finish.')
) if epoch % cf.test_every == 0: acc = 0 for _, (img_batch, label_batch) in enumerate(test_loader): label_preds = model.test_batch_supervised(img_batch) acc += datasets.accuracy(label_preds, label_batch) metrics["acc"].append(acc / len(test_loader)) print("\nTest @ epoch {} / Accuracy: {:.4f}".format( epoch, acc / len(test_loader))) utils.save_json(metrics, cf.logdir + "metrics.json") if __name__ == "__main__": cf = utils.AttrDict() cf.seeds = [0] for seed in cf.seeds: # experiment params cf.seed = seed cf.n_epochs = 30 cf.test_every = 1 cf.log_every = 100 cf.logdir = f"data/supervised/{seed}/" # dataset params cf.train_size = None cf.test_size = None cf.label_scale = None