def train(gpu, region_id, mode, checkpoint_path): """ main training/evaluation method """ # ------------ # model & data # ------------ params = cf.get_params(region_id=region_id, collapse_time=True) data = DataModule(params['data_params'], params['training_params']) model = load_model(Model, params, checkpoint_path) # ------------ # trainer # ------------ trainer = get_trainer(gpu) print_training(params['data_params']) # ------------ # train & final validation # ------------ if mode == 'train': print("-----------------") print("-- TRAIN MODE ---") print("-----------------") trainer.fit(model, data) # validate do_test(trainer, model, data.val_dataloader())
def add_slot_embs_to_slu_embs(slot_embs_file, slu_embs_file): from src.slu.datareader import datareader from src.utils import init_experiment from config import get_params with open(slot_embs_file, "rb") as f: slot_embs_dict = pickle.load(f) slu_embs = np.load(slu_embs_file) params = get_params() logger = init_experiment(params, logger_filename=params.logger_filename) _, vocab = datareader(use_label_encoder=True) new_slu_embs = np.zeros( (vocab.n_words, 400)) # 400: word + char level embs # copy previous embeddings prev_length = len(slu_embs) new_slu_embs[:prev_length, :] = slu_embs for slot_name in slot_list: emb = None index = vocab.word2index[slot_name] if index < prev_length: continue for domain, slot_embs in slot_embs_dict.items(): slot_list_based_on_domain = domain2slot[domain] if slot_name in slot_list_based_on_domain: slot_index = slot_list_based_on_domain.index(slot_name) emb = slot_embs[slot_index] break assert emb is not None new_slu_embs[index] = emb np.save("../data/snips/emb/slu_word_char_embs_with_slotembs.npy", new_slu_embs)
def gen_embs_for_vocab(): from src.datareader import datareader from src.utils import load_embedding, init_experiment from config import get_params params = get_params() logger = init_experiment(params, logger_filename=params.logger_filename) _, vocab = datareader() embedding = load_embedding(vocab, 300, "/data/sh/glove.6B.300d.txt", "/data/sh/coachdata/snips/emb/oov_embs.txt") np.save("/data/sh/coachdata/snips/emb/slu_embs.npy", embedding)
def gen_embs_for_vocab(): from src.slu.datareader import datareader from src.utils import load_embedding, init_experiment from config import get_params params = get_params() logger = init_experiment(params, logger_filename=params.logger_filename) _, vocab = datareader() embedding = load_embedding(vocab, 300, "PATH_OF_THE_WIKI_EN_VEC", "../data/snips/emb/oov_embs.txt") np.save("../data/snips/emb/slu_embs.npy", embedding)
def get_data_iterator(region_id, data_path, splits_path, data_split='test', collapse_time=True, batch_size=32, shuffle=False, num_workers=0): """ Creates an iterator for data in region 'region_id' for the days in `splits_path` """ params = cf.get_params(region_id=region_id, data_path=data_path, splits_path=splits_path) params['data_params']['collapse_time'] = collapse_time ds = create_dataset(data_split, params['data_params']) dataloader = DataLoader(ds, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) data_splits, test_sequences = data_utils.read_splits(params['data_params']['train_splits'], params['data_params']['test_splits']) test_dates = data_splits[data_splits.split=='test'].id_date.sort_values().values return iter(dataloader), test_dates, params
def build_coefs(n_tasks, n_sources=1, overlap=100, seed=None, positive=False, hemi="lh", illustration=False, labels_type="any", dataset="camcan", spacing="ico4"): params = cfg.get_params(dataset) data_path = params["data_path"] labels = np.load(data_path + "label/labels-%s-%s-%s.npy" % (labels_type, spacing, hemi)) n_labels, n_features = labels.shape rng = np.random.RandomState(seed) # simulate random activation (one per label) coefs = np.zeros((n_features, n_tasks_max)) if overlap < 0. or overlap > 100: raise ValueError("Overlap must be in 0-100%. Got %s." % overlap) frac = Fraction(overlap, 100) denom = frac.denominator numer = frac.numerator for l in range(n_sources): labels_idx, = np.where(labels[l]) n_source_in_label = len(labels_idx) choices = np.arange(n_source_in_label) permutation = rng.permutation(choices) ido = permutation[0] if positive: sign = 1 else: sign = (-1)**rng.randint(2) idx_tasks_o = np.arange(n_tasks_max) mod = idx_tasks_o % denom < numer idx_tasks_o = idx_tasks_o[mod] vals = 10. * (2 + rng.rand(len(idx_tasks_o))) coefs[labels_idx[ido], idx_tasks_o] = sign * vals idx_no = list(set(np.arange(n_tasks_max)) - set(idx_tasks_o)) no = len(idx_no) if no: choices_no = permutation[1:no + 1] vals = 10. * (2 + rng.rand(no)) idno = rng.choice(choices_no, size=no) coefs[labels_idx[idno], np.array(idx_no)] = sign * vals coefs = coefs[:, :n_tasks] return coefs
def train(model, trainloader): params = get_params() ITER_LOG = params.ITER_LOG device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") now = datetime.datetime.now() optimizer = torch.optim.AdamW(model.parameters(), lr=params.LEARNING_RATE) criterion = nn.BCELoss() losses = [] for epoch in tqdm(range(params.N_EPOCHS)): loss_log = 0.0 y_true = [] y_pred = [] for i, data in enumerate(trainloader, 0): mels, labels = data[0].to(device), data[1].to(device) pred = model(mels.unsqueeze(-1).permute(0, 3, 1, 2)) optimizer.zero_grad() loss = criterion(pred.float(), labels.float()) loss_log += loss.item() loss.backward() optimizer.step() pred = np.round(pred.to('cpu').detach()) target = np.round(labels.to('cpu').detach()) y_pred.extend(pred.tolist()) y_true.extend(target.tolist()) if i % ITER_LOG == ITER_LOG - 1: # wandb.log({"loss": loss_log / ITER_LOG}) print('[%d, %5d] Running loss: %.3f' % (epoch + 1, i + 1, loss_log / ITER_LOG)) losses.append(loss_log / ITER_LOG) loss_log = 0.0 print('time:', datetime.datetime.now() - now) now = datetime.datetime.now() # print('Acc:\t', accuracy(pred.to('cpu').detach(), labels.to('cpu').detach()).item()) print('Acc:\t', accuracy_score(y_true, y_pred)) plt.plot(list(range(params.N_EPOCHS)), losses) plt.show() print('Finished Training') return model
def main(args): seed = args.seed torch.manual_seed(seed) torch.cuda.manual_seed(seed) np.random.seed(seed) params = get_params(args.dataset) params.update(vars(args)) pprint.pprint(params) net = Solver(params) if args.train: net.train() else: net.viz_traverse(0)
def main(): params = get_params() set_random_seed(params.RANDOM_SEED) parse_data() data = DatasetNorm('cutted_data') train_set, test_set = torch.utils.data.random_split( data, [data.__len__() - 100, 100]) trainloader = DataLoader(dataset=train_set, batch_size=params.BATCH_SIZE, shuffle=True, num_workers=8) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") tcnn = TempoCNN().to(device) wandb.init(project="tcnn") config = wandb.config config.learning_rate = 0.001 wandb.watch(tcnn) if not params.LOAD_MODEL: model = train(tcnn, trainloader) save_model(model) else: model = load_model().to(device) testloader = DataLoader(dataset=test_set, batch_size=params.BATCH_SIZE, shuffle=True) iters = 0 loss = 0.0 cr_loss = nn.BCELoss() for i, data in enumerate(testloader, 0): tcnn.eval() mels, labels = data[0].to(device), data[1].to(device) pred = model(mels.unsqueeze(-1).permute(0, 3, 1, 2)).to('cpu').detach() res = accuracy(pred, labels) print(res) loss += cr_loss(pred.float(), labels.float().to('cpu').detach()).item() iters += 1 print(loss / iters)
def combine_word_with_char_embs_for_vocab(wordembs_file): from src.slu.datareader import datareader from src.utils import init_experiment from config import get_params import torchtext char_ngram_model = torchtext.vocab.CharNGram() params = get_params() logger = init_experiment(params, logger_filename=params.logger_filename) _, vocab = datareader() embedding = np.load(wordembs_file) word_char_embs = np.zeros((vocab.n_words, 400)) for index, word in vocab.index2word.items(): word_emb = embedding[index] char_emb = char_ngram_model[word].squeeze(0).numpy() word_char_embs[index] = np.concatenate((word_emb, char_emb), axis=-1) np.save("../data/snips/emb/slu_word_char_embs.npy", word_char_embs)
def build_dataset(coefs, std=0.2, seed=None, same_design=False, randomize_subjects=False, hemi="lh", dataset="camcan", age_min=0, age_max=30, spacing="ico4"): """Build multi-task regression data.""" params = cfg.get_params(dataset) data_path = params["data_path"] rng = np.random.RandomState(seed) n_features, n_tasks = coefs.shape subjects = cfg.get_subjects_list(dataset, age_min, age_max)[:n_tasks] if randomize_subjects: subjects = rng.permutation(subjects) if same_design: s_id = rng.randint(0, n_tasks) subjects = n_tasks * [subjects[s_id]] x_names = [ data_path + "leadfields/X_%s_%s_%s.npy" % (s, hemi, spacing) for s in subjects ] X = np.stack([np.load(x_name) for x_name in x_names], axis=0) X = X.astype(np.float64) y = [x.dot(coef) for x, coef in zip(X, coefs.T)] y = np.array(y) y *= 1e4 # get Y in fT/cm n_samples = y.shape[1] std *= np.std(y, axis=1).mean() noise = std * rng.randn(n_tasks_max, n_samples) y += noise[:n_tasks] X = X[:n_tasks] y = y[:n_tasks] return X, y
seed=TP['shuffle_seed']) X_train = data_set['x_train'] X_test = data_set['x_test'] Y_train = data_set['y_train'] Y_test = data_set['y_test'] total_counts = np.sum(Y_train, axis=0) + np.sum(Y_test, axis=0) train_gen = ImageDataGenerator(horizontal_flip=True, vertical_flip=True, rotation_range=180, zoom_range=(1, 1.2), preprocessing_function=xcept_preproc) test_gen = ImageDataGenerator(preprocessing_function=xcept_preproc) trials = Trials() algo = partial(tpe.suggest, n_startup_jobs=TP['n_rand_hp_iters']) argmin = fmin(xcept_net, space=get_params(MP, TP), algo=algo, max_evals=TP['n_total_hp_iters'], trials=trials) end_time = dt.now() print_end_details(start_time, end_time) print("Evalutation of best performing model:") print(trials.best_trial['result']['loss']) with open(op.join(ckpt_dir, 'trials_{}.pkl'.format(start_time)), "wb") as pkl_file: pickle.dump(trials, pkl_file)
print('For {}ing, found {} {} images'.format(fold, n_fnames, sub_fold)) if fold == 'test': total_test_images += n_fnames if TP['steps_per_test_epo'] is None: TP['steps_per_test_epo'] = int(np.ceil(total_test_images / DF['flow_from_dir']['batch_size']) + 1) ################################### # Set up generators ################################### train_gen = ImageDataGenerator(preprocessing_function=xcept_preproc, **DF['image_data_generator']) test_gen = ImageDataGenerator(preprocessing_function=xcept_preproc) ############################################################ # Run training with hyperparam optimization (using hyperopt) ############################################################ trials = Trials() algo = partial(tpe.suggest, n_startup_jobs=TP['n_rand_hp_iters']) argmin = fmin(xcept_net, space=get_params(MP, TP), algo=algo, max_evals=TP['n_total_hp_iters'], trials=trials) end_time = dt.now() print_end_details(start_time, end_time) print("Evalutation of best performing model:") print(trials.best_trial['result']['loss']) with open(op.join(ckpt_dir, 'trials_{}.pkl'.format(start_time)), "wb") as pkl_file: pickle.dump(trials, pkl_file) sup
# 保存至文件 def save_words(words, path): of = open(path, 'w', encoding='UTF-8') for idx, w in enumerate(words): of.write(str(w) + '\n') of.close() # 词向量训练方法 def word2vec_proc(params): line_sentence = LineSentence(config.words_file) model = word2vec.Word2Vec(line_sentence, size=params['vector_dim'], window=params['window_size'], min_count=params['min_frequency'], workers=params['workers'], sg=params['use_skip_gram'], hs=params['use_hierarchical_softmax'], negative=params['negative_size'], iter=params['pre_proc_epochs']) return model def fasttext_proc(params): line_sentence = LineSentence(config.words_file) model = fasttext.FastText(line_sentence, size=params['vector_dim'], window=params['window_size'], min_count=params['min_frequency'], workers=params['workers'], sg=params['use_skip_gram'], hs=params['use_hierarchical_softmax'], negative=params['negative_size'], iter=params['pre_proc_epochs']) return model if __name__ == '__main__': pre_processing(config.get_params())
def main(argv): # set fixed random seed, load config files tf.random.set_seed(RANDOM_SEED) # using mix precision or not if MIXPRECISION: policy = mixed_precision.Policy('mixed_float16') mixed_precision.set_policy(policy) # get params for model train_iter, input_size, num_cls, lrs_schedule_params, loss_params, parser_params, model_params = get_params( FLAGS.name) # ----------------------------------------------------------------- # set up Grappler for graph optimization # Ref: https://www.tensorflow.org/guide/graph_optimization @contextlib.contextmanager def options(opts): old_opts = tf.config.optimizer.get_experimental_options() tf.config.optimizer.set_experimental_options(opts) try: yield finally: tf.config.optimizer.set_experimental_options(old_opts) # ----------------------------------------------------------------- # Creating the instance of the model specified. logging.info("Creating the model instance of YOLACT") model = Yolact(**model_params) # add weight decay for layer in model.layers: if isinstance(layer, tf.keras.layers.Conv2D) or isinstance( layer, tf.keras.layers.Dense): layer.add_loss(lambda: tf.keras.regularizers.l2(FLAGS.weight_decay) (layer.kernel)) if hasattr(layer, 'bias_regularizer') and layer.use_bias: layer.add_loss(lambda: tf.keras.regularizers.l2(FLAGS.weight_decay) (layer.bias)) # ----------------------------------------------------------------- # Creating dataloaders for training and validation logging.info("Creating the dataloader from: %s..." % FLAGS.tfrecord_dir) dateset = ObjectDetectionDataset(dataset_name=FLAGS.name, tfrecord_dir=os.path.join( FLAGS.tfrecord_dir, FLAGS.name), anchor_instance=model.anchor_instance, **parser_params) train_dataset = dateset.get_dataloader(subset='train', batch_size=FLAGS.batch_size) valid_dataset = dateset.get_dataloader(subset='val', batch_size=1) # count number of valid data for progress bar # Todo any better way to do it? num_val = 0 for _ in valid_dataset: num_val += 1 # ----------------------------------------------------------------- # Choose the Optimizor, Loss Function, and Metrics, learning rate schedule lr_schedule = learning_rate_schedule.Yolact_LearningRateSchedule( **lrs_schedule_params) logging.info("Initiate the Optimizer and Loss function...") optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=FLAGS.momentum) criterion = loss_yolact.YOLACTLoss(**loss_params) train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32) loc = tf.keras.metrics.Mean('loc_loss', dtype=tf.float32) conf = tf.keras.metrics.Mean('conf_loss', dtype=tf.float32) mask = tf.keras.metrics.Mean('mask_loss', dtype=tf.float32) seg = tf.keras.metrics.Mean('seg_loss', dtype=tf.float32) # ----------------------------------------------------------------- # Setup the TensorBoard for better visualization # Ref: https://www.tensorflow.org/tensorboard/get_started logging.info("Setup the TensorBoard...") current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = './logs/gradient_tape/' + current_time + '/train' test_log_dir = './logs/gradient_tape/' + current_time + '/test' train_summary_writer = tf.summary.create_file_writer(train_log_dir) test_summary_writer = tf.summary.create_file_writer(test_log_dir) # ----------------------------------------------------------------- # Start the Training and Validation Process logging.info("Start the training process...") # setup checkpoints manager checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, model=model) manager = tf.train.CheckpointManager(checkpoint, directory="./checkpoints", max_to_keep=5) # restore from latest checkpoint and iteration status = checkpoint.restore(manager.latest_checkpoint) if manager.latest_checkpoint: logging.info("Restored from {}".format(manager.latest_checkpoint)) else: logging.info("Initializing from scratch.") best_masks_map = 0. iterations = checkpoint.step.numpy() for image, labels in train_dataset: # check iteration and change the learning rate if iterations > train_iter: break checkpoint.step.assign_add(1) iterations += 1 with options({ 'constant_folding': True, 'layout_optimize': True, 'loop_optimization': True, 'arithmetic_optimization': True, 'remapping': True }): loc_loss, conf_loss, mask_loss, seg_loss = train_step( model, criterion, train_loss, optimizer, image, labels, num_cls) loc.update_state(loc_loss) conf.update_state(conf_loss) mask.update_state(mask_loss) seg.update_state(seg_loss) with train_summary_writer.as_default(): tf.summary.scalar('Total loss', train_loss.result(), step=iterations) tf.summary.scalar('Loc loss', loc.result(), step=iterations) tf.summary.scalar('Conf loss', conf.result(), step=iterations) tf.summary.scalar('Mask loss', mask.result(), step=iterations) tf.summary.scalar('Seg loss', seg.result(), step=iterations) if iterations and iterations % FLAGS.print_interval == 0: tf.print( "Iteration {}, LR: {}, Total Loss: {}, B: {}, C: {}, M: {}, S:{} " .format(iterations, optimizer._decayed_lr(var_dtype=tf.float32), train_loss.result(), loc.result(), conf.result(), mask.result(), seg.result())) if iterations and iterations % FLAGS.save_interval == 0: # save checkpoint save_path = manager.save() logging.info("Saved checkpoint for step {}: {}".format( int(checkpoint.step), save_path)) # validation and print mAP table all_map = evaluate(model, valid_dataset, num_val, num_cls) box_map, mask_map = all_map['box']['all'], all_map['mask']['all'] tf.print(f"box mAP:{box_map}, mask mAP:{mask_map}") with test_summary_writer.as_default(): tf.summary.scalar('Box mAP', box_map, step=iterations) tf.summary.scalar('Mask mAP', mask_map, step=iterations) # Saving the weights: if mask_map > best_masks_map: best_masks_map = mask_map model.save_weights( f'{FLAGS.weights}/weights_{FLAGS.name}_{str(best_masks_map)}.h5' ) # reset the metrics train_loss.reset_states() loc.reset_states() conf.reset_states() mask.reset_states() seg.reset_states()
def computeCost(self, rootDic, noAGV, k, u, utilObj, prevTask): #rootDic = sys.argv[1] #taskList = np.arange(100) #q = taskList #state = 'INITIALISE' #itr = 0 currTask = u #trainObj = training(rootDic, noAGV) #obsObj = utility(rootDic) optim = torch.optim.Adam(self.Model.parameters()) #lastTask = 0 # if state == 'START': # currTask = random.randint(0, 100) # if (currTask in q): # q= q[q != currTask] # #q.remove() # state = 'EXECUTE' # elif (currTask not in q): # state = 'START' # if state == 'EXECUTE': #itr = itr +1 obs = utilObj.readObs( prevTask) # this iteration have to be calculated from planner # The last task is to be decided by the planner #allsTensor, allobsTensor = \ estimatedCost = self.doTrain( k, obs, currTask, prevTask, utilObj) #itr, obs, currTask, prevTask, obsObj print('estimated Cost' + str(estimatedCost)) #utilObj.storeObs(k, currTask) params = config.get_params() if (k >= 5): if k % params["timestep"] == 0: print("Calculating Loss") allsTensor = torch.Tensor(self.k, 1, 3) allsTensor = torch.cat(self.allS, dim=1) #out=) allobsTensor = torch.Tensor(self.k, 1, 3) allobsTensor = torch.cat(self.allObs, dim=1) L = self.loss_fn(allsTensor, allobsTensor) print('Loss =' + str(L)) # sys.exit() L.backward() self.allObs.clear() self.allS.clear() optim.step() optim.zero_grad() #end if np.savetxt('estimate.xls', self.saveEstimate, delimiter=',') np.savetxt('observations.xls', self.saveObs, delimiter=',') #self.stateDict = collections.OrderedDict(sorted(self.stateDict.items())) return estimatedCost
import tensorflow as tf from config import PASCAL_CLASSES, COLORS, get_params, ROOT_DIR from data.coco_dataset import ObjectDetectionDataset from utils import learning_rate_schedule from utils.utils import postprocess, denormalize_image from yolact import Yolact # Todo Add your custom dataset tf.random.set_seed(1234) NAME_OF_DATASET = "pascal" CLASS_NAMES = PASCAL_CLASSES # ----------------------------------------------------------------------------------------------- # create model and dataloader train_iter, input_size, num_cls, lrs_schedule_params, loss_params, parser_params, model_params = get_params( NAME_OF_DATASET) model = Yolact(**model_params) dateset = ObjectDetectionDataset(dataset_name=NAME_OF_DATASET, tfrecord_dir=os.path.join( ROOT_DIR, "data", NAME_OF_DATASET), anchor_instance=model.anchor_instance, **parser_params) train_dataset = dateset.get_dataloader(subset='train', batch_size=1) valid_dataset = dateset.get_dataloader(subset='val', batch_size=1) # ----------------------------------------------------------------------------------------------- # Restore CheckPoints # Choose the Optimizor, Loss Function, and Metrics, learning rate schedule lr_schedule = learning_rate_schedule.Yolact_LearningRateSchedule( **lrs_schedule_params) optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9)
def scheduling(self): params = config.get_params() state = 'INITIALISE' while self.itr <= params["repNo"]: if state == 'INITIALISE': utilObj = utility(self.dirName, self.ownNo) if (self.ownNo == 1): lastTask = 2 utilObj.storeCurrLoc(self.ownNo, lastTask) #utilObj.storeObs(self.itr , lastTask) utilObj.storeObs(k=1, currTask=lastTask, estCost=0) elif (self.ownNo == 2): lastTask = 8 utilObj.storeCurrLoc(self.ownNo, lastTask) utilObj.storeObs(k=1, currTask=lastTask, estCost=0) #utilObj.storeCurrLoc(self.ownNo, 8) elif (self.ownNo == 3): lastTask = 12 utilObj.storeCurrLoc(self.ownNo, lastTask) utilObj.storeObs(k=1, currTask=lastTask, estCost=0) #utilObj.storeCurrLoc(self.ownNo, 12) elif (self.ownNo == 4): lastTask = 7 utilObj.storeCurrLoc(self.ownNo, lastTask) utilObj.storeObs(k=1, currTask=lastTask, estCost=0) #utilObj.storeCurrLoc(self.ownNo, 7) #end if state = 'START' if state == 'START': self.itr += 1 currLoc = utilObj.getCurrLoc(self.ownNo) print("Itr: ", self.itr) print("currLoc at AGV level: ", currLoc) stateDict, lenTaskSeq = self.planning(currLoc, utilObj, params) stateDict = collections.OrderedDict(sorted(stateDict.items())) #print(stateDict) df = pd.DataFrame({ key: pd.Series(value) for key, value in stateDict.items() }) df.to_csv('estState.xlsx', encoding='utf-8', index=False) # #df = pd.DataFrame(data=stateDict, index=[0]) #df = (df.T) #print(df) #df.to_excel() #print("Tasksequence of AGV", self.ownNo, self.taskSequence) self.completedTask_lst[self.itr] = self.taskSequence state = "COMPLETE" if state == 'COMPLETE': lastTask = self.endTask utilObj.storeCurrLoc(self.ownNo, lastTask) #utilObj.storeObs(self.lenT, self.endTask) outtxt1 = 'AGV: ' + str(self.ownNo) + ' ' + 'ITR: ' + str( self.itr) + ' ' + 'TASKS: ' + str( self.completedTask_lst) + '\n' self.fid1 = open(self.f1, 'a') self.fid1.write(outtxt1) self.fid1.close() state = 'START'
# parse arguments parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--mode', dest='mode', type=int, default=2, help='run mode - (0-train+test, 1-train only, 2-test only, 3-val only)') parser.add_argument('--nlayers', dest='nlayers', type=int, default=3, help='Number of reader layers') parser.add_argument('--dataset', dest='dataset', type=str, default='cmrc', help='Dataset - cmrc') parser.add_argument('--seed', dest='seed', type=int, default=36, help='Seed for different experiments with same settings') parser.add_argument('--gating_fn', dest='gating_fn', type=str, default='T.mul', help='Gating function (T.mul || Tsum || Tconcat)') args = parser.parse_args() cmd = vars(args) params = get_params(cmd['dataset']) params.update(cmd) np.random.seed(params['seed']) random.seed(params['seed']) # save directory w2v_filename = params['word2vec'].split('/')[-1].split('.')[0] if params['word2vec'] else 'None' save_path = ('experiments/mul_bpe_1k cmrc_nhid128_nlayers3_dropout0.5_wiki_chardim100_train1_seed36_use-feat1_gfT.mul') if not os.path.exists(save_path): os.makedirs(save_path) # train if params['mode']<2: train.main(save_path, params) # test
def depth_meansubtract(params): for dir in params["dataset"]: if dir ==-1: continue normalizer=52492 sbt=params["depth_mean"] im_type='depth' im_type_to='mean_depth' new_dir=dir[0]+im_type_to+"/" if os.path.exists(new_dir): rmtree(new_dir) if not os.path.exists(new_dir): os.makedirs(new_dir) full_path=dir[0]+'/'+im_type+'/*.png' lst=glob.glob(full_path) for f in lst: img = Image.open(f) arr1= numpy.array(img,theano.config.floatX) arr2=np.zeros_like(arr1) arr2[arr1.nonzero()]=sbt arr1=(arr1-arr2)/normalizer f=new_dir+os.path.basename(f).replace(".png","") np.save(f,arr1) print("data set converted %s"%(dir[0])) else: print("data set has already proccessed %s"%(dir[0])) print "Depth data proccessing completed" params=config.get_params() depth_meansubtract(params)
def result_proc(text): """ 对预测结果做最后处理 :param text: 单条预测结果 :return: """ # text = text.lstrip(' ,!。') text = text.replace(' ', '') text = text.strip() if '<end>' in text: text = text[:text.index('<end>')] return text test_csv = pd.read_csv(config.test_set, encoding="UTF-8") # 赋值结果 test_csv['Prediction'] = results # 提取ID和预测结果两列 test_df = test_csv[['QID', 'Prediction']] # 结果处理 test_df['Prediction'] = test_df['Prediction'].apply(result_proc) # 保存结果 test_df.to_csv(config.inference_result_path, index=None, sep=',') print('已保存文件至{}'.format(config.inference_result_path)) if __name__ == '__main__': params = config.get_params() params['batch_size'] = 8 params['beam_size'] = 8 beam_search(params)
from config import get_params from yolact import Yolact name = "coco" train_iter, input_size, num_cls, lrs_schedule_params, loss_params, parser_params, model_params = get_params( name) model = Yolact(**model_params) model.build(input_shape=(2, 550, 550, 3)) model.summary()
import torch from torch import nn from config import get_params params = get_params() class MultiFilterBlock(nn.Module): def __init__(self, in_channels, out_channels, pool_size, n_parallels=4): super(MultiFilterBlock, self).__init__() self.pooling = nn.Sequential(nn.AvgPool2d(kernel_size=(pool_size, 1)), nn.BatchNorm2d(in_channels)) convs = [] filter = 4 for i in range(n_parallels): convs.append( nn.Conv2d(in_channels=in_channels, out_channels=26, kernel_size=(1, filter), padding=filter // 2)) filter += 4 self.conv_modules = nn.ModuleList(convs) self.bottleneck = nn.Conv2d(in_channels=26, out_channels=out_channels, kernel_size=1) self.elu = nn.ELU() def forward(self, x): output = self.pooling(x) parallel_output = []
steps_per_val = len( data_dict['x_test']) // (MP['batch_size'] * MP['n_gpus']) ######################### # Generate the generators ######################### train_gen = ImageDataGenerator(**IMP) test_gen = ImageDataGenerator() ###################################################### # Define Hyperopt optimization and run training trials ###################################################### trials = Trials() algo = partial(tpe.suggest, n_startup_jobs=MP['n_rand_hp_iters']) argmin = fmin(run_deeplab_trial, space=get_params(MP), algo=algo, max_evals=MP['n_total_hp_iters'], trials=trials) ############################### # End of training cleanup ############################### end_time = dt.now() print_end_details(start_time, end_time) print("Evalutation of best performing model:") print(trials.best_trial['result']['loss']) # Dump trials object for safe-keeping with open(op.join(ckpt_dir, 'trials_{}.pkl'.format(start_time)), "wb") as pkl_file: