def __init__(self, dataset_path, batch_size, mode, x_keys, y_keys, split_name, valid_split_number=0, p=None): # Setting the random generator seed np.random.seed(cfg.RANDOM_SEED) self.images_path = os.path.join(dataset_path, 'images') if p is None: self.annotations_path = os.path.join(dataset_path, 'annotations', '100') self.missing_annotations_path = os.path.join( dataset_path, 'missing_organs', '100') else: self.annotations_path = os.path.join(dataset_path, 'annotations', str(p)) self.missing_annotations_path = os.path.join( dataset_path, 'missing_organs', str(p)) self.valid_split_number = valid_split_number self.split_name = split_name Dataset.__init__(self, dataset_path, batch_size, mode, x_keys, y_keys, p)
def train(net_type, generator_fn_str, dataset_file, build_net_fn, featurized=True): d = Dataset(dataset_file + 'train.pgn') generator_fn = getattr(d, generator_fn_str) d_test = Dataset(dataset_file + 'test.pgn') X_val, y_val = d_test.load(generator_fn.__name__, featurized = featurized, refresh = False, board = net_type) board_num_channels = X_val[0].shape[1] if net_type == 'to' else X_val[0].shape[0] model = build_net_fn(board_num_channels=board_num_channels, net_type=net_type) start_time = str(int(time.time())) try: plot_model(model, start_time, net_type) except: print("Skipping plot") from keras.callbacks import ModelCheckpoint checkpointer = ModelCheckpoint( filepath = get_filename_for_saving(start_time, net_type), verbose = 2, save_best_only = True) model.fit_generator(generator_fn(featurized=featurized, board=net_type), samples_per_epoch = SAMPLES_PER_EPOCH, nb_epoch = NUMBER_EPOCHS, callbacks = [checkpointer], validation_data = (X_val, y_val), verbose = VERBOSE_LEVEL)
def example_test(model, images_dir="./data/images", annotations_dir="./data/annotations", demo_dir="./examples", name="example01.png"): if not (os.path.exists(demo_dir)): os.mkdir(demo_dir) dataset = Dataset(training=False, images_dir=images_dir, annotations_dir=annotations_dir) image_name = np.random.choice(dataset.get_images_list()) image_path = os.path.join(images_dir, image_name + ".jpg") mask_path = os.path.join(annotations_dir, "trimaps", image_name + ".png") orig_img = cv2.imread(image_path)[..., ::-1] true_mask = (cv2.imread(mask_path) * 100)[..., ::-1] pred_mask = process(image_path, model) labels = ["Original Image", "True Mask", "Predicted Mask"] collection = [orig_img, true_mask, pred_mask] creat_example(collection, labels, demo_dir, name, 3)
def main(): """ Creates a temporary file for the given input which is used to create a dataset, that is then evaluated on the given model. The generated summary is printed to standard out. """ args, unknown_args = prepare_arg_parser().parse_known_args() model_file = args.model_file with suppress_stdout_stderr(): model, _optimizer, vocab, _stats, cfg = train.load_model( model_file, unknown_args ) _, filename = tempfile.mkstemp() try: with open(filename, "a") as f: input_ = sys.stdin.read() article = preprocess.parse(input_) print(f"{article}\tSUMMARY_STUB", file=f) with suppress_stdout_stderr(): dataset = Dataset(filename, vocab, cfg) batch = next(dataset.generator(1, cfg.pointer)) # don't enforce any min lengths (useful for short cmdline summaries") setattr(cfg, "min_summary_length", 1) bs = BeamSearch(model, cfg=cfg) summary = evaluate.batch_to_text(bs, batch)[0] print(f"SUMMARY:\n{summary}") finally: os.remove(filename)
class ParseState(object): """Support object for read().""" def __init__(self, config=defaults, name=None): self.config = config self.dataset = Dataset(name=name) self.document = Document() self.texts = [] self.tags = [] def sentence_break(self): if len(self.texts) == 0: return if self.config.iobes: self.tags = iob_to_iobes(self.tags) tokens = [Token(t, g) for t, g in zip(self.texts, self.tags)] self.document.add_child(Sentence(tokens=tokens)) self.texts = [] self.tags = [] def document_break(self): self.sentence_break() if len(self.document) == 0: return self.dataset.add_child(self.document) self.document = Document() def finish(self): self.document_break()
def main(args): dataset = Dataset(batch_size) class_num = dataset.class_num() image_batch, label_batch = dataset.get_train_batch() image_batch = tf.reshape(image_batch, [-1, image_size, image_size, 3]) glaph_ = Graph(batch_size, class_num) train_op = glaph_.inference(image_batch, label_batch) gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=gpu_memory_fraction) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # with sess.as_default(): # sess.run(iterator.initializer) for epoch in range(epoch_num): i_batch, l_batch = sess.run([image_batch, label_batch]) # logger.debug('i_batch: %s, l_batch: %s' % (i_batch, l_batch)) _, loss_t, loss_p_adv, loss_p, los_re = sess.run(train_op) logger.debug('loss_t: %s, loss_p_adv: %s, loss_p: %s, los_re: %s' % (loss_t, loss_p_adv, loss_p, los_re)) sess.close()
def __init__(self, model, config): self.model = model self.lr = config['lr'] self.epoches = config['epochs'] self.batches = config['batchs'] self.samples = config['samples'] self.dataset = Dataset(self.samples, self.batches)
def eval(model, loss, dataset, params, batch_size=None): model.eval() if batch_size is None: batch_size = params.batch_size total_loss = 0. total_token = 0. hidden = model.init_hidden(batch_size) eval_data = Dataset(dataset, params.vocab) start_time = time.time() with torch.no_grad(): for bidx, batch in enumerate( eval_data.batcher( batch_size, params.num_steps, )): x, t = batch x = x.to(params.device) t = t.to(params.device) hidden = repackage_hidden(hidden) logits, hidden = model(x, state=hidden) gloss = loss(logits.view(-1, logits.size(-1)), t.view(-1)) num_token = (x > 0).float().sum() total_loss += gloss.item() * num_token total_token += num_token return total_loss / total_token, time.time() - start_time
def __init__(self, model, optimizer, vocab, cfg, stats=defaultdict(int)): """ Create a trainer instance for the given model. This includes creating datasets based on the `input_file` and `valid_file` in the given `cfg`. :param model: The Seq2Seq model to train :param optimizer: The optimizer for the `model` :param vocab: A `Vocabulary` instance to be used :param cfg: The current Config from which we get epochs/batch_size/files etc. :param stats: A dict with values such as epoch/running_avg_loss etc. when resuming training """ self.cfg = cfg self.dataset = Dataset(cfg.train_file, vocab, cfg) self.validation_dataset = Dataset(cfg.valid_file, vocab, cfg, evaluation=True) self.model = model self.optimizer = optimizer self.validator = cfg.validator.lower() # if cfg.validate_every == 0, we validate once every epoch self.validate_every = ( math.ceil(len(self.dataset) / cfg.batch_size) # batches/epoch if cfg.validate_every == 0 else cfg.validate_every) self.rouge_valid = self.validator == "rouge" self.loss_valid = self.validator == "loss" self.scheduler = ( None if cfg.learning_rate_decay >= 1.0 else ReduceLROnPlateau( optimizer, mode="min" if self.validator == "loss" else "max", factor=cfg.learning_rate_decay, patience=cfg.learning_rate_patience, verbose=True, )) self.coverage_loss_weight_decay = cfg.coverage_loss_weight_decay assert self.validator in ["loss", "rouge"] self.early_stopping = cfg.early_stopping self.patience = cfg.patience self.epoch = stats["epoch"] self.iteration = stats["iteration"] self.running_avg_loss = stats["running_avg_loss"] self.running_avg_cov_loss = stats["running_avg_cov_loss"] self.best_validation_score = stats["best_validation_score"] self.current_validation_score = stats.get("current_validation_score", 0) self.current_patience = stats.get("current_patience", 0) self.model_identifier = stats["model_identifier"] self.time_training = stats["time_training"] # Updated and managed from train function and context self.training_start_time = None self.writer = None self.pbar = None
def __init__(self, embeddings, evalita=None, twitter=False, debug=False): """ This class define the neural network model as well as its operations for training, prediction and evaluation :param embeddings: Embeddings :param evalita: (optional) tuple => (evalita_test_path, evalita_train_path) :param twitter: (optional) to enable twitter char '#' '@' in dataset :param debug: (optional) to enable debug info and logs """ # initialization parameters self.hidden_size = embeddings.embed_dim self._embeddings = embeddings self.dataset = Dataset(embeddings, evalita, twitter) self.ntags = len(self.dataset.labels) # fixed parameters # TODO it is necessary to find a right forget_bias and learning_rate self.forget_bias = 1.0 # initializing future variables self.embeddings = None self.labels = None self.output = None self.predictions = None self.logits = None self.loss = None self.transition_params = None # dropout keep probability self.keep_prob = None # utils self.batch_size = BATCH_SIZE self.labels_len = self.dataset.max_sequence # training utilities self.optimizer = None self.train_op = None # initializing feed dictioanry self.feed_dict = dict() # initializer self.init_g = None self.init_l = None # saver - initialization after variable init self.saver = None # debug self.debug = debug # tensorboard self.file_writer = tf.summary.FileWriter(LOG_DIR) self.summary = None # metrics self.metrics = None
def evaluate(model_path, batch_size=128): print("Evaluating model {}".format(model_path)) model = load_model(model_path) ds = Dataset() X, y = ds.get_test_data() gen = DataLoader(X, y, IMG_WIDTH, IMG_HEIGHT, batch_size) metrics = model.evaluate_generator(gen, steps=len(gen), workers=8) pprint(dict(zip(model.metrics_names, metrics)))
def load_train_val(cls, normalize=True): ds = loaders.load_cifar10(cls.data_path, 'train') if normalize: cls.mean, cls.std = pp.get_normalization_statistics(ds.images) ds = Dataset(pp.normalize(ds.images, cls.mean, cls.std), ds.labels, ds.class_count) ds.shuffle() ds_train, ds_val = ds.split(0, int(ds.size * 0.8)) return ds_train, ds_val
def predict(df): basepath = '/media/user1/preprocessed' features = [] truth_label = [] df.sort() print(df) label = np.zeros((1, )) for name in df: print(name) label[0] = 0 vector = np.zeros(800) if 'COR' in name or 'SUB' in name: label[0] = 1 labels_df = pd.read_csv('./Book1.csv') path = labels_df['ID'].tolist() if name in path: label[0] = 1 basedir = os.path.normpath(basepath) print(basedir) files = glob(basedir + '/' + name + '*.npy') files.sort() l = len(files) if l == 0: break for i in range(l): max_pro = 0 img = np.load(files[i]) s = img.shape if s[0] != 224: img = misc.imresize(img, (224, 224), 'bilinear') img = np.stack((img, img, img), axis=2) img = img[np.newaxis, :, :, :] test_generator = Dataset( img, label, augment=False, shuffle=False, input_form='t1', seed=seed, ) test_generator.reset() test_results = evaluate.get_results(model, test_generator) probabilities = list( evaluate.transform_binary_probabilities(test_results)) vector[i] = probabilities[0] features.append(vector) truth_label.append(label[0]) return features, truth_label
def nfold_training(self, N=N_FOLD, model_path=MODEL_CKPT): """ Ad-hoc routine to run nfold cross validation over tweets dataset :param N: int, (optional) number of folds :param model_path: string, (optional) path to model """ # re instantiating dataset self.dataset = Dataset(self._embeddings, twitter=TWITTER_CHARS, n_fold=N) metrics = [] nfold_counter = 0 # running nfold for training, testing_batch in zip(self.dataset.trainingset, self.dataset.testingset): nfold_counter += 1 print("---- N FOLD RUN ", nfold_counter, "----") # instantiating a new session with tf.Session() as sess: # preparing model sess.run(self.init_g) sess.run(self.init_l) # restoring previous checkpoint self.restore_model(sess, path=model_path) # batching for i in range(0, int(len(training) / BATCH_SIZE) + 1): print("[TRAIN TWEETS] RUN ", i) # retrieving training batch training_batch = training[i * BATCH_SIZE:(i + 1) * BATCH_SIZE] # feeding dictionary (sentences, labels), sequence_lengths = self.dataset.build_batch( training_batch, self._embeddings.vocabulary, self.dataset.labels, self.dataset.max_sequence, self.dataset.word_vector_len) self.set_feed_dictionary(embeddings=(sentences, sequence_lengths), labels=labels) # training step self.training_step(sess) # evaluating print("[EVALUATION TWEETS] running...") metrics.append(self.evaluate_model(sess, testing=testing_batch)) if LOG_METRICS is not None: self.metrics.write_log_metric_nfold(metrics, N, LOG_METRICS)
def __init__(self): self.dataset = Dataset() self.cap = None self.showMoments = False self.showEvaluation = False self.bufferFace = Buffer(self.ROLLING_WINDOW_LENGTH) self.bufferLeftEye = Buffer(self.ROLLING_WINDOW_LENGTH) self.bufferRightEye = Buffer(self.ROLLING_WINDOW_LENGTH)
def prepare(args): logger = logging.getLogger("rc") logger.info('train test split...') # train_test_split(args.all_file, args.train_file, args.test_file, args.train_rate) sen_data = Dataset(args) sen_data.build_vocab() with open(args.data_path, 'wb') as fout: pickle.dump(sen_data, fout) logger.info('Done with preparing!')
def build_data(args): datafile = h5py.File(args.datafile, 'r') train_inputs = datafile['train_inputs'] # TODO stupid file name train_targs = datafile['train_targets'] valid_inputs = datafile['valid_inputs'] valid_targs = datafile['valid_targets'] train = Dataset(train_inputs, train_targs, args.batch_size) valid = Dataset(valid_inputs, valid_targs, args.batch_size) return {'train':train, 'valid':valid}, \ {'gram_size':datafile['gram_size'][0] - 1, 'vocab_size':datafile['vocab_size'][0], \ 'hid_size':args.d_hid, 'emb_size':args.d_emb}
def get_data(self, data_path): self.data_path = data_path train_path = data_path + '/test.csv' # train_path = data_path + '/train.csv' test_path = data_path + '/test.csv' self.train_data = Dataset(train_path) self.test_data = Dataset(test_path) self.input_dim = self.train_data.x_dim self.num_label = self.train_data.num_label self.num_samples = self.train_data.__len__()
def main(params): filepath = params['filepath'] date = params['date'] forecastcol = params['forecastcolumn'] epochs = params['epochs'] bs = params['batch_size'] ws = params['sequence_length'] #read the filepath df = pd.read_csv(filepath, sep=",") dataobj = Dataset(df, date, forecastcol) #normal trian and eval split #this split doesn't belong to model data preparation train_split, eval_split = dataobj.normal_split() model_train, model_labels = model_data_prep(train_split, ws) logging.info("Train Data Shape:{}".format(model_train.shape)) logging.info("Train Label Shape:{}".format(model_labels.shape)) #call a model file logging.info("============= MODEL TRAINING STARTED =============") network, modeldump = model.conv1d(model_train, model_labels, epochs=epochs, batch_size = bs) #model.plot_loss(modeldump) predictions = model.forecast_fn(network, model_train[-1], len(eval_split), eval_split) assert len(eval_split) == len(predictions), "Length Mismatch between Actuals and Predictions" score = model.evaluate_model_performance(eval_split, predictions) #mse square #sve the model model_name = 'conv1d_{}.h5'.format(datetime.datetime.now().strftime('%d-%m-%Y')) network.save(model_name) #plot the figure plt.figure(figsize=(15, 7)) plt.plot(eval_split, color='orange', label='Actual Values') plt.plot(predictions, color='red', label='Forecasted Values') plt.legend() plt.title("Model Error Rate:{0:2f}".format(score)) plt.xticks(rotation=90) plt.savefig("conv1d_model_forecast.png") plt.show() logging.info("Model Score on Test Data:{}".format(score))
def detect(path,**kwargs): d = Dataset('./data') image = d.read(path) global rgb rgb = d.read(path) pupil = find_pupil(image, 15)[0] e = Ellipse() e._axes = (pupil[2], pupil[2]) e._center = pupil[0:2] x, y = circle_perimeter(pupil[0], pupil[1], pupil[2]) #rgb[x,y] = 255 return e, image
def main(epoch_count=1): from data import Dataset, loaders from data.preparers import Iccv09Preparer import ioutils from ioutils import console print("Loading and deterministically shuffling data...") data_path = os.path.join( ioutils.path.find_ancestor(os.path.dirname(__file__), 'projects'), 'datasets/cifar-10-batches-py') ds = loaders.load_cifar10_train(data_path) labels = np.array([np.bincount(l.flat).argmax() for l in ds.labels]) ds = Dataset(ds.images, labels, ds.class_count) ds.shuffle(order_determining_number=0.5) print("Splitting dataset...") ds_train, ds_val = ds.split(0, int(ds.size * 0.8)) print(ds_train.size, ds_val.size) print("Initializing model...") model = ClfBaselineB( input_shape=ds.image_shape, class_count=ds.class_count, batch_size=128, learning_rate_policy=1e-6, #learning_rate_policy={ # 'boundaries': [3, 5, 7], # 'values': [10**-i for i in range(1, 5)] #}, ortho_penalty=0, use_multiclass_hinge_loss=False, training_log_period=100) def handle_step(i): text = console.read_line(impatient=True, discard_non_last=True) if text == 'd': viz.display(ds_val, lambda im: model.predict([im])[0]) elif text == 'q': return True return False model.training_step_event_handler = handle_step #viz = visualization.Visualizer() print("Starting training and validation loop...") #model.test(ds_val) #ds_train = ds_train[:128] for i in range(epoch_count): model.train(ds_train, epoch_count=1) model.test(ds_train, "Training") model.test(ds_val, "Validation") model.save_state()
def single_image_upload(): # Access the user-sent image from the request object. fname = request.files['input-file'].filename fpath = os.path.join("imgs", fname) request.files['input-file'].save(fpath) # Create a dataset to contain the image. ds = Dataset(name='dataset_' + str(int(random() * 1000000)), prefix='./imgs', batch_size=1, images=True) ds.initialize(sources=[fname]) ds_id = STATE.add_dataset(ds) return json.dumps({'datasetId': ds_id})
def load_dataset(file_name): dataset = Dataset(logger, file_name) columns = ['date', 'security', 'feature'] dataset.drop_na(columns) dataset.factorize_label() dataset.sort_date() return dataset
def main(args): # metadata for training trajectory_length = 20 prediction_length = trajectory_length // 2 print("************* Loading Dataset ***************") dataset = Dataset() dataset.load_data(args.dataset) training_data, testing_data = dataset.get_train_validation_batch( trajectory_length) # # reduce the size of training data.. if args.truncated: print("Using truncated data") training_data = training_data[:10] testing_data = testing_data[:10] # experiment configs # experiment_embedding_size = [64] # experiment_hidden_size = [128] experiment_embedding_size = args.embedding_size experiment_hidden_size = args.hidden_size for embedding_size in experiment_embedding_size: for hidden_size in experiment_hidden_size: social_model = SocialModel(hidden_size=hidden_size, embedding_size=embedding_size) experiment(social_model, training_data, testing_data, hidden_size, embedding_size, args, model_name='social_lstm') # lstm_model = VanillaLSTMModel(hidden_size, embedding_size) # experiment( # lstm_model, training_data, testing_data, # hidden_size = hidden_size, # embedding_size = embedding_size, # num_epochs = 100, # lr = args.lr, # model_name = 'vanilla_lstm', # ) print("done!")
def validate(model_hdf5, net_type, generator_fn_str, dataset_file, featurized=True): from keras.models import load_model import data d_test = Dataset(dataset_file + 'test.pgn') X_val, y_val = d_test.load(generator_fn_str, featurized = featurized, refresh = False, board = "both") boards = data.board_from_state(X_val) if net_type == "from": model_from = load_model("saved/" + model_hdf5) y_hat_from = model_from.predict(X_val) num_correct = 0 for i in range(len(boards)): if y_val[0][i,np.argmax(y_hat_from[i])] > 0: num_correct += 1 print(num_correct / len(boards)) elif net_type == "to": model_to = load_model("saved/" + model_hdf5) y_hat_to = model_to.predict([X_val, y_val[0].reshape(y_val[0].shape[0],1,X_val.shape[2],X_val.shape[3])]) num_correct = 0 for i in range(len(boards)): if y_val[1][i,np.argmax(y_hat_to[i])] > 0: num_correct += 1 print(num_correct / len(boards)) elif net_type == "from_to": model_from = load_model("saved/" + model_hdf5[0]) model_to = load_model("saved/" + model_hdf5[1]) y_hat_from = model_from.predict(X_val) for i in range(len(boards)): from_square = np.argmax(y_hat_from[i]) y_max_from = np.zeros((1,1,X_val.shape[2],X_val.shape[3])) y_max_from.flat[from_square] = 1 y_hat_to = model_to.predict([np.expand_dims(X_val[i], 0), y_max_from]) to_square = np.argmax(y_hat_to) move_attempt = data.move_from_action(from_square, to_square) if boards[i].is_legal(move_attempt): print("YAY") else: print("BOO") print(move_attempt) move = data.move_from_action(np.argmax(y_val[0]), np.argmax(y_val[1])) print(move)
def treino(self, input_train): self.train = input_train p = Parameters() d = Dataset() with tf.Session(graph = self.graph) as session: # Tensorboard area self._loss_gen = tf.placeholder(tf.float32, shape=()) self._loss_dis = tf.placeholder(tf.float32, shape=()) self.img_pl = tf.placeholder(tf.float32, shape=self.shape_out) self.score_summary_op = tf.summary.merge([ tf.summary.scalar('Generator_loss', self._loss_gen), tf.summary.scalar('Discriminator_loss_real', self._loss_dis), tf.summary.image('Generated_images', self.img_pl, 128) ]) logdir = self.param.TENSORBOARD_DIR + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/" self.writer = tf.summary.FileWriter(logdir, session.graph) # weight initialization session.run(tf.global_variables_initializer()) saver = tf.train.Saver() # full optimization for epoch in range(self.param.NUM_EPOCHS_FULL): print('Epoch: '+ str(epoch+1), end=' ') self._training_epoch(session, epoch+1) path_model = self.param.LOG_DIR_MODEL + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '_gan.ckpt' saver.save(session, path_model) print("The model has saved in: " + path_model)
def main(): path = argv[1] dr_config = Config(constants.DREAM_CONFIG) with open(path, 'rb') as f: dr_model = torch.load(f) bc = BasketConstructor(constants.RAW_DATA_DIR, constants.FEAT_DATA_DIR) ub_basket = bc.get_baskets('prior', reconstruct=False) # this function needs a sample input to infer types baskets, lens, users = Dataset(ub_basket)[0: dr_config.batch_size] baskets, lens, users = sort_batch_of_lists(baskets, lens, users) baskets = pad_batch_of_lists(baskets, lens[0]) dr_hidden = dr_model.init_hidden(dr_config.batch_size) ub_seqs = [] # users' basket sequence for ubaskets in baskets: x = dr_model.embed_baskets(ubaskets) ub_seqs.append(torch.cat(x, 0).unsqueeze(0)) ub_seqs = torch.cat(ub_seqs, 0) r_model = dr_model.rnn print("Converting model of type: " + str(type(dr_model))) m_in = (ub_seqs, dr_hidden) m_out = r_model(*m_in) torch.onnx.export(r_model, ub_seqs, './models/model.onnx', verbose=True)
def main(): torch_tvm.enable(opt_level=3, device_type="gpu", device="cuda", host="llvm") path = argv[1] dr_config = Config(constants.DREAM_CONFIG) with open(path, 'rb') as f: dr_model = torch.load(f) bc = BasketConstructor(constants.RAW_DATA_DIR, constants.FEAT_DATA_DIR) ub_basket = bc.get_baskets('prior', reconstruct=False) # this function needs a sample input to infer types baskets, lens, users = Dataset(ub_basket)[0:dr_config.batch_size] baskets, lens, users = sort_batch_of_lists(baskets, lens, users) baskets = pad_batch_of_lists(baskets, lens[0]) dr_hidden = dr_model.init_hidden(dr_config.batch_size) ub_seqs = [] # users' basket sequence for ubaskets in baskets: x = dr_model.embed_baskets(ubaskets) ub_seqs.append(torch.cat(x, 0).unsqueeze(0)) ub_seqs = torch.cat(ub_seqs, 0) arg = [ub_seqs, dr_hidden] relay_graph = torch_tvm.to_relay(dr_model.rnn, arg)
def main(config_file,mode,distributed): config = check_params(config_file) if mode in ["Train","train"]: train_dataset = Dataset(config["train_params"]["input_path"],config["train_params"]["imsize"]) if distributed: import horovod as hvd hvd.init() if hvd.rank()==0: writer = setup_tensorboard(get_params(config["train_params"],"tensorboard_location","./summary/")) train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas=hvd.size(), rank=hvd.rank()) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=config["train_params"]["batch_size"], sampler=train_sampler, shuffle=True) model = ConvLSTM(**config["model_params"]) optimizer = hvd.DistributedOptimizer(model.optimizer, named_parameters=model.named_parameters()) hvd.broadcast_parameters(model.state_dict(), root_rank=0) train_distributed(model,train_loader,optimizer,config,writer) else: writer = setup_tensorboard(get_params(config["train_params"], "tensorboard_location", "./summary/")) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=config["train_params"]["batch_size"], shuffle = True) model = ConvLSTM(**config["model_params"]) train(model,train_loader,model.optimizer,config,writer) elif mode in ["infer","Infer"]: model = ConvLSTM(**config["model_params"]) model.load_state_dict(config["infer_params"]["model_save_path"]) output_file = open(config["infer_params"]["output_path"])
def train(auto_stop, steps=50, threshold=0.5): dataset = Dataset() discriminator_feeder = TrainFeeder(dataset) generator_feeder = TrainFeeder(dataset) discriminator = Discriminator(len(dataset.ci2n)).cuda() generator = Generator(len(dataset.ci2n)).cuda() criterion = torch.nn.BCEWithLogitsLoss(size_average=False) discriminator_optimizer = torch.optim.Adam(discriminator.parameters(), lr=1e-4) generator_optimizer = torch.optim.Adam(generator.parameters(), lr=1e-4) discriminator_feeder.prepare('train') generator_feeder.prepare('train') if os.path.isfile(ckpt_path): ckpt = torch.load(ckpt_path) discriminator.load_state_dict(ckpt['discriminator']) generator.load_state_dict(ckpt['generator']) discriminator_optimizer.load_state_dict(ckpt['discriminator_optimizer']) generator_optimizer.load_state_dict(ckpt['generator_optimizer']) discriminator_feeder.load_state(ckpt['discriminator_feeder']) generator_feeder.load_state(ckpt['generator_feeder']) loss = 1 while True: #run_generator_epoch(generator, discriminator, generator_feeder, criterion, generator_optimizer, 0.2, 100) loss = run_discriminator_epoch(generator if loss < threshold else None, discriminator, discriminator_feeder, criterion, discriminator_optimizer, steps) if loss < threshold: run_generator_epoch(generator, discriminator, generator_feeder, criterion, generator_optimizer, 0.2, 100) utils.mkdir(config.checkpoint_folder) torch.save({ 'discriminator': discriminator.state_dict(), 'generator': generator.state_dict(), 'discriminator_optimizer': discriminator_optimizer.state_dict(), 'generator_optimizer': generator_optimizer.state_dict(), 'discriminator_feeder': discriminator_feeder.state(), 'generator_feeder': generator_feeder.state() }, ckpt_path) print('MODEL SAVED.')
def train(dataset_path, ratio=(1, 1)): g_loss_sum = d_loss_sum = 0 for i, batch in enumerate(Dataset(dataset_path).batches(64)): real = torch.ones(64, 1).to(device) fake = torch.zeros(64, 1).to(device) # train g z = torch.randn(64, 64, requires_grad=True).to(device) gen_x = g(z) gen_y = d(gen_x) g_loss = loss_func(gen_y, real) g_opt.zero_grad() (g_loss * ratio[0]).backward() g_opt.step() # train d real_x = batch.reshape(64, 1, 64, 64) real_x = torch.tensor(real_x, requires_grad=True).to(device) real_y = d(real_x) fake_y = d(gen_x.detach()) d_loss = (loss_func(real_y, real) + loss_func(fake_y, fake)) / 2 d_opt.zero_grad() (d_loss * ratio[1]).backward() d_opt.step() g_loss_sum += g_loss.item() d_loss_sum += d_loss.item() if (i + 1) % 100 == 0: print(f'g: {g_loss_sum / 100}, d: {d_loss_sum / 100}') g_loss_sum = d_loss_sum = 0
def detect(path,**kwargs): d = Dataset('./data') image = d.read(path) global rgb rgb = d.read(path) pupil = find_pupil(image, 15)[0] e = Ellipse() e._axes = (pupil[2], pupil[2]) e._center = pupil[0:2] #img, points, ellipse = find_iris(image, pupil, **kwargs) x, y = circle_perimeter(pupil[0], pupil[1], pupil[2]) rgb[x,y] = 255 #ex, ey = ellipse.center #major, minor = ellipse.axes #orientation = ellipse.orientation #imshow(rgb) #x, y = ellipse_perimeter(int(ex), int(ey), int(major), int(minor), orientation) #rgb[x,y] = (220, 40, 40) #imshow(rgb) return e, image
import numpy as np import matplotlib.pyplot as plt from skimage import data, filter, color from skimage.transform import hough_circle from skimage.feature import peak_local_max from skimage.draw import circle_perimeter from skimage.util import img_as_ubyte from data import Dataset # Load picture and detect edges d = Dataset('data') image = d.read(d.images[0]) edges = filter.canny(image, sigma=2, low_threshold=10, high_threshold=50) fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(6, 6)) # Detect two radii hough_radii = np.arange(15, 30, 2) hough_res = hough_circle(edges, hough_radii) centers = [] accums = [] radii = [] for radius, h in zip(hough_radii, hough_res): # For each radius, extract two circles peaks = peak_local_max(h, num_peaks=2) centers.extend(peaks) accums.extend(h[peaks[:, 0], peaks[:, 1]])
def __init__(self, config=defaults, name=None): self.config = config self.dataset = Dataset(name=name) self.document = Document() self.texts = [] self.tags = []