def __init__(self, logging_path, C=2.5, gamma=0.001, threshold=1e-12, p_poly=1): self.logging_path = logging_path initialize_logger(logging_path) self.C = C self.gamma = gamma self.data_path = join(getcwd(), 'project2', 'Data') self.process_dataset() self.train_len = len(self.train_x) self.lower_bound = np.zeros((len(self.train_x), 1)) self.upper_bound = np.full((len(self.train_x), 1), C) self.threshold = threshold self.p_poly = p_poly k_poly = self.poly_kernel(self.train_x, self.train_x, self.p_poly) y1 = np.eye(self.train_len) * self.train_y2 y4 = np.eye(self.train_len) * self.train_y4 y6 = np.eye(self.train_len) * self.train_y6 self.P1 = np.dot((np.dot(y1, k_poly)), y1) self.P4 = np.dot((np.dot(y4, k_poly)), y4) self.P6 = np.dot((np.dot(y6, k_poly)), y6) self.h = np.concatenate([self.lower_bound, self.upper_bound], 0) self.e = -1 * np.ones((self.train_len, 1)) self.Gp, self.Gn = np.eye(self.train_len), -np.eye(self.train_len) self.G = np.concatenate([self.Gn, self.Gp], 0) print(f'C = {self.C}, gamma = {self.gamma}')
def __init__(self, path, noise_dim=100, input_dim=(28, 28, 1), optimizer='adam_beta', batch_size=128, visualize=True): initialize_logger() configure_tf() self.path = path self.name = 'gan' self.input_dim = input_dim self.z_dim = noise_dim self.batch_size = batch_size self.train_x, self.train_y = load_mnist() logging.info('Dataset is loaded') self.optimizer = optimizer self.discriminator_lr = 1e-5 self.generator_lr = self.discriminator_lr / 2 self.weight_initialization = RandomNormal(mean=0., stddev=0.02, seed=0) self.epochs = 6000 self.sample_every_n_steps = 200 self.discriminator_losses, self.generator_losses = [], [] self._build_discriminator_network() logging.info('Discriminator model is built') self._build_generator_network() logging.info('Generator model is built') self._build_adversarial_network() logging.info('GAN is built') if visualize: print(self.model.summary()) print(self.generator.summary()) print(self.discriminator.summary())
def main(): """ Parse Lighthouse JSON for failing elements and capture images of those elements """ args = get_args() input_file = args.input_file assets_dir = args.assets_dir output_dir = os.path.dirname(args.input_file) sleep = args.sleep initialize_logger('capture', output_dir) logging.info('Starting image creation...') if args.driver == 'firefox': driver = get_firefox_driver() elif args.driver == 'chrome': driver = get_chrome_driver() else: raise ValueError("Driver must be one of: firefox, chrome") try: with open(input_file, encoding='utf-8') as json_file: data = json.load(json_file) detect_full_html_loaded(driver) capture_screenshot(assets_dir, data['finalUrl'], sleep, driver) for sel in identifier_generator( data, 'color-contrast', 'link-name', 'button-name', 'image-alt', 'input-image-alt', 'label', 'accesskeys', 'frame-title', 'duplicate-id', 'list', 'listitem', 'definition-list', 'dlitem', 'aria-allowed-attr', 'aria-required-attr', 'aria-required-children', 'aria-required-parent', 'aria-roles', 'aria-valid-attr-value', 'aria-valid-attr'): capture_element_pic(input_file, assets_dir, data['finalUrl'], sel, sleep, driver) finally: driver.quit() logging.info('Image creation complete in: "' + assets_dir + '"')
def main(): """ Parse Lighthouse JSON and convert to Markdown """ args = get_args() input_file = args.input_file output_file = args.output_file output_dir = os.path.dirname(args.input_file) paths = list() if args.user_template_path: user_template_path = args.user_template_path paths.append(user_template_path) else: paths.append(template_path) loader = jinja2.FileSystemLoader(paths) env = jinja2.Environment(loader=loader) template = loader.load(env, 'index.md') rendered = template.render({ 'data': preprocess_data(read_input(input_file)), 'generate_img_filename': generate_img_filename, }) write_output(args.output_file, rendered, force_stdout=args.e or not output_file) initialize_logger('markdown', output_dir) logging.info('Markdown convertion complete in: ' + args.output_file)
def main(): input_dim = 6 spatial_dims = [0, 1, 2] args = utils.read_args() experiment_dir = utils.get_experiment_dir(args.name, args.run) utils.initialize_experiment_if_needed(experiment_dir, args.evaluate) # Logger will print to stdout and logfile utils.initialize_logger(experiment_dir) # Optionally restore arguments from previous training # Useful if training is interrupted if not args.evaluate: try: args = utils.load_args(experiment_dir) except: args.best_tpr = 0.0 args.nb_epochs_complete = 0 # Track in case training interrupted utils.save_args(experiment_dir, args) # Save initial args net = utils.create_or_restore_model(experiment_dir, args.nb_hidden, args.nb_layer, input_dim, spatial_dims) if torch.cuda.is_available(): net = net.cuda() logging.warning("Training on GPU") logging.info("GPU type:\n{}".format(torch.cuda.get_device_name(0))) criterion = nn.functional.binary_cross_entropy if not args.evaluate: assert (args.train_file != None) assert (args.val_file != None) train_loader = construct_loader(args.train_file, args.nb_train, args.batch_size, shuffle=True) valid_loader = construct_loader(args.val_file, args.nb_val, args.batch_size) logging.info("Training on {} samples.".format( len(train_loader) * args.batch_size)) logging.info("Validate on {} samples.".format( len(valid_loader) * args.batch_size)) train(net, criterion, args, experiment_dir, train_loader, valid_loader) # Perform evaluation over test set try: net = utils.load_best_model(experiment_dir) logging.warning("\nBest model loaded for evaluation on test set.") except: logging.warning( "\nCould not load best model for test set. Using current.") assert (args.test_file != None) test_loader = construct_loader(args.test_file, args.nb_test, args.batch_size) test_stats = evaluate(net, criterion, experiment_dir, args, test_loader, TEST_NAME)
def main(): """ Pass arguments, check csv validity, and add hash """ args = get_args() input_file = args.input_file output_file = args.output_file initialize_logger('add_hash') try: if check_header(input_file, ): add_hash(input_file, output_file) except Exception as ex: logging.error(ex)
def main(): """ Pass in arguments """ args = get_args() new = args.new old = args.old diff = args.diff initialize_logger('add_hash') try: if check_header(old) and check_header(new): identify_diffs(old, new, diff) except Exception as ex: logging.error(ex)
def __init__(self, logging_path, C=2.5, gamma=0.01): initialize_logger(logging_path) self.C = C self.gamma = gamma self.logging_path = logging_path self.data_path = os.path.join(os.getcwd(), 'project2', 'Data') self.train_x, self.train_y, self.test_x, self.test_y = load_mnist( self.data_path, kind='train') logging.info('Dataset is loaded.') self.bias = 0 self.hessian_mat = np.zeros(shape=(self.train_y.shape[0], self.train_y.shape[0])) self.e = -1. * np.ones(shape=(self.train_y.shape[0], 1)) self.lambda_star = None print(f'C = {self.C}, gamma = {self.gamma}')
def main(): """ Pass arguments, check csv validity, and add hash """ args = get_args() input_file = args.input_file output_file = args.output_file output_dir = os.path.dirname(args.input_file) initialize_logger('add_hash', output_dir) if output_file == "-": # yes_or_no("Are you sure you want to add hashes to the '%s' file? (keeping a backup is recommended)" % (input_file)) output_file = tempfile.gettempdir() + 'tmp.csv' try: if check_header(input_file, ['url'], ['hash']): add_hash(input_file, output_file) os.remove(input_file) os.rename(output_file, input_file) except Exception as ex: logging.error(ex)
def main(): """ Pass in arguments """ args = get_args() new = args.new old = args.old diff = args.diff output_dir = os.path.dirname(args.new) initialize_logger('identify_diffs', output_dir) if check_header(old, ['url', 'hash', 'comply'], []) and check_header( new, ['url', 'hash', 'comply'], ['diff']): try: identify_diffs(old, new, diff) if diff == "-": # yes_or_no("Are you sure you want to add the data to the existing '%s' file? (keeping a backup is recommended)" % (input_file)) os.remove(new) os.rename(diff, new) logging.info("Updated '%s' file with diff" % (new)) else: logging.info("Created new '%s' file with diff" % (new)) except Exception as ex: logging.error(ex)
def main(): cudnn.benchmark = True base_path = '/NSL/data/images/HyperspectralImages/ICVL/' # Dataset val_data = DatasetFromHdf5(base_path + '/testclean_si50_st80.h5') print(len(val_data)) # Data Loader (Input Pipeline) val_loader = DataLoader(dataset=val_data, num_workers=1, batch_size=1, shuffle=False, pin_memory=True) # Model model_path = base_path + 'hscnn_5layer_dim10_93.pkl' result_path = base_path + '/test_results/' var_name = 'rad' save_point = torch.load(model_path) model_param = save_point['state_dict'] model = resblock(conv_relu_res_relu_block, 16, 3, 31) model = nn.DataParallel(model) model.load_state_dict(model_param) model = model.cuda() model.eval() model_path = base_path if not os.path.exists(model_path): os.makedirs(model_path) loss_csv = open(os.path.join(model_path, 'loss.csv'), 'w+') log_dir = os.path.join(model_path, 'train.log') logger = initialize_logger(log_dir) test_loss = validate(val_loader, model, rrmse_loss) print("Test Loss: %.9f " % (test_loss)) # save loss record_loss(loss_csv, test_loss)
# histogram equlization hist_eq=False # pre train pre_train=False #%% ########## log path2logs='./output/logs/' now = datetime.datetime.now() info='log_trainval_' suffix = info + '_' + str(now.strftime("%Y-%m-%d-%H-%M")) # Direct the output to a log file and to screen loggerFileName = os.path.join(path2logs, suffix + '.txt') utils.initialize_logger(loggerFileName) #%% # random data generator datagen = ImageDataGenerator(featurewise_center=False, samplewise_center=False, rescale=1/255., #preprocessing_function=preprocess_input, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=45, width_shift_range=0.1, height_shift_range=0.1,
# Hyper-parameters parser.add_argument("--window", type=int, default=10) # Training vars parser.add_argument("--epochs", type=int, default=10) parser.add_argument("--sample", type=float, default=1e-5) params = vars(parser.parse_args()) return params if __name__ == "__main__": np.random.seed(0) initialize_logger() params = parse_arg() (file_path, train_o_matic_file_path, sew_dir_path, output_file_path, mapping_file, filtered_corpus_output, model_path, tab_file_path) = directory_variables() sentences = parse_datasets(file_path, train_o_matic_file_path, sew_dir_path, output_file_path, mapping_file) w2v_model = build_model(sentences, params["window"], params["sample"]) w2v_model = train_multiple_corpus(sew_dir_path, w2v_model, tab_file_path, params)
desc = """Script performs NBA stats searching.""" parser = argparse.ArgumentParser(description=desc) parser.add_argument("-f", "--file", default=False, required=True, help="Input file with NBA data.") args = parser.parse_args() path_to_file = args.file # Is file with data exist? if not os.path.exists(path_to_file): LOG.error("No file with data.") sys.exit(1) return path_to_file def main(fpath): """test""" LOG.info("Starting script") LOG.debug(fpath) # To fix the break line verification in the end of csv file on start. # Uncomment line below # add_csv_string(fpath) statwork = NBAStats(fpath, LOG) statwork.main() if __name__ == '__main__': LOG = initialize_logger("logs", "nba", "INFO") sys.exit(main(parse_arg()))
def main(): cudnn.benchmark = True # Dataset train_data = DatasetFromHdf5('./Data/train_Material_.h5') print(len(train_data)) val_data = DatasetFromHdf5('./Data/valid_Material_.h5') print(len(val_data)) # Data Loader (Input Pipeline) train_data_loader = DataLoader(dataset=train_data, num_workers=1, batch_size=64, shuffle=True, pin_memory=True) val_loader = DataLoader(dataset=val_data, num_workers=1, batch_size=1, shuffle=False, pin_memory=True) # Model model = resblock(conv_bn_relu_res_block, 10, 25, 25) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) if torch.cuda.is_available(): model.cuda() # Parameters, Loss and Optimizer start_epoch = 0 end_epoch = 100 init_lr = 0.0001 iteration = 0 record_test_loss = 1000 # criterion_RRMSE = torch.nn.L1Loss() criterion_RRMSE = rrmse_loss criterion_Angle = Angle_Loss criterion_MSE = torch.nn.MSELoss() criterion_SSIM = pytorch_msssim.SSIM() # criterion_Div = Divergence_Loss criterion_Div = torch.nn.KLDivLoss() optimizer = torch.optim.Adam(model.parameters(), lr=init_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.01) model_path = './models/' if not os.path.exists(model_path): os.makedirs(model_path) loss_csv = open(os.path.join(model_path, 'loss_material.csv'), 'w+') log_dir = os.path.join(model_path, 'train_material.log') logger = initialize_logger(log_dir) # Resume resume_file = '' if resume_file: if os.path.isfile(resume_file): print("=> loading checkpoint '{}'".format(resume_file)) checkpoint = torch.load(resume_file) start_epoch = checkpoint['epoch'] iteration = checkpoint['iter'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) for epoch in range(start_epoch + 1, end_epoch): start_time = time.time() train_loss, iteration, lr = train(train_data_loader, model, criterion_MSE, criterion_RRMSE, criterion_Angle, criterion_SSIM, criterion_Div, optimizer, iteration, init_lr, end_epoch, epoch) test_loss, loss_angle, loss_reconstruct, loss_SSIM, loss_Div = validate( val_loader, model, criterion_MSE, criterion_RRMSE, criterion_Angle, criterion_SSIM, criterion_Div) # xxx_loss = validate_save(val_loader, model, criterion_MSE, criterion_RRMSE, epoch) save_checkpoint_material(model_path, epoch, iteration, model, optimizer) # print loss end_time = time.time() epoch_time = end_time - start_time print( "Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f , Angle Loss: %.9f, Recon Loss: %.9f, SSIM Loss: %.9f , Div Loss: %.9f" % (epoch, iteration, epoch_time, lr, train_loss, test_loss, loss_angle, loss_reconstruct, loss_SSIM, loss_Div)) # save loss record_loss(loss_csv, epoch, iteration, epoch_time, lr, train_loss, test_loss) logger.info( "Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f, Angle Loss: %.9f, Recon Loss: %.9f, SSIM Loss: %.9f, Div Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, test_loss, loss_angle, loss_reconstruct, loss_SSIM, loss_Div))
def main(): cudnn.benchmark = True # load dataset print("\nloading dataset ...") train_data1 = HyperDatasetTrain1(mode='train') train_data2 = HyperDatasetTrain2(mode='train') train_data3 = HyperDatasetTrain3(mode='train') train_data4 = HyperDatasetTrain4(mode='train') print("Train1:%d,Train2:%d,Train3:%d,Train4:%d," % ( len(train_data1), len(train_data2), len(train_data3), len(train_data4), )) val_data = HyperDatasetValid(mode='valid') print("Validation set samples: ", len(val_data)) # Data Loader (Input Pipeline) train_loader1 = DataLoader(dataset=train_data1, batch_size=opt.batchSize, shuffle=True, num_workers=2, pin_memory=True, drop_last=True) train_loader2 = DataLoader(dataset=train_data2, batch_size=opt.batchSize, shuffle=True, num_workers=2, pin_memory=True, drop_last=True) train_loader3 = DataLoader(dataset=train_data3, batch_size=opt.batchSize, shuffle=True, num_workers=2, pin_memory=True, drop_last=True) train_loader4 = DataLoader(dataset=train_data4, batch_size=opt.batchSize, shuffle=True, num_workers=2, pin_memory=True, drop_last=True) train_loader = [train_loader1, train_loader2, train_loader3, train_loader4] val_loader = DataLoader(dataset=val_data, batch_size=1, shuffle=False, num_workers=2, pin_memory=True) # model print("\nbuilding models_baseline ...") model = AWAN(3, 31, 200, 8) print('Parameters number is ', sum(param.numel() for param in model.parameters())) criterion_train = LossTrainCSS() criterion_valid = Loss_valid() if torch.cuda.device_count() > 1: model = nn.DataParallel(model) # batchsize integer times if torch.cuda.is_available(): model.cuda() criterion_train.cuda() criterion_valid.cuda() # Parameters, Loss and Optimizer start_epoch = 0 iteration = 0 record_val_loss = 1000 optimizer = optim.Adam(model.parameters(), lr=opt.init_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0) # visualzation if not os.path.exists(opt.outf): os.makedirs(opt.outf) loss_csv = open(os.path.join(opt.outf, 'loss.csv'), 'a+') log_dir = os.path.join(opt.outf, 'train.log') logger = initialize_logger(log_dir) # Resume # resume_file = opt.outf + '/net_10epoch.pth' resume_file = '' if resume_file: if os.path.isfile(resume_file): print("=> loading checkpoint '{}'".format(resume_file)) checkpoint = torch.load(resume_file) start_epoch = checkpoint['epoch'] iteration = checkpoint['iter'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) # start epoch for epoch in range(start_epoch + 1, opt.end_epoch): start_time = time.time() train_loss, iteration, lr = train(train_loader, model, criterion_train, optimizer, epoch, iteration, opt.init_lr, opt.decay_power, opt.trade_off) val_loss = validate(val_loader, model, criterion_valid) # Save model if torch.abs(val_loss - record_val_loss) < 0.0001 or val_loss < record_val_loss: save_checkpoint(opt.outf, epoch, iteration, model, optimizer) if val_loss < record_val_loss: record_val_loss = val_loss # print loss end_time = time.time() epoch_time = end_time - start_time print( "Epoch [%02d], Iter[%06d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, val_loss)) # save loss record_loss(loss_csv, epoch, iteration, epoch_time, lr, train_loss, val_loss) logger.info( "Epoch [%02d], Iter[%06d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, val_loss))
import click import logging from queue import Queue from threading import Thread import utils from exporters import Exporter from collector import Collector # Python 3 is required! if sys.version_info[0] < 3: sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n") sys.exit(1) # Logging utils.initialize_logger("log/") # Load config right at the start config = utils.load_config() # Create the core CLI launcher @click.group() def cli(): """Welcome to AtmoPi, your very own DIY Weather station""" pass ####################################### ### Verify command ### #######################################
def main(): #https://drive.google.com/file/d/1QxQxf2dzfSbvCgWlI9VuxyBgfmQyCmfE/view?usp=sharing - train data #https://drive.google.com/file/d/11INkjd_ajT-RSCSFqfB7reLI6_m1jCAC/view?usp=sharing - val data #https://drive.google.com/file/d/1m0EZaRjla2o_eL3hOd7UMkSwoME5mF4A/view?usp=sharing - extra val data cudnn.benchmark = True # train_data = DatasetFromHdf5('C:/Users/alawy/Desktop/Training/Training-shadesofgrey/train_tbands.h5') train_data = DatasetFromHdf5('/storage/train_cropped14.h5') print(len(train_data)) val_data_extra = DatasetFromHdf5('/storage/valid_extra99.h5') val_data = DatasetFromHdf5('/storage/valid_cropped89.h5') new_val=[] new_val.append(val_data) new_val.append(val_data_extra) print(len(new_val)) print('con') val_new = data.ConcatDataset(new_val) print(len(val_new)) # Data Loader (Input Pipeline) train_data_loader = DataLoader(dataset=train_data, num_workers=4, batch_size=512, shuffle=True, pin_memory=True) val_loader = DataLoader(dataset=val_new, num_workers=1, batch_size=1, shuffle=False, pin_memory=True) # Dataset # torch.set_num_threads(12) # Model model = resblock(conv_relu_res_relu_block, 16, 3, 25) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) if torch.cuda.is_available(): model = model.to('cuda') # Parameters, Loss and Optimizer start_epoch = 0 end_epoch = 1000 init_lr = 0.0002 iteration = 0 record_test_loss = 1000 criterion = rrmse_loss #optimizer=torch.optim.AdamW(model.parameters(), lr=init_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0) optimizer=torch.optim.Adam(model.parameters(), lr=init_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.01) # model_path = '/storage/models-crop/' model_path = './models-crop/' if not os.path.exists(model_path): os.makedirs(model_path) loss_csv = open(os.path.join(model_path,'loss.csv'), 'w+') log_dir = os.path.join(model_path,'train.log') logger = initialize_logger(log_dir) # Resume resume_file = '' #resume_file = '/storage/notebooks/r9h1kyhq8oth90j/models/hscnn_5layer_dim10_69.pkl' #resume_file = '/storage/notebooks/r9h1kyhq8oth90j/models-crop/hscnn_5layer_dim10_95.pkl' if resume_file: if os.path.isfile(resume_file): print("=> loading checkpoint '{}'".format(resume_file)) checkpoint = torch.load(resume_file) start_epoch = checkpoint['epoch'] iteration = checkpoint['iter'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) for epoch in range(start_epoch+1, end_epoch): start_time = time.time() train_loss, iteration, lr = train(train_data_loader, model, criterion, optimizer, iteration, init_lr, end_epoch) test_loss = validate(val_loader, model, criterion) # Save model if test_loss < record_test_loss: record_test_loss = test_loss save_checkpoint(model_path, epoch, iteration, model, optimizer) else: save_checkpoint(model_path, epoch, iteration, model, optimizer) # print loss end_time = time.time() epoch_time = end_time - start_time print ("Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " %(epoch, iteration, epoch_time, lr, train_loss, test_loss)) # save loss record_loss(loss_csv,epoch, iteration, epoch_time, lr, train_loss, test_loss) logger.info("Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " %(epoch, iteration, epoch_time, lr, train_loss, test_loss)) gc.collect()
def main(**kwargs): # 1. Parse command line arguments. opt._parse(kwargs) # 2. Visdom # vis = Visualizer(env=opt.env) # 3. GPU settings # n_gpu = utils.set_gpu('0,1') # 4. Configure model logging.info('==> Traing model for clothing type: {}'.format(opt.category)) cudnn.benchmark = True net = getattr(models, opt.model)(opt) # 5. Initialize logger cur_time = time.strftime('%Y-%m-%dT%H:%M:%S', timm.localtime()) initialize_logger(f'{opt.category}_{opt.model}_{cur_time}') # 6. Initialize checkpoints directory lr = opt.lr start_epoch = 1 best_val_loss = float('inf') if opt.load_checkpoint_path: logging.info('==> Resuming from checkpoint...') checkpoint = torch.load(opt.load_checkpoint_path) start_epoch = checkpoint['epoch'] + 1 lr = checkpoint['lr'] best_val_loss = checkpoint['best_val_loss'] net.load_state_dict(checkpoint['state_dict']) # 7. Data setup train_dataset = FashionAIKeypoints(opt, phase='train') logging.info('Train sample number: {}'.format(len(train_dataset))) train_loader = DataLoader(train_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, collate_fn=train_dataset.collate_fn, pin_memory=True) val_dataset = FashionAIKeypoints(opt, phase='val') logging.info('Val sample number: {}'.format(len(val_dataset))) val_loader = DataLoader(val_dataset, batch_size=opt.batch_size, shuffle=False, num_workers=opt.num_workers, collate_fn=val_dataset.collate_fn, pin_memory=True) net = net.cuda() # net = DataParallel(net) loss = CPNLoss() loss = loss.cuda() # 8. Loss, optimizer and LR scheduler optimizer = torch.optim.SGD(net.parameters(), lr, momentum=0.9, weight_decay=1e-4) lrs = LRScheduler(lr, patience=3, factor=0.1, min_lr=0.01 * lr, best_loss=best_val_loss) # 9. Training loop for epoch in range(start_epoch, opt.max_epochs + 1): # Training logging.info("Start training loop...") train_metrics, train_time = train(train_loader, net, loss, optimizer, lr) # Validating logging.info("Start validating loop...") with torch.no_grad(): val_metrics, val_time = validate(val_loader, net, loss) log_model(epoch, lr, train_metrics, train_time, val_metrics, val_time) val_loss = np.mean(val_metrics[:, 0]) lr = lrs.update_by_rule(val_loss) # Save checkpoints if val_loss < best_val_loss or epoch % 10 == 0 or lr is None: if val_loss < best_val_loss: best_val_loss = val_loss state_dict = net.module.state_dict() for key in state_dict.keys(): state_dict[key] = state_dict[key].cpu() torch.save( { 'epoch': epoch, 'save_dir': opt.checkpoint_path, 'state_dict': state_dict, 'lr': lr, 'best_val_loss': best_val_loss }, opt.checkpoint_path / 'kpt_{}_{:03d}.ckpt'.format(opt.category, epoch)) if lr is None: logging.info('Training is early-stopped') break
def main(): cudnn.benchmark = True base_path = '/NSL/data/images/HyperspectralImages/ICVL/' # Dataset train_data = DatasetFromHdf5(base_path + '/train.h5') print(len(train_data)) val_data = DatasetFromHdf5(base_path + '/valid.h5') print(len(val_data)) # Data Loader (Input Pipeline) train_data_loader = DataLoader(dataset=train_data, num_workers=1, batch_size=64, shuffle=True, pin_memory=True) val_loader = DataLoader(dataset=val_data, num_workers=1, batch_size=1, shuffle=False, pin_memory=True) # Model model = resblock(conv_batch_relu_res_block, 16, 3, 31) if torch.cuda.device_count() > 1: model = nn.DataParallel(model) if torch.cuda.is_available(): model.cuda() # Parameters, Loss and Optimizer start_epoch = 0 end_epoch = 1000 init_lr = 0.0002 iteration = 0 record_test_loss = 1000 criterion = rrmse_loss optimizer = torch.optim.Adam(model.parameters(), lr=init_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0) model_path = base_path + '/models/' if not os.path.exists(model_path): os.makedirs(model_path) loss_csv = open(os.path.join(model_path, 'loss.csv'), 'w+') log_dir = os.path.join(model_path, 'train.log') logger = initialize_logger(log_dir) # Resume resume_file = '' if resume_file: if os.path.isfile(resume_file): print("=> loading checkpoint '{}'".format(resume_file)) checkpoint = torch.load(resume_file) start_epoch = checkpoint['epoch'] iteration = checkpoint['iter'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) for epoch in range(start_epoch + 1, end_epoch): print("epoch [%d]" % (epoch)) start_time = time.time() train_loss, iteration, lr = train(train_data_loader, model, criterion, optimizer, iteration, init_lr, end_epoch) print("train done! epoch [%d]" % (epoch)) test_loss = validate(val_loader, model, criterion) print("test done! epoch [%d]" % (epoch)) # Save model if test_loss < record_test_loss: record_test_loss = test_loss save_checkpoint(model_path, epoch, iteration, model, optimizer) # print loss end_time = time.time() epoch_time = end_time - start_time print( "Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, test_loss)) # save loss record_loss(loss_csv, epoch, iteration, epoch_time, lr, train_loss, test_loss) logger.info( "Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, test_loss))
"""Web / UI Server for Soundwave""" import json import os from datetime import datetime import cherrypy import cherrypy.wsgiserver import requests from flask import Flask, request, render_template, g, send_from_directory from config import SOUNDWAVE_API, SOUNDWAVE_HOST, SOUNDWAVE_PORT from utils import get_soundwave_url, json_loads_byteified, \ initialize_logger, get_table_data, query_soundwave, get_reservation_rows # Initializations logger = initialize_logger() app = Flask(__name__, template_folder='templates', static_folder='static') @app.before_request def log_request(): """Request logger.""" if "static" not in request.path: logger.info("\npath: %s \nhttp-method: %s \nheaders: %s", str(request.path), str(request.method), str(request.headers)) g.start = datetime.now()
def main(): cudnn.benchmark = True train_data = Dataset_cave_train('./data/train') print('number of train data: ', len(train_data)) val_data = Dataset_cave_val('./data/test') print('number of validate data: ', len(val_data)) # Model model = Net(HSI_num_residuals=args.HSI_num_residuals, RGB_num_residuals=args.RGB_num_residuals) # multi-GPU setup device = torch.device("cuda:0, 1" if torch.cuda.is_available() else "cpu") model = nn.DataParallel(model) model = model.to(device=device, dtype=torch.float) # float32 model.apply(weights_init_kaiming) # Parameters, Loss and Optimizer start_epoch = 0 end_epoch = 501 init_lr = 0.0002 iteration = 0 criterion = MyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=init_lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0) # scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=False, # threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08) model_path = args.model_path if not os.path.exists(model_path): os.makedirs(model_path) loss_csv = open(os.path.join(model_path, 'loss.csv'), 'w+') log_dir = os.path.join(model_path, 'train.log') logger = initialize_logger(log_dir) # Resume resume_file = '' if resume_file: if os.path.isfile(resume_file): print("=> loading checkpoint '{}'".format(resume_file)) checkpoint = torch.load(resume_file) # start_epoch = checkpoint['epoch'] # iteration = checkpoint['iter'] model.load_state_dict(checkpoint['state_dict']) # optimizer.load_state_dict(checkpoint['optimizer']) for epoch in range(start_epoch + 1, end_epoch): train_data = Dataset_cave_train('./data/train') train_data_loader = DataLoader( dataset=train_data, num_workers=8, batch_size=16, shuffle=True, pin_memory=True, ) val_data = Dataset_cave_val('./data/test') val_data_loader = DataLoader(dataset=val_data, num_workers=8, batch_size=16, shuffle=False, pin_memory=True) start_time = time.time() train_loss, iteration = train(train_data_loader, model, criterion, optimizer, iteration, device) val_loss = validate(val_data_loader, model, criterion, device) # Save model if epoch % 100 == 0: save_checkpoint(model_path, epoch, iteration, model, optimizer) # # Update learning rate for param_group in optimizer.param_groups: lr = param_group['lr'] # scheduler.step(val_loss) # print loss end_time = time.time() epoch_time = end_time - start_time print( "Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, val_loss)) # save loss record_loss(loss_csv, epoch, iteration, epoch_time, lr, train_loss, val_loss) # 调用record_方法:将epoch等6个指标写到csv文件中 logger.info( "Epoch [%d], Iter[%d], Time:%.9f, learning rate : %.9f, Train Loss: %.9f Test Loss: %.9f " % (epoch, iteration, epoch_time, lr, train_loss, val_loss))