def post_process(Arguments): ''' This is simply a distibutor for all functions involved in postprocessing POSE results from previous calculations, which is required to make sense of most POSE output ''' if Arguments.PostProcess == "Validate": validate(Arguments) if Arguments.PostProcess == "Predict": predict(Arguments) return
def Kvalidate(): train_fld = os.path.join(os.getcwd(),'bbc') topics = set() for topic in os.listdir(train_fld): topics.add(topic) all_docs = list() for topic in topics: topic_dir = os.path.join(train_fld,topic) for file in os.listdir(topic_dir): file_path = os.path.join(topic_dir,file) document = open(file_path,encoding='latin-1').readlines() all_docs.append(Document(topic,document[0],document[1:])) #80% file used for training validate().validate(all_docs) #60% file used for testing validate().validate(all_docs,k=0.4) #50% file used for testing validate().validate(all_docs,k=0.5) #40% file used for testing validate().validate(all_docs,k=0.6)
def post_process(Arguments): ''' This is simply a distibutor for all functions involved in postprocessing POSE results from previous calculations, which is required to make sense of most POSE output ''' if Arguments.PostProcess == "Validate": validate(Arguments) if Arguments.PostProcess == "Predict": predict(Arguments) return
def run_batch(cfg): directory = cfg.get('data', 'response_file') get_capabilities_docs = cfg.get('data', 'get_capabilities') output_dir = cfg.get('data', 'output_dir') max_features = cfg.get('other', 'max_features_for_validation') for file_path in glob.glob(directory + '*.json'): file = Path(file_path).stem logging.info("file {} begins".format(file)) config = configparser.ConfigParser() config.add_section('data') config.add_section('other') config.set('data', 'response_file', file_path) config.set('data', 'get_capabilities', get_capabilities_docs + file.split("_")[0] + ".xml") config.set('data', 'raster_output_path', output_dir + file + ".tif") config.set('data', 'output_dir', output_dir) config.set('data', 'raster_output_path', output_dir + file + ".tif") config.set('data', 'binary_raster_output_path', output_dir + "bin_" + file + ".tif") config.set('data', 'validation_raster_output_path', output_dir + "val_" + file + ".tif") config.set('other', 'max_features_for_validation', max_features) process = Process(config) process.run_algorithm() # validation of the result. validate(process.url, process.layer_name, process.input_data.crs.name, process.layer_bbox, process.bin_raster_path, process.val_raster_output_path, process.service_type, process.service_version, process.max_features_for_validation) logging.info("file {} done \n \n".format(file))
def run_batch(cfg): """For each file in folder given in configuration, generates a configuration file, creates an instance of the process.py class, runs algorithm which produces results and validates these results using the validate module. Parameters ---------- cfg : ConfigParser object """ directory = cfg.get('data', 'response_file') get_capabilities_docs = cfg.get('data', 'get_capabilities') output_dir = cfg.get('data', 'output_dir') input_first_axis_direction = cfg.get('input', 'first_axis_direction') resolution = cfg.get('result', 'resolution') output_crs = cfg.get('result', 'output_crs') output_first_axis_direction = cfg.get('result', 'first_axis_direction') max_raster_size = cfg.get('other', 'max_raster_size') max_features = cfg.get('other', 'max_features_for_validation') signal.signal(signal.SIGALRM, handler) for file_path in glob.glob(directory + '*.json'): file = Path(file_path).stem logging.info("file {} begins".format(file)) config = configparser.ConfigParser() config.add_section('data') config.add_section('other') config.add_section('result') config.add_section('input') config.set('data', 'response_file', file_path) config.set('data', 'get_capabilities', get_capabilities_docs + file.split("_")[0] + ".xml") config.set('data', 'raster_output_path', output_dir + file + ".tif") config.set('data', 'output_dir', output_dir) config.set('data', 'raster_output_path', output_dir + file + ".tif") config.set('data', 'binary_raster_output_path', output_dir + "bin_" + file + ".tif") config.set('data', 'validation_raster_output_path', output_dir + "val_" + file + ".tif") config.set('input', 'first_axis_direction', input_first_axis_direction) config.set('result', 'resolution', resolution) config.set('result', 'output_crs', output_crs) config.set('result', 'first_axis_direction', output_first_axis_direction) config.set('other', 'max_raster_size', max_raster_size) config.set('other', 'max_features_for_validation', max_features) try: process = Process(config) process.run_algorithm() signal.alarm(180) try: # validation of the result. validate(process.url, process.layer_name, process.crs.crs_code, process.layer_bbox, process.bin_raster_path, process.val_raster_output_path, process.service_type, process.service_version, process.max_features_for_validation, process.flip_features, process.data_bounds, process.service) except Exception as e: print(e) signal.alarm(0) logging.info("File '{}' done. \n \n".format(file)) except Exception as e: logging.info("File '{}' failed.\nError message: '{}' \n \n".format( file, e))
import torch import torchvision from torchvision import transforms import Model from Validate import validate device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print('Running on', device) teacher = Model.ResNet18() teacher.load_state_dict(torch.load('Teacher.pt')) val_transformer = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform = transforms.Compose([transforms.ToTensor()]) test_set = torchvision.datasets.CIFAR10(root="~/data", train=False, download=False, transform=val_transformer) test_loader = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False, num_workers=4) loss, acc = validate(teacher, device, test_loader) print(acc)
def run_algorithm(self): a = Algorithm(self.raster, self.input_data, self.service_type, self.result) return a.solve(self.output_raster_path, self.bin_raster_path) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("path_to_config", help="Path to the file containing configuration.") args = parser.parse_args() config = configparser.ConfigParser() data = config.read(args.path_to_config) if len(data) == 0: raise Exception("Configuration file not found.") process = Process(config) process.run_algorithm() # validation of the result. validate(process.url, process.layer_name, process.input_data.crs.name, process.layer_bbox, process.bin_raster_path, process.val_raster_output_path, process.service_type, process.service_version, process.max_features_for_validation)
solve(self.features, self.raster, self.bin_raster_path) self.data_bounds = convert_to_vector_format(self.crs, self.output_dir, self.resolution, self.bin_raster_path, self.output_crs, self.url, self.layer_name) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("path_to_config", help="Path to the file containing configuration.") args = parser.parse_args() config = configparser.ConfigParser() data = config.read(args.path_to_config) if len(data) == 0: raise Exception("Configuration file not found.") process = Process(config) process.run_algorithm() # validation of the result. validate(process.url, process.layer_name, process.crs.crs_code, process.layer_bbox, process.bin_raster_path, process.val_raster_output_path, process.service_type, process.service_version, process.max_features_for_validation, process.flip_features, process.data_bounds, process.service)
test_set, batch_size=128, shuffle=False, num_workers=4 ) node1 = Node(Model.LeNet5(), train_loader) optimizer = torch.optim.SGD( node1.model.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4 ) epochs = trange(60) loss = 0 acc = 0 acc_best = 0 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1) for epoch in epochs: description = "Total Process (the {:d}-epoch): Loss = {:.4f} Accuracy = {:.2f}%".format( epoch + 1, loss, acc ) epochs.set_description(description) train(node1, device, optimizer) loss, acc = validate(node1.model, device, test_loader) scheduler.step() msg = "Validation [{:d}-epoch]: val_Loss = {:.4f} val_Accuracy = {:.2f}%\n".format( epoch + 1, loss, acc ) epochs.write(msg) if acc > acc_best: acc_best = acc epochs.write("A Better Accuracy: {:.2f}%! Model Saved!\n".format(acc_best)) torch.save(node1.model.state_dict(), "Node1_LeNet5.pt") print("Finished! The Best Accuracy: {:.2f}%! Model Saved!\n".format(acc_best))