def post_process(Arguments):
'''
This is simply a distibutor for all functions involved in
postprocessing POSE results from previous calculations, which
is required to make sense of most POSE output
'''
if Arguments.PostProcess == "Validate":
validate(Arguments)
if Arguments.PostProcess == "Predict":
predict(Arguments)
return
def Kvalidate():
train_fld = os.path.join(os.getcwd(),'bbc')
topics = set()
for topic in os.listdir(train_fld):
topics.add(topic)
all_docs = list()
for topic in topics:
topic_dir = os.path.join(train_fld,topic)
for file in os.listdir(topic_dir):
file_path = os.path.join(topic_dir,file)
document = open(file_path,encoding='latin-1').readlines()
all_docs.append(Document(topic,document[0],document[1:]))
#80% file used for training
validate().validate(all_docs)
#60% file used for testing
validate().validate(all_docs,k=0.4)
#50% file used for testing
validate().validate(all_docs,k=0.5)
#40% file used for testing
validate().validate(all_docs,k=0.6)
def post_process(Arguments):
'''
This is simply a distibutor for all functions involved in
postprocessing POSE results from previous calculations, which
is required to make sense of most POSE output
'''
if Arguments.PostProcess == "Validate":
validate(Arguments)
if Arguments.PostProcess == "Predict":
predict(Arguments)
return
def run_batch(cfg):
directory = cfg.get('data', 'response_file')
get_capabilities_docs = cfg.get('data', 'get_capabilities')
output_dir = cfg.get('data', 'output_dir')
max_features = cfg.get('other', 'max_features_for_validation')
for file_path in glob.glob(directory + '*.json'):
file = Path(file_path).stem
logging.info("file {} begins".format(file))
config = configparser.ConfigParser()
config.add_section('data')
config.add_section('other')
config.set('data', 'response_file', file_path)
config.set('data', 'get_capabilities',
get_capabilities_docs + file.split("_")[0] + ".xml")
config.set('data', 'raster_output_path', output_dir + file + ".tif")
config.set('data', 'output_dir', output_dir)
config.set('data', 'raster_output_path', output_dir + file + ".tif")
config.set('data', 'binary_raster_output_path',
output_dir + "bin_" + file + ".tif")
config.set('data', 'validation_raster_output_path',
output_dir + "val_" + file + ".tif")
config.set('other', 'max_features_for_validation', max_features)
process = Process(config)
process.run_algorithm()
# validation of the result.
validate(process.url, process.layer_name, process.input_data.crs.name,
process.layer_bbox, process.bin_raster_path,
process.val_raster_output_path, process.service_type,
process.service_version, process.max_features_for_validation)
logging.info("file {} done \n \n".format(file))
def run_batch(cfg):
"""For each file in folder given in configuration,
generates a configuration file, creates an instance
of the process.py class, runs algorithm which produces
results and validates these results using the validate
module.
Parameters
----------
cfg : ConfigParser object
"""
directory = cfg.get('data', 'response_file')
get_capabilities_docs = cfg.get('data', 'get_capabilities')
output_dir = cfg.get('data', 'output_dir')
input_first_axis_direction = cfg.get('input', 'first_axis_direction')
resolution = cfg.get('result', 'resolution')
output_crs = cfg.get('result', 'output_crs')
output_first_axis_direction = cfg.get('result', 'first_axis_direction')
max_raster_size = cfg.get('other', 'max_raster_size')
max_features = cfg.get('other', 'max_features_for_validation')
signal.signal(signal.SIGALRM, handler)
for file_path in glob.glob(directory + '*.json'):
file = Path(file_path).stem
logging.info("file {} begins".format(file))
config = configparser.ConfigParser()
config.add_section('data')
config.add_section('other')
config.add_section('result')
config.add_section('input')
config.set('data', 'response_file', file_path)
config.set('data', 'get_capabilities',
get_capabilities_docs + file.split("_")[0] + ".xml")
config.set('data', 'raster_output_path', output_dir + file + ".tif")
config.set('data', 'output_dir', output_dir)
config.set('data', 'raster_output_path', output_dir + file + ".tif")
config.set('data', 'binary_raster_output_path',
output_dir + "bin_" + file + ".tif")
config.set('data', 'validation_raster_output_path',
output_dir + "val_" + file + ".tif")
config.set('input', 'first_axis_direction', input_first_axis_direction)
config.set('result', 'resolution', resolution)
config.set('result', 'output_crs', output_crs)
config.set('result', 'first_axis_direction',
output_first_axis_direction)
config.set('other', 'max_raster_size', max_raster_size)
config.set('other', 'max_features_for_validation', max_features)
try:
process = Process(config)
process.run_algorithm()
signal.alarm(180)
try:
# validation of the result.
validate(process.url, process.layer_name, process.crs.crs_code,
process.layer_bbox, process.bin_raster_path,
process.val_raster_output_path, process.service_type,
process.service_version,
process.max_features_for_validation,
process.flip_features, process.data_bounds,
process.service)
except Exception as e:
print(e)
signal.alarm(0)
logging.info("File '{}' done. \n \n".format(file))
except Exception as e:
logging.info("File '{}' failed.\nError message: '{}' \n \n".format(
file, e))
import torch
import torchvision
from torchvision import transforms
import Model
from Validate import validate
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print('Running on', device)
teacher = Model.ResNet18()
teacher.load_state_dict(torch.load('Teacher.pt'))
val_transformer = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
transform = transforms.Compose([transforms.ToTensor()])
test_set = torchvision.datasets.CIFAR10(root="~/data", train=False, download=False, transform=val_transformer)
test_loader = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False, num_workers=4)
loss, acc = validate(teacher, device, test_loader)
print(acc)
def run_algorithm(self):
a = Algorithm(self.raster, self.input_data, self.service_type,
self.result)
return a.solve(self.output_raster_path, self.bin_raster_path)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("path_to_config",
help="Path to the file containing configuration.")
args = parser.parse_args()
config = configparser.ConfigParser()
data = config.read(args.path_to_config)
if len(data) == 0:
raise Exception("Configuration file not found.")
process = Process(config)
process.run_algorithm()
# validation of the result.
validate(process.url, process.layer_name, process.input_data.crs.name,
process.layer_bbox, process.bin_raster_path,
process.val_raster_output_path, process.service_type,
process.service_version, process.max_features_for_validation)
solve(self.features, self.raster, self.bin_raster_path)
self.data_bounds = convert_to_vector_format(self.crs, self.output_dir,
self.resolution,
self.bin_raster_path,
self.output_crs, self.url,
self.layer_name)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("path_to_config",
help="Path to the file containing configuration.")
args = parser.parse_args()
config = configparser.ConfigParser()
data = config.read(args.path_to_config)
if len(data) == 0:
raise Exception("Configuration file not found.")
process = Process(config)
process.run_algorithm()
# validation of the result.
validate(process.url, process.layer_name, process.crs.crs_code,
process.layer_bbox, process.bin_raster_path,
process.val_raster_output_path, process.service_type,
process.service_version, process.max_features_for_validation,
process.flip_features, process.data_bounds, process.service)
test_set, batch_size=128, shuffle=False, num_workers=4
)
node1 = Node(Model.LeNet5(), train_loader)
optimizer = torch.optim.SGD(
node1.model.parameters(), lr=1e-3, momentum=0.9, weight_decay=5e-4
)
epochs = trange(60)
loss = 0
acc = 0
acc_best = 0
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)
for epoch in epochs:
description = "Total Process (the {:d}-epoch): Loss = {:.4f} Accuracy = {:.2f}%".format(
epoch + 1, loss, acc
)
epochs.set_description(description)
train(node1, device, optimizer)
loss, acc = validate(node1.model, device, test_loader)
scheduler.step()
msg = "Validation [{:d}-epoch]: val_Loss = {:.4f} val_Accuracy = {:.2f}%\n".format(
epoch + 1, loss, acc
)
epochs.write(msg)
if acc > acc_best:
acc_best = acc
epochs.write("A Better Accuracy: {:.2f}%! Model Saved!\n".format(acc_best))
torch.save(node1.model.state_dict(), "Node1_LeNet5.pt")
print("Finished! The Best Accuracy: {:.2f}%! Model Saved!\n".format(acc_best))