def score(
self,
bundle,
bundle_size=32,
verbose=True,
):
loader = Loader(bundle)
data = loader.data_test(bundle_size)
averages_mse = 0.0
averages_psnr = 0.0
for i in range(0, len(data[0])):
image = numpy.array(data[0][i])
image *= 255
image = Image.fromarray(numpy.uint8(image))
image = image.convert('RGB')
image = image.resize((
int(image.size[0] * self._scale_factor),
int(image.size[1] * self._scale_factor),
), self._scale_type)
image = numpy.array(image)
image = image.astype('float32')
image /= 255
averages_mse += mse_np(data[1][i], image)
averages_psnr += psnr_np(data[1][i], image)
return [
averages_mse / len(data[0]),
averages_psnr / len(data[0]),
]
def test(
self,
bundle,
bundle_size=32,
verbose=True,
):
loader = Loader(bundle)
data = loader.data_test(bundle_size)
result = []
for image in data[0]:
image = numpy.array(image)
image *= 255
image = Image.fromarray(numpy.uint8(image))
image = image.convert('RGB')
image = image.resize((
int(image.size[0] * self._scale_factor),
int(image.size[1] * self._scale_factor),
), self._scale_type)
image = numpy.array(image)
image = image.astype('float32')
image /= 255
result.append(image)
keeper = Keeper(bundle, self.name())
for i in range(0, len(data[0])):
keeper.save(f'{bundle}-{i+1}', data[1][i], data[0][i], result[i])
def main(_):
loader = Loader(FLAGS.data_dir, FLAGS.data, FLAGS.batch_size)
print("# of data: {}".format(loader.data_num))
with tf.Session() as sess:
lsgan = LSGAN([FLAGS.batch_size, 112, 112, 3])
sess.run(tf.global_variables_initializer())
for epoch in range(10000):
loader.reset()
for step in range(int(loader.batch_num / FLAGS.d)):
if (step == 0 and epoch % 1 == 100):
utils.visualize(sess.run(lsgan.gen_img), epoch)
for _ in range(FLAGS.d):
batch = np.asarray(loader.next_batch(), dtype=np.float32)
batch = (batch - 127.5) / 127.5
#print("{}".format(batch.shape))
feed = {lsgan.X: batch}
_ = sess.run(lsgan.d_train_op, feed_dict=feed)
#utils.visualize(batch, (epoch+1)*100)
#cv2.namedWindow("window")
#cv2.imshow("window", cv2.cvtColor(batch[0], cv2.COLOR_RGB2BGR))
#cv2.waitKey(0)
#cv2.destroyAllWindows()
_ = sess.run(lsgan.g_train_op)
def train(self, config=None):
#mnist = input_data.read_data_sets("/tmp/tensorflow/mnist/input_dat", one_hot=True)
loader = Loader(config.data_dir, config.data, config.batch_size)
loaded = False
if not config.reset:
loaded, global_step = self.restore(config.checkpoint_dir)
if not loaded:
tf.global_variables_initializer().run()
global_step = 0
d_losses = []
g_losses = []
steps = []
gif = []
for epoch in range(config.epoch):
loader.reset()
#for idx in range(config.step):
for idx in range(loader.batch_num):
#batch_X, _ = mnist.train.next_batch(config.batch_size)
#batch_X = batch_X.reshape([-1]+self.in_dim)
batch_X = np.asarray(loader.next_batch(), dtype=np.float32)
#batch_X = (batch_X*255.-127.5)/127.5
batch_X = (batch_X - 127.5) / 127.5
batch_z = np.random.uniform(-1, 1,
[config.batch_size, self.z_dim])
_, d_loss = self.sess.run([self.d_train_op, self.d_loss],
feed_dict={
self.X: batch_X,
self.z: batch_z
})
_, g_loss = self.sess.run([self.g_train_op, self.g_loss],
feed_dict={self.z: batch_z})
d_losses.append(d_loss)
g_losses.append(g_loss)
steps.append(global_step)
global_step += 1
print(" [Epoch {}] d_loss:{}, g_loss:{}".format(
epoch, d_loss, g_loss))
batch_z = np.random.uniform(-1, 1, [config.batch_size, self.z_dim])
imgs = self.sess.run(self.sampler, feed_dict={self.z: batch_z})
gif.append(visualize(imgs, epoch, config.data))
self.save("{}_{}".format(config.checkpoint_dir, config.data),
global_step,
model_name="dcgan")
plot({
'd_loss': d_losses,
'g_loss': g_losses
},
steps,
title="DCGAN loss ({})".format(config.data),
x_label="Step",
y_label="Loss")
save_gif(gif, "gen_img_{}".format(config.data))
def setUp(self):
loader = Loader(True)
distances, costs = loader.load_dataset_a()
self.problem = TSPProblem(distances, costs)
self.problem.generation_limit = 10
self.problem.population_size = 100
population = TSPPopulation(self.problem)
self.pool = population.children
self.pool.append(deepcopy(self.pool[0]))
def score(
self,
bundle,
bundle_size=32,
verbose=True,
):
loader = Loader(bundle)
return self.model().evaluate_generator(
loader.gen_data_test(bundle_size),
steps=1,
verbose=verbose,
)
def setUp(self):
print("Setting up")
loader = Loader(True)
distances, costs, front = loader.load_dataset_b()
self.problem = TSPProblem(distances, costs)
self.problem.generation_limit = 10
self.problem.population_size = 1000
population = TSPPopulation(self.problem)
population.evaluate_fitnesses()
self.pool = population.children
self.pool.append(deepcopy(self.pool[0]))
self.pool.append(deepcopy(self.pool[0]))
self.pool.append(deepcopy(self.pool[0]))
def score(
self,
bundle,
bundle_size=32,
verbose=True,
scale_factor=4,
):
loader = Loader(bundle, scale_factor, Image.BICUBIC)
return self.model().evaluate_generator(
loader.gen_data_test(bundle_size),
steps=1,
verbose=verbose,
)
def label(self, report_pairs, reports_path, extract_impression):
"""Label the provided report(s)."""
# Load reports in place.
loader = Loader(report_pairs, reports_path, extract_impression)
loader.load()
# Extract observation mentions in place.
self.extractor.extract(loader.collection)
# Classify mentions in place.
self.classifier.classify(loader.collection)
# output mentions/categories/negation/attributes
attributes = self.aggregator.getAttributeOutput(loader.collection)
# # Aggregate mentions to obtain one set of labels for each report.
# labels = aggregator.aggregate(loader.collection)
return loader.reports, attributes
def test_clean_json_linkedIssues_ConnectionError_real_data(self) -> None:
with patch('base_class.requests') as mock_requests:
from test_extraction_V2 import TestExtractor, NAME
from deployment_log_UAT import DeployExtractor
from barros_request import BarrosExtractor
dic = dict()
dic['T2L-249'] = grab_tickets_json['T2L-249']
mock_requests.get.side_effect = [
ConnectionError, ConnectionError, ConnectionError,
ConnectionError, ConnectionError, ConnectionError
]
with self.assertRaises(
ConnectionError
): # testing 2 linked tickets so the asynchronous call is triggered
TestExtractor(NAME, False).clean_json(dic)
with self.assertRaises(
ConnectionError
): # testing 2 linked tickets so the asynchronous call is triggered
DeployExtractor(NAME, False).clean_json(dic)
with self.assertRaises(
ConnectionError
): # testing 2 linked tickets so the asynchronous call is triggered
BarrosExtractor(NAME, False).clean_json(dic)
configuration = Loader.grab_configuration(self)
creds = f"Basic {configuration.u}"
mock_requests.assert_has_calls([
call.get(self.TEST_call_2,
headers={
'Authorization': creds,
'Content-Type': 'application/json'
},
verify=False)
])
def get_train_test_data(self):
train_data, test_data = [], []
train_target, test_target = [], []
train_dict, test_dict = Loader.build_train_test()
for key, value in train_dict.items():
for v in value:
ids = []
length = self.max_len
for i in range(length):
if i < len(v.split()):
ids.append(self.word_idx_map[v.split()[i]])
else:
ids.append(0)
train_data.append(ids)
target = [0, 0, 0, 0, 0, 0, 0]
target[self.c2id[key]] = 1
train_target.append(target)
for key, value in test_dict.items():
for v in value:
ids = []
length = self.max_len
for i in range(length):
if i < len(v.split()):
ids.append(self.word_idx_map[v.split()[i]])
else:
ids.append(0)
test_data.append(ids)
target = [0, 0, 0, 0, 0, 0, 0]
target[self.c2id[key]] = 1
test_target.append(target)
return train_data, train_target, test_data, test_target
def generate(self):
# self.reset()
file_id = 0
files = ['town.txt', 'dungeon1.txt', 'dungeon2.txt', 'dungeon3.txt']
grid = Loader.load_grid(files[file_id])
grid.extend(['.' * self.width] * (self.height - len(grid)))
for row in grid:
if len(row) < self.width:
row.extend('.' * (self.width - len(row)))
self.zone = Zone(grid, [], [], self._temperature)
self.zone.recommended_stairs_coords.append((10, 5))
self.zone.place_unit_by_name('white rat', 22, 7,
[Item(i) for i in ['cotton sweater']])
self.zone.place_item(Item('hearty meal', 22, 10))
self.zone.place_unit_by_name('coyote', 32, 2)
self.zone.place_unit_by_name('coyote', 33, 2)
self.zone.place_item(Item('stick', 33, 3))
self.zone.place_unit_by_name('coyote', 34, 2)
self.zone.place_unit_by_name('coyote', 35, 2)
self.zone.place_item(Item('hearty meal', 12, 4))
self.zone.place_item(Item('barrel top', 12, 3))
self.zone.place_item(Item('lit torch', 18, 3))
self.zone.place_item(Item('glass shard', 16, 10))
self.zone.place_item(Item('green puddle', 15, 7))
def process_batch(self, batch):
# Grab a batch, shuffled according to the provided seed. Note that
# i-th image: samples[i][0], i-th mask: samples[i][1]
samples = Loader.get_batch(self.image_paths, self.batch_size, batch,
None)
samples.astype(float)
# Cast samples into torch.FloatTensor for interaction with U-Net
samples = torch.from_numpy(samples)
samples = samples.float()
# Cast into a CUDA tensor, if GPUs are available
if torch.cuda.is_available():
samples = samples.cuda()
# Isolate images and their masks
samples_images = samples[:, 0]
samples_masks = samples[:, 1]
# Reshape for interaction with U-Net
samples_images = samples_images.unsqueeze(1)
source = samples_images
samples_masks = samples_masks.unsqueeze(1)
# Run inputs through the model
output = self.model(samples_images)
# Clamp the target for proper interaction with BCELoss
target = torch.clamp(samples_masks, min=0, max=1)
del samples
return source, output, target
def test(
self,
bundle,
bundle_size=32,
verbose=True,
):
loader = Loader(bundle)
data = loader.data_test(bundle_size)
result = self.model().predict(
data[0],
batch_size=bundle_size,
verbose=verbose,
)
keeper = Keeper(bundle, self.name())
for i in range(0, len(data[0])):
keeper.save(f'{bundle}-{i+1}', data[1][i], data[0][i], result[i])
def __init__(self, side_length, batch_size, seed, image_paths, state_dict):
self.side_length = side_length
self.batch_size = batch_size
self.seed = seed
self.image_paths = glob.glob(image_paths)
self.batches = Tester.get_number_of_batches(self.image_paths, self.batch_size)
self.model = UNet()
self.loader = Loader(self.side_length)
self.state_dict = state_dict
def train(
self,
bundle,
bundle_size=32,
steps_per_epoch=128,
epochs=8,
verbose=True,
scale_factor=4,
):
loader = Loader(bundle, scale_factor, Image.BICUBIC)
history = self.model().fit_generator(
loader.gen_data_train(bundle_size),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=verbose,
)
path_history = path.join(self._path_history, self.name() + '.hi')
with open(path_history, 'wb') as file_hi:
dump(history.history, file_hi)
def train(
self,
bundle,
bundle_size=32,
steps_per_epoch=128,
epochs=8,
verbose=True,
):
loader = Loader(bundle)
history = self.model().fit_generator(
loader.gen_data_train(bundle_size),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=verbose,
validation_data=loader.gen_data_test(bundle_size),
validation_steps=steps_per_epoch,
)
path_history = path.join(self._path_history, self.name() + '.hi')
with open(path_history, 'wb') as file_hi:
dump(history.history, file_hi)
def create_ticket(payload):
loader = Loader()
jira = atlassian.Jira(url="https://jira/",
username=loader.cfg.user,
password=loader.cfg.password)
response = jira.issue_create(fields=payload)
key = response["key"]
jira.issue_transition(key, "In Use")
logging.getLogger(__name__).info(f"{key} created")
return key
def train_save(self):
train_data, train_target, test_data, test_target = self.get_train_test_data(
)
print(len(train_data), len(train_target))
print(len(test_data), len(test_target))
train_data = np.array(train_data)
test_data = np.array(test_data)
print(len(train_data))
best = -999999
for i in range(10):
self.model.fit(train_data,
train_target,
batch_size=20,
nb_epoch=1,
shuffle=True)
score = self.model.evaluate(test_data, test_target, batch_size=20)
print(score)
if score[1] > best:
print('forward!!')
utils.forward(self)
best = score[1]
Loader.save_model(self.model, self.model_path, "lstm",
self.params_path)
def __init__(self, side_length, batch_size, epochs, learning_rate,
momentum_parameter, seed, image_paths, state_dict,
train_val_split):
self.side_length = side_length
self.batch_size = batch_size
self.epochs = epochs
self.learning_rate = learning_rate
self.momentum_parameter = momentum_parameter
self.seed = seed
self.image_paths = glob.glob(image_paths)
self.batches = Trainer.get_number_of_batches(self.image_paths,
self.batch_size)
self.model = UNet()
self.loader = Loader(self.side_length)
self.state_dict = state_dict
self.train_val_split = train_val_split
self.train_size = int(np.floor((self.train_val_split * self.batches)))
def test_clean_json_patch_ConnectionError(self) -> None:
with patch('jira_routine_V2.requests') as mock_requests:
from jira_routine_V2 import LogExtractor, NAME
dic = dict()
dic['T2L-249'] = grab_tickets_json['T2L-249']
mock_requests.get.side_effect = [
ConnectionError, ConnectionError, ConnectionError,
ConnectionError, ConnectionError, ConnectionError
]
with self.assertRaises(ConnectionError):
LogExtractor(NAME, False).clean_json(dic)
configuration = Loader.grab_configuration(self)
creds = f"Basic {configuration.u}"
mock_requests.assert_has_calls([
call.get(self.TEST_call_1,
headers={
'Authorization': creds,
'Content-Type': 'application/json'
},
verify=False)
])
def __init__(self, load=None):
self.model_path = "../../data/lstm_model.json"
self.params_path = "../../data/lstm_params.h5"
self.vocab, self.max_len = Loader.build_vocab()
print("data loaded!")
print("vocab size: " + str(len(self.vocab)))
print("max sentence length: " + str(self.max_len))
self.w2v = Loader.load_word_vec(self.vocab)
print("word2vec loaded!")
print("num words already in word2vec: " + str(len(self.w2v)))
Loader.add_unknown_words(self.w2v, self.vocab)
self.W, self.word_idx_map = Loader.get_W(self.w2v)
self.c2id, self.id2c = Loader.build_class()
print(self.c2id)
if load:
self.model = Loader.load_model(self.model_path, "lstm",
self.params_path)
return
self.model = Sequential()
self.model.add(
Embedding(len(self.word_idx_map) + 1, 300, weights=[self.W]))
self.model.add(
recurrent.LSTM(output_dim=100,
activation='tanh',
dropout_W=0,
dropout_U=0))
#self.model.add(convolutional.Convolution1D(100, 3, activation='tanh', border_mode='same'))
self.model.add(pooling.GlobalMaxPooling1D())
#self.model.add(Dropout(0.2))
self.model.add(Dense(7))
self.model.add(Activation('softmax'))
print(self.model.summary())
rmsprop = keras.optimizers.rmsprop(lr=0.002)
self.model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=["accuracy"])
def _open(self, fileName):
self.clearLabels()
self.view.clear()
loader = Loader()
d = loader.loadDicom(fileName)
if loader.isImage():
self.view = ImageView()
elif loader.isVolume():
self.view = DicomView()
elif loader.isVideo():
self.view = VideoView()
self.initViewSlot()
self.setCentralWidget(self.view)
self.view.addMenu(self.actions.editMenu)
self.view.loadImage(d)
self.zoom_widget.setValue(100)
self.actions.saveAs.setEnabled(False)
self.toggleDrawMode(type.Mode_polygon)
self.loader = loader
def main(output_dir, model_name, batch_size, num_workers, augment_epoch,
unaugment_epoch, device, label_type, confidence, num_classes,
epochs_per_save, teacher_noise, data_config, model_config):
assert label_type in ["soft", "hard"]
# setup for directory and log
output_dir = os.path.join(output_dir, model_name)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
os.chmod(output_dir, 0o775)
if hasattr(print, "set_log_dir"):
print.set_log_dir(output_dir)
print(f"==========Settings==========")
print(f"batch size: {batch_size}")
print(f"augment epoch: {augment_epoch}")
print(f"unaugment epoch: {unaugment_epoch}")
print(f"label type: {label_type}")
print(f"confidence: {confidence}")
print(f"models: {model_config['models']}")
print(f"epochs: {model_config['epochs']}")
print(f"ratio: {model_config['ratio']}")
print(f"learning rate: {model_config['learning_rate']}")
print(f"lr_decay_rate: {model_config['lr_decay_rate']}")
print(f"lr_decay_epoch: {model_config['lr_decay_epoch']}")
print(f"N, M: {data_config['N']}, {data_config['M']}")
print(f"teacher noise: {teacher_noise}")
print(f"dropout_prob: {model_config['dropout_prob']}")
print(f"stoc_depth_prob: {model_config['stochastic_depth_prob']}")
print("============================")
logger = Logger(os.path.join(output_dir, 'logs'))
# dataset, dataloader
dataset = load_dataset(**data_config)
dataloaders = {}
dataloaders['train_unaugmented'] = DataLoader(dataset['train_unaugmented'],
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
dataloaders['train_augmented'] = DataLoader(dataset['train_augmented'],
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
dataloaders['test'] = DataLoader(dataset['test'],
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
# load model
loader = Loader(**model_config, device=device, num_classes=num_classes)
(teacher, student, optimizer, lr_scheduler, start_epoch, end_epoch, phase,
best_acc, best_epoch, best_model) = loader.load(0)
model_name = model_config["models"][phase]
ratio = model_config["ratio"][phase]
save_path = os.path.join(output_dir, f"{model_name}.pt")
logger.set_model_name(model_name)
############################
###### Training phase ######
############################
if teacher is None:
dataloaders['train'] = dataloaders[
'train_augmented'] if teacher_noise else dataloaders[
'train_unaugmented']
else:
dataloaders['train'] = NS_DataLoader(dataloaders, dataset, teacher,
device, label_type, confidence,
ratio, num_workers, batch_size,
num_classes, print)
while True:
teacher = train_model(student, optimizer, lr_scheduler, dataloaders,
start_epoch, end_epoch, best_acc, best_epoch,
best_model, device, logger, phase,
epochs_per_save, save_path, augment_epoch,
unaugment_epoch)
phase += 1
if phase < len(model_config["models"]):
(_, student, optimizer, lr_scheduler, start_epoch, end_epoch, _,
best_acc, best_epoch, best_model) = loader.load(phase)
model_name = model_config["models"][phase]
ratio = model_config["ratio"][phase]
logger.set_model_name(model_name)
save_path = os.path.join(output_dir, f"{model_name}.pt")
dataloaders['train'] = NS_DataLoader(dataloaders, dataset, teacher,
device, label_type,
confidence, ratio,
num_workers, batch_size,
num_classes, print)
del teacher
else:
break
#!/usr/bin/env python
# coding:utf8
import ctypes
from utils import Loader
from utils import convert_data
import numpy as np
import api
mv_lib = Loader.get_lib()
class TableHandler(object):
'''`TableHandler` is an interface to sync different kinds of values.
If you are not writing python code based on theano or lasagne, you are
supposed to sync models (for initialization) and gradients (during
training) so as to let multiverso help you manage the models in distributed
environments.
Otherwise, you'd better use the classes in `multiverso.theano_ext` or
`multiverso.theano_ext.lasagne_ext`
'''
def __init__(self, size, init_value=None):
raise NotImplementedError("You must implement the __init__ method.")
def get(self, size):
raise NotImplementedError("You must implement the get method.")
def add(self, data, sync=False):
raise NotImplementedError("You must implement the add method.")
#!/usr/bin/env python
# coding:utf8
import ctypes
from utils import Loader
import numpy as np
mv_lib = Loader.get_lib()
def init(sync=False):
'''Initialize mutliverso.
This should be called only once before training at the beginning of the
whole project.
If sync is True, a sync server will be created. Otherwise an async server
will be created.
'''
args = [""] # the first argument will be ignored. So we put a placeholder here
if sync:
args.append("-sync=true")
n = len(args)
args_type = ctypes.c_char_p * n
mv_lib.MV_Init(ctypes.pointer(ctypes.c_int(n)), args_type(*[ctypes.c_char_p(arg) for arg in args]))
def shutdown():
'''Set a barrier for all workers to wait.
Workers will wait until all workers reach a specific barrier.
def load():
loader = Loader()
jira = atlassian.Jira(url="https://jira",
username=loader.cfg.user,
password=loader.cfg.password)
return jira
if Model_Flag == 2:
data_fp = '../data/PreProcessingData/cikm_spanish_train.csv'
feature_version = 0
Not(config_fp).extract(data_fp=data_fp, feature_version=0)
WordMatchShare(config_fp).extract(data_fp=data_fp, feature_version=0)
Length(config_fp).extract(data_fp=data_fp, feature_version=0)
NgramJaccardCoef(config_fp).extract(data_fp=data_fp, feature_version=0)
long_common_sequence(config_fp).extract(data_fp=data_fp, feature_version=0)
fuzz_partial_token_set_ratio(config_fp).extract(data_fp=data_fp, feature_version=0)
fuzz_partial_token_sort_ratio(config_fp).extract(data_fp=data_fp, feature_version=0)
TFIDFSpanish(config_fp, data_fp).extract(data_fp=data_fp, feature_version=0)
Label(config_fp).extract(data_fp=data_fp, feature_version=0)
if Model_Flag == 3:
DataSpanishSenPair = Loader(config_fp).loadAllData()
print(DataSpanishSenPair.keys())
print(DataSpanishSenPair['Labels'])
print(len(DataSpanishSenPair['Labels']))
print(sum(DataSpanishSenPair['Labels']))
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
data = DataSpanishSenPair['Features'].assign(L = DataSpanishSenPair['Labels'])
data = data.drop(['Length_0_0', 'Length_0_1', 'Length_0_2', 'Length_0_3', 'Not_0_0', 'Not_0_1',
'Not_0_2', 'Not_0_3', 'TFIDFSpanish_0_0',
'TFIDFSpanish_0_1', 'TFIDFSpanish_0_2', 'TFIDFSpanish_0_3'], axis = 1)
data.to_csv("spanishMatching_data")
n_gpu = 0
os.environ["CUDA_VISIBLE_DEVICES"] = str(n_gpu)
n_classes = 11
batch_size = 4
epochs = 150
width = 448
height = 448
channels = 3
lr = 1.5e-4
name_best_model = 'weights/camvid/best'
dataset_path = 'Datasets/camvid'
preprocess_mode = 'imagenet' #possible values 'imagenet', 'normalize',None
loader = Loader.Loader(dataFolderPath=dataset_path, n_classes=n_classes, problemType='segmentation',
width=width, height=height, channels=channels, median_frequency=0.0)
# build model and optimizer
model = Segception.Efficient(num_classes=n_classes, weights='imagenet', input_shape=(None, None, channels))
# optimizer
learning_rate = tfe.Variable(lr)
optimizer = tf.train.AdamOptimizer(learning_rate)
# Init models (optional, just for get_params function)
init_model(model, input_shape=(batch_size, width, height, channels))
variables_to_restore = model.variables
variables_to_save = model.variables
variables_to_optimize = model.variables
class EARunner:
def __init__(self):
logging.basicConfig(level=logging.DEBUG)
self.loader = Loader(False)
self.run_problem()
def run_problem(self):
distances, costs = self.loader.load_dataset_a()
problem = TSPProblem(distances, costs)
save_path = str(problem.population_size) + ' ' \
+ str(problem.generation_limit) + ' ' \
+ str(problem.crossover_rate) + ' ' \
+ str(problem.mutation_rate) + ' report'
self.run(problem, plot=True)
# self.run(problem, plot=True, save_path="../results/" + save_path)
def run_true_front(self):
distances, costs, front = self.loader.load_dataset_b()
problem = TSPProblem(distances, costs)
self.run(problem, plot=True, true_front=front)
def load_results(self):
paths = ["../results/50 4000 0.7 0.05 report-0.pickle",
"../results/100 2000 0.8 0.01 report-0.pickle",
"../results/200 1000 0.8 0.05 report-1.pickle"]
self.load_results_stats(paths)
self.load_results_plot(paths)
@staticmethod
def run(problem, true_front=None, plot=True, save_path=None):
"""
:param problem:
:param plot:
:param true_front: actual optimal front (for comparison with discovered/calculated front)
:param save_path: Save the first front of the final population to file with the given path
:return:
"""
# Generate the initial population
population = TSPPopulation(problem)
logging.info("Generations: %s, Pop. size: %s, Cross. rate: %s, Mut. rate: %s",
problem.generation_limit,
problem.population_size,
problem.crossover_rate, problem.mutation_rate)
fronts = []
def main_loop():
while population.generation < problem.generation_limit:
population.generation += 1
population.evaluate_fitnesses() # Calculate total cost and total distance for each route/individual
population.select_adults()
population.select_parents()
population.reproduce()
if population.generation % (problem.generation_limit / 5) == 0:
logging.info("\t\t Generation %s/%s", population.generation, problem.generation_limit)
fronts.append(population.get_front(0))
logging.info("\tExecution time: %s", timeit.timeit(main_loop, number=1))
logging.info("\t(Min/Max) Distance: %s/%s; Cost: %s/%s",
TSPPopulation.min_fitness(population.adults, 0).fitnesses[0],
TSPPopulation.max_fitness(population.adults, 0).fitnesses[0],
TSPPopulation.min_fitness(population.adults, 1).fitnesses[1],
TSPPopulation.max_fitness(population.adults, 1).fitnesses[1])
if save_path:
with open(save_path + "-" + str(np.random.randint(10)) + '.pickle', 'wb') as f:
pickle.dump(population.get_front(0), f)
if plot:
EARunner.plot([population.adults], save_path=save_path)
EARunner.plot([population.get_front(0)],
name='Fitnesses, final Pareto-front',
save_path=save_path)
# EARunner.plot(fronts, true_front=true_front, dash=True,
# name='Fitnesses, final Pareto-front per 20% progress', save_path=save_path)
plt.show()
@staticmethod
def plot(pools, true_front=None, dash=False, name='Fitnesses', save_path=None):
"""
:param true_front:
:param pools: NOT instance of TSPPopulations, but a list of lists of individuals (lists of population.adults)
:param dash: dash lines between each individual in each pool
:param name: Plot legend
:param save_path:
:return:
"""
marker = itertools.cycle(('o', ',', '+', '.', '*'))
color = itertools.cycle(('b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'))
if dash:
linestyle = "--"
for pool_i in range(len(pools)):
pools[pool_i] = sorted(pools[pool_i],
key=lambda ind: ind.fitnesses[0])
else:
linestyle = ""
plt.figure()
plt.title(name)
for i, pool in enumerate(pools):
c = next(color)
plt.plot([individual.fitnesses[0] for individual in pool],
[individual.fitnesses[1] for individual in pool],
marker=next(marker), linestyle=linestyle, color=c,
label=str((i + 1) * 20) + "%-" + str(len(pool))
+ "sols-" + str(TSPPopulation.n_unique(pool)) + "uniq")
min_dist = TSPPopulation.min_fitness(pool, 0).fitnesses
max_dist = TSPPopulation.max_fitness(pool, 0).fitnesses
min_cost = TSPPopulation.min_fitness(pool, 1).fitnesses
max_cost = TSPPopulation.max_fitness(pool, 1).fitnesses
if not dash:
c = 'r'
plt.plot([min_dist[0]], [min_dist[1]], marker='D', color=c)
plt.plot([max_dist[0]], [max_dist[1]], marker='D', color=c)
plt.plot([min_cost[0]], [min_cost[1]], marker='D', color=c)
plt.plot([max_cost[0]], [max_cost[1]], marker='D', color=c)
if true_front is not None:
plt.plot([i[0] for i in true_front], [i[1] for i in true_front],
linestyle="--", label="True front")
# if dash:
# plt.legend(loc="best")
plt.xlabel("Distance")
plt.xticks(np.arange(30000, 120001, 10000))
plt.ylabel("Cost")
plt.yticks(np.arange(300, 1401, 100))
if save_path:
plt.savefig(save_path + "-" + str(np.random.randint(10)) + ".png")
@staticmethod
def load_results_plot(paths):
populations = []
for path in paths:
with open(path, 'rb') as f:
population = pickle.load(f)
populations.append(population)
EARunner.plot(populations, dash=True,
name="Final pareto fronts, 3 configurations")
plt.show()
@staticmethod
def load_results_stats(paths):
for path in paths:
with open(path, 'rb') as f:
population = pickle.load(f)
logging.info("\t(Min/Max) Distance: %s/%s; Cost: %s/%s",
TSPPopulation.min_fitness(population, 0).fitnesses[0],
TSPPopulation.max_fitness(population, 0).fitnesses[0],
TSPPopulation.min_fitness(population, 1).fitnesses[1],
TSPPopulation.max_fitness(population, 1).fitnesses[1])
def __init__(self):
logging.basicConfig(level=logging.DEBUG)
self.loader = Loader(False)
self.run_problem()