def main():
"""
Run all the worlds in the benchmark and tabulate their performance.
"""
performance = []
performance.append(tester.test(World_grid_1D))
performance.append(tester.test(World_grid_1D_delay))
performance.append(tester.test(World_grid_1D_chase))
performance.append(tester.test(World_grid_1D_ms))
performance.append(tester.test(World_grid_1D_noise))
performance.append(tester.test(World_grid_2D))
performance.append(tester.test(World_grid_2D_dc))
performance.append(tester.test(World_image_1D))
performance.append(tester.test(World_image_2D))
performance.append(tester.test(World_fruit))
print('Individual benchmark scores:')
scores = []
for score in performance:
print(' {0:.2}, {1}'.format(score[0], score[1]))
scores.append(score[0])
print('Overall benchmark score: {0:.2}'.format(np.mean(scores)))
# Block the program, displaying all plots.
# When the plot windows are closed, the program closes.
plt.show()
def test_one_layer_gaussian():
"""Test variational EM with pathwise gradients."""
config = """
learning_rate: 0.1
n_iterations: 100
gradient:
estimator: pathwise
n_samples: 7
batch_size: 1
layer_1:
latent_distribution: gaussian
p_z_variance: 1.
size: 3
layer_0:
weight_distribution: point_mass
p_w_variance: 1.
data_size: 1
"""
data = np.array([[3.3]])
def test_posterior_predictive(sample: np.array) -> None:
print('data:', data)
print('posterior predictive sample:', sample)
np.testing.assert_allclose(sample, data, rtol=1e-1)
tester.test(
config, data=data, test_fn=test_posterior_predictive)
def test_latent_layer():
"""Test matching q(z) to p(z) where p is Gaussian."""
mean = 5.
config = """
n_iterations: 50
learning_rate: 0.1
gradient:
estimator: pathwise
n_samples: 1
batch_size: 1
layer_1:
latent_distribution: gaussian
size: 1
p_z_mean: {}
p_z_variance: 1.
""".format(mean)
def test_posterior_predictive(sample: np.array) -> None:
print('posterior predictive mean:', sample)
print('prior mean:', mean)
np.testing.assert_allclose(sample, mean, rtol=1e-1)
tester.test(config,
data=np.array([np.nan]),
test_fn=test_posterior_predictive)
def test_two_layer_poisson_score():
"""Test variational EM with score function gradients and gaussian latents."""
config = """
learning_rate: 0.1
n_iterations: 100
gradient:
estimator: score_function
n_samples: 32
batch_size: 1
layer_2:
latent_distribution: poisson
size: 2
layer_1:
latent_distribution: poisson
weight_distribution: point_mass
size: 3
layer_0:
weight_distribution: point_mass
data_size: 1
"""
data = np.array([[-3.2]])
def test_posterior_predictive(sample: np.array) -> None:
print('data:', data)
print('posterior predictive sample:', sample)
np.testing.assert_allclose(sample, data, rtol=0.3)
tester.test(
config, data=data, test_fn=test_posterior_predictive)
def main(_):
if "absl.logging" in sys.modules:
import absl.logging
absl.logging.set_verbosity("info")
absl.logging.set_stderrthreshold("info")
config = FLAGS.config
print(config)
# Set the seed
torch.manual_seed(config.seed)
np.random.seed(config.seed)
# Check if in the correct branch
# group_name = config["model"][: config["model"].find("sa")]
# if group_name not in ["z2", "mz2", "p4", "p4m"]:
# raise ValueError(
# "Mlp_encoding is required for rotations finer than 90 degrees. Please change to the mlp_encoding branch."
# )
# initialize weight and bias
os.environ["WANDB_API_KEY"] = "691777d26bb25439a75be52632da71d865d3a671"
if not config.train:
os.environ["WANDB_MODE"] = "dryrun"
wandb.init(
project="equivariant-attention",
config=config,
group=config["dataset"],
entity="equivatt_team",
)
# Define the device to be used and move model to that device
config["device"] = (
"cuda:0" if (config.device == "cuda" and torch.cuda.is_available()) else "cpu"
)
model = get_model(config)
# Define transforms and create dataloaders
dataloaders = dataset.get_dataset(config, num_workers=4)
# Create model directory and instantiate config.path
model_path(config)
if config.pretrained:
# Load model state dict
model.module.load_state_dict(torch.load(config.path), strict=False)
# Train the model
if config.train:
# Print arguments (Sanity check)
print(config)
print(datetime.datetime.now())
# Train the model
trainer.train(model, dataloaders, config)
# Test model
tester.test(model, dataloaders["test"], config)
def test_gaussian_score_multivariate_data():
config = """
learning_rate: 0.1
n_iterations: 100
gradient:
estimator: score_function
n_samples: 16
batch_size: 3
layer_1:
latent_distribution: gaussian
size: 7
layer_0:
weight_distribution: point_mass
data_size: 3
"""
data = np.array([[20.3, -30.3, 15.3], [-30.3, -40.4, 23.5],
[15., -20.3, 28.9]])
def test_posterior_predictive(sample: np.array, data: np.array) -> None:
print('----')
print('data:', data)
print('posterior predictive sample:', sample)
np.testing.assert_allclose(sample, np.expand_dims(data, 0), rtol=0.2)
tester.test(config, data=data, test_fn=test_posterior_predictive)
def main():
"""
Run all the worlds in the benchmark and tabulate their performance.
"""
performance = []
performance.append(tester.test(World_grid_1D))
performance.append(tester.test(World_grid_1D_delay))
performance.append(tester.test(World_grid_1D_chase))
performance.append(tester.test(World_grid_1D_ms))
performance.append(tester.test(World_grid_1D_noise))
performance.append(tester.test(World_grid_2D))
performance.append(tester.test(World_grid_2D_dc))
performance.append(tester.test(World_image_1D))
performance.append(tester.test(World_image_2D))
performance.append(tester.test(World_fruit))
print('Individual benchmark scores:')
scores = []
for score in performance:
print(' {0:.2}, {1}'.format(score[0], score[1]))
scores.append(score[0])
print('Overall benchmark score: {0:.2}'.format(np.mean(scores)))
# Block the program, displaying all plots.
# When the plot windows are closed, the program closes.
plt.show()
def main():
"""Main function to run model."""
config = get_config(os.environ)
sys.path.append(os.path.join('tasks', config.task_folder))
# pylint: disable=import-error
from trainer import train
from tester import test
# pylint: enable=import-error
if config.is_distributed:
torch.cuda.set_device(config.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
logger = setup_logger(config.work_dir, distributed_rank=get_rank())
logger.info(f'Using {config.num_gpus} GPUs.')
logger.info(f'Collecting environment info:{get_env_info()}')
logger.info(f'------------------------------')
logger.info(f'Running configurations:')
for key, val in config.__dict__.items():
logger.info(f' {key}: {val}')
logger.info(f'------------------------------')
if config.run_mode == 'train':
train(config, logger)
elif config.run_mode == 'test':
test(config, logger)
def main():
parser = argparse.ArgumentParser(
description="checkPy: a simple python testing framework")
parser.add_argument(
"-m",
action="store",
dest="module",
help=
"provide a module name or path to run all tests from the module, or target a module for a specific test"
)
parser.add_argument(
"-d",
action="store",
dest="githubLink",
help="download tests from a Github repository and exit")
parser.add_argument("-update",
action="store_true",
help="update all downloaded tests and exit")
parser.add_argument("-list",
action="store_true",
help="list all download locations and exit")
parser.add_argument("-clean",
action="store_true",
help="remove all tests from the tests folder and exit")
parser.add_argument("file",
action="store",
nargs="?",
help="name of file to be tested")
args = parser.parse_args()
rootPath = os.sep.join(
os.path.abspath(os.path.dirname(__file__)).split(os.sep)[:-1])
if rootPath not in sys.path:
sys.path.append(rootPath)
if args.githubLink:
downloader.download(args.githubLink)
return
if args.update:
downloader.update()
return
if args.list:
downloader.list()
return
if args.clean:
downloader.clean()
return
if args.file and args.module:
tester.test(args.file, module=args.module)
elif args.file and not args.module:
tester.test(args.file)
elif not args.file and args.module:
tester.testModule(args.module)
else:
parser.print_help()
return
def test_multivariate_data():
"""Test variational EM with score function gradients."""
config = """
learning_rate: 0.1
n_iterations: 100
print_every: 100
gradient:
estimator: score_function
n_samples: 16
batch_size: 1
layer_1:
latent_distribution: gaussian
size: 1
layer_0:
weight_distribution: point_mass
data_size: 3
"""
# shape [n_data, data_size]
data = np.array([[30.3, -10., 5.]])
def test_posterior_predictive(sample: np.array) -> None:
print('data:', data)
print('posterior predictive sample:', sample)
np.testing.assert_allclose(sample, data, rtol=1e-1)
tester.test(
config, data=data, test_fn=test_posterior_predictive)
def main(args):
# Create directories
if not os.path.exists("./logs"):
os.makedirs("./logs")
if not os.path.exists("./pytorch_models"):
os.makedirs("./pytorch_models")
# Set logs
tb_writer = SummaryWriter('./logs/tb_{0}'.format(args.log_name))
log = set_log(args)
# Create env
env = make_env(log, args)
# Set seeds
env.seed(args.seed)
torch.manual_seed(args.seed)
np.random.seed(args.seed)
# Initialize policy
agent = set_policy(env, tb_writer, log, args, name=args.algorithm)
if args.test:
from tester import test
test(agent=agent, env=env, log=log, tb_writer=tb_writer, args=args)
else:
from trainer import train
train(agent=agent, env=env, log=log, tb_writer=tb_writer, args=args)
def test_latent_poisson_layer():
"""Test matching q(z) to p(z) where p is Poisson."""
mean = 5.
config = """
n_iterations: 100
learning_rate: 0.1
gradient:
estimator: score_function
n_samples: 16
batch_size: 1
layer_1:
latent_distribution: poisson
size: 1
p_z_mean: {}
""".format(mean)
def test_posterior_predictive(sample: np.array) -> None:
print('posterior predictive sample:', sample)
print('prior mean:', mean)
np.testing.assert_allclose(sample, mean, rtol=1e-1)
tester.test(config,
data=np.array([np.nan]),
test_fn=test_posterior_predictive)
def test_3(self):
t = 0
conjuntos_de_processos = [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]]
n = 11
atributos = [[1, 3, 5, 1], [2, 3, 3, 2], [3, 3, 5, 3], [4, 2, 1, 4],
[5, 2, 1, 5], [6, 3, 5, 6], [7, 1, 5, 7], [8, 2, 1, 8],
[9, 1, 3, 9], [10, 2, 2, 10], [11, 4, 5, 11]]
tester.test(t, conjuntos_de_processos, n, atributos, 3)
def test_5(self):
n = 1
conjuntos = [
[1, 2, 3, 4, 5],
]
m = 4
operacoes = [(1, 2), (3, 4), (1, 5), (3, 5)]
tester.test(n, conjuntos, m, operacoes, 5)
def model_process(count, model):
opt = parse_opts()
if opt.root_path != '':
opt.video_path = os.path.join(opt.root_path, opt.video_path)
opt.annotation_path = os.path.join(opt.root_path, opt.annotation_path)
opt.result_path = os.path.join(opt.root_path, opt.result_path)
if opt.resume_path:
opt.resume_path = os.path.join(opt.root_path, opt.resume_path)
if opt.pretrain_path:
opt.pretrain_path = os.path.join(opt.root_path, opt.pretrain_path)
opt.scales = [opt.initial_scale]
for i in range(1, opt.n_scales):
opt.scales.append(opt.scales[-1] * opt.scale_step)
#opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
opt.mean = get_mean(opt.norm_value, dataset=opt.mean_dataset)
opt.std = get_std(opt.norm_value)
#print(opt)
#print(opt.result_path)
with open(os.path.join(opt.result_path, 'opts.json'), 'w') as opt_file:
json.dump(vars(opt), opt_file)
torch.manual_seed(opt.manual_seed)
#print(model)
criterion = nn.CrossEntropyLoss()
if not opt.no_cuda:
criterion = criterion.cuda()
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
print('testing is run')
if opt.test:
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = VideoID()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
tester.test(count, test_loader, model, opt, test_data.class_names)
def main(argv=None):
writer = Writer(RESULTS_DIR)
trainer = Trainer(RESULTS_DIR, 'train', writer)
tester = Tester(RESULTS_DIR, 'valid', writer)
step, _ = tester.test(EVAL_STEP_NUM)
while (step < LAST_STEP):
lr = learning_rate(step)
step, _ = trainer.train(lr, EVAL_FREQUENCY, step, RESTORING_FILE)
tester.test(EVAL_STEP_NUM, step)
def main(argv=None):
writer = Writer(RESULTS_DIR)
trainer = Trainer(RESULTS_DIR, 'train', writer)
tester = Tester(RESULTS_DIR, 'valid', writer)
step, _ = tester.test(EVAL_STEP_NUM)
while (step < LAST_STEP):
lr = learning_rate(step)
step, _ = trainer.train(lr, EVAL_FREQUENCY, step, RESTORING_FILE)
tester.test(EVAL_STEP_NUM, step)
def main(argv=None):
writer = Writer(RESULTS_DIR)
tester = Tester(RESULTS_DIR, 'valid', writer)
status_file = os.path.join(RESULTS_DIR, 'checkpoint')
last_update = 0
while True:
cur_time = os.stat(status_file).st_mtime
if (cur_time > last_update):
tester.test(EVAL_STEP_NUM)
last_update = cur_time
time.sleep(INTERVAL)
def test_4(self):
n = 5
conjuntos = [
[1],
[2],
[3],
[4],
[5],
]
m = 4
operacoes = [(1, 2), (3, 4), (1, 5), (3, 5)]
tester.test(n, conjuntos, m, operacoes, 4)
def main():
print_header("START", CONFIG, level=0)
data = load(CONFIG)
print(data.head())
data.to_csv("./outputs/data.csv", index=False)
describe(data, CONFIG)
test(data, CONFIG)
forecast(data, CONFIG)
predict(data, CONFIG)
report(data, CONFIG)
print_header("DONE", CONFIG, level=0)
def test(fileName):
"""
Run tests for a single file
"""
import caches
caches.clearAllCaches()
import tester
tester.test(fileName)
try:
if __IPYTHON__:
import matplotlib.pyplot
matplotlib.pyplot.close("all")
except:
pass
def run():
train_file = 'trained_models/english.crfsuite'
train_data_percent = 1
blocks = int(1 / train_data_percent)
max_c = 5
all_config = 2**max_c
best_f1 = 0
best_config = 0
print("N, F1, P, R ")
for c in range(all_config):
config = get_feature_configuration(c, max_c)
total_f1 = 0
for i in range(blocks):
train_d = get_eng_train_data(train_data_percent, i)
test_d = get_eng_test_data(train_data_percent, i)
train(train_d, train_file, config)
f1 = test(test_d, train_file, config)
total_f1 += f1
#print(f1)
total_f1 /= blocks
#print("Config = " + str(config))
#print("Number = " + str(c+1))
#print("Final F1 = " + str(total_f1))
#print("#############")
#print(str(c+1) + ", " + str(total_f1) + ", " + str(total_f1) + ", " + )
if total_f1 > best_f1:
best_f1 = total_f1
best_config = config
print("Best config = " + str(best_config))
print("Best F1 = " + str(best_f1))
def train(args, model, device, data, input, target, test_input, test_target, optimizer, epoch):
device = torch.device("cuda") # Sending to GPU
def closure():
optimizer.zero_grad() #Reset grads
out = model(input) # Passing batch through model
loss = criterion(out, target)
print(f"Loss: {loss.item()}")
loss.backward() # Backprop
return loss
optimizer.step(closure) # Pass through optimizer
y, future = test(model, test_input, test_target)
# Display the graphs
plt.figure(figsize = (30 , 10))
plt.title("Predictions")
plt.xlabel('x')
plt.ylabel('y')
plt.xticks()
plt.yticks()
def draw(yi, color):
plt.plot(np.arange(input.size(1)), yi[:input.size(1)], color, linewidth
= 2.0)
plt.plot(np.arange(input.size(1), input.size(1)+future),
yi[input.size(1):], color + ':', linewidth = 2.0)
draw(y[0], 'r')
draw(y[1], 'g')
draw(y[2], 'b')
plt.savefig(f"outputs/predict_{epoch}.png")
plt.close()
def test_strip_neg(self):
for fname, out_prefix in get_progs(r"(strip_\w+).in", 2):
with self.subTest(name=fname):
ret, *_ = tester.test(["cat"], 1., 0, False, False,
f"tests{sep}{fname}",
f"tests{sep}{out_prefix}.out")
self.assertEqual(ret, "WA")
def main():
# data
train_dataloader, test_dataloader = data.mnist_dataloader()
# model
model = models.CNN_Net().to(cfg.device)
# train
trainer.train(model, train_dataloader, test_dataloader)
# test
model.load_state_dict(torch.load(cfg.best_model_path))
tester.test(model, test_dataloader)
# metrics
evaluate.eval()
def index(request):
context = {'jsdata' : ""}
fobj = request.FILES # here you get the files needed
if(len(fobj) != 0):
f = fobj['sentFile']
os.remove('tmp/test_image.jpg')
path = default_storage.save('tmp/test_image.jpg', ContentFile(f.read()))
jsdata = json.dumps({"res" : tester.test('tmp/test_image.jpg')})
pr = tester.test('tmp/test_image.jpg')
context = {'jsres' : pr[0], 'jstreat' : pr[1]}
return render(request, 'diagnosis/result.html', context)
else:
return render(request, 'diagnosis/index.html', context)
def test_strict(self):
for _, fname, expected_ret in get_progs(r"(prog\w*_([A-Z]+)).py", 3):
with self.subTest(name=fname):
prefix = f"tests{sep}{fname}"
ret, *_ = tester.test([executable, prefix + ".py"], 1.,
16 * 1024**2, False, False,
prefix + ".in", prefix + ".out")
self.assertEqual(ret, expected_ret)
def evaluation(net, train_loader, test_loader, optimizer, epochs, device,
train_acc, train_losses, test_acc, test_losses):
# initialising cumulative train and test metrics which will store per epoch metrics
# cum_train_acc = []
# cum_train_losses = []
# cum_test_acc = []
# cum_test_losses = []
# net = model.Cifar10_Net(norm_type = 'BN').to(device)
# scheduler = StepLR(optimizer, step_size=6, gamma=0.1)
for epoch in range(1, epochs + 1):
print('\n Epoch:', epoch)
trainer.train(net, device, train_loader, optimizer, epoch, train_acc,
train_losses)
# scheduler.step()
tester.test(net, device, test_loader, test_acc, test_losses)
def main(file: str = None):
if file is None:
print("Default config file will be used")
file = "config.json"
print("Using default config.json file")
config = load_config(file)
asyncio.get_event_loop().run_until_complete(test(config))
def __call__(self, f):
"""
When the object is called like a function, we run the method
specified by self.method on the dataset in file 'f'.
Args:
f: path to file where the network corresponding the the ground
truth community structure lies.
Returns:
A string of test results.
"""
print(self.method)
print(f)
G = initialize_graph(f)
known = tester.parse(self.truth)
known -= 1
exporter = Exporter(f, G.n, False)
arguments = Arguments(exporter, None, None, 0.02,
False, False, self.method)
if self.method == Method.prop:
labelprop.propagate(G, arguments)
found = arguments.exporter.comlist[:, -1]
numcoms = len(np.unique(found))
test_results = tester.test(found, known)
else:
community_detect(G, arguments)
hierarchy = arguments.exporter.comlist[:, 1:] # Exclude the 0...n col
colresult = np.empty(shape=(hierarchy.shape[1], 4))
lengths = []
for j, column in enumerate(hierarchy.T):
lengths.append(len(np.unique(column)))
colresult[j, :] = tester.test(column, known)
idx = get_best_column(colresult)
test_results = colresult[idx, :]
numcoms = lengths[idx]
return format(os.path.basename(f), test_results[0], test_results[1],
test_results[2], test_results[3], numcoms,
len(np.unique(known)), str(arguments.method).split('.')[-1])
def get_data(x_min,x_max,x_step):
tpr_points = []
fpr_points = []
ppv_points = []
for i in range(((x_max-x_min)//x_step)):
f,t,p = tester.test(x_min+(x_step*i))
tpr_points.append(t)
fpr_points.append(f)
ppv_points.append(p)
print("Total_Progress ="+str((i*1.0/((x_max-x_min)//x_step))))
return (tpr_points,fpr_points,ppv_points)
def main():
N_RUNS = 7
benchmark_lifespan = 1e4
overall_performance = []
# Run all the worlds in the benchmark and tabulate their performance
for i in range(N_RUNS):
performance = []
world = World_grid_1D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_1D_ms(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_1D_noise(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_2D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_2D_dc(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_image_1D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_image_2D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
print "Individual benchmark scores: ", performance
total = 0
for val in performance:
total += val
mean_performance = total / len(performance)
overall_performance.append(mean_performance)
print "Overall benchmark score, ", i, "th run: ", mean_performance
print "All overall benchmark scores: ", overall_performance
# Automatically throw away the 2 highest and 2 lowest values
# if you choose N_RUNS to be 7 or more.
if N_RUNS >= 7:
for i in range(2):
highest_val = -10**6
lowest_val = 10**6
for indx in range(len(overall_performance)):
if overall_performance[indx] > highest_val:
highest_val = overall_performance[indx]
if overall_performance[indx] < lowest_val:
lowest_val = overall_performance[indx]
overall_performance.remove(highest_val)
overall_performance.remove(lowest_val)
# Find the average of what's left
sum_so_far = 0.
for indx in range(len(overall_performance)):
sum_so_far += overall_performance[indx]
typical_performance = sum_so_far / len(overall_performance)
print "Typical performance score: ", typical_performance
# Block the program, displaying all plots.
# When the plot windows are closed, the program closes.
plt.show()
def main():
N_RUNS = 7
benchmark_lifespan = 1e4
overall_performance = []
# Run all the worlds in the benchmark and tabulate their performance
for i in range(N_RUNS):
performance = []
world = World_grid_1D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_1D_ms(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_1D_noise(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_2D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_grid_2D_dc(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_image_1D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
world = World_image_2D(lifespan=benchmark_lifespan)
performance.append(tester.test(world, show=False))
print "Individual benchmark scores: " , performance
total = 0
for val in performance:
total += val
mean_performance = total / len(performance)
overall_performance.append(mean_performance)
print "Overall benchmark score, ", i , "th run: ", mean_performance
print "All overall benchmark scores: ", overall_performance
# Automatically throw away the 2 highest and 2 lowest values
# if you choose N_RUNS to be 7 or more.
if N_RUNS >= 7:
for i in range(2):
highest_val = -10 ** 6
lowest_val = 10 ** 6
for indx in range(len(overall_performance)):
if overall_performance[indx] > highest_val:
highest_val = overall_performance[indx]
if overall_performance[indx] < lowest_val:
lowest_val = overall_performance[indx]
overall_performance.remove(highest_val)
overall_performance.remove(lowest_val)
# Find the average of what's left
sum_so_far = 0.
for indx in range(len(overall_performance)):
sum_so_far += overall_performance[indx]
typical_performance = sum_so_far / len(overall_performance)
print "Typical performance score: ", typical_performance
# Block the program, displaying all plots.
# When the plot windows are closed, the program closes.
plt.show()
def run_tests(self):
exe_loc = self.root_dir + '/SDPSolver.exe'
CLI.fancy_print('Enter Implementation type',
'[1] Kiarash\'s implementation',
'[2] Derivative method (Keivan & Hamidreza)',
'[3] Generalized eigenvalue method (Hamidreza)')
imp_type = input()
test_folder = CLI.select_dirs(
self.root_dir + "/SDPA/testSet",
"Select one of the test directories below:")
CLI.list_dirs(
self.root_dir + "/SDPA/testSet/" + test_folder,
"Enter test name you wish to run (Regex is also supported e.g. \"*.dat-s\")",
TestFormats.regex_format())
tests_reg = input()
try:
try:
os.mkdir(self.root_dir + "/out")
except:
pass
sv_dir = os.getcwd()
os.chdir(self.root_dir)
os.chdir('build')
executable_path = os.getcwd()
os.chdir(self.root_dir + '/SDPA/testSet')
os.chdir(test_folder)
tests_dir = os.getcwd()
os.chdir(self.root_dir + "/out")
output_path = os.getcwd()
os.chdir(sv_dir)
test(executable_path=executable_path,
tests_dir=tests_dir,
output_path=output_path,
test_reg=tests_reg,
implementation_type=imp_type)
except:
print('problem in running tester')
self.back_to_main()
return
def main(argv=None):
writer = Writer(RESULTS_DIR)
trainer = Trainer(RESULTS_DIR, 'train', writer)
tester = Tester(RESULTS_DIR, 'valid', writer)
params_file = os.path.join(RESULTS_DIR, PARAMS_FILE)
if (os.path.isfile(params_file)):
with open(params_file, 'r') as handle:
params = json.load(handle)
else:
params = {}
params['min_test_step'], params['min_test_loss'] = tester.test(EVAL_STEP_NUM)
params['step'] = params['min_test_step']
params['unchanged'] = 0
params['num_decays'] = 0
params['learning_rate'] = LEARNING_RATE
while (params['num_decays'] <= MAX_DECAYS):
params['step'], _ = trainer.train(params['learning_rate'], EVAL_FREQUENCY,
params['step'], RESTORING_FILE)
_, test_loss = tester.test(EVAL_STEP_NUM, params['step'])
if (test_loss < params['min_test_loss']):
params['min_test_loss'] = test_loss
params['min_test_step'] = params['step']
params['unchanged'] = 0
else:
params['unchanged'] += EVAL_FREQUENCY
if (params['unchanged'] >= PATIENCE):
params['learning_rate'] *= DECAY_FACTOR
params['num_decays'] += 1
params['step'] = params['min_test_step']
params['unchanged'] = 0
with open(params_file, 'w') as handle:
json.dump(params, handle, indent=2)
print(params)
def test(batch, remote, debug, dependency = []):
params = cache.get("batch/%s/params" % batch, remote)
numEpisodes = params['episodes']['num']
i_ = range(numEpisodes)
f = lambda i : tester.test(batch, params, i, remote, debug)
logging.info("running %s test instances" % len(i_))
if (remote):
k_ = cloud.map(f, i_, _label = "%s/test" % batch, _depends_on = dependency, _type = 'c1', _max_runtime = 30)
logging.info("k_ %s" % k_)
return k_
else:
results = map(f, i_)
return results
#
# refsets = collections.defaultdict(set)
# testsets = collections.defaultdict(set)
#
# for i, (feats, label) in enumerate(testfeats):
# refsets[label].add(i)
# observed = classifier.classify(feats)
# testsets[observed].add(i)
#
#
# print '#### POSITIVE ####'
# print 'pos precision:', nltk.metrics.precision(refsets['pos'], testsets['pos'])
# print 'pos recall:', nltk.metrics.recall(refsets['pos'], testsets['pos'])
# print 'pos F-measure:', nltk.metrics.f_measure(refsets['pos'], testsets['pos'])
# print
# print '#### NEGATIVE ####'
# print 'neg precision:', nltk.metrics.precision(refsets['neg'], testsets['neg'])
# print 'neg recall:', nltk.metrics.recall(refsets['neg'], testsets['neg'])
# print 'neg F-measure:', nltk.metrics.f_measure(refsets['neg'], testsets['neg'])
#
# print '--------------------'
# print 'Classifier Accuracy:', util.accuracy(classifier, testfeats)
# classifier.show_most_informative_features()
if __name__ == "__main__":
# example train and tester.test to display accuracies
train(train_samples=2000, wordcount_samples=1000, wordcount_range=2000, force_update=False, verbose=True)
from tester import test
test()
def train(self, trainInput, trainTarget, validInput=None, validTarget=None):
self.initFolder()
trainOpt = self.trainOpt
if validInput is None and validTarget is None:
X, T, VX, VT = self.initData(\
trainInput, trainTarget, \
split=self.trainOpt['needValid'])
else:
X = trainInput
T = trainTarget
VX = validInput
VT = validTarget
N = X.shape[0]
numEpoch = trainOpt['numEpoch']
calcError = trainOpt['calcError']
numExPerBat = trainOpt['batchSize']
progressWriter = ProgressWriter(N, width=80)
logger = Logger(self, csv=trainOpt['writeRecord'])
logger.logMsg('Trainer ' + self.name)
plotter = Plotter(self)
bestVscore = None
bestTscore = None
bestEpoch = 0
nAfterBest = 0
stop = False
# Train loop through epochs
for epoch in range(0, numEpoch):
E = 0
correct = 0
total = 0
if trainOpt['shuffle']:
X, T = vt.shuffleData(X, T, self.random)
batchStart = 0
while batchStart < N:
# Batch info
batchEnd = min(N, batchStart + numExPerBat)
numExThisBat = batchEnd - batchStart
# Write progress bar
if trainOpt['progress']:
progressWriter.increment(amount=numExThisBat)
# Forward
Y_bat = self.model.forward(X[batchStart:batchEnd], dropout=True)
T_bat = T[batchStart:batchEnd]
# Loss
Etmp, dEdY = self.model.getCost(Y_bat, T_bat)
E += Etmp * numExThisBat / float(N)
# Backward
self.model.backward(dEdY)
# Update
self.model.updateWeights()
# Prediction error
if calcError:
rate_, correct_, total_ = \
tester.calcRate(self.model, Y_bat, T_bat)
correct += correct_
total += total_
batchStart += numExPerBat
# Store train statistics
if calcError:
rate = correct / float(total)
self.rate[epoch] = rate
self.loss[epoch] = E
if not trainOpt.has_key('criterion'):
Tscore = E
else:
if trainOpt['criterion'] == 'loss':
Tscore = E
elif trainOpt['criterion'] == 'rate':
Tscore = 1 - rate
else:
raise Exception('Unknown stopping criterion "%s"' % \
trainOpt['criterion'])
# Run validation
if trainOpt['needValid']:
VY = tester.test(self.model, VX)
VE, dVE = self.model.getCost(VY, VT)
self.validLoss[epoch] = VE
if calcError:
Vrate, correct, total = tester.calcRate(self.model, VY, VT)
self.validRate[epoch] = Vrate
# Check stopping criterion
if not trainOpt.has_key('criterion'):
Vscore = VE
else:
if trainOpt['criterion'] == 'loss':
Vscore = VE
elif trainOpt['criterion'] == 'rate':
Vscore = 1 - Vrate
else:
raise Exception('Unknown stopping criterion "%s"' % \
trainOpt['criterion'])
if (bestVscore is None) or (Vscore < bestVscore):
bestVscore = Vscore
bestTscore = Tscore
nAfterBest = 0
bestEpoch = epoch
# Save trainer if VE is best
if trainOpt['saveModel']:
self.save()
else:
nAfterBest += 1
# Stop training if above patience level
if nAfterBest > trainOpt['patience']:
print 'Patience level reached, early stop.'
print 'Will stop at score ', bestTscore
stop = True
else:
if trainOpt['saveModel']:
self.save()
if trainOpt.has_key('stopScore') and \
Tscore < trainOpt['stopScore']:
print 'Training score is lower than %.4f , ealy stop.' % \
trainOpt['stopScore']
stop = True
# Anneal learning rate
self.model.updateLearningParams(epoch)
# Print statistics
logger.logTrainStats()
if trainOpt['needValid']:
print 'BT: %.4f' % bestTscore
# Plot train curves
if trainOpt['plotFigs']:
plotter.plot()
# Terminate
if stop:
break
# Record final epoch number
self.stoppedTrainScore = bestTscore
self.stoppedEpoch = bestEpoch if trainOpt['needValid'] else epoch
def main(argv=None):
hyper_file = os.path.join(RESULTS_DIR, HYPER_FILE)
if (os.path.isfile(hyper_file)):
with open(hyper_file, 'r') as handle:
hyper = json.load(handle)
else:
hyper = {}
hyper['min_test_step'] = LAYERS_NUM
hyper['step'] = hyper['min_test_step']
hyper['unchanged'] = 0
hyper['restfile'] = RESTORING_FILE
while (hyper['unchanged'] < HYPER_PATIENCE):
results_dir = os.path.join(RESULTS_DIR, str(hyper['step']))
writer = Writer(results_dir)
trainer = Trainer(results_dir, 'train', writer, hyper['step'])
tester = Tester(results_dir, 'valid', writer, hyper['step'])
params_file = os.path.join(results_dir, PARAMS_FILE)
if (os.path.isfile(params_file)):
with open(params_file, 'r') as handle:
params = json.load(handle)
else:
params = {}
params['min_test_step'], params['min_test_loss'] = tester.test(EVAL_STEP_NUM)
params['step'] = params['min_test_step']
params['unchanged'] = 0
params['num_decays'] = 0
params['learning_rate'] = LEARNING_RATE
if ('min_test_loss' not in hyper):
hyper['min_test_loss'] = params['min_test_loss']
while (params['num_decays'] <= MAX_DECAYS):
params['step'], _ = trainer.train(params['learning_rate'], EVAL_FREQUENCY,
params['step'], hyper['restfile'])
_, test_loss = tester.test(EVAL_STEP_NUM, params['step'])
if (test_loss < params['min_test_loss']):
params['min_test_loss'] = test_loss
params['min_test_step'] = params['step']
params['unchanged'] = 0
else:
params['unchanged'] += EVAL_FREQUENCY
if (params['unchanged'] >= PATIENCE):
params['learning_rate'] *= DECAY_FACTOR
params['num_decays'] += 1
params['step'] = params['min_test_step']
params['unchanged'] = 0
with open(params_file, 'w') as handle:
json.dump(params, handle, indent=2)
print(params)
#tester.test(step_num=None, init_step=params['min_test_step'])
if (params['min_test_loss'] < hyper['min_test_loss']):
hyper['min_test_loss'] = params['min_test_loss']
hyper['min_test_step'] = hyper['step']
hyper['unchanged'] = 0
else:
hyper['unchanged'] += 1
hyper['restfile'] = os.path.join(results_dir, model_file(params['min_test_step']))
hyper['step'] += 2
with open(hyper_file, 'w') as handle:
json.dump(hyper, handle, indent=2)
print(hyper)
print('\n NEW HYPER PARAMETER: %d' %hyper['step'])
def train(
self,
trainInput,
trainTarget,
trainInputWeights=None,
validInput=None,
validTarget=None,
validInputWeights=None):
self.initFolder()
trainOpt = self.trainOpt
if validInput is None and validTarget is None:
X, T, VX, VT = self.initData(\
trainInput, trainTarget, \
split=self.trainOpt['needValid'])
else:
X = trainInput
T = trainTarget
VX = validInput
VT = validTarget
N = X.shape[0]
print 'Epoch size:', N
numEpoch = trainOpt['numEpoch']
calcError = trainOpt['calcError']
numExPerBat = trainOpt['batchSize']
print 'Batch size:', numExPerBat
numBatPerStep = trainOpt['stepSize'] \
if trainOpt.has_key('stepSize') \
else int(np.ceil(N / float(numExPerBat)))
print 'Step size:', numBatPerStep
numExPerStep = numExPerBat * numBatPerStep \
if trainOpt.has_key('stepSize') \
else N
print 'Examples per step:', numExPerStep
numStepPerEpoch = int(np.ceil(
N / float(numExPerStep))) \
if trainOpt.has_key('stepSize') \
else 1
print 'Steps per epoch:', numStepPerEpoch
progressWriter = ProgressWriter(numExPerStep, width=80)
logger = Logger(self, csv=trainOpt['writeRecord'])
logger.logMsg('Trainer ' + self.name)
plotter = Plotter(self)
bestVscore = None
bestTscore = None
bestStep = 0
totalBat = 0
step = 0
totalStep = 0
nAfterBest = 0
stop = False
self.loss = np.zeros((numStepPerEpoch * numEpoch))
self.validLoss = np.zeros((numStepPerEpoch * numEpoch))
self.rate = np.zeros((numStepPerEpoch * numEpoch))
self.validRate = np.zeros((numStepPerEpoch * numEpoch))
# Train loop through epochs
for epoch in range(0, numEpoch):
self.epoch = epoch
epochE = 0
epochCorrect = 0
epochTotal = 0
# Shuffle data
if trainOpt['shuffle']:
X, T = vt.shuffleData(X, T, self.random)
# Every step, validate
for step in range(0, numStepPerEpoch):
stepStart = step * numExPerStep
stepEnd = min((step + 1) * numExPerStep, N)
numExThisStep = stepEnd - stepStart
E = 0
correct = 0
total = 0
self.totalStep = totalStep
# Every batch forward-backward
for batch in range(0, numBatPerStep):
batchStart = stepStart + batch * numExPerBat
if batchStart > N:
break
batchEnd = min(
stepStart + (batch + 1) * numExPerBat, stepEnd)
numExThisBat = batchEnd - batchStart
self.totalBatch = totalBat
if trainOpt['progress']:
progressWriter.increment(amount=numExThisBat)
# Forward
Y_bat = self.model.forward(
X[batchStart:batchEnd], dropout=True)
T_bat = T[batchStart:batchEnd]
# Loss
Etmp, dEdY = self.model.getCost(
Y_bat, T_bat, weights=trainInputWeights)
E += Etmp * numExThisBat / float(numExThisStep)
epochE += Etmp * numExThisBat / float(N)
# Backward
self.model.backward(dEdY)
# Update
self.model.updateWeights()
# Prediction error
if calcError:
rate_, correct_, total_ = \
tester.calcRate(self.model, Y_bat, T_bat)
correct += correct_
total += total_
epochCorrect += correct_
epochTotal += total_
totalBat += 1
# Store train statistics
if calcError:
rate = correct / float(total)
self.rate[totalStep] = rate
self.loss[totalStep] = E
# Early stop
if not trainOpt.has_key('criterion'):
Tscore = E
else:
if trainOpt['criterion'] == 'loss':
Tscore = E
elif trainOpt['criterion'] == 'rate':
Tscore = 1 - rate
else:
raise Exception('Unknown stopping criterion "%s"' % \
trainOpt['criterion'])
# Run validation
if trainOpt['needValid']:
VY = tester.test(self.model, VX)
VE, dVE = self.model.getCost(
VY, VT, weights=validInputWeights)
self.validLoss[totalStep] = VE
if calcError:
Vrate, correct, total = tester.calcRate(
self.model, VY, VT)
self.validRate[totalStep] = Vrate
# Check stopping criterion
if not trainOpt.has_key('criterion'):
Vscore = VE
else:
if trainOpt['criterion'] == 'loss':
Vscore = VE
elif trainOpt['criterion'] == 'rate':
Vscore = 1 - Vrate
else:
raise Exception(
'Unknown stopping criterion "%s"' % \
trainOpt['criterion'])
if (bestVscore is None) or (Vscore < bestVscore):
bestVscore = Vscore
bestTscore = Tscore
nAfterBest = 0
bestStep = totalStep
# Save trainer if VE is best
if trainOpt['saveModel']:
self.save()
else:
nAfterBest += 1
# Stop training if above patience level
if nAfterBest > trainOpt['patience']:
print 'Patience level reached, early stop.'
print 'Will stop at score ', bestTscore
stop = True
else:
if trainOpt['saveModel']:
self.save()
if trainOpt.has_key('stopScore') and \
Tscore < trainOpt['stopScore']:
print \
'Training score is lower than %.4f , ealy stop.' % \
trainOpt['stopScore']
stop = True
logger.logTrainStats()
if trainOpt['needValid']:
print 'P: %d' % nAfterBest,
print self.name
if stop:
break
# Store train statistics
if calcError:
epochRate = epochCorrect / float(epochTotal)
print 'Epoch Final: %d TE: %.4f TR:%.4f' % \
(epoch, epochE, epochRate)
# Anneal learning rate
self.model.updateLearningParams(epoch)
# Plot train curves
if trainOpt['plotFigs']:
plotter.plot()
# Terminate
if stop:
break
# Report best train score
self.stoppedTrainScore = bestTscore
def test_missing_is_stale():
import os, dagger
os.system('touch tmp1')
n1 = dagger.node('tmp1')
n1.update()
n2 = dagger.node('tmp_missing')
n2.update()
return n1.stale == None and n2.stale
#############################################
tests = [
test_add,
test_format_abs,
test_format_date,
test_format_time,
test_format_base,
test_format_base_dot,
test_format_root,
test_format_root_dot,
test_update_time,
test_update_hash,
test_missing_is_stale,
]
from tester import test
import sys
sys.exit( not test(tests=tests) )
def test(args):
import tester
tester.test()
from tester import test
# D. sort_last
# Given a list of non-empty tuples, return a list sorted in increasing
# order by the last element in each tuple.
# e.g. [(1, 7), (1, 3), (3, 4, 5), (2, 2)] yields
# [(2, 2), (1, 3), (3, 4, 5), (1, 7)]
# Hint: use a custom key= function to extract the last element form each tuple.
def sort_last(tuples):
# +++your code here+++
return
test(sort_last([(1, 3), (3, 2), (2, 1)]),
[(2, 1), (3, 2), (1, 3)])
test(sort_last([(2, 3), (1, 2), (3, 1)]),
[(3, 1), (1, 2), (2, 3)])
test(sort_last([(1, 7), (1, 3), (3, 4, 5), (2, 2)]),
[(2, 2), (1, 3), (3, 4, 5), (1, 7)])
# E. Given two lists sorted in increasing order, create and return a merged
# list of all the elements in sorted order. You may modify the passed in lists.
# Ideally, the solution should work in "linear" time, making a single
# pass of both lists.
def linear_merge(list1, list2):
# +++your code here+++
return
from tester import test
# E. verbing
# Given a string, if its length is at least 3,
# add 'ing' to its end.
# Unless it already ends in 'ing', in which case
# add 'ly' instead.
# If the string length is less than 3, leave it unchanged.
# Return the resulting string.
def verbing(s):
# +++your code here+++
return
test(verbing('hail'), 'hailing')
test(verbing('swiming'), 'swimingly')
test(verbing('do'), 'do')
# F. not_bad
# Given a string, find the first appearance of the
# substring 'not' and 'bad'. If the 'bad' follows
# the 'not', replace the whole 'not'...'bad' substring
# with 'good'.
# Return the resulting string.
# So 'This dinner is not that bad!' yields:
# This dinner is good!
def not_bad(s):
# +++your code here+++
return
from tester import test # A. donuts # Given an int count of a number of donuts, return a string # of the form 'Number of donuts: <count>', where <count> is the number # passed in. However, if the count is 10 or more, then use the word 'many' # instead of the actual count. # So donuts(5) returns 'Number of donuts: 5' # and donuts(23) returns 'Number of donuts: many' def donuts(count): # +++your code here+++ return test(donuts(4), 'Number of donuts: 4') test(donuts(9), 'Number of donuts: 9') test(donuts(10), 'Number of donuts: many') test(donuts(99), 'Number of donuts: many') # B. both_ends # Given a string s, return a string made of the first 2 # and the last 2 chars of the original string, # so 'spring' yields 'spng'. However, if the string length # is less than 2, return instead the empty string. def both_ends(s): # +++your code here+++ return
delete('missing')
try: os.remove('6')
except: pass
d = dagger.dagger()
d.add('1', ['2','3'])
d.add('2', ['missing'])
d.add('4', ['5'])
d.add('5', ['6'])
d.run(allpaths=True)
it = d.iter(['4'])
return it.next() == ['4']
#############################################
tests = [
test_iter,
test_iterator_all,
test_iterator_names,
test_missing,
test_top_fresh,
test_top_stale,
]
from tester import test
import sys
if __name__=='__main__':
test(tests=tests)
# sys.exit( not test(tests=tests) )
from tester import test # A. match_ends # Given a list of strings, return the count of the number of # strings where the string length is 2 or more and the first # and last chars of the string are the same. # Note: python does not have a ++ operator, but += works. def match_ends(words): # +++your code here+++ return test(match_ends(['aba', 'xyz', 'aa', 'x', 'bbb']), 3) test(match_ends(['', 'x', 'xy', 'xyx', 'xx']), 2) test(match_ends(['aaa', 'be', 'abc', 'hello']), 1) # B. front_x # Given a list of strings, return a list with the strings # in sorted order, except group all the strings that begin with 'x' first. # e.g. ['mix', 'xyz', 'apple', 'xanadu', 'aardvark'] yields # ['xanadu', 'xyz', 'aardvark', 'apple', 'mix'] # Hint: this can be done by making 2 lists and sorting each of them # before combining them. def front_x(words): # +++your code here+++ return test(front_x(['bbb', 'ccc', 'axx', 'xzz', 'xaa']),
def OnCheckPSP(self, event):
"Find defects and errors, if complete, change to the next phase"
evt_id = event.GetId()
if evt_id == ID_COMPILE:
self.SetPSPPhase('compile')
elif evt_id == ID_TEST:
self.SetPSPPhase('test')
if self.active_child:
phase = self.GetPSPPhase()
defects = [] # static checks and failed tests
errors = [] # sanity checks (planning & postmortem)
if phase == "planning":
# check plan summary completeness
for phase, times in self.psptimetable.cells.items():
if not times['plan']:
errors.append("Complete %s estimate time!" % phase)
elif phase == "design" or phase == "code":
#TODO: review checklist?
pass
elif phase == "compile":
# run "static" chekers to find coding defects (pep8, pyflakes)
import checker
defects.extend(checker.check(self.active_child.GetFilename()))
elif phase == "test":
# run doctests to find defects
import tester
defects.extend(tester.test(self.active_child.GetFilename()))
elif phase == "postmortem":
# check that all defects are fixed
for defect in self.psp_defect_list.data.values():
if not defect['remove_phase']:
errors.append("Defect %(number)s not fixed!" % defect)
# add found defects (highlight them in the editor window)
line_numbers = set()
for defect in defects:
self.NotifyDefect(**defect)
errors.append("Defect found: %(summary)s" % defect)
if defect['lineno'] is not None:
line_numbers.add(defect['lineno'])
self.active_child.HighlightLines(line_numbers)
# show errors
if errors:
dlg = wx.MessageDialog(self, "\n".join(errors),
"PSP Check Phase Errors", wx.ICON_EXCLAMATION | wx.OK)
dlg.ShowModal()
dlg.Destroy()
self._mgr.GetPane("psp_defects").Show(True)
self._mgr.Update()
# phase completed? project completed?
if not defects and not errors:
i = PSP_PHASES.index(phase) + 1
if i < len(PSP_PHASES):
phase = PSP_PHASES[i]
else:
phase = ""
self.OnStopPSP(event)
self.SetPSPPhase(phase)
else:
dlg = wx.MessageDialog(self, "No active file, cannot check it.\n"
"Change PSP phase manually if desired.",
"PSP Check Phase Errors", wx.ICON_EXCLAMATION)
dlg.ShowModal()
dlg.Destroy()
