def run_benchmark(name, arguments_list, precision, num_runs, platform, device):
binary = "./clblast_client_x" + name
# Loops over sub-benchmarks per benchmark
results = []
for arguments in arguments_list:
# Sets the arguments
constant_arguments = ["-warm_up", "-q", "-no_abbrv", "-cblas 0", "-cublas 0"]
common_arguments = ["-precision %d" % precision, "-runs %d" % num_runs]
opencl_arguments = ["-platform %d" % platform, "-device %d" % device]
all_arguments = opencl_arguments + common_arguments + constant_arguments
for name, value in arguments.items():
all_arguments.append("-" + name + " " + str(value))
# Calls the binary and parses the results
benchmark_output = utils.run_binary(binary, all_arguments)
result = utils.parse_results(benchmark_output)
# For half-precision: also runs single-precision for comparison
if precision == 16:
all_arguments = [arg if arg != "-precision 16" else "-precision 32" for arg in all_arguments]
benchmark_output = utils.run_binary(binary, all_arguments)
result_extra = utils.parse_results(benchmark_output)
for index in range(len(min(result, result_extra))):
result[index]["GBs_1_FP32"] = result_extra[index]["GBs_1"]
result[index]["GBs_2"] = result_extra[index]["GBs_2"]
result[index]["GFLOPS_1_FP32"] = result_extra[index]["GFLOPS_1"]
result[index]["GFLOPS_2"] = result_extra[index]["GFLOPS_2"]
results.extend(result)
return results
def main(args):
expectations, input_files, columns = pre_process_args(args)
successful_dest_folder = args.successful_dest_folder
failed_dest_folder = args.failed_dest_folder
files_handler = FilesHandler(input_files=input_files)
files_path = files_handler.get_files_path()
validator = FileValidator(columns=columns, expectations=expectations)
logging.info("Starting the validation files process.. ")
p = Pool(processes=cpu_count())
r = list(
tqdm(p.imap(validator.validate, files_path), total=len(files_path)))
p.close()
p.join()
sucessful_expectations, failed_expectations = parse_results(r)
write_results(sucessful_expectations, success=True)
write_results(failed_expectations, success=False)
if (successful_dest_folder != None):
for expectation in sucessful_expectations:
file_name = expectation["file_path"].split("/")[-1]
files_handler.move_file(file_name, successful_dest_folder)
if (failed_dest_folder != None):
for expectation in failed_expectations:
file_name = expectation["file_path"].split("/")[-1]
files_handler.move_file(file_name, failed_dest_folder)
if (files_handler.is_from_storage):
files_handler.clean_temp_folder()
def test_run_axe(self, base_url, selenium):
"""Run axe against base_url and verify JSON output."""
selenium.get(base_url)
# Include axe-core API
selenium.execute_script(open('./src/axe.min.js').read())
# Script to run axe API against page
selenium.execute_script(open('./src/script.js').read())
# Delay while JavaScript runs
time.sleep(1)
# Get JSON results
result = selenium.find_element_by_id('axe-result').text
# --- Is there a better way to pass data from JS to python? --- #
# Parse JSON
parsed = json.loads(result)
global test_results
test_results = utils.parse_results(parsed['violations'])
# Create file to JSON response from axe
file = open('result.json', 'w+')
# And pretty-print to file
file.write(json.dumps(parsed['violations'], indent=4))
file.close
assert len(test_results) != 0, test_results
def do_POST(self):
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length)
is_valid = self.check_request_data(post_data, self.headers)
if not is_valid:
self.send_error(400, 'Invalid message body')
else:
try:
post_data = post_data.split('\r\n')
test_results = parse_results(post_data)
if test_results: PatchServerHandler.update(test_results, self.headers)
self._set_headers()
except KeyError:
self.send_error('400', 'Invalid message structure')
def parse(self):
tested_patches = os.listdir(self.builds_path)
logger.info('Parsing results for the following patches %s' % tested_patches)
result_dict = {}
for log_folder in os.listdir(self.results_path):
log_path = os.path.join(self.results_path, log_folder)
with open('%s/ica.log' % log_path, 'r') as log_content:
logger.info('Parsing logs from %s' % log_folder)
results = parse_results(log_content, tested_patches)
for patch_name, result in results.items():
if result != 'Passed':
logger.warning('%s failed on %s' % (patch_name, log_folder))
build_path = os.path.join(self.builds_path, patch_name)
move(build_path, self.failures_path)
def parse(self):
tested_patches = os.listdir(self.builds_path)
logger.info('Parsing results for the following patches %s' % tested_patches)
result_dict = {}
for log_folder in os.listdir(self.results_path):
log_path = os.path.join(self.results_path, log_folder)
with open('%s/ica.log' % log_path, 'r') as log_content:
logger.info('Parsing logs from %s' % log_folder)
results = parse_results(log_content, tested_patches)
for patch_name, result in results.items():
if result != 'Passed':
logger.warning('%s failed on %s' % (patch_name, log_folder))
build_path = os.path.join(self.builds_path, patch_name)
move(build_path, self.failures_path)
def do_POST(self):
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length)
is_valid = self.check_request_data(post_data, self.headers)
if not is_valid:
self.send_error(400, 'Invalid message body')
else:
try:
post_data = post_data.split('\r\n')
test_results = parse_results(post_data)
if test_results:
PatchServerHandler.update(test_results, self.headers)
self._set_headers()
except KeyError:
self.send_error('400', 'Invalid message structure')
def main():
config = RetrainConfig()
main_proc = not config.distributed or config.local_rank == 0
if config.distributed:
torch.cuda.set_device(config.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method=config.dist_url,
rank=config.local_rank,
world_size=config.world_size)
if main_proc:
os.makedirs(config.output_path, exist_ok=True)
if config.distributed:
torch.distributed.barrier()
logger = utils.get_logger(os.path.join(config.output_path, 'search.log'))
if main_proc:
config.print_params(logger.info)
utils.reset_seed(config.seed)
loaders, samplers = get_augment_datasets(config)
train_loader, valid_loader = loaders
train_sampler, valid_sampler = samplers
model = Model(config.dataset,
config.layers,
in_channels=config.input_channels,
channels=config.init_channels,
retrain=True).cuda()
if config.label_smooth > 0:
criterion = utils.CrossEntropyLabelSmooth(config.n_classes,
config.label_smooth)
else:
criterion = nn.CrossEntropyLoss()
fixed_arc_path = os.path.join(config.output_path, config.arc_checkpoint)
with open(fixed_arc_path, "r") as f:
fixed_arc = json.load(f)
fixed_arc = utils.encode_tensor(fixed_arc, torch.device("cuda"))
genotypes = utils.parse_results(fixed_arc, n_nodes=4)
genotypes_dict = {i: genotypes for i in range(3)}
apply_fixed_architecture(model, fixed_arc_path)
param_size = utils.param_size(
model, criterion,
[3, 32, 32] if 'cifar' in config.dataset else [3, 224, 224])
if main_proc:
logger.info("Param size: %.6f", param_size)
logger.info("Genotype: %s", genotypes)
# change training hyper parameters according to cell type
if 'cifar' in config.dataset:
if param_size < 3.0:
config.weight_decay = 3e-4
config.drop_path_prob = 0.2
elif 3.0 < param_size < 3.5:
config.weight_decay = 3e-4
config.drop_path_prob = 0.3
else:
config.weight_decay = 5e-4
config.drop_path_prob = 0.3
if config.distributed:
apex.parallel.convert_syncbn_model(model)
model = DistributedDataParallel(model, delay_allreduce=True)
optimizer = torch.optim.SGD(model.parameters(),
config.lr,
momentum=config.momentum,
weight_decay=config.weight_decay)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
config.epochs,
eta_min=1E-6)
best_top1 = best_top5 = 0.
for epoch in range(config.epochs):
drop_prob = config.drop_path_prob * epoch / config.epochs
if config.distributed:
model.module.drop_path_prob(drop_prob)
else:
model.drop_path_prob(drop_prob)
# training
if config.distributed:
train_sampler.set_epoch(epoch)
train(logger, config, train_loader, model, optimizer, criterion, epoch,
main_proc)
# validation
top1, top5 = validate(logger, config, valid_loader, model, criterion,
epoch, main_proc)
best_top1 = max(best_top1, top1)
best_top5 = max(best_top5, top5)
lr_scheduler.step()
logger.info("Final best Prec@1 = %.4f Prec@5 = %.4f", best_top1, best_top5)
def main():
config = RetrainConfig()
main_proc = not config.distributed or config.local_rank == 0
if config.distributed:
torch.cuda.set_device(config.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method=config.dist_url,
rank=config.local_rank,
world_size=config.world_size)
if main_proc:
os.makedirs(config.output_path, exist_ok=True)
if config.distributed:
torch.distributed.barrier()
logger = utils.get_logger(os.path.join(config.output_path, 'search.log'))
if main_proc:
config.print_params(logger.info)
utils.reset_seed(config.seed)
loaders, samplers = get_augment_datasets(config)
train_loader, valid_loader = loaders
train_sampler, valid_sampler = samplers
train_loader = CyclicIterator(train_loader, train_sampler)
# valid_loader = CyclicIterator(valid_loader, valid_sampler, False)
model = Model(config.dataset,
config.layers,
in_channels=config.input_channels,
channels=config.init_channels,
retrain=True).cuda()
if config.label_smooth > 0:
criterion = utils.CrossEntropyLabelSmooth(config.n_classes,
config.label_smooth)
else:
criterion = nn.CrossEntropyLoss()
fixed_arc_path = os.path.join('', config.arc_checkpoint)
with open(fixed_arc_path, "r") as f:
fixed_arc = json.load(f)
fixed_arc = utils.encode_tensor(fixed_arc, torch.device("cuda"))
genotypes = utils.parse_results(fixed_arc, n_nodes=4)
genotypes_dict = {i: genotypes for i in range(3)}
apply_fixed_architecture(model, fixed_arc_path)
param_size = utils.param_size(model, criterion, [3, 512, 512])
if main_proc:
logger.info("Param size: %.6f", param_size)
logger.info("Genotype: %s", genotypes)
# change training hyper parameters according to cell type
if 'cifar' in config.dataset:
if param_size < 3.0:
config.weight_decay = 3e-4
config.drop_path_prob = 0.2
elif 3.0 < param_size < 3.5:
config.weight_decay = 3e-4
config.drop_path_prob = 0.3
else:
config.weight_decay = 5e-4
config.drop_path_prob = 0.3
if config.distributed:
apex.parallel.convert_syncbn_model(model)
model = DistributedDataParallel(model, delay_allreduce=True)
optimizer = torch.optim.AdamW(model.parameters(), config.lr)
# lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, config.epochs, eta_min=1E-6)
best_top1 = 0.
epoch = 0
try:
checkpoint = torch.load(config.model_checkpoint)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']
model.eval()
print("----------------------------")
print("MODEL LOADED FROM CHECKPOINT" + config.model_checkpoint)
print("----------------------------")
except:
print("----------------------------")
print("MODEL NOT LOADED FROM CHECKPOINT")
print("----------------------------")
pass
# for epoch in range(0, epoch):
# lr_scheduler.step()
for epoch in range(epoch, config.epochs):
drop_prob = config.drop_path_prob * epoch / config.epochs
if config.distributed:
model.module.drop_path_prob(drop_prob)
else:
model.drop_path_prob(drop_prob)
# training
if config.distributed:
train_sampler.set_epoch(epoch)
train(logger, config, train_loader, model, optimizer, criterion, epoch,
main_proc)
if (epoch % config.log_frequency == 0):
# validation
top1 = validate(logger, config, valid_loader, model, criterion,
epoch, main_proc)
best_top1 = max(best_top1, top1)
# lr_scheduler.step()
logger.info("Final best Prec@1 = %.4f", best_top1)
def plot_genotype(self, results, logger):
genotypes = parse_results(results, self.n_nodes)
logger.info(genotypes)
return genotypes
def test_parse_results_invalid(invalid_data_frame):
invalid_df, cols = invalid_data_frame
try:
result = parse_results(invalid_df)
except Exception as e:
assert type(e) == KeyError
def test_parse_results_valid(valid_data_frame):
result = parse_results(valid_data_frame)
num_cols = len(result[0])
num_rows = len(result)
assert valid_data_frame.shape == (num_rows, num_cols)
