def validate_checkpoints(self_args, args):
set_library_seeds(args.seed)
config = bert_config_from_args(args)
checkpoint_paths = [p for p in Path(
args.checkpoint_dir).rglob(self_args.model_search_string)]
if len(checkpoint_paths) < 1:
raise FileNotFoundError(
f"Did not recursively find any checkpoints at path: {args.checkpoint_dir}")
# Load an initial model to configure the IO tensors and the session
args.onnx_checkpoint = checkpoint_paths[0]
initializers = bert_pretrained_initialisers(config, args)
results = perform_validations(self_args.num_processes,
checkpoint_paths,
args,
config,
initializers,
self_args.available_ipus)
with open(os.path.join(self_args.checkpoint_dir, "validation_result.json"), 'w') as fh:
json.dump(results, fh, indent=4)
return results
def test_activation_function(mode, phase, momentum, micro_batch_size,
batch_serialization_factor):
set_library_seeds(0)
popart_act_function, pytorch_activation = ACTIVATIONS["Gelu"]
config = BertConfig(vocab_length=128,
micro_batch_size=micro_batch_size,
hidden_size=768,
sequence_length=128,
popart_dtype="FLOAT",
no_dropout=True,
activation_type=str(popart_act_function))
data, outputs, proto, post_proto = popart_result_and_model(
config,
mode,
batch_serialization_factor,
is_bwd=False if phase is 'fwd' else True,
momentum=momentum)
inputs = [
data.reshape(config.micro_batch_size, config.sequence_length,
config.hidden_size)
]
# ------------------- PyTorch -------------------------
torch_config = TorchBertConfig(config.vocab_length,
config.hidden_size,
config.num_layers,
config.attention_heads,
layer_norm_eps=config.layer_norm_eps,
hidden_dropout_prob=0.,
hidden_act=pytorch_activation)
torch_output, torch_model = pytorch_result_and_model(
torch_config,
inputs,
proto,
mode,
is_bwd=False if phase is 'fwd' else True,
momentum=momentum)
check_tensors(torch_output, outputs, margin=7e-6)
if phase is 'bwd':
check_model(torch_model,
post_proto,
TORCH_TO_ONNX[mode],
transform=TRANSPOSE_WEIGHTS,
margin=7e-6)
def finetune_checkpoints(self_args, args):
set_library_seeds(args.seed)
config = bert_config_from_args(args)
checkpoint_paths = glob.glob(
os.path.join(self_args.checkpoint_dir, self_args.model_search_string))
checkpoint_paths.sort()
os.makedirs(os.path.join(self_args.checkpoint_dir, "squad_output"),
exist_ok=True)
if len(checkpoint_paths) < 1:
raise FileNotFoundError(
f"Did not find any checkpoints at path: {args.checkpoint_dir}")
# Load an initial model to configure the IO tensors and the session
args.onnx_checkpoint = checkpoint_paths[0]
initializers = bert_pretrained_initialisers(config, args)
training_run(args, config, initializers, checkpoint_paths)
logger.info("Fine-Tuning Complete")
def run_embedding_layer(args):
set_library_seeds(args.seed)
config = bert_config_from_args(args)
initializers = bert_pretrained_initialisers(config, args)
logger.info("Building Model")
# Specifying ai.onnx opset9 for the slice syntax
# TODO: Change slice to opset10
model = Bert(config,
builder=popart.Builder(opsets={
"ai.onnx": 9,
"ai.onnx.ml": 1,
"ai.graphcore": 1
}),
initializers=initializers,
execution_mode=args.execution_mode)
# If config.host_embedding is enabled, indices and positions will have the matrices instead of the index vector.
indices, positions, segments, masks, labels = bert_add_inputs(args, model)
logits = tuple([model.embedding(indices, positions, segments)])
if args.inference:
outputs = bert_add_logit_outputs(model, logits)
writer = None
dataset = get_bert_dataset(
model, args, [indices, positions, segments, masks, labels])
data_flow = popart.DataFlow(dataset.batches_per_step, outputs)
iteration = Iteration(
args,
steps_per_epoch=len(dataset),
writer=writer,
recording_steps=args.aggregate_metrics_over_steps)
request_ipus = bert_required_ipus(args, model)
device = acquire_device(args, request_ipus)
session, anchors = bert_inference_session(model, args, data_flow,
device)
logger.info("Inference Started")
inputs = [indices, positions, segments, *masks]
"""bert_infer_loop(args, session,
dataset, inputs, logits, anchors,
iteration)"""
save_results = args.task == "SQUAD" and not (args.synthetic_data
or args.generated_data)
start_times = defaultdict(list)
end_times = defaultdict(list)
# Create the stepio once outside of the inference loop:
static_data = {}
if args.low_latency_inference and args.task == "SQUAD":
stepio = create_callback_stepio(static_data, anchors, start_times,
end_times,
dataset.batches_per_step)
else:
stepio = None
output = []
logger.info(dataset)
for data in dataset:
static_data.update({t: data[t] for t in inputs})
result = bert_process_infer_data(args, session, static_data,
anchors, logits, iteration,
start_times, end_times, stepio)
if save_results:
output.append(result)
break
device.detach()
return output
return None
