def benchmark_torch(model_name: str, seq_len: int, batch_size: int, n: int,
num_threads: int):
import torch
import transformers
import contexttimer
import benchmark_helper
torch.set_num_threads(num_threads)
torch.set_grad_enabled(False)
if model_name == "bert":
cfg = transformers.BertConfig()
model = transformers.BertModel(cfg)
elif model_name == "albert":
cfg = transformers.AlbertConfig()
model = transformers.AlbertModel(cfg)
elif model_name == "roberta":
cfg = transformers.RobertaConfig()
model = transformers.RobertaModel(cfg)
else:
raise (f"benchmark does not support {model_name}")
model.eval()
cfg = model.config # type: transformers.BertConfig
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long)
benchmark_helper.run_model(lambda: model(input_ids), False, n, batch_size,
seq_len, "torch", num_threads)
def benchmark_torch(model: str, seq_len: int, batch_size: int, n: int):
import torch
import transformers
import contexttimer
import benchmark_helper
if not torch.cuda.is_available():
print("cuda is not available for torch")
return
test_device = torch.device('cuda:0')
torch.set_grad_enabled(False)
model_id = "bert-base-uncased"
model = transformers.BertModel.from_pretrained(
model_id) # type: transformers.BertModel
model.eval()
model.to(test_device)
cfg = model.config # type: transformers.BertConfig
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long,
device=test_device)
benchmark_helper.run_model(lambda: model(input_ids), True, n, batch_size,
seq_len, "torch")
def benchmark_turbo_transformers(model_name: str, seq_len: int,
batch_size: int, n: int, enable_random: bool,
max_seq_len: int, min_seq_len: int,
num_threads: int, use_gpu: bool,
enable_mem_opt: bool):
import torch
import transformers
import turbo_transformers
import benchmark_helper
test_device = torch.device('cuda:0') if use_gpu else torch.device('cpu:0')
cfg = None
torch.set_grad_enabled(False)
if model_name == "bert":
cfg = transformers.BertConfig()
model = transformers.BertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.BertModel.from_torch(model, backend="turbo")
elif model_name == "albert":
cfg = transformers.AlbertConfig(hidden_size=768,
num_attention_heads=12,
intermediate_size=3072)
model = transformers.AlbertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.AlbertModel.from_torch(model)
elif model_name == "roberta":
cfg = transformers.RobertaConfig()
model = transformers.RobertaModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.RobertaModel.from_torch(model)
elif model_name == "distilbert":
cfg = transformers.DistilBertConfig()
model = transformers.DistilBertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.DistilBertModel.from_torch(model)
else:
raise (f"benchmark does not support {model_name}")
turbo_transformers.set_num_threads(num_threads)
if enable_random:
if enable_mem_opt:
turbo_transformers.reset_allocator_schema("model-aware")
benchmark_helper.run_variable_model(model, use_gpu, n, max_seq_len,
min_seq_len, "turbo", num_threads,
cfg, enable_mem_opt, model_name)
if enable_mem_opt:
turbo_transformers.reset_allocator_schema("naive")
else:
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long,
device=test_device)
benchmark_helper.run_model(lambda: model(input_ids), use_gpu, n,
batch_size, seq_len, "turbo", num_threads,
enable_mem_opt, model_name)
def benchmark_turbo_transformers(model_name: str, seq_len: int,
batch_size: int, n: int, enable_random: bool,
max_seq_len: int, min_seq_len: int,
num_threads: int, use_gpu: bool):
import torch
import transformers
import contexttimer
import turbo_transformers
import benchmark_helper
test_device = torch.device('cuda:0') if use_gpu else torch.device('cpu:0')
if use_gpu:
print("using GPU")
else:
print("using CPU")
cfg = None
torch.set_grad_enabled(False)
if model_name == "bert":
cfg = transformers.BertConfig()
model = transformers.BertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.BertModel.from_torch(model)
elif model_name == "albert":
cfg = transformers.AlbertConfig()
model = transformers.AlbertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.AlbertModel.from_torch(model)
elif model_name == "roberta":
cfg = transformers.RobertaConfig()
model = transformers.RobertaModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.RobertaModel.from_torch(model)
else:
raise (f"benchmark does not support {model_name}")
turbo_transformers.set_num_threads(num_threads)
if enable_random:
benchmark_helper.run_variable_model(model, use_gpu, n, max_seq_len,
min_seq_len, "turbo", num_threads,
cfg)
else:
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long,
device=test_device)
benchmark_helper.run_model(lambda: model(input_ids), use_gpu, n,
batch_size, seq_len, "turbo", num_threads)
def benchmark_turbo_transformers(model_name: str, seq_len: int,
batch_size: int, n: int):
import torch
import transformers
import contexttimer
import turbo_transformers
import benchmark_helper
if not torch.cuda.is_available():
print("cuda is not available for torch")
return
test_device = torch.device('cuda:0')
if model_name == "bert":
cfg = transformers.BertConfig()
model = transformers.BertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.BertModel.from_torch(model)
elif model_name == "albert":
cfg = transformers.AlbertConfig()
model = transformers.AlbertModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.AlbertModel.from_torch(model)
elif model_name == "roberta":
cfg = transformers.RobertaConfig()
model = transformers.RobertaModel(cfg)
model.to(test_device)
model.eval()
model = turbo_transformers.RobertaModel.from_torch(model)
else:
raise (f"benchmark does not support {model_name}")
cfg = model.config # type: transformers.BertConfig
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long,
device=test_device)
benchmark_helper.run_model(lambda: model(input_ids), True, n, batch_size,
seq_len, "turbo")
def benchmark_torch(model_name: str, seq_len: int, batch_size: int, n: int,
enable_random: bool, max_seq_len: int, min_seq_len: int,
num_threads: int, use_gpu: bool, enable_mem_opt: bool):
import torch
import transformers
import benchmark_helper
test_device = torch.device('cuda:0') if use_gpu else torch.device('cpu:0')
torch.set_grad_enabled(False)
torch.set_num_threads(num_threads)
cfg = None
if model_name == "bert":
cfg = transformers.BertConfig()
model = transformers.BertModel(cfg)
elif model_name == "albert":
cfg = transformers.AlbertConfig()
model = transformers.AlbertModel(cfg)
elif model_name == "roberta":
cfg = transformers.RobertaConfig()
model = transformers.RobertaModel(cfg)
elif model_name == "distilbert":
cfg = transformers.DistilBertConfig()
model = transformers.DistilBertModel(cfg)
else:
raise (f"benchmark does not support {model_name}")
model.eval()
model.to(test_device)
# cfg = model.config # type: transformers.BertConfig
if enable_random:
benchmark_helper.run_variable_model(model, use_gpu, n, max_seq_len,
min_seq_len, "torch", num_threads,
cfg, enable_mem_opt, model_name)
else:
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long,
device=test_device)
benchmark_helper.run_model(lambda: model(input_ids), use_gpu, n,
batch_size, seq_len, "torch", num_threads,
enable_mem_opt, model_name)
def benchmark_torch(model: str, seq_len: int, batch_size: int, n: int,
num_threads: int):
import torch
import transformers
import contexttimer
import benchmark_helper
torch.set_num_threads(num_threads)
torch.set_grad_enabled(False)
model_id = "bert-base-uncased"
model = transformers.BertModel.from_pretrained(
model_id) # type: transformers.BertModel
model.eval()
cfg = model.config # type: transformers.BertConfig
input_ids = torch.randint(low=0,
high=cfg.vocab_size - 1,
size=(batch_size, seq_len),
dtype=torch.long)
benchmark_helper.run_model(lambda: model(input_ids), False, n, batch_size,
seq_len, "torch", num_threads)
