def run():
'''
Run benchmark.
'''
tf.enable_eager_execution()
file_mgr = FileManager()
results = []
with tf.device('/cpu:0'), file_mgr.open_training(MODEL_NAME) as file:
# Load and process batch.
X = [None] * BATCH_SIZE
y = [None] * BATCH_SIZE
file_iter = iter(file)
for i in range(BATCH_SIZE):
X[i], y[i] = ujson.loads(next(file_iter))
assert len(X) == BATCH_SIZE
assert len(y) == BATCH_SIZE
assert list(filter(lambda x: x is None, X)) == []
assert list(filter(lambda x: x is None, y)) == []
# Convert each pair of lists to tensors.
stack_times = []
concat_times = []
convert_times = []
# Hack: the first call (stack/concat/convert) runs significantly slower, resulting in imbalanced results. We try
# to fix this by running an unprofiled stack(). This does seem to make the subsequent calls perform more evenly.
# Most likely, the first call causes TF to initialise the GPU, so by calling stack() we are avoiding this.
tf.stack(X)
tf.stack(y)
for _ in range(BENCH_ITER):
# Profile tf.stack().
with prof(None, resources=['time'], log_level=None) as p:
tf.stack(X)
tf.stack(y)
time = p.profilers['time'].compute_delta()
time = round(time * 1000)
stack_times.append(time)
# Profile tf.concat().
with prof(None, resources=['time'], log_level=None) as p:
tf.concat(X, axis=0)
tf.concat(y, axis=0)
time = p.profilers['time'].compute_delta()
time = round(time * 1000)
concat_times.append(time)
# Profile tf.convert_to_tensor().
with prof(None, resources=['time'], log_level=None) as p:
tf.convert_to_tensor(X)
tf.convert_to_tensor(y)
time = p.profilers['time'].compute_delta()
time = round(time * 1000)
convert_times.append(time)
results.append(BenchmarkResult('stack', stack_times))
results.append(BenchmarkResult('concat', concat_times))
results.append(BenchmarkResult('convert', convert_times))
return results
def bench_with_module(module, file):
'''
Benchmark ujson or json module.
'''
#file.seek(0)
count = 0
with prof(None, resources=['time','memory'], log_level=None) as p:
for line in file:
module.loads(line)
if count >= MAX_RECORDS:
break
count += 1
return p.profilers['time'].compute_delta()
def _train_fold(X, y, params, train, test, fold_num):
random.seed(1)
np.random.seed(1)
tf.set_random_seed(1)
acc = -np.inf
loss = np.inf
with prof('Fold {}/{}: acc={}%, loss={}',
fold_num,
params['kfolds'],
lambda: acc * 100,
lambda: loss,
log_level='info',
start_msg='Fold {}/{}'.format(fold_num, params['kfolds'])):
# Train and cross-validate.
_, acc, loss = train_model(X, y, params, train, test)
return acc, loss
def select_params(config, file_mgr, model_name, codecs):
'''
Select hyperparameters by gridsearch with cross-validation.
'''
log.info('Selecting hyperparameters')
grid = parameter_grid(config['grid'])
if len(grid) == 0:
log.error('No parameters to tune. Stopping.')
exit(0)
fit_num = 1
num_fits = len(grid)
best_params = None
best_acc = -np.inf
acc = 0
loss = 0
for grid_params in grid:
with prof('Grid {}/{}: acc={}%, loss={}',
fit_num,
num_fits,
lambda: 100 * acc,
lambda: round(loss, 4),
log_level='info',
start_msg='Grid {}/{}: {}'.format(fit_num, num_fits,
grid_params)):
params = dict(config['model'])
params.update(grid_params)
try:
queue = mp.Queue()
process = mp.Process(target=tune_model,
args=(queue, config, params, file_mgr,
model_name, codecs))
process.start()
acc, loss = queue.get()
finally:
process.join()
# Select model by accuracy.
if acc > best_acc:
best_acc = acc
best_params = params
fit_num += 1
assert best_params is not None
log.info('Best accuracy was {}% with parameters {}'.format(
round(best_acc * 100, 2), best_params))
return best_params
def _do_load_training(file, codecs, block_size, max_records, line_num=1):
'''
Load up to `max_records` from `file` using blocks of `block_size` bytes.
:returns: A tuple of the training inputs and labels, or None if there are no records left in the file.
'''
with prof('Loaded batch ({} records)', lambda: num_lines):
num_lines = 0
x_codec, y_codec = codecs
# Blocks are likely to end partway through a record, so we read more data than we need and discard any
# excess records. We save the file position so we can calculate the amount of data actually used and seek to
# the beginning of the discarded record(s) for the next read.
file_pos = file.tell()
data = ''
num_lines = 0
while data.count('\n') <= max_records:
block = file.read(block_size)
if not block:
break
data += block
if not data:
return None
# Split on newline and discard records above the maximum.
lines = data.split('\n')
num_lines = min(len(lines), max_records)
# Process the records and rewind to account for any extra records read.
X = [None]*num_lines
y = [None]*num_lines
len_lines = 0
for i in range(num_lines):
line_num += i
len_lines += len(lines[i]) + 1 # +1 to account for newline stripped by str.split().
opcode, disasm = lines[i].split('|')
X[i] = x_codec.encode(opcode)
y[i] = y_codec.encode(disasm)
file.seek(file_pos + len_lines)
return tf.convert_to_tensor(X), tf.convert_to_tensor(y)