def __init__(self, cfg, dirname):
self.parser = ChunkParser(cfg, dirname)
self.cfg = cfg
self.xsize = cfg.xsize
self.ysize = cfg.ysize
self.input_channels = cfg.input_channels
self.input_features = cfg.input_features
self.policy_map = cfg.policy_map
def main():
if len(sys.argv) != 2:
print("Usage: {} config.yaml".format(sys.argv[0]))
return 1
cfg = yaml.safe_load(open(sys.argv[1], 'r').read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks)
num_train = int(num_chunks*cfg['dataset']['train_ratio'])
shuffle_size = cfg['training']['shuffle_size']
ChunkParser.BATCH_SIZE = cfg['training']['batch_size']
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
#bench_parser = ChunkParser(FileDataSrc(chunks[:1000]), shuffle_size=1<<14, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
#benchmark(bench_parser)
train_parser = ChunkParser(FileDataSrc(chunks[:num_train]),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
#benchmark(train_parser)
dataset = tf.data.Dataset.from_generator(
train_parser.parse, output_types=(tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
train_iterator = dataset.make_one_shot_iterator()
test_parser = ChunkParser(FileDataSrc(chunks[num_train:]), batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
test_parser.parse, output_types=(tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
test_iterator = dataset.make_one_shot_iterator()
tfprocess = TFProcess(cfg)
tfprocess.init(dataset, train_iterator, test_iterator)
if os.path.exists(os.path.join(root_dir, 'checkpoint')):
cp = get_checkpoint(root_dir)
tfprocess.restore(cp)
# Sweeps through all test chunks statistically
num_evals = int(round(((num_chunks-num_train) * (200 / SKIP)) / ChunkParser.BATCH_SIZE))
print("Using {} evaluation batches".format(num_evals))
# while True:
for _ in range(cfg['training']['total_steps']):
tfprocess.process(ChunkParser.BATCH_SIZE, num_evals)
def extract_data(parser: ChunkParser, chunkdata):
lst = []
gen = parser.sample_record(chunkdata)
for s in gen:
(planes, probs,
winner), (ver, probs2, planes, us_ooo, us_oo, them_ooo, them_oo, stm,
rule50_plane, move_count, winner,
planes1) = parser.convert_v3_to_tuple(s, return_planes=True)
shape = {'planes': planes1, 'probs': probs, 'winner': winner}
lst.append(shape)
return lst
def main(args):
train_data_prefix = args.pop(0)
chunks = get_chunks(train_data_prefix)
print("Found {0} chunks".format(len(chunks)))
if not chunks:
return
# The following assumes positions from one game are not
# spread through chunks.
random.shuffle(chunks)
training, test = split_chunks(chunks, 0.1)
print("Training with {0} chunks, validating on {1} chunks".format(
len(training), len(test)))
train_parser = ChunkParser(FileDataSrc(training),
shuffle_size=1 << 19,
sample=DOWN_SAMPLE,
batch_size=BATCH_SIZE)
#benchmark(train_parser)
dataset = tf.data.Dataset.from_generator(train_parser.parse,
output_types=(tf.string,
tf.string,
tf.string))
dataset = dataset.map(_parse_function)
dataset = dataset.prefetch(4)
train_iterator = dataset.make_one_shot_iterator()
test_parser = ChunkParser(FileDataSrc(test),
shuffle_size=1 << 19,
sample=DOWN_SAMPLE,
batch_size=BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(test_parser.parse,
output_types=(tf.string,
tf.string,
tf.string))
dataset = dataset.map(_parse_function)
dataset = dataset.prefetch(4)
test_iterator = dataset.make_one_shot_iterator()
tfprocess = TFProcess()
tfprocess.init(dataset, train_iterator, test_iterator)
#benchmark1(tfprocess)
if args:
restore_file = args.pop(0)
tfprocess.restore(restore_file)
while True:
tfprocess.process(BATCH_SIZE)
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks)
train_ratio = cfg['dataset']['train_ratio']
num_train = int(num_chunks * train_ratio)
shuffle_size = cfg['training']['shuffle_size']
ChunkParser.BATCH_SIZE = cfg['training']['batch_size']
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
train_parser = ChunkParser(FileDataSrc(chunks[:num_train]),
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(train_parser.parse,
output_types=(tf.string,
tf.string,
tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
train_iterator = dataset.make_one_shot_iterator()
shuffle_size = int(shuffle_size * (1.0 - train_ratio))
test_parser = ChunkParser(FileDataSrc(chunks[num_train:]),
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(test_parser.parse,
output_types=(tf.string,
tf.string,
tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
test_iterator = dataset.make_one_shot_iterator()
tfprocess = TFProcess(cfg)
tfprocess.init(dataset, train_iterator, test_iterator)
if os.path.exists(os.path.join(root_dir, 'checkpoint')):
cp = get_checkpoint(root_dir)
tfprocess.restore(cp)
# Sweeps through all test chunks statistically
num_evals = (num_chunks - num_train) * 10 // ChunkParser.BATCH_SIZE
print("Using {} evaluation batches".format(num_evals))
for _ in range(cfg['training']['total_steps']):
tfprocess.process(ChunkParser.BATCH_SIZE, num_evals)
tfprocess.save_leelaz_weights(cmd.output)
tfprocess.session.close()
train_parser.shutdown()
test_parser.shutdown()
def chunk_parser(q_in, q_out, shuffle_size, chunk_size):
"""
Parse input chunks from 'q_in', shuffle, and put
chunks of moves in v2 format into 'q_out'
Each output chunk contains 'chunk_size' moves.
Moves are shuffled in a buffer of 'shuffle_size' moves.
(A 2^20 items shuffle buffer is ~ 2.2GB of RAM).
"""
workers = max(1, mp.cpu_count() - 2)
parse = ChunkParser(QueueChunkSrc(q_in),
shuffle_size=shuffle_size,
workers=workers)
gen = parse.v2_gen()
while True:
s = list(itertools.islice(gen, chunk_size))
if not len(s):
break
s = b''.join(s)
q_out.put(s)
q_out.put('STOP')
def chunk_parser(q_in, q_out, shuffle_size, chunk_size):
"""
Parse input chunks from 'q_in', shuffle, and put
chunks of moves in v2 format into 'q_out'
Each output chunk contains 'chunk_size' moves.
Moves are shuffled in a buffer of 'shuffle_size' moves.
(A 2^20 items shuffle buffer is ~ 2.2GB of RAM).
"""
workers = max(1, mp.cpu_count() - 2)
parse = ChunkParser(QueueChunkSrc(q_in),
shuffle_size=shuffle_size,
workers=workers)
gen = parse.v2_gen()
while True:
s = list(itertools.islice(gen, chunk_size))
if not len(s):
break
s = b''.join(s)
q_out.put(s)
q_out.put('STOP')
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks)
train_ratio = cfg['dataset']['train_ratio']
num_train = int(num_chunks*train_ratio)
shuffle_size = cfg['training']['shuffle_size']
ChunkParser.BATCH_SIZE = cfg['training']['batch_size']
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
train_parser = ChunkParser(FileDataSrc(chunks[:num_train]),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
train_parser.parse, output_types=(tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
train_iterator = dataset.make_one_shot_iterator()
shuffle_size = int(shuffle_size*(1.0-train_ratio))
test_parser = ChunkParser(FileDataSrc(chunks[num_train:]),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
test_parser.parse, output_types=(tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
test_iterator = dataset.make_one_shot_iterator()
tfprocess = TFProcess(cfg)
tfprocess.init(dataset, train_iterator, test_iterator)
if os.path.exists(os.path.join(root_dir, 'checkpoint')):
cp = get_checkpoint(root_dir)
tfprocess.restore(cp)
# Sweeps through all test chunks statistically
num_evals = (num_chunks-num_train)*10 // ChunkParser.BATCH_SIZE
print("Using {} evaluation batches".format(num_evals))
for _ in range(cfg['training']['total_steps']):
tfprocess.process(ChunkParser.BATCH_SIZE, num_evals)
tfprocess.save_leelaz_weights(cmd.output)
tfprocess.session.close()
train_parser.shutdown()
test_parser.shutdown()
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
train_ratio = cfg['dataset']['train_ratio']
num_train = int(num_chunks*train_ratio)
num_test = num_chunks - num_train
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'], num_train)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks)
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
shuffle_size = cfg['training']['shuffle_size']
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
# Load data with split batch size, which will be combined to the total batch size in tfprocess.
ChunkParser.BATCH_SIZE = split_batch_size
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
train_parser = ChunkParser(FileDataSrc(train_chunks),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
train_parser.parse, output_types=(tf.string, tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
train_iterator = dataset.make_one_shot_iterator()
shuffle_size = int(shuffle_size*(1.0-train_ratio))
test_parser = ChunkParser(FileDataSrc(test_chunks),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
test_parser.parse, output_types=(tf.string, tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
test_iterator = dataset.make_one_shot_iterator()
tfprocess = TFProcess(cfg)
tfprocess.init(dataset, train_iterator, test_iterator)
if os.path.exists(os.path.join(root_dir, 'checkpoint')):
cp = tf.train.latest_checkpoint(root_dir)
tfprocess.restore(cp)
# If number of test positions is not given
# sweeps through all test chunks statistically
# Assumes average of 10 samples per test game.
# For simplicity, testing can use the split batch size instead of total batch size.
# This does not affect results, because test results are simple averages that are independent of batch size.
num_evals = cfg['training'].get('num_test_positions', num_test * 10)
num_evals = max(1, num_evals // ChunkParser.BATCH_SIZE)
print("Using {} evaluation batches".format(num_evals))
tfprocess.process_loop(total_batch_size, num_evals, batch_splits=batch_splits)
if cmd.output is not None:
tfprocess.save_leelaz_weights(cmd.output)
tfprocess.session.close()
train_parser.shutdown()
test_parser.shutdown()
def main():
parser = argparse.ArgumentParser(
description='Train network from game data.')
parser.add_argument("blockspref",
help="Number of blocks",
nargs='?',
type=int)
parser.add_argument("filterspref",
help="Number of filters",
nargs='?',
type=int)
parser.add_argument("trainpref",
help='Training file prefix',
nargs='?',
type=str)
parser.add_argument("restorepref",
help='Training snapshot prefix',
nargs='?',
type=str)
parser.add_argument("--blocks", '-b', help="Number of blocks", type=int)
parser.add_argument("--filters", '-f', help="Number of filters", type=int)
parser.add_argument("--train", '-t', help="Training file prefix", type=str)
parser.add_argument("--test", help="Test file prefix", type=str)
parser.add_argument("--restore",
type=str,
help="Prefix of tensorflow snapshot to restore from")
parser.add_argument(
"--logbase",
default='leelalogs',
type=str,
help="Log file prefix (for tensorboard) (default: %(default)s)")
parser.add_argument(
"--sample",
default=DOWN_SAMPLE,
type=int,
help="Rate of data down-sampling to use (default: %(default)d)")
args = parser.parse_args()
blocks = args.blocks or args.blockspref
filters = args.filters or args.filterspref
train_data_prefix = args.train or args.trainpref
restore_prefix = args.restore or args.restorepref
if not blocks or not filters:
print("Must supply number of blocks and filters")
return
training = get_chunks(train_data_prefix)
if not args.test:
# Generate test by taking 10% of the training chunks.
random.shuffle(training)
print("here1")
training, test = split_chunks(training, 0.1)
else:
test = get_chunks(args.test)
if not training:
print("No data to train on!")
return
print("Training with {0} chunks, validating on {1} chunks".format(
len(training), len(test)))
train_parser = ChunkParser(
FileDataSrc(training),
shuffle_size=1 << 20, # 2.2GB of RAM.
sample=args.sample,
batch_size=RAM_BATCH_SIZE).parse()
test_parser = ChunkParser(FileDataSrc(test),
shuffle_size=1 << 19,
sample=args.sample,
batch_size=RAM_BATCH_SIZE).parse()
tfprocess = TFProcess(blocks, filters)
tfprocess.init(RAM_BATCH_SIZE,
logbase=args.logbase,
macrobatch=BATCH_SIZE // RAM_BATCH_SIZE)
#benchmark1(tfprocess)
if restore_prefix:
tfprocess.restore(restore_prefix)
tfprocess.process(train_parser, test_parser)
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
allow_less = cfg['dataset'].get('allow_less_chunks', False)
train_ratio = cfg['dataset']['train_ratio']
experimental_parser = cfg['dataset'].get('experimental_v5_only_dataset',
False)
# num_train = int(num_chunks * train_ratio)
# we just need to use one data loader, just put everything into train
num_train = int(num_chunks)
num_test = num_chunks - num_train
sort_type = cfg['dataset'].get('sort_type', 'mtime')
if sort_type == 'mtime':
sort_key_fn = os.path.getmtime
elif sort_type == 'number':
sort_key_fn = game_number_for_name
elif sort_type == 'name':
sort_key_fn = identity_function
else:
raise ValueError('Unknown dataset sort_type: {}'.format(sort_type))
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'],
num_train, allow_less, sort_key_fn)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test,
allow_less, sort_key_fn)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks,
allow_less, sort_key_fn)
if allow_less:
num_train = int(len(chunks) * train_ratio)
num_test = len(chunks) - num_train
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
# shuffle_size = cfg['training']['shuffle_size']
shuffle_size = 1
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
train_workers = cfg['dataset'].get('train_workers', None)
test_workers = cfg['dataset'].get('test_workers', None)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
# Load data with split batch size, which will be combined to the total batch size in tfprocess.
ChunkParser.BATCH_SIZE = split_batch_size
value_focus_min = cfg['training'].get('value_focus_min', 1)
value_focus_slope = cfg['training'].get('value_focus_slope', 0)
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
tfprocess = TFProcess(cfg)
experimental_reads = max(2, mp.cpu_count() - 2) // 2
extractor = select_extractor(tfprocess.INPUT_MODE)
if experimental_parser and (value_focus_min != 1
or value_focus_slope != 0):
raise ValueError(
'Experimental parser does not support non-default value \
focus parameters.')
def read(x):
return tf.data.FixedLengthRecordDataset(
x,
8308,
compression_type='GZIP',
num_parallel_reads=experimental_reads)
if experimental_parser:
# train_dataset = tf.data.Dataset.from_tensor_slices(train_chunks).shuffle(len(train_chunks)).repeat().batch(256)\
train_dataset = tf.data.Dataset.from_tensor_slices(train_chunks).repeat().batch(256)\
.interleave(read, num_parallel_calls=1)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.batch(split_batch_size).map(extractor)
# .shuffle(shuffle_size)\
# .batch(split_batch_size).map(extractor)
else:
train_parser = ChunkParser(train_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
value_focus_min=value_focus_min,
value_focus_slope=value_focus_slope,
workers=train_workers)
train_dataset = tf.data.Dataset.from_generator(
train_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
train_dataset = train_dataset.map(ChunkParser.parse_function)
shuffle_size = int(shuffle_size * (1.0 - train_ratio))
if experimental_parser:
# test_dataset = tf.data.Dataset.from_tensor_slices(test_chunks).shuffle(len(test_chunks)).repeat().batch(256)\
test_dataset = tf.data.Dataset.from_tensor_slices(test_chunks).repeat().batch(256)\
.interleave(read, num_parallel_calls=2)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.batch(split_batch_size).map(extractor)
# .shuffle(shuffle_size)\
# .batch(split_batch_size).map(extractor)
else:
# no value focus for test_parser
test_parser = ChunkParser(test_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
workers=test_workers)
test_dataset = tf.data.Dataset.from_generator(
test_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
test_dataset = test_dataset.map(ChunkParser.parse_function)
validation_dataset = None
if 'input_validation' in cfg['dataset']:
valid_chunks = get_all_chunks(cfg['dataset']['input_validation'])
validation_dataset = tf.data.FixedLengthRecordDataset(valid_chunks, 8308, compression_type='GZIP', num_parallel_reads=experimental_reads)\
.batch(split_batch_size, drop_remainder=True).map(extractor)
if tfprocess.strategy is None: #Mirrored strategy appends prefetch itself with a value depending on number of replicas
train_dataset = train_dataset.prefetch(4)
test_dataset = test_dataset.prefetch(4)
if validation_dataset is not None:
validation_dataset = validation_dataset.prefetch(4)
else:
options = tf.data.Options()
options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF
train_dataset = train_dataset.with_options(options)
test_dataset = test_dataset.with_options(options)
if validation_dataset is not None:
validation_dataset = validation_dataset.with_options(options)
##########################
# Custom Additions #
##########################
tfprocess.init_v2(train_dataset, test_dataset, validation_dataset)
# load net from weights file given in yaml config
tfprocess.replace_weights_v2(proto_filename=cmd.net, ignore_errors=False)
tfprocess.model.summary()
for layer_name, path in zip(cmd.layer, cmd.path):
# sort data files
train_chunks = sorted(train_chunks)
# create predictor that gives access to specific intermediate layer
layer = tfprocess.model.get_layer(layer_name)
earlyPredictor = tf.keras.models.Model(
tfprocess.model.inputs,
[tfprocess.model.inputs, tfprocess.model.outputs, layer.output])
# create custom iterator which doesn't shuffle the data etc
custom_parse_gen = train_parser.custom_parse(train_chunks)
turn_counter = 0
custom_iter = iter(custom_parse_gen)
# prepare dataframe
df = pd.DataFrame()
# iterate entire dataset generator / iterator
for data in custom_iter: #i in range(30):
# data = next(custom_iter)
planes, probs, winner, best_q = train_parser.custom_get_batch(data)
x = planes
print('predicting...')
_, _, layer_results = earlyPredictor.predict(x)
# append to dataframe
# df = df.append(pd.DataFrame(activation_31.reshape(-1,128*8*8)))
shape_tuple = (-1, np.prod(layer.output_shape[1:]))
df = df.append(pd.DataFrame(layer_results.reshape(shape_tuple)))
turn_counter += len(x)
df.info()
df.to_csv(path)
print('done')
train_parser.shutdown()
test_parser.shutdown()
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks)
train_ratio = cfg['dataset']['train_ratio']
num_train = int(num_chunks*train_ratio)
shuffle_size = cfg['training']['shuffle_size']
ChunkParser.BATCH_SIZE = cfg['training']['batch_size']
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
train_parser = ChunkParser(FileDataSrc(chunks[:num_train]),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
train_parser.parse, output_types=(tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
train_iterator = dataset.make_one_shot_iterator()
shuffle_size = int(shuffle_size*(1.0-train_ratio))
test_parser = ChunkParser(FileDataSrc(chunks[num_train:]),
shuffle_size=shuffle_size, sample=SKIP, batch_size=ChunkParser.BATCH_SIZE)
dataset = tf.data.Dataset.from_generator(
test_parser.parse, output_types=(tf.string, tf.string, tf.string))
dataset = dataset.map(ChunkParser.parse_function)
dataset = dataset.prefetch(4)
test_iterator = dataset.make_one_shot_iterator()
tfprocess = TFProcess(cfg)
tfprocess.init(dataset, train_iterator, test_iterator)
if os.path.exists(os.path.join(root_dir, 'checkpoint')):
cp = get_checkpoint(root_dir)
tfprocess.restore(cp)
# Sweeps through all test chunks statistically
num_evals = (num_chunks-num_train)*10 // ChunkParser.BATCH_SIZE
print("Using {} evaluation batches".format(num_evals))
for _ in range(cfg['training']['total_steps']):
tfprocess.process(ChunkParser.BATCH_SIZE, num_evals)
tfprocess.save_leelaz_weights('/tmp/weights.txt')
with open('/tmp/weights.txt', 'rb') as f:
m = hashlib.sha256()
w = f.read()
m.update(w)
digest = m.hexdigest()
filename = '/tmp/{}.gz'.format(digest)
with gzip.open(filename, 'wb') as f:
f.write(w)
if cmd.upload:
metadata = {'training_id':'1', 'layers':cfg['model']['residual_blocks'],
'filters':cfg['model']['filters']}
print("\nUploading `{}'...".format(digest[:8]), end='')
upload(cmd.upload, metadata, filename)
print("[done]\n")
else:
print("\nStored `{}'\n".format(filename))
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
allow_less = cfg['dataset'].get('allow_less_chunks', False)
train_ratio = cfg['dataset']['train_ratio']
num_train = int(num_chunks * train_ratio)
num_test = num_chunks - num_train
sort_type = cfg['dataset'].get('sort_type', 'mtime')
if sort_type == 'mtime':
sort_key_fn = os.path.getmtime
elif sort_type == 'number':
sort_key_fn = game_number_for_name
elif sort_type == 'name':
sort_key_fn = identity_function
else:
raise ValueError('Unknown dataset sort_type: {}'.format(sort_type))
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'],
num_train, allow_less, sort_key_fn)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test,
allow_less, sort_key_fn)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks,
allow_less, sort_key_fn)
if allow_less:
num_train = int(len(chunks) * train_ratio)
num_test = len(chunks) - num_train
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
shuffle_size = cfg['training']['shuffle_size']
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
train_workers = cfg['dataset'].get('train_workers', None)
test_workers = cfg['dataset'].get('test_workers', None)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
diff_focus_min = cfg['training'].get('diff_focus_min', 1)
diff_focus_slope = cfg['training'].get('diff_focus_slope', 0)
diff_focus_q_weight = cfg['training'].get('diff_focus_q_weight', 6.0)
diff_focus_pol_scale = cfg['training'].get('diff_focus_pol_scale', 3.5)
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
train_parser = ChunkParser(train_chunks,
get_input_mode(cfg),
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=split_batch_size,
diff_focus_min=diff_focus_min,
diff_focus_slope=diff_focus_slope,
diff_focus_q_weight=diff_focus_q_weight,
diff_focus_pol_scale=diff_focus_pol_scale,
workers=train_workers)
test_shuffle_size = int(shuffle_size * (1.0 - train_ratio))
# no diff focus for test_parser
test_parser = ChunkParser(test_chunks,
get_input_mode(cfg),
shuffle_size=test_shuffle_size,
sample=SKIP,
batch_size=split_batch_size,
workers=test_workers)
if 'input_validation' in cfg['dataset']:
valid_chunks = get_all_chunks(cfg['dataset']['input_validation'])
validation_parser = ChunkParser(valid_chunks,
get_input_mode(cfg),
sample=1,
batch_size=split_batch_size,
workers=0)
import tensorflow as tf
from chunkparsefunc import parse_function
from tfprocess import TFProcess
tfprocess = TFProcess(cfg)
train_dataset = tf.data.Dataset.from_generator(
train_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string, tf.string))
train_dataset = train_dataset.map(parse_function)
test_dataset = tf.data.Dataset.from_generator(
test_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string, tf.string))
test_dataset = test_dataset.map(parse_function)
validation_dataset = None
if 'input_validation' in cfg['dataset']:
validation_dataset = tf.data.Dataset.from_generator(
validation_parser.sequential,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
validation_dataset = validation_dataset.map(parse_function)
if tfprocess.strategy is None: #Mirrored strategy appends prefetch itself with a value depending on number of replicas
train_dataset = train_dataset.prefetch(4)
test_dataset = test_dataset.prefetch(4)
if validation_dataset is not None:
validation_dataset = validation_dataset.prefetch(4)
else:
options = tf.data.Options()
options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF
train_dataset = train_dataset.with_options(options)
test_dataset = test_dataset.with_options(options)
if validation_dataset is not None:
validation_dataset = validation_dataset.with_options(options)
tfprocess.init(train_dataset, test_dataset, validation_dataset)
tfprocess.restore()
# If number of test positions is not given
# sweeps through all test chunks statistically
# Assumes average of 10 samples per test game.
# For simplicity, testing can use the split batch size instead of total batch size.
# This does not affect results, because test results are simple averages that are independent of batch size.
num_evals = cfg['training'].get('num_test_positions',
len(test_chunks) * 10)
num_evals = max(1, num_evals // split_batch_size)
print("Using {} evaluation batches".format(num_evals))
tfprocess.total_batch_size = total_batch_size
tfprocess.process_loop(total_batch_size,
num_evals,
batch_splits=batch_splits)
if cmd.output is not None:
if cfg['training'].get('swa_output', False):
tfprocess.save_swa_weights(cmd.output)
else:
tfprocess.save_leelaz_weights(cmd.output)
train_parser.shutdown()
test_parser.shutdown()
class DataSet():
def __init__(self, cfg, dirname):
self.parser = ChunkParser(cfg, dirname)
self.cfg = cfg
self.xsize = cfg.xsize
self.ysize = cfg.ysize
self.input_channels = cfg.input_channels
self.input_features = cfg.input_features
self.policy_map = cfg.policy_map
def get_x(self, idx):
return idx % self.xsize
def get_y(self, idx):
return idx // self.xsize
def __getitem__(self, idx):
b, s = self.parser[idx]
data = self.parser.unpack_v1(b, s)
input_planes = np.zeros((self.input_channels, self.ysize, self.xsize))
input_features = np.zeros(self.input_features)
pol = np.zeros(self.policy_map * self.ysize * self.xsize)
wdl = np.zeros(3)
stm = np.zeros(1)
symmetry = bool(np.random.choice(2, 1)[0])
# input planes
for i in range(7):
start = data.ACCUMULATE[i]
num = data.PIECES_NUMBER[i]
for n in range(num):
cp_idx = data.current_pieces[start + n]
if symmetry:
cp_idx = symmetry_index[cp_idx]
if cp_idx != -1:
x = self.get_x(cp_idx)
y = self.get_y(cp_idx)
input_planes[i][y][x] = 1
op_idx = data.other_pieces[start + n]
if symmetry:
op_idx = symmetry_index[op_idx]
if op_idx != -1:
x = self.get_x(op_idx)
y = self.get_y(op_idx)
input_planes[i + 7][y][x] = 1
if data.tomove == 1:
input_planes[14][:] = 1
else:
input_planes[15][:] = 1
# input features
input_features[0] = data.plies / 30
input_features[1] = data.rule50_remaining / 30
if data.repetitions >= 1:
input_features[2] = 1
if data.repetitions >= 2:
input_features[3] = 1
# probabilities
for idx, p in zip(data.policyindex, data.probabilities):
prob_idx = idx
if symmetry:
prob_idx = symmetry_maps[prob_idx]
assert prob_idx != -1, "Invalid probabilities"
pol[prob_idx] = p
# winrate
stm = data.result
wdl[1 - data.result] = 1
return (torch.tensor(input_planes).float(),
torch.tensor(input_features).float(),
torch.tensor(pol).float(), torch.tensor(wdl).float(),
torch.tensor(stm).float())
def __len__(self):
return len(self.parser)
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
allow_less = cfg['dataset'].get('allow_less_chunks', False)
train_ratio = cfg['dataset']['train_ratio']
experimental_parser = cfg['dataset'].get('experimental_v5_only_dataset',
False)
# num_train = int(num_chunks * train_ratio)
# we just need to use one data loader, just put everything into train
num_train = int(num_chunks)
num_test = num_chunks - num_train
sort_type = cfg['dataset'].get('sort_type', 'mtime')
if sort_type == 'mtime':
sort_key_fn = os.path.getmtime
elif sort_type == 'number':
sort_key_fn = game_number_for_name
elif sort_type == 'name':
sort_key_fn = identity_function
else:
raise ValueError('Unknown dataset sort_type: {}'.format(sort_type))
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'],
num_train, allow_less, sort_key_fn)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test,
allow_less, sort_key_fn)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks,
allow_less, sort_key_fn)
if allow_less:
num_train = int(len(chunks) * train_ratio)
num_test = len(chunks) - num_train
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
# shuffle_size = cfg['training']['shuffle_size']
shuffle_size = 1
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
train_workers = cfg['dataset'].get('train_workers', None)
test_workers = cfg['dataset'].get('test_workers', None)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
# Load data with split batch size, which will be combined to the total batch size in tfprocess.
ChunkParser.BATCH_SIZE = split_batch_size
value_focus_min = cfg['training'].get('value_focus_min', 1)
value_focus_slope = cfg['training'].get('value_focus_slope', 0)
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
tfprocess = TFProcess(cfg)
experimental_reads = max(2, mp.cpu_count() - 2) // 2
extractor = select_extractor(tfprocess.INPUT_MODE)
if experimental_parser and (value_focus_min != 1
or value_focus_slope != 0):
raise ValueError(
'Experimental parser does not support non-default value \
focus parameters.')
def read(x):
return tf.data.FixedLengthRecordDataset(
x,
8308,
compression_type='GZIP',
num_parallel_reads=experimental_reads)
if experimental_parser:
# train_dataset = tf.data.Dataset.from_tensor_slices(train_chunks).shuffle(len(train_chunks)).repeat().batch(256)\
train_dataset = tf.data.Dataset.from_tensor_slices(train_chunks).repeat().batch(256)\
.interleave(read, num_parallel_calls=1)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.batch(split_batch_size).map(extractor)
# .shuffle(shuffle_size)\
# .batch(split_batch_size).map(extractor)
else:
train_parser = ChunkParser(train_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
value_focus_min=value_focus_min,
value_focus_slope=value_focus_slope,
workers=train_workers)
train_dataset = tf.data.Dataset.from_generator(
train_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
train_dataset = train_dataset.map(ChunkParser.parse_function)
shuffle_size = int(shuffle_size * (1.0 - train_ratio))
if experimental_parser:
# test_dataset = tf.data.Dataset.from_tensor_slices(test_chunks).shuffle(len(test_chunks)).repeat().batch(256)\
test_dataset = tf.data.Dataset.from_tensor_slices(test_chunks).repeat().batch(256)\
.interleave(read, num_parallel_calls=2)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.batch(split_batch_size).map(extractor)
# .shuffle(shuffle_size)\
# .batch(split_batch_size).map(extractor)
else:
# no value focus for test_parser
test_parser = ChunkParser(test_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
workers=test_workers)
test_dataset = tf.data.Dataset.from_generator(
test_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
test_dataset = test_dataset.map(ChunkParser.parse_function)
validation_dataset = None
if 'input_validation' in cfg['dataset']:
valid_chunks = get_all_chunks(cfg['dataset']['input_validation'])
validation_dataset = tf.data.FixedLengthRecordDataset(valid_chunks, 8308, compression_type='GZIP', num_parallel_reads=experimental_reads)\
.batch(split_batch_size, drop_remainder=True).map(extractor)
if tfprocess.strategy is None: #Mirrored strategy appends prefetch itself with a value depending on number of replicas
train_dataset = train_dataset.prefetch(4)
test_dataset = test_dataset.prefetch(4)
if validation_dataset is not None:
validation_dataset = validation_dataset.prefetch(4)
else:
options = tf.data.Options()
options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF
train_dataset = train_dataset.with_options(options)
test_dataset = test_dataset.with_options(options)
if validation_dataset is not None:
validation_dataset = validation_dataset.with_options(options)
tfprocess.init_v2(train_dataset, test_dataset, validation_dataset)
# load net from weights file given in yaml config
tfprocess.replace_weights_v2(proto_filename=cmd.net, ignore_errors=False)
train_chunks = sorted(train_chunks)
custom_parse_gen = train_parser.custom_parse(train_chunks)
print(train_chunks)
counter = 0
custom_iter = iter(custom_parse_gen)
for data in custom_iter:#i in range(30):
# data = next(custom_iter)
planes, probs, winner, best_q = train_parser.custom_get_batch(data)
print(planes.shape)
x = planes
# TODO make sure no shuffling happens. the following output should clearly show the first few moves of the game w.r.t. pawn placement
for i in range(len(x)):
counter += 1
print('move no.:', counter)
print(x[i, 0].reshape(8,8))
print()
print()
# TODO
# print(tfprocess.model.summary())
# print(tfprocess.train_dataset)
# data_iter = iter(tfprocess.train_dataset)
# nxt = next(data_iter)
# x, _, _, _, _ = nxt
# x2_0_r = tf.reshape(x[0], [1, 112, 64])
# pred = tfprocess.model.predict(x)
earlyPredictor = tf.keras.models.Model(tfprocess.model.inputs, [tfprocess.model.inputs, tfprocess.model.outputs, tfprocess.model.get_layer('activation_31').output])
early_pred_single = earlyPredictor.predict(x)
# print(np.array(early_pred_single[0]).shape)
input = np.array(early_pred_single[0])
print(input.shape)
# print(early_pred_single[0]) # input
# print(early_pred_single[1]) # output
# print(early_pred_single[2]) # intermediate layer
# print(train_parser.sample_record())
# print(next(tfprocess.train_iter))
# tfprocess.restore_v2()
# If number of test positions is not given
# sweeps through all test chunks statistically
# Assumes average of 10 samples per test game.
# For simplicity, testing can use the split batch size instead of total batch size.
# This does not affect results, because test results are simple averages that are independent of batch size.
# num_evals = cfg['training'].get('num_test_positions',
# len(test_chunks) * 10)
# num_evals = max(1, num_evals // ChunkParser.BATCH_SIZE)
# print("Using {} evaluation batches".format(num_evals))
# tfprocess.total_batch_size = total_batch_size
# tfprocess.process_loop_v2(total_batch_size,
# num_evals,
# batch_splits=batch_splits)
# if cmd.output is not None:
# if cfg['training'].get('swa_output', False):
# tfprocess.save_swa_weights_v2(cmd.output)
# else:
# tfprocess.save_leelaz_weights_v2(cmd.output)
train_parser.shutdown()
test_parser.shutdown()
def read_data_sets(filenames,
cfg=None,
fake_data=False,
one_hot=False,
dtype=tf.float32,
reshape=True,
validation_size=5000,
seed=None):
if cfg is None:
cfg = yaml.safe_load(FLAGS.cfg.read())
tf.logging.info(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
train_ratio = cfg['dataset']['train_ratio']
num_train = int(num_chunks * train_ratio)
num_test = num_chunks - num_train
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'],
num_train)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks)
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
shuffle_size = cfg['training']['shuffle_size']
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
# Load data with split batch size, which will be combined to the total batch size in tfprocess.
ChunkParser.BATCH_SIZE = split_batch_size
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
t_chunks = FileDataSrc(train_chunks)
train_parser = ChunkParser(t_chunks,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
auto_start_workers=False)
final_train_data = []
tf.logging.info('Loading training dataset')
for chunkdata in t_chunks:
if len(final_train_data) > FLAGS.record_count:
break
lst = extract_data(train_parser, chunkdata)
for i in lst:
tf.logging.debug('{}: {:4}'.format('train', len(final_train_data)))
final_train_data.append(i)
shuffle_size = int(shuffle_size * (1.0 - train_ratio))
tt_chunks = FileDataSrc(test_chunks)
test_parser = ChunkParser(tt_chunks,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
auto_start_workers=False)
final_test_data = []
tf.logging.info('Loading testing dataset')
for chunkdata in tt_chunks:
if len(final_test_data) > FLAGS.record_count:
break
lst = extract_data(test_parser, chunkdata)
for i in lst:
tf.logging.debug('{}: {:4}'.format('test', len(final_test_data)))
final_test_data.append(i)
train_parser.shutdown()
test_parser.shutdown()
datasets = Dataset(train_data=final_train_data, test_data=final_test_data)
return datasets
8308,
compression_type='GZIP',
num_parallel_reads=experimental_reads)
if experimental_parser:
train_dataset = tf.data.Dataset.from_tensor_slices(train_chunks).shuffle(len(train_chunks)).repeat().batch(256)\
.interleave(read, num_parallel_calls=2)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.shuffle(shuffle_size)\
.batch(split_batch_size).map(extractor)
else:
train_parser = ChunkParser(train_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
value_focus_min=value_focus_min,
value_focus_slope=value_focus_slope,
workers=train_workers)
train_dataset = tf.data.Dataset.from_generator(
train_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string, tf.string))
train_dataset = train_dataset.map(ChunkParser.parse_function)
shuffle_size = int(shuffle_size * (1.0 - train_ratio))
if experimental_parser:
test_dataset = tf.data.Dataset.from_tensor_slices(test_chunks).shuffle(len(test_chunks)).repeat().batch(256)\
.interleave(read, num_parallel_calls=2)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.shuffle(shuffle_size)\
.batch(split_batch_size).map(extractor)
def main():
parser = argparse.ArgumentParser(
description='Train network from game data.')
parser.add_argument("blockspref",
help="Number of blocks", nargs='?', type=int)
parser.add_argument("filterspref",
help="Number of filters", nargs='?', type=int)
parser.add_argument("trainpref",
help='Training file prefix', nargs='?', type=str)
parser.add_argument("restorepref",
help='Training snapshot prefix', nargs='?', type=str)
parser.add_argument("--blocks", '-b',
help="Number of blocks", type=int)
parser.add_argument("--filters", '-f',
help="Number of filters", type=int)
parser.add_argument("--train", '-t',
help="Training file prefix", type=str)
parser.add_argument("--test", help="Test file prefix", type=str)
parser.add_argument("--restore", type=str,
help="Prefix of tensorflow snapshot to restore from")
parser.add_argument("--logbase", default='leelalogs', type=str,
help="Log file prefix (for tensorboard) (default: %(default)s)")
parser.add_argument("--sample", default=DOWN_SAMPLE, type=int,
help="Rate of data down-sampling to use (default: %(default)d)")
parser.add_argument("--bufferbits", default=TRAIN_SHUFFLE_BITS, type=int,
help="Train shuffle-buffer size in bits (default: %(default)d)")
parser.add_argument("--rate", default=LEARN_RATE, type=float,
help="Learning rate (default: %(default)f)")
parser.add_argument("--steps", default=TRAINING_STEPS, type=int,
help="Training step before writing a network (default: %(default)d)")
parser.add_argument("--maxsteps", default=MAX_TRAINING_STEPS, type=int,
help="Terminates after this many steps (default: %(default)d)")
parser.add_argument("--maxkeep", default=MAX_SAVER_TO_KEEP, type=int,
help="Keeps meta files for at most this many networks (default: %(default)d)")
parser.add_argument("--policyloss", default=POLICY_LOSS_WT, type=float,
help="Coefficient for policy term in loss function (default: %(default)f)")
parser.add_argument("--mseloss", default=MSE_LOSS_WT, type=float,
help="Coefficient for mse term in loss function (default: %(default)f)")
parser.add_argument("--regloss", default=REG_LOSS_WT, type=float,
help="Coefficient for regularizing term in loss function (default: %(default)f)")
args = parser.parse_args()
blocks = args.blocks or args.blockspref
filters = args.filters or args.filterspref
train_data_prefix = args.train or args.trainpref
restore_prefix = args.restore or args.restorepref
if not blocks or not filters:
print("Must supply number of blocks and filters")
return
training = get_chunks(train_data_prefix)
if not args.test:
# Generate test by taking 10% of the training chunks.
random.shuffle(training)
training, test = split_chunks(training, 0.1)
else:
test = get_chunks(args.test)
if not training:
print("No data to train on!")
return
print("Training with {0} chunks, validating on {1} chunks".format(
len(training), len(test)))
train_parser = ChunkParser(FileDataSrc(training),
shuffle_size=1<<args.bufferbits, # was 20 -- 2.2GB of RAM.
sample=args.sample,
batch_size=RAM_BATCH_SIZE).parse()
test_parser = ChunkParser(FileDataSrc(test),
shuffle_size=1<<(args.bufferbits-3), # was 19
sample=args.sample,
batch_size=RAM_BATCH_SIZE).parse()
tfprocess = TFProcess(blocks, filters,
args.rate, args.steps, args.maxsteps, args.maxkeep,
args.policyloss, args.mseloss, args.regloss)
tfprocess.init(RAM_BATCH_SIZE,
logbase=args.logbase,
macrobatch=BATCH_SIZE // RAM_BATCH_SIZE)
#benchmark1(tfprocess)
if restore_prefix:
tfprocess.restore(restore_prefix)
tfprocess.process(train_parser, test_parser)
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
allow_less = cfg['dataset'].get('allow_less_chunks', False)
train_ratio = cfg['dataset']['train_ratio']
experimental_parser = cfg['dataset'].get('experimental_v5_only_dataset',
False)
num_train = int(num_chunks * train_ratio)
num_test = num_chunks - num_train
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'],
num_train, allow_less)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test,
allow_less)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks,
allow_less)
if allow_less:
num_train = int(len(chunks) * train_ratio)
num_test = len(chunks) - num_train
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
shuffle_size = cfg['training']['shuffle_size']
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
train_workers = cfg['dataset'].get('train_workers', None)
test_workers = cfg['dataset'].get('test_workers', None)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
# Load data with split batch size, which will be combined to the total batch size in tfprocess.
ChunkParser.BATCH_SIZE = split_batch_size
root_dir = os.path.join(cfg['training']['path'], cfg['name'])
if not os.path.exists(root_dir):
os.makedirs(root_dir)
tfprocess = TFProcess(cfg)
experimental_reads = max(2, mp.cpu_count() - 2) // 2
extractor = select_extractor(tfprocess.INPUT_MODE)
def read(x):
return tf.data.FixedLengthRecordDataset(
x,
8308,
compression_type='GZIP',
num_parallel_reads=experimental_reads)
if experimental_parser:
train_dataset = tf.data.Dataset.from_tensor_slices(train_chunks).shuffle(len(train_chunks)).repeat().batch(256)\
.interleave(read, num_parallel_calls=2)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.shuffle(shuffle_size)\
.batch(split_batch_size).map(extractor).prefetch(4)
else:
train_parser = ChunkParser(train_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
workers=train_workers)
train_dataset = tf.data.Dataset.from_generator(
train_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
train_dataset = train_dataset.map(ChunkParser.parse_function)
train_dataset = train_dataset.prefetch(4)
shuffle_size = int(shuffle_size * (1.0 - train_ratio))
if experimental_parser:
test_dataset = tf.data.Dataset.from_tensor_slices(test_chunks).shuffle(len(test_chunks)).repeat().batch(256)\
.interleave(read, num_parallel_calls=2)\
.batch(SKIP_MULTIPLE*SKIP).map(semi_sample).unbatch()\
.shuffle(shuffle_size)\
.batch(split_batch_size).map(extractor).prefetch(4)
else:
test_parser = ChunkParser(test_chunks,
tfprocess.INPUT_MODE,
shuffle_size=shuffle_size,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
workers=test_workers)
test_dataset = tf.data.Dataset.from_generator(
test_parser.parse,
output_types=(tf.string, tf.string, tf.string, tf.string,
tf.string))
test_dataset = test_dataset.map(ChunkParser.parse_function)
test_dataset = test_dataset.prefetch(4)
validation_dataset = None
if 'input_validation' in cfg['dataset']:
valid_chunks = get_all_chunks(cfg['dataset']['input_validation'])
validation_dataset = tf.data.FixedLengthRecordDataset(valid_chunks, 8308, compression_type='GZIP', num_parallel_reads=experimental_reads)\
.batch(split_batch_size, drop_remainder=True).map(extractor).prefetch(4)
tfprocess.init_v2(train_dataset, test_dataset, validation_dataset)
tfprocess.restore_v2()
# If number of test positions is not given
# sweeps through all test chunks statistically
# Assumes average of 10 samples per test game.
# For simplicity, testing can use the split batch size instead of total batch size.
# This does not affect results, because test results are simple averages that are independent of batch size.
num_evals = cfg['training'].get('num_test_positions',
len(test_chunks) * 10)
num_evals = max(1, num_evals // ChunkParser.BATCH_SIZE)
print("Using {} evaluation batches".format(num_evals))
tfprocess.process_loop_v2(total_batch_size,
num_evals,
batch_splits=batch_splits)
if cmd.output is not None:
if cfg['training'].get('swa_output', False):
tfprocess.save_swa_weights_v2(cmd.output)
else:
tfprocess.save_leelaz_weights_v2(cmd.output)
train_parser.shutdown()
test_parser.shutdown()
def main(cmd):
cfg = yaml.safe_load(cmd.cfg.read())
print(yaml.dump(cfg, default_flow_style=False))
num_chunks = cfg['dataset']['num_chunks']
allow_less = cfg['dataset'].get('allow_less_chunks', False)
train_ratio = cfg['dataset']['train_ratio']
experimental_parser = cfg['dataset'].get('experimental_v5_only_dataset',
False)
num_train = int(num_chunks * train_ratio)
num_test = num_chunks - num_train
if 'input_test' in cfg['dataset']:
train_chunks = get_latest_chunks(cfg['dataset']['input_train'],
num_train, allow_less)
test_chunks = get_latest_chunks(cfg['dataset']['input_test'], num_test,
allow_less)
else:
chunks = get_latest_chunks(cfg['dataset']['input'], num_chunks,
allow_less)
if allow_less:
num_train = int(len(chunks) * train_ratio)
num_test = len(chunks) - num_train
train_chunks = chunks[:num_train]
test_chunks = chunks[num_train:]
shuffle_size = cfg['training']['shuffle_size']
total_batch_size = cfg['training']['batch_size']
batch_splits = cfg['training'].get('num_batch_splits', 1)
if total_batch_size % batch_splits != 0:
raise ValueError('num_batch_splits must divide batch_size evenly')
split_batch_size = total_batch_size // batch_splits
# Load data with split batch size, which will be combined to the total batch size in tfprocess.
ChunkParser.BATCH_SIZE = split_batch_size
tfprocess = TFProcess(cfg)
train_parser = ChunkParser(train_chunks,
tfprocess.INPUT_MODE,
shuffle_size=10000,
sample=SKIP,
batch_size=ChunkParser.BATCH_SIZE,
workers=4)
batch_gen = train_parser.parse()
device = th.device('cuda:0' if th.cuda.is_available() else 'cpu')
model = th.nn.Sequential(th.nn.Conv2d(112, 128, 3, padding=1, bias=False),
th.nn.ReLU(), ResidualBlock(), ResidualBlock(),
ResidualBlock(), ResidualBlock(), ResidualBlock(),
ResidualBlock(),
th.nn.Conv2d(128, 32, 1, bias=False),
th.nn.ReLU(), th.nn.Flatten(),
th.nn.Linear(2048, 1858, bias=False))
model.train()
model = model.to(device)
optimizer = th.optim.SGD(model.parameters(), lr=0.01)
train_batches = 100000
for i in range(train_batches):
# print(f'getting data {i} ...')
x, y, z, q, m = next(batch_gen)
x, y, z, q, m = ChunkParser.parse_function(x, y, z, q, m)
x = x.numpy()
x = th.Tensor(x)
x = x.reshape((-1, 112, 8, 8))
y = th.Tensor(y.numpy())
x, y = x.to(device), y.to(device)
optimizer.zero_grad()
policy = model(x)
loss = th.mean(
th.sum(-th.log_softmax(policy, 1) * th.nn.functional.relu(y), 1))
loss.backward()
optimizer.step()
if i % 10 == 0:
print(f'step = {i}, loss = {loss.item()}')
train_parser.shutdown()