def validate_feature_planes(verbose, dataset, model_features):
"""Verify that dataset's features match the model's expected features.
"""
if 'features' in dataset:
dataset_features = dataset['features'][()]
dataset_features = dataset_features.split(",")
if len(dataset_features) != len(model_features) or \
any(df != mf for (df, mf) in zip(dataset_features, model_features)):
raise ValueError(
"Model JSON file expects features \n\t%s\n"
"But dataset contains \n\t%s" %
("\n\t".join(model_features), "\n\t".join(dataset_features)))
elif verbose:
print(
"Verified that dataset features and model features exactly match."
)
else:
# Cannot check each feature, but can check number of planes.
n_dataset_planes = dataset["states"].shape[1]
tmp_preprocess = Preprocess(model_features)
n_model_planes = tmp_preprocess.get_output_dimension()
if n_dataset_planes != n_model_planes:
raise ValueError(
"Model JSON file expects a total of %d planes from features \n\t%s\n"
"But dataset contains %d planes" %
(n_model_planes, "\n\t".join(model_features),
n_dataset_planes))
elif verbose:
print(
"Verified agreement of number of model and dataset feature planes, but cannot "
"verify exact match using old dataset format.")
class GameConverter:
def __init__(self, features):
self.feature_processor = Preprocess(features)
self.n_features = self.feature_processor.get_output_dimension()
def convert_game(self, file_name, bd_size):
"""Read the given SGF file into an iterable of (input,output) pairs
for neural network training
Each input is a GameState converted into one-hot neural net features
Each output is an action as an (x,y) pair (passes are skipped)
If this game's size does not match bd_size, a SizeMismatchError is raised
"""
with open(file_name, 'r') as file_object:
state_action_iterator = sgf_iter_states(file_object.read(),
include_end=False)
for (state, move, player) in state_action_iterator:
if state.get_size() != bd_size:
raise SizeMismatchError()
if move != go.PASS:
nn_input = self.feature_processor.state_to_tensor(state)
yield (nn_input, move)
def sgfs_to_hdf5(self,
sgf_files,
hdf5_file,
bd_size=19,
ignore_errors=True,
verbose=False):
"""Convert all files in the iterable sgf_files into an hdf5 group to be stored in hdf5_file
Arguments:
- sgf_files : an iterable of relative or absolute paths to SGF files
- hdf5_file : the name of the HDF5 where features will be saved
- bd_size : side length of board of games that are loaded
- ignore_errors : if True, issues a Warning when there is an unknown
exception rather than halting. Note that sgf.ParseException and
go.IllegalMove exceptions are always skipped
The resulting file has the following properties:
states : dataset with shape (n_data, n_features, board width, board height)
actions : dataset with shape (n_data, 2) (actions are stored as x,y tuples of
where the move was played)
file_offsets : group mapping from filenames to tuples of (index, length)
For example, to find what positions in the dataset come from 'test.sgf':
index, length = file_offsets['test.sgf']
test_states = states[index:index+length]
test_actions = actions[index:index+length]
"""
# make a hidden temporary file in case of a crash.
# on success, this is renamed to hdf5_file
tmp_file = os.path.join(os.path.dirname(hdf5_file),
".tmp." + os.path.basename(hdf5_file))
h5f = h5.File(tmp_file, 'w')
try:
# see http://docs.h5py.org/en/latest/high/group.html#Group.create_dataset
states = h5f.require_dataset(
'states',
dtype=np.uint8,
shape=(1, self.n_features, bd_size, bd_size),
maxshape=(None, self.n_features, bd_size,
bd_size), # 'None' == arbitrary size
exact=
False, # allow non-uint8 datasets to be loaded, coerced to uint8
chunks=(64, self.n_features, bd_size,
bd_size), # approximately 1MB chunks
compression="lzf")
actions = h5f.require_dataset('actions',
dtype=np.uint8,
shape=(1, 2),
maxshape=(None, 2),
exact=False,
chunks=(1024, 2),
compression="lzf")
# 'file_offsets' is an HDF5 group so that 'file_name in file_offsets' is fast
file_offsets = h5f.require_group('file_offsets')
# Store comma-separated list of feature planes in the scalar field 'features'. The
# string can be retrieved using h5py's scalar indexing: h5f['features'][()]
h5f['features'] = np.string_(','.join(
self.feature_processor.get_feature_list()))
if verbose:
print("created HDF5 dataset in {}".format(tmp_file))
next_idx = 0
for file_name in sgf_files:
if verbose:
print(file_name)
# count number of state/action pairs yielded by this game
n_pairs = 0
file_start_idx = next_idx
try:
for state, move in self.convert_game(file_name, bd_size):
if next_idx >= len(states):
states.resize((next_idx + 1, self.n_features,
bd_size, bd_size))
actions.resize((next_idx + 1, 2))
states[next_idx] = state
actions[next_idx] = move
n_pairs += 1
next_idx += 1
except go.IllegalMove:
warnings.warn("Illegal Move encountered in %s\n"
"\tdropping the remainder of the game" %
file_name)
except sgf.ParseException:
warnings.warn("Could not parse %s\n\tdropping game" %
file_name)
except SizeMismatchError:
warnings.warn("Skipping %s; wrong board size" % file_name)
except Exception as e:
# catch everything else
if ignore_errors:
warnings.warn("Unkown exception with file %s\n\t%s" %
(file_name, e),
stacklevel=2)
else:
raise e
finally:
if n_pairs > 0:
# '/' has special meaning in HDF5 key names, so they
# are replaced with ':' here
file_name_key = file_name.replace('/', ':')
file_offsets[file_name_key] = [file_start_idx, n_pairs]
if verbose:
print("\t%d state/action pairs extracted" %
n_pairs)
elif verbose:
print("\t-no usable data-")
except Exception as e:
print("sgfs_to_hdf5 failed")
os.remove(tmp_file)
raise e
if verbose:
print("finished. renaming %s to %s" % (tmp_file, hdf5_file))
# processing complete; rename tmp_file to hdf5_file
h5f.close()
os.rename(tmp_file, hdf5_file)
class NeuralNetBase(object):
"""Base class for neural network classes handling feature processing, construction
of a 'forward' function, etc.
"""
# keep track of subclasses to make generic saving/loading cleaner.
# subclasses can be 'registered' with the @neuralnet decorator
subclasses = {}
def __init__(self, feature_list, **kwargs):
"""create a neural net object that preprocesses according to feature_list and uses
a neural network specified by keyword arguments (using subclass' create_network())
optional argument: init_network (boolean). If set to False, skips initializing
self.model and self.forward and the calling function should set them.
"""
defaults = {"board": 19}
defaults.update(kwargs)
self.preprocessor = Preprocess(feature_list, size=defaults["board"])
kwargs["input_dim"] = self.preprocessor.get_output_dimension()
if kwargs.get('init_network', True):
# self.__class__ refers to the subclass so that subclasses only
# need to override create_network()
self.model = self.__class__.create_network(**kwargs)
# self.forward is a lambda function wrapping a Keras function
self.forward = self._model_forward()
def _model_forward(self):
"""Construct a function using the current keras backend that, when given a batch
of inputs, simply processes them forward and returns the output
This is as opposed to model.compile(), which takes a loss function
and training method.
c.f. https://github.com/fchollet/keras/issues/1426
"""
# The uses_learning_phase property is True if the model contains layers that behave
# differently during training and testing, e.g. Dropout or BatchNormalization.
# In these cases, K.learning_phase() is a reference to a backend variable that should
# be set to 0 when using the network in prediction mode and is automatically set to 1
# during training.
if self.model.uses_learning_phase:
forward_function = K.function(
[self.model.input, K.learning_phase()], [self.model.output])
# the forward_function returns a list of tensors
# the first [0] gets the front tensor.
return lambda inpt: forward_function([inpt, 0])[0]
else:
# identical but without a second input argument for the learning phase
forward_function = K.function([self.model.input],
[self.model.output])
return lambda inpt: forward_function([inpt])[0]
@staticmethod
def load_model(json_file):
"""create a new neural net object from the architecture specified in json_file
"""
with open(json_file, 'r') as f:
object_specs = json.load(f)
# Create object; may be a subclass of networks saved in specs['class']
class_name = object_specs.get('class', 'CNNPolicy')
try:
network_class = NeuralNetBase.subclasses[class_name]
except KeyError:
raise ValueError(
"Unknown neural network type in json file: {}\n"
"(was it registered with the @neuralnet decorator?)".format(
class_name))
# create new object
new_net = network_class(object_specs['feature_list'],
init_network=False)
new_net.model = model_from_json(object_specs['keras_model'],
custom_objects={'Bias': Bias})
if 'weights_file' in object_specs:
new_net.model.load_weights(object_specs['weights_file'])
new_net.forward = new_net._model_forward()
return new_net
def save_model(self, json_file, weights_file=None):
"""write the network model and preprocessing features to the specified file
If a weights_file (.hdf5 extension) is also specified, model weights are also
saved to that file and will be reloaded automatically in a call to load_model
"""
# this looks odd because we are serializing a model with json as a string
# then making that the value of an object which is then serialized as
# json again.
# It's not as crazy as it looks. A Network has 2 moving parts - the
# feature preprocessing and the neural net, each of which gets a top-level
# entry in the saved file. Keras just happens to serialize models with JSON
# as well. Note how this format makes load_model fairly clean as well.
object_specs = {
'class': self.__class__.__name__,
'keras_model': self.model.to_json(),
'feature_list': self.preprocessor.get_feature_list()
}
if weights_file is not None:
self.model.save_weights(weights_file)
object_specs['weights_file'] = weights_file
# use the json module to write object_specs to file
with open(json_file, 'w') as f:
json.dump(object_specs, f)