def __init__(self, version):
self.version = version
# Construct a fake parser just to get access to it's variables
self.parser = chunkparser.ChunkParser(chunkparser.ChunkDataSrc([]),
workers=1)
self.NUM_HIST = 8
self.NUM_PIECE_TYPES = 6
self.V3_NUM_PLANES = self.NUM_PIECE_TYPES * 2 + 1 # = 13 (6*2 us/them pieces, rep1 (no rep2))
self.NUM_PLANES = self.V3_NUM_PLANES
self.NUM_REALS = 7 # 4 castling, 1 color, 1 50rule, 1 movecount
self.NUM_OUTPUTS = 2 # policy, value
self.NUM_PLANES_BYTES = self.NUM_PLANES * 4
self.NUM_PLANES_BYTES = self.NUM_PLANES * 4
self.NUM_PLANES_BYTES = self.NUM_PLANES * 4
self.V3_NUM_POLICY_MOVES = 1858 # (7432 bytes)
self.NUM_POLICY_MOVES = self.V3_NUM_POLICY_MOVES
self.init_structs()
self.init_move_map()
self.history = []
self.probs = []
for history in range(self.NUM_HIST):
self.history.append(Board())
self.us_ooo = 0
self.us_oo = 0
self.them_ooo = 0
self.them_oo = 0
self.us_black = 0
self.rule50_count = 0
self.winner = None
def main(args):
for filename in args.files:
#print("Parsing {}".format(filename))
with gzip.open(filename, 'rb') as f:
chunkdata = f.read()
if chunkdata[0:4] == b'\1\0\0\0':
print("Invalid version")
elif chunkdata[0:4] == VERSION3:
#print("debug Version3")
for i in range(0, len(chunkdata), V3_BYTES):
ts = TrainingStep(3)
ts.display_v2_or_v3(i // V3_BYTES,
chunkdata[i:i + V3_BYTES])
else:
parser = chunkparser.ChunkParser(chunkparser.ChunkDataSrc(
[chunkdata]),
workers=1)
gen1 = parser.convert_chunkdata_to_v2(chunkdata)
ply = 1
for t1 in gen1:
ts = TrainingStep(2)
ts.display_v2_or_v3(ply, t1)
ply += 1
# TODO maybe detect new games and reset ply count
# It's informational only
for _ in parser.parse():
# TODO: What is happening here?
#print("debug drain", len(_))
pass
def main(args):
convert = len(args.output) > 0
if convert and not os.path.exists(args.output):
os.makedirs(args.output)
print("Created '{}'".format(args.output))
for filename in args.files:
foutname = os.path.basename(filename)[:-3] + "-v3.gz"
with gzip.open(foutname, 'wb') as fout:
#print("Parsing {}".format(filename))
with gzip.open(filename, 'rb') as f:
chunkdata = f.read()
if chunkdata[0:4] == b'\1\0\0\0':
print("Invalid version")
elif chunkdata[0:4] == VERSION2:
#print("debug Version2")
for i in range(0, len(chunkdata), V2_BYTES):
ts = TrainingStep(2)
if args.display:
ts.display_v2_or_v3(i // V2_BYTES,
chunkdata[i:i + V2_BYTES])
if convert:
ts.convert_v2_to_v3(chunkdata[i:i + V2_BYTES],
fout)
elif chunkdata[0:4] == VERSION3:
#print("debug Version3")
for i in range(0, len(chunkdata), V3_BYTES):
ts = TrainingStep(3)
if args.display:
ts.display_v2_or_v3(i // V3_BYTES,
chunkdata[i:i + V3_BYTES])
if convert:
raise Exception("Convert from V3 to V3?")
else:
parser = chunkparser.ChunkParser(chunkparser.ChunkDataSrc(
[chunkdata]),
workers=1)
gen1 = parser.convert_chunkdata_to_v2(chunkdata)
ply = 1
for t1 in gen1:
ts = TrainingStep(2)
if args.display:
ts.display_v2_or_v3(ply, t1)
if convert:
ts.convert_v2_to_v3(t1, fout)
ply += 1
# TODO maybe detect new games and reset ply count
# It's informational only
for _ in parser.parse():
# TODO: What is happening here?
#print("debug drain", len(_))
pass
if convert:
shutil.move(foutname, os.path.join(args.output, foutname))
else:
os.remove(foutname)
def __init__(self, version):
self.version = version
# Construct a fake parser just to get access to it's variables
self.parser = chunkparser.ChunkParser(chunkparser.ChunkDataSrc([]),
workers=1)
self.NUM_HIST = 8
self.NUM_PIECE_TYPES = 6
self.V2_NUM_PLANES = self.NUM_PIECE_TYPES * 2 + 2 # = 14 (6*2 us/them pieces, rep1, rep2)
self.V3_NUM_PLANES = self.NUM_PIECE_TYPES * 2 + 1 # = 13 (6*2 us/them pieces, rep1 (no rep2))
if self.version <= 2:
self.NUM_PLANES = self.V2_NUM_PLANES
else:
self.NUM_PLANES = self.V3_NUM_PLANES
self.NUM_REALS = 7 # 4 castling, 1 color, 1 50rule, 1 movecount
self.NUM_OUTPUTS = 2 # policy, value
self.NUM_PLANES_BYTES = self.NUM_PLANES * 4
self.NUM_PLANES_BYTES = self.NUM_PLANES * 4
self.NUM_PLANES_BYTES = self.NUM_PLANES * 4
self.V2_NUM_POLICY_MOVES = 1924 # (7696 bytes)
self.V3_NUM_POLICY_MOVES = 1858 # (7432 bytes)
if self.version <= 2:
self.NUM_POLICY_MOVES = self.V2_NUM_POLICY_MOVES
else:
self.NUM_POLICY_MOVES = self.V3_NUM_POLICY_MOVES
# This one is used for V1 text
self.DATA_ITEM_LINES = self.NUM_HIST * self.NUM_PLANES + self.NUM_REALS + self.NUM_OUTPUTS # 121
# Note: The C++ code adds 1 unused REAL to make the number even.
self.init_structs()
self.init_move_map()
self.history = []
self.probs = []
for history in range(self.NUM_HIST):
self.history.append(Board())
self.us_ooo = 0
self.us_oo = 0
self.them_ooo = 0
self.them_oo = 0
self.us_black = 0
self.rule50_count = 0
#!/usr/bin/env python3
import sys
import chunkparser
import numpy as np
import gzip
with gzip.open(sys.argv[1], 'rb') as f:
v1 = f.read()
with gzip.open(sys.argv[2], 'rb') as f:
v2 = f.read()
parser = chunkparser.ChunkParser(chunkparser.ChunkDataSrc([v1, v2]), workers=1)
gen1 = parser.convert_chunkdata_to_v2(v1)
gen2 = parser.convert_chunkdata_to_v2(v2)
for t1 in gen1:
t2 = next(gen2)
t1 = parser.convert_v2_to_tuple(t1)
t2 = parser.convert_v2_to_tuple(t2)
p1 = np.frombuffer(t1[1], dtype=np.float32, count=1924)
p2 = np.frombuffer(t2[1], dtype=np.float32, count=1924)
pl1 = np.frombuffer(t1[0], dtype=np.uint8, count=120*8*8)
pl2 = np.frombuffer(t2[0], dtype=np.uint8, count=120*8*8)
assert((pl1[0] == pl2[0]).all())
assert(t1[2] == t2[2])
a = np.argsort(p1[p1>0])
b = np.argsort(p2[p2>0])
assert((a == b).all())