class Board():
#board = np.zeros((19,19))
timeStart = time.time()
#moves = [[Stone() for j in range(19)] for i in range(19)]
#print (board)
countTrain = 0
countTest = 0
max = 3672204
rootDir = sys.argv[1]
# name of DB:
#inputDB = sys.argv[2]
#dbNameTrain = sys.argv[2]
#dbNameTest = sys.argv[3]
train = True
writeData = np.array([], dtype=np.uint8).reshape(0, 4, 21, 21)
writeSequence = np.array([])
for dirName, subdirList, fileList in os.walk(rootDir):
print('Found directory: %s' % dirName)
for fname in fileList:
try:
print('\t%s' % fname)
# /home/nathan/Go_Database/expanded-GoGod/rotation-7/69/1994-02-17g_7.sgf new troublesome one
# 2015-03-23b_0.sgf is the troublesome one
if fname.endswith('.sgf'):
path = os.path.join(dirName, fname)
f = open(path, 'r')
else:
continue
sgfString = f.read()
game = SGFsequence(sgfString)
board = Board(19)
# legit moves
onesA = np.ones((len(game.moves)))
# random moves
# random number of random moves
zerosLen = random.randrange(len(onesA) * 2)
zerosA = np.zeros(zerosLen)
sequence = np.append(onesA, (zerosA))
#make a random sequence between random and pro moves
#so we will know the total length
#this is also the labels
np.random.shuffle(sequence)
batchLen = len(sequence)
#lets write all the data for a game at once
data = np.empty((batchLen, 4, 21, 21), dtype=np.uint8)
labels = np.empty((batchLen, 1, 21, 21), dtype=np.uint8)
# game[i].order[0][0][0] this is how you get X
# game[i].order[0][0][1] this is how you get Y
# process move by move
i = 0
j = 0
randGame = True
path = countTrain / 100000
for item in sequence:
label = np.zeros((21, 21), dtype=np.uint8)
#just tring single stream learning so commenting out siamese thing for now
#writeData2 = lmdbReadWrite2.getRandDataPoint( max, inputDB)[0] #read from db
#moves = random.randrange(280)
if (path >= i and randGame):
if (i > 0):
moves = i - 1
else:
moves = 0
elif (randGame):
moves = random.randrange(path, i)
if (not item and not randGame):
#randBoard = Board(19)
#for randMove in range(0,moves):
#randomPosition = random.sample((randBoard.libertySet),1)
#x = randomPosition[0] % 19
#y = randomPosition[0] / 19
#randBoard.place_stone(x, y, 1 if randMove % 2 else -1, i-1)
randomPosition = random.sample((board.libertySet), 1)
#print("randompos: %d" % randomPosition[0])
x = randomPosition[0] % 19
y = randomPosition[0] / 19
writeData1 = board.poof_stone(x, y, 0 if i % 2 else -1,
i)
label[y + 1, x + 1] = 100 #-1
elif (not item): # randgame alternative move here
randBoard = Board(19)
for orderMove in range(0, moves):
x = game.moves[orderMove][
0] #randomPosition[0] % 19
y = game.moves[orderMove][
1] #randomPosition[0] / 19
writeData1 = randBoard.place_stone(
x, y, 1 if orderMove % 2 else -1, i - 1)
for randMove in range(moves, i + 1):
randomPosition = random.sample(
(randBoard.libertySet), 1)
x = randomPosition[0] % 19
y = randomPosition[0] / 19
writeData1 = randBoard.place_stone(
x, y, 1 if randMove % 2 else -1, i - 1)
label[y + 1, x + 1] = 100 #-1
else:
label.fill(127)
x = game.moves[i][0]
y = game.moves[i][1]
#x, y = board.__board_to_ints__(xchar, ychar)
writeData1 = board.place_stone(x, y,
0 if i % 2 else -1, i)
i = i + 1
#print("i = %s" % i)
#array2 = board.print_board()
#array3 = board.print_weight()
#array1 = board.print_board_liberty()
#print(writeData1[0,0,0].dtype)
#print(writeData1.shape)
transpose1 = writeData1.transpose((2, 0, 1))
#print(transpose1.shape)
#print(label.shape)
#labelTranspose = label.transpose((2, 0, 1))
#print(labelTranspose.shape)
#transpose2 = writeData2 #.transpose((2, 0, 1))
#concatedData = np.concatenate((transpose1, transpose2))
data[j, :, :, :] = transpose1 #concatedData
labels[j, 0, :, :] = label
#print("J: %s ~~~~~~~~~~~~~~~~~~~~~~`" % j)
#print(data[j,0,:,:])
#print(labels[j,0,:,:])
#array2 = board.print_board()
#array1 = board.print_board_liberty()
#np.set_printoptions(threshold=np.nan)
#print(transpose1)
#print(transpose2)
#print(concatedData)
j = j + 1
# def writeToDB(data, label, index, database):
#print(data.shape)
train = train + 1
writeData = np.concatenate((data, writeData))
writeSequence = np.concatenate((sequence, writeSequence))
if (train >= 30):
if (countTrain > 6000000):
error = 1 / 0
print("writing to database")
print(countTrain)
print(countTest)
print(data.shape)
print(writeData.shape)
rng_state = np.random.get_state()
np.random.shuffle(writeData)
np.random.set_state(rng_state)
np.random.shuffle(labels)
length = len(labels)
part = length / 50
lmdbReadWrite2.writeToDB(
writeData[part:], None, countTrain,
"shuf2_trainD_%s_%s_randboard:%s" %
(sys.argv[1], countTrain / 7000000, randGame))
lmdbReadWrite2.writeToDB(
labels[part:], None, countTrain,
"shuf2_trainL_%s_%s_randboard:%s" %
(sys.argv[1], countTrain / 7000000, randGame))
countTrain = countTrain + length - part
lmdbReadWrite2.writeToDB(
writeData[:part], None, countTest,
"shuf2_testD_%s_%s_randboard:%s" %
(sys.argv[1], countTest / 140000, randGame))
lmdbReadWrite2.writeToDB(
labels[:part], None, countTest,
"shuf2_testL_%s_%s_randboard:%s" %
(sys.argv[1], countTest / 140000, randGame))
countTest = countTest + part
train = 0
writeData = np.array([],
dtype=np.uint8).reshape(0, 4, 21, 21)
writeSequence = np.array([])
#print("writing")
#print(j)
#if(train < 50):
# lmdbReadWrite2.writeToDB(data, sequence, countTrain, "train_%s_%s" % (sys.argv[1], countTrain/7000000))
# countTrain = countTrain + len(sequence)
# train = train + 1;
#else:
# lmdbReadWrite2.writeToDB(data, sequence, countTest, "test_%s_%s" % (sys.argv[1], countTest/140000))
# countTest = countTest + len(sequence)
# train = 0;
except:
print('Error in \t%s' % fname)
#count = count + len(sequence)
#trainLabel = 'pro_vs_random_moves_train_data'
#print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
#print("move")
#print(count)
#print("class")
#print(label)
#print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
#a, b, c, d, e = np.dsplit(writeData, 5)
#arrayPrint(a)
#arrayPrint(b)
#arrayPrint(c)
#arrayPrint(d)
#arrayPrint(e)
#array3 = board.print_weight()
#array4 = board.print_info()
#array5 = board.print_guide()
# 5 arrays
#print(j)
#print("first:")
#print(data.shape[0])
#print("second:")
#print(data.shape[1])
#print("third:")
#print(data.shape[2])
#print("fourth")
#print(data.shape[3])
#print(data[j][0])
#addmove()
print(time.time() - timeStart)
class Board():
#board = np.zeros((19,19))
timeStart = time.time()
#moves = [[Stone() for j in range(19)] for i in range(19)]
#print (board)
countTrain = 0
countTest = 0
max = 3672204
rootDir = sys.argv[1]
# name of DB:
#inputDB = sys.argv[2]
#dbNameTrain = sys.argv[2]
#dbNameTest = sys.argv[3]
train = True
for dirName, subdirList, fileList in os.walk(rootDir):
print('Found directory: %s' % dirName)
for fname in fileList:
try:
print('\t%s' % fname)
# /home/nathan/Go_Database/expanded-GoGod/rotation-7/69/1994-02-17g_7.sgf new troublesome one
# 2015-03-23b_0.sgf is the troublesome one
if fname.endswith('.sgf'):
path = os.path.join(dirName, fname)
f = open(path, 'r')
else:
continue
sgfString = f.read()
game = SGFsequence(sgfString)
board = Board(19)
onesA = np.ones((len(game.moves)))
zerosA = np.zeros((len(game.moves)))
sequence = np.append(onesA, (zerosA))
#make a random sequence between random and pro moves
#so we will know the total length
#this is also the labels
np.random.shuffle(sequence)
batchLen = len(sequence)
#lets write all the data for a game at once
data = np.empty((batchLen, 4, 21, 21), dtype=np.uint8)
# game[i].order[0][0][0] this is how you get X
# game[i].order[0][0][1] this is how you get Y
# process move by move
i = 0
j = 0
path = countTrain / 100000
for item in sequence:
#just tring single stream learning so commenting out siamese thing for now
#writeData2 = lmdbReadWrite2.getRandDataPoint( max, inputDB)[0] #read from db
if (path >= i):
if (i > 0):
moves = i - 1
else:
moves = 0
else:
moves = random.randrange(path, i)
#print("moves = %s" % moves)
#print(" i = %s" % i)
if (not item):
randBoard = Board(19)
for orderMove in range(0, moves):
#print("ordermove: %s" % orderMove)
#print(game.moves)
#randomPosition = random.sample((randBoard.libertySet),1)
x = game.moves[orderMove][
0] #randomPosition[0] % 19
y = game.moves[orderMove][
1] #randomPosition[0] / 19
#print("x: %s" % x)
#print("y: %s" % y)
#randBoard.print_board()
#print((set(range(361)) - randBoard.libertySet))
writeData1 = randBoard.place_stone(
x, y, 1 if orderMove % 2 else -1, i - 1)
for randMove in range(moves, i + 1):
#print("randmove: %s" % randMove)
#print((set(range(361)) - randBoard.libertySet))
randomPosition = random.sample(
(randBoard.libertySet), 1)
#print("randomPos: %s " % randomPosition)
x = randomPosition[0] % 19
y = randomPosition[0] / 19
writeData1 = randBoard.place_stone(
x, y, 1 if randMove % 2 else -1, i - 1)
#randomPosition = random.sample((board.libertySet),1)
#print("randompos: %d" % randomPosition[0])
#x = randomPosition[0] % 19
#y = randomPosition[0] / 19
#writeData1 = board.poof_stone(x, y, 1 if i % 2 else -1, i)
else:
x = game.moves[i][0]
y = game.moves[i][1]
#x, y = board.__board_to_ints__(xchar, ychar)
writeData1 = board.place_stone(x, y,
1 if i % 2 else -1, i)
i = i + 1
#print("i = %s" % i)
#array2 = board.print_board()
#array3 = board.print_weight()
#array1 = board.print_board_liberty()
#print(writeData1[0,0,0].dtype)
transpose1 = writeData1.transpose((2, 0, 1))
#transpose2 = writeData2 #.transpose((2, 0, 1))
#concatedData = np.concatenate((transpose1, transpose2))
data[j, :, :, :] = transpose1 #concatedData
#array2 = board.print_board()
#array1 = board.print_board_liberty()
#np.set_printoptions(threshold=np.nan)
#print(transpose1)
#print(transpose2)
#print(concatedData)
j = j + 1
# def writeToDB(data, label, index, database):
#print("writing")
print(countTrain)
print(countTest)
print(data.shape)
#print(j)
if (train < 50):
lmdbReadWrite2.writeToDB(
data, sequence, countTrain, "halfrand_train_%s_%s" %
(sys.argv[1], countTrain / 1000000))
countTrain = countTrain + len(sequence)
train = train + 1
else:
lmdbReadWrite2.writeToDB(
data, sequence, countTest, "halfrand_test_%s_%s" %
(sys.argv[1], countTest / 20000))
countTest = countTest + len(sequence)
train = 0
except:
print('Error in \t%s' % fname)
#count = count + len(sequence)
#trainLabel = 'pro_vs_random_moves_train_data'
#print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
#print("move")
#print(count)
#print("class")
#print(label)
#print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
#a, b, c, d, e = np.dsplit(writeData, 5)
#arrayPrint(a)
#arrayPrint(b)
#arrayPrint(c)
#arrayPrint(d)
#arrayPrint(e)
#array3 = board.print_weight()
#array4 = board.print_info()
#array5 = board.print_guide()
# 5 arrays
#print(j)
#print("first:")
#print(data.shape[0])
#print("second:")
#print(data.shape[1])
#print("third:")
#print(data.shape[2])
#print("fourth")
#print(data.shape[3])
#print(data[j][0])
#addmove()
print(time.time() - timeStart)