def reportsApiTest3():
policy = PytorchPolicy(32, 1, 32, 3, 64, Connect4GameState(7, 6, 4),
"cuda:0", "torch.optim.adamw.AdamW", {
"lr": 0.001,
"weight_decay": 0.0001
})
encoded = encodeToBson(policy.store())
print("Encoded network into " + str(len(encoded)))
response = requests.post(
url=
"http://127.0.0.1:8042/api/networks/c48b01d7-18e8-4df0-9c8a-d9886473bb49/"
+ policy.getUUID(),
data=encoded)
response.raise_for_status()
response = requests.get(
url="http://127.0.0.1:8042/api/networks/download/" + policy.getUUID(),
stream=True)
response.raise_for_status()
redownloaded = decodeFromBson(response.raw.data)
prevId = policy.getUUID()
policy.reset()
policy.load(redownloaded)
print(policy.getUUID(), prevId)
def test_dicts(self):
root = dict()
root["A"] = 1
root["B"] = [1, 2, 3]
transfered = decodeFromBson(encodeToBson(root))
self.assertEqual(transfered, root)
def getNetworkFile(networkId):
networkData = encodeToBson(networks.downloadNetwork(network))
ff = tempfile.NamedTemporaryFile(suffix=".x0", mode="wb+")
ff.write(networkData)
ff.flush()
#logMsg("Using tempfile for network:", ff.name)
return ff
def pushResults(self):
logMsg("Started poll results thread")
while True:
while len(self.resultsQueue) == 0:
time.sleep(0.2)
nextResult = self.resultsQueue[0]
del self.resultsQueue[0]
resultId = nextResult["id"]
resultData = encodeToBson(nextResult["data"])
postBytes(self.command + "/checkin/" + resultId, "", resultData)
def saveNetwork(net):
uid = str(uuid.uuid4())
uuidBytes = stringToBytes(uid)
ublen = np.array([uuidBytes.shape[0]], dtype=np.uint8)
buffer = io.BytesIO()
torch.save(net.state_dict(), buffer)
modelBytes = np.frombuffer(buffer.getvalue(), dtype=np.uint8)
result = encodeToBson(np.concatenate((ublen, uuidBytes, modelBytes)))
with open(uid + ".network", "w+b") as f:
f.write(result)
def reportsApiTest2():
reps = [makeReport() for _ in range(1000)]
encStart = time.time()
repEnc = encodeToBson(reps)
print("Encoding time taken per state:",
int(((time.time() - encStart) / len(reps)) * 1000000), "us")
print("===")
reportId = requests.post(
url=
"http://127.0.0.1:8042/api/state/test/4492b3fc-0989-497d-b78c-5d97f148bfd4",
data=repEnc).json()
print(reportId)
def test_network_posting(self):
run = self.postARun()
policy = PytorchPolicy(32, 1, 32, 3, 64, Connect4GameState(7, 6, 4),
"cuda:0", "torch.optim.adamw.AdamW", {
"lr": 0.001,
"weight_decay": 0.0001
})
postBytes(urlBase + "api/networks/" + run["id"] + "/" +
policy.getUUID(),
config["secret"],
encodeToBson(policy.store()),
retries=1)
networkList = requestJson(urlBase + "api/networks/list/" + run["id"],
config["secret"],
retries=1)
self.assertEqual(len(networkList), 1)
self.assertEqual(networkList[0]["id"], policy.getUUID())
redownloaded = decodeFromBson(
requestBytes(urlBase + "api/networks/download/" + policy.getUUID(),
config["secret"],
retries=1))
game = Connect4GameState(7, 6, 4)
game = game.playMove(np.random.randint(7))
game = game.playMove(np.random.randint(7))
policy.isRandom = False
forwardResultPre = policy.forward([game])[0]
preUUID = policy.getUUID()
policy.reset()
policy.isRandom = False
forwardResultReset = policy.forward([game])[0]
policy.load(redownloaded)
policy.isRandom = False
forwardResultPost = policy.forward([game])[0]
self.assertEqual(preUUID, policy.getUUID())
self.assertTrue(np.all(forwardResultPre[0] == forwardResultPost[0]))
self.assertTrue(np.all(forwardResultPre[1] == forwardResultPost[1]))
self.assertFalse(np.all(forwardResultPre[0] == forwardResultReset[0]))
self.assertFalse(np.all(forwardResultPre[1] == forwardResultReset[1]))
def reportsApiTest():
report = makeReport()
# print(report)
# enc = encodeToBson(report)
# print("\nEncoded into %i bytes: " % len(enc), enc)
# print("\nDecoded:\n", decodeFromBson(enc))
reps = [makeReport() for _ in range(1000)]
print(reps[0])
encStart = time.time()
repEnc = encodeToBson(reps)
print("Encoding time taken:", time.time() - encStart)
print("Encoded %i reports into %i kbyte" % (len(reps), 1 +
(len(repEnc) / 1000)))
reportId = requests.post(url="http://127.0.0.1:8042/api/reports/",
data=repEnc).json()
print("Posted report of %i bytes and got response: %s" %
(len(repEnc), reportId))
response = requests.get(url="http://127.0.0.1:8042/api/reports/" +
reportId,
stream=True)
reponse.raise_for_status()
redownloaded = response.raw.data
print("Get report gave us %i bytes" % len(redownloaded))
decStart = time.time()
repDec = decodeFromBson(redownloaded)
print("Decode time taken:", time.time() - decStart)
print("Decoded %i" % len(repDec))
print(repDec[0])
def test_deepNumpyDicts(self):
root = dict()
root["A"] = 1
root["B"] = dict()
root["B"]["FOO"] = [1, 2, 3]
root["C"] = dict()
root["C"]["data"] = [np.array([3.1415, -3.1415], dtype=np.float32), 42]
root["D"] = [dict(), dict()]
root["D"][0]["A"] = 42
root["D"][1]["B"] = "hello world"
enc = encodeToBson(root)
transfered = decodeFromBson(enc)
self.assertEqual(transfered["A"], root["A"])
self.assertEqual(transfered["B"], root["B"])
self.assertTrue("C" in transfered and "data" in transfered["C"]
and isinstance(transfered["C"]["data"], list))
self.assertTrue(
np.sum(transfered["C"]["data"][0] == root["C"]["data"][0]) == 2)
self.assertEqual(transfered["C"]["data"][1], root["C"]["data"][1])
self.assertTrue(transfered["D"], root["D"])
def reportFinishedPackage(commandHost, workerName, run, secret, pending):
reportId = postBytes(commandHost + "/api/state/" + workerName + "/" + run,
secret,
encodeToBson(pending),
expectResponse=True)
def test_state_posting(self):
run1 = self.postARun()
run2 = self.postARun()
states1 = [makeReport() for _ in range(123)]
states2 = [makeReport() for _ in range(456)]
report1Id = requests.post(url=urlBase + "api/state/test/" + run1["id"],
data=encodeToBson(states1),
headers={
"secret": config["secret"]
}).json()
report2Id = requests.post(url=urlBase + "api/state/test2/" +
run2["id"],
data=encodeToBson(states2),
headers={
"secret": config["secret"]
}).json()
listRun1 = requestJson(urlBase + "api/state/list/" + run1["id"],
config["secret"],
retries=1)
listRun2 = requestJson(urlBase + "api/state/list/" + run2["id"],
config["secret"],
retries=1)
self.assertEqual(len(listRun1), 1)
self.assertEqual(len(listRun2), 1)
self.assertEqual(listRun1[0]["id"], report1Id)
self.assertEqual(listRun2[0]["id"], report2Id)
self.assertEqual(listRun1[0]["worker"], "test")
self.assertEqual(listRun2[0]["worker"], "test2")
self.assertEqual(listRun1[0]["packageSize"], len(states1))
self.assertEqual(listRun2[0]["packageSize"], len(states2))
states1Downloaded = decodeFromBson(
requestBytes(urlBase + "api/state/download/" + report1Id,
config["secret"],
retries=1))
states2Downloaded = decodeFromBson(
requestBytes(urlBase + "api/state/download/" + report2Id,
config["secret"],
retries=1))
self.assertEqual(len(states1), len(states1Downloaded))
self.assertEqual(len(states2), len(states2Downloaded))
def checkStatesEqual(statesPre, statesPost):
for pre, post in zip(statesPre, statesPost):
piPre = pre["policyIterated"]
piPost = post["policyIterated"]
stPre = pre["state"]
stPost = post["state"]
del pre["policyIterated"]
del post["policyIterated"]
del pre["state"]
del post["state"]
self.assertDictEqual(pre, post)
self.assertTrue(np.all(piPre == piPost))
self.assertTrue(np.all(stPre == stPost))
checkStatesEqual(states1, states1Downloaded)
checkStatesEqual(states2, states2Downloaded)
def test_numpy_tuples(self):
data1 = (np.random.dirichlet([0.9] * 2),
np.random.dirichlet([0.9] * 2))
data2 = decodeFromBson(encodeToBson(data1))
self.assertTrue(np.all(data1[0] == data2[0]))
self.assertTrue(np.all(data1[1] == data2[1]))
def test_numpyList(self):
lst = [np.array([1, 2, 3]), np.array([4, 5, 6])]
transfered = decodeFromBson(encodeToBson(lst))
self.assertEqual(len(transfered), 2)
self.assertEqual(transfered[0].tolist(), [1, 2, 3])
self.assertEqual(transfered[1].tolist(), [4, 5, 6])
def test_number(self):
number = 42
enc = encodeToBson(number)
dec = decodeFromBson(enc)
self.assertEqual(number, dec)
def test_numpy(self):
data = np.array([1.23456789, 9.87654321])
transfered = decodeFromBson(encodeToBson(data))
self.assertEqual(len(data), len(transfered))
self.assertEqual(np.sum(data == transfered), len(data))
def test_SimpleList(self):
lst = [1, 2, 3]
enc = encodeToBson(lst)
dec = decodeFromBson(enc)
self.assertEqual(lst, dec)
def uploadNetwork(self, policy):
newPolicyEncoded = encodeToBson(policy.store())
self.uploadEncodedNetwork(policy.getUUID(), newPolicyEncoded)