def test_simple_path(self):
node0 = Node(node_id=0, label="L1")
node1 = Node(node_id=1, label="L1")
node2 = Node(node_id=2, label="L1")
edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1})
edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2})
redis_graph.add_node(node0)
redis_graph.add_node(node1)
redis_graph.add_node(node2)
redis_graph.add_edge(edge01)
redis_graph.add_edge(edge12)
redis_graph.flush()
# Rewrite the edges with IDs instead of node values to match how they are returned.
edge01 = Edge(0, "R1", 1, edge_id=0, properties={'value': 1})
edge12 = Edge(1, "R1", 2, edge_id=1, properties={'value': 2})
path01 = Path.new_empty_path().add_node(node0).add_edge(
edge01).add_node(node1)
path12 = Path.new_empty_path().add_node(node1).add_edge(
edge12).add_node(node2)
expected_results = [[path01], [path12]]
query = "MATCH p=(:L1)-[:R1]->(:L1) RETURN p"
query_info = QueryInfo(query=query,
description="Tests simple paths",
expected_result=expected_results)
self._assert_resultset_and_expected_mutually_included(
redis_graph.query(query), query_info)
def test_variable_length_path(self):
node0 = Node(node_id=0, label="L1")
node1 = Node(node_id=1, label="L1")
node2 = Node(node_id=2, label="L1")
edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1})
edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2})
redis_graph.add_node(node0)
redis_graph.add_node(node1)
redis_graph.add_node(node2)
redis_graph.add_edge(edge01)
redis_graph.add_edge(edge12)
redis_graph.flush()
path01 = Path.new_empty_path().add_node(node0).add_edge(
edge01).add_node(node1)
path12 = Path.new_empty_path().add_node(node1).add_edge(
edge12).add_node(node2)
path02 = Path.new_empty_path().add_node(node0).add_edge(
edge01).add_node(node1).add_edge(edge12).add_node(node2)
expected_results = [[path01], [path12], [path02]]
query = "MATCH p=(:L1)-[:R1*]->(:L1) RETURN p"
query_info = QueryInfo(query=query,
description="Tests variable length paths",
expected_result=expected_results)
self._assert_resultset_and_expected_mutually_included(
redis_graph.query(query), query_info)
def test_bi_directional_path(self):
node0 = Node(node_id=0, label="L1")
node1 = Node(node_id=1, label="L1")
node2 = Node(node_id=2, label="L1")
edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1})
edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2})
redis_graph.add_node(node0)
redis_graph.add_node(node1)
redis_graph.add_node(node2)
redis_graph.add_edge(edge01)
redis_graph.add_edge(edge12)
redis_graph.flush()
# Reverse direction edges which are not part of the graph. Read only values.
edge10 = Edge(1, "R1", 0, edge_id = 0 , properties={'value':1})
edge21 = Edge(2, "R1", 1, edge_id = 1 , properties={'value':2})
path010 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1).add_edge(edge10).add_node(node0)
path0121 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1).add_edge(edge12) \
.add_node(node2).add_edge(edge21).add_node(node1)
path01210 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1).add_edge(edge12) \
.add_node(node2).add_edge(edge21).add_node(node1).add_edge(edge10).add_node(node0)
path121 = Path.new_empty_path().add_node(node1).add_edge(edge12).add_node(node2).add_edge(edge21).add_node(node1)
path1210 = Path.new_empty_path().add_node(node1).add_edge(edge12).add_node(node2).add_edge(edge21) \
.add_node(node1).add_edge(edge10).add_node(node0)
expected_results=[[path010], [path0121], [path01210], [path121], [path1210]]
query = "MATCH p=(:L1)-[:R1*]->(:L1)<-[:R1*]-() RETURN p"
query_info = QueryInfo(query = query, description="Tests bi directional variable length paths", \
expected_result = expected_results)
self._assert_resultset_and_expected_mutually_included(redis_graph.query(query), query_info)
def save_playthrough_step_info_to_redis(gamename,
step_num,
obs,
rewards,
dones,
infos,
cmds,
redisbasekey=REDIS_FTWC_PLAYTHROUGHS,
ptid=f'eatmeal_42',
do_write=False,
redis=None,
redis_ops=0):
print(
f"{'++' if do_write else '..'} step:[{step_num}] save_playthrough_step_info: {gamename} ({ptid})"
)
step_json = playthrough_step_to_json(cmds, dones, infos, obs, rewards,
step_num)
if do_write:
step_key = list(step_json.keys())[0] # get the first (and only) key
redis.jsonset(f'{redisbasekey}:{gamename}',
Path(f".{ptid}.{step_key}"), step_json[step_key])
redis_ops += 1
# print(f"[{step_num}] **** REWARD: {rewards} SCORE: {infos['game_score']} DONE: {done} ***********************\n")
# print("--------------------------- Feedback: ---------------------------\n", infos['feedback'][0])
# print("--------------------------- Description: ------------------------\n", infos['description'][0])
# print("--------------------------- Inventory: --------------------------\n", infos['inventory'][0])
# print("--------------------------- Admissible Commands: ----------------\n", infos['admissible_commands'][0])
# print("--------------------------- Oracle Next Action: -----------------\n", infos['tw_o_step'][0])
# #print()
# print("--------------------------- Facts: ------------------------------\n", infos['facts'][0])
return redis_ops, step_json
def test_path_comparison(self):
node0 = Node(node_id=0, label="L1")
node1 = Node(node_id=1, label="L1")
node2 = Node(node_id=2, label="L1")
edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1})
edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2})
redis_graph.add_node(node0)
redis_graph.add_node(node1)
redis_graph.add_node(node2)
redis_graph.add_edge(edge01)
redis_graph.add_edge(edge12)
redis_graph.flush()
# Rewrite the edges with IDs instead of node values to match how they are returned.
edge01 = Edge(0, "R1", 1, edge_id=0, properties={'value': 1})
edge12 = Edge(1, "R1", 2, edge_id=1, properties={'value': 2})
path01 = Path.new_empty_path().add_node(node0).add_edge(
edge01).add_node(node1)
path12 = Path.new_empty_path().add_node(node1).add_edge(
edge12).add_node(node2)
# Test a path equality filter
query = "MATCH p1 = (:L1)-[:R1]->(:L1) MATCH p2 = (:L1)-[:R1]->(:L1) WHERE p1 = p2 RETURN p1"
expected_results = [[path01], [path12]]
query_info = QueryInfo(query=query,
description="Test path equality",
expected_result=expected_results)
self._assert_resultset_and_expected_mutually_included(
redis_graph.query(query), query_info)
# Test a path inequality filter
query = "MATCH p1 = (:L1)-[:R1]->(:L1) MATCH p2 = (:L1)-[:R1]->(:L1) WHERE p1 <> p2 RETURN DISTINCT p1, p2"
expected_results = [[path01, path12], [path12, path01]]
query_info = QueryInfo(query=query,
description="Test path inequality",
expected_result=expected_results)
self._assert_resultset_and_expected_mutually_included(
redis_graph.query(query), query_info)
def retrieve_playthrough_nsteps(
gamename,
redis=None, # redis server: expected to be a RedisJSON client connection
ptid=playthrough_id(), # default playthrough, can optionally specify
redisbasekey=REDIS_FTWC_PLAYTHROUGHS,
randseed=DEFAULT_PTHRU_SEED):
nsteps = _rj.jsonobjlen(f'{redisbasekey}:{gamename}', Path(f".{ptid}"))
if redis is None:
_rj.close()
return nsteps
def test_zero_length_path(self):
node0 = Node(node_id=0, label="L1")
node1 = Node(node_id=1, label="L2")
edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1})
redis_graph.add_node(node0)
redis_graph.add_node(node1)
redis_graph.add_edge(edge01)
redis_graph.flush()
path01 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1)
expected_results=[[path01]]
query = "MATCH p=(:L1)-[*0..]->()-[]->(:L2) RETURN p"
query_info = QueryInfo(query = query, description="Tests path with zero length variable length paths", \
expected_result = expected_results)
self._assert_resultset_and_expected_mutually_included(redis_graph.query(query), query_info)
def test_path(self):
redis_graph = Graph('social', self.r)
node0 = Node(node_id=0, label="L1")
node1 = Node(node_id=1, label="L1")
edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1})
redis_graph.add_node(node0)
redis_graph.add_node(node1)
redis_graph.add_edge(edge01)
redis_graph.flush()
path01 = Path.new_empty_path().add_node(node0).add_edge(
edge01).add_node(node1)
expected_results = [[path01]]
query = "MATCH p=(:L1)-[:R1]->(:L1) RETURN p ORDER BY p"
result = redis_graph.query(query)
self.assertEqual(expected_results, result.result_set)
# All done, remove graph.
redis_graph.delete()
def retrieve_playthrough_json(
gamename,
redis=None, # redis server: expected to be a RedisJSON client connection
ptid=playthrough_id(), # default playthrough, can optionally specify
redisbasekey=REDIS_FTWC_PLAYTHROUGHS,
randseed=DEFAULT_PTHRU_SEED):
if redis is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
assert isinstance(redis, Client)
_rj = redis
jsonobj = _rj.jsonget(f'{redisbasekey}:{gamename}', Path('.' + ptid))
if redis is None:
_rj.close()
step_array = [
] # convert json dict data (with redundant keys) to an array for convenience
for i, step_key in enumerate(list(jsonobj.keys())):
assert step_key == format_stepkey(
i) # format_stepkey() from twutils.playthroughs
step_array.append(jsonobj[step_key])
assert len(step_array) == len(jsonobj.keys())
return step_array
def save_playthrough_to_redis(gamename,
gamedir=None,
redisbasekey=REDIS_FTWC_PLAYTHROUGHS,
randseed=DEFAULT_PTHRU_SEED,
goal_type=GOAL_MEAL,
do_write=False,
redis=None,
skip_existing=True):
if gamedir is None:
if redis:
gamedir = get_dir_for_game(redis, gamename)
else:
gamedir = TW_TRAINING_DIR # best guess, we'll see later if it's there or not
if gamename.endswith(".z8") or gamefile.endswith(".ulx"):
_gamefile = f"{gamedir}/{gamename}"
else:
_gamefile = f"{gamedir}/{gamename}.z8"
if not os.path.exists(_gamefile):
_gamefile = f"{gamedir}/{gamename}.ulx"
redis_ops = 0
num_steps = 0
step_array = [
] # convert json dict data (with redundant keys) to an array for convenience
ptid = playthrough_id(
objective_name=goal_type, seed=randseed
) # playtrough ID (which of potentially different) for this gamename
if not redis:
do_write = False
if do_write:
if not redis.exists(f'{redisbasekey}:{gamename}'):
redis_ops += 1
redis.jsonset(f'{redisbasekey}:{gamename}', Path.rootPath(),
{ptid: {}})
elif skip_existing:
if redis.jsonobjlen(f'{redisbasekey}:{gamename}',
Path('.' + ptid)): # if exists and non-empty
print(f"SKIPPED EXISTING playthrough {gamename}")
return num_steps, redis_ops
_dones = [0]
_rewards = [0]
next_cmds = ['start']
gymenv, _obs, _infos = start_game_for_playthrough(_gamefile)
redis_ops, playthru_step_data = save_playthrough_step_info_to_redis(
gamename,
num_steps,
_obs,
_rewards,
_dones,
_infos,
next_cmds,
redisbasekey=redisbasekey,
ptid=ptid,
redis=redis,
do_write=do_write,
redis_ops=redis_ops)
step_array.append(playthru_step_data[format_stepkey(num_steps)])
next_cmds = _infos['tw_o_step']
while not _dones[0] and num_steps < MAX_PLAYTHROUGH_STEPS + 1:
num_steps += 1
_obs, _rewards, _dones, _infos = step_game_for_playthrough(
gymenv, next_cmds)
redis_ops, playthru_step_data = save_playthrough_step_info_to_redis(
gamename,
num_steps,
_obs,
_rewards,
_dones,
_infos,
next_cmds,
redisbasekey=redisbasekey,
ptid=ptid,
redis=redis,
do_write=do_write,
redis_ops=redis_ops)
step_array.append(playthru_step_data[format_stepkey(num_steps)])
next_cmds = _infos['tw_o_step']
gymenv.close()
print(
f"----------------- {gamename} playthrough steps: {num_steps} Redis writes {redis_ops} ----------------"
)
return num_steps, redis_ops, step_array
def extracted_data_to_redis(game_names=None,
xtract_dir=XTRACT_DIR,
redis=None):
do_write = False
if not game_names:
game_names = []
itercount = 0
room_count = 0
desc_count = 0
failed_count = 0
max_desc_wlen = 0
max_desc_wlen = 0
max_desc_str = ''
redis_ops = 0
for bn in game_names: #[0:10]:
if do_write:
if not redis.exists(f'{REDIS_EXTRACTED_DATA}:{bn}'):
redis_ops += 1
redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}', Path.rootPath(),
{'room': {}})
xdir = xtract_dir + '/' + bn
itercount += 1
data_files = os.listdir(xdir)
# suffixes = ['.room_state', '.desc']
suffix = '.room_state'
filtered_files = list(
filter(lambda fname: fname.endswith(suffix), data_files))
sorted_files = list(
map(lambda fname: fname.split('_', maxsplit=2), filtered_files))
for i, tup in enumerate(sorted_files):
tup[1] = int(tup[1]) # convert room numbers from ascii to int
tup[2] = tup[2].split('.')[0] # the room name
tup.append(
filtered_files[i]
) # add the original file name (NOTE: tup is a list, not a tuple)
# sort by room number, not alphabetically
sorted_files = sorted(sorted_files)
# print(sorted_files[0:3])
for (_, i, room, f) in sorted_files:
obj_lines = []
room_count += 1
with open(xdir + '/' + f, 'r') as room_state_file:
state_str = room_state_file.read()
room_name = f.split('.')[0]
with open(xdir + '/' + room_name + '.desc', 'r') as desc_file:
desc_lines = [line.strip() for line in desc_file]
desc_str = ' '.join(desc_lines)
if len(desc_str):
desc_count += 1
if len(desc_str) > len(max_desc_str):
if len(desc_str.split(' ')) > max_desc_wlen:
max_desc_wlen = len(desc_str.split(' '))
max_desc_str = desc_str
jsonobj = {
'room_name': room,
'room_id': i,
'init_descr': desc_str,
'init_state': state_str
}
if do_write:
redis_ops += 1
redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}',
Path(f'.room.{room}'), jsonobj)
#print(bn, room.upper(), f)
print("Processed", itercount, "games, ", room_count, "rooms.")
print(redis_ops, "redis write ops")
print("max room description length =", len(max_desc_str), "# words=",
max_desc_wlen)
assert desc_count == room_count
