def test_simple_path(self): node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L1") node2 = Node(node_id=2, label="L1") edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1}) edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2}) redis_graph.add_node(node0) redis_graph.add_node(node1) redis_graph.add_node(node2) redis_graph.add_edge(edge01) redis_graph.add_edge(edge12) redis_graph.flush() # Rewrite the edges with IDs instead of node values to match how they are returned. edge01 = Edge(0, "R1", 1, edge_id=0, properties={'value': 1}) edge12 = Edge(1, "R1", 2, edge_id=1, properties={'value': 2}) path01 = Path.new_empty_path().add_node(node0).add_edge( edge01).add_node(node1) path12 = Path.new_empty_path().add_node(node1).add_edge( edge12).add_node(node2) expected_results = [[path01], [path12]] query = "MATCH p=(:L1)-[:R1]->(:L1) RETURN p" query_info = QueryInfo(query=query, description="Tests simple paths", expected_result=expected_results) self._assert_resultset_and_expected_mutually_included( redis_graph.query(query), query_info)
def test_variable_length_path(self): node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L1") node2 = Node(node_id=2, label="L1") edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1}) edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2}) redis_graph.add_node(node0) redis_graph.add_node(node1) redis_graph.add_node(node2) redis_graph.add_edge(edge01) redis_graph.add_edge(edge12) redis_graph.flush() path01 = Path.new_empty_path().add_node(node0).add_edge( edge01).add_node(node1) path12 = Path.new_empty_path().add_node(node1).add_edge( edge12).add_node(node2) path02 = Path.new_empty_path().add_node(node0).add_edge( edge01).add_node(node1).add_edge(edge12).add_node(node2) expected_results = [[path01], [path12], [path02]] query = "MATCH p=(:L1)-[:R1*]->(:L1) RETURN p" query_info = QueryInfo(query=query, description="Tests variable length paths", expected_result=expected_results) self._assert_resultset_and_expected_mutually_included( redis_graph.query(query), query_info)
def test_bi_directional_path(self): node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L1") node2 = Node(node_id=2, label="L1") edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1}) edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2}) redis_graph.add_node(node0) redis_graph.add_node(node1) redis_graph.add_node(node2) redis_graph.add_edge(edge01) redis_graph.add_edge(edge12) redis_graph.flush() # Reverse direction edges which are not part of the graph. Read only values. edge10 = Edge(1, "R1", 0, edge_id = 0 , properties={'value':1}) edge21 = Edge(2, "R1", 1, edge_id = 1 , properties={'value':2}) path010 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1).add_edge(edge10).add_node(node0) path0121 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1).add_edge(edge12) \ .add_node(node2).add_edge(edge21).add_node(node1) path01210 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1).add_edge(edge12) \ .add_node(node2).add_edge(edge21).add_node(node1).add_edge(edge10).add_node(node0) path121 = Path.new_empty_path().add_node(node1).add_edge(edge12).add_node(node2).add_edge(edge21).add_node(node1) path1210 = Path.new_empty_path().add_node(node1).add_edge(edge12).add_node(node2).add_edge(edge21) \ .add_node(node1).add_edge(edge10).add_node(node0) expected_results=[[path010], [path0121], [path01210], [path121], [path1210]] query = "MATCH p=(:L1)-[:R1*]->(:L1)<-[:R1*]-() RETURN p" query_info = QueryInfo(query = query, description="Tests bi directional variable length paths", \ expected_result = expected_results) self._assert_resultset_and_expected_mutually_included(redis_graph.query(query), query_info)
def save_playthrough_step_info_to_redis(gamename, step_num, obs, rewards, dones, infos, cmds, redisbasekey=REDIS_FTWC_PLAYTHROUGHS, ptid=f'eatmeal_42', do_write=False, redis=None, redis_ops=0): print( f"{'++' if do_write else '..'} step:[{step_num}] save_playthrough_step_info: {gamename} ({ptid})" ) step_json = playthrough_step_to_json(cmds, dones, infos, obs, rewards, step_num) if do_write: step_key = list(step_json.keys())[0] # get the first (and only) key redis.jsonset(f'{redisbasekey}:{gamename}', Path(f".{ptid}.{step_key}"), step_json[step_key]) redis_ops += 1 # print(f"[{step_num}] **** REWARD: {rewards} SCORE: {infos['game_score']} DONE: {done} ***********************\n") # print("--------------------------- Feedback: ---------------------------\n", infos['feedback'][0]) # print("--------------------------- Description: ------------------------\n", infos['description'][0]) # print("--------------------------- Inventory: --------------------------\n", infos['inventory'][0]) # print("--------------------------- Admissible Commands: ----------------\n", infos['admissible_commands'][0]) # print("--------------------------- Oracle Next Action: -----------------\n", infos['tw_o_step'][0]) # #print() # print("--------------------------- Facts: ------------------------------\n", infos['facts'][0]) return redis_ops, step_json
def test_path_comparison(self): node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L1") node2 = Node(node_id=2, label="L1") edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1}) edge12 = Edge(node1, "R1", node2, edge_id=1, properties={'value': 2}) redis_graph.add_node(node0) redis_graph.add_node(node1) redis_graph.add_node(node2) redis_graph.add_edge(edge01) redis_graph.add_edge(edge12) redis_graph.flush() # Rewrite the edges with IDs instead of node values to match how they are returned. edge01 = Edge(0, "R1", 1, edge_id=0, properties={'value': 1}) edge12 = Edge(1, "R1", 2, edge_id=1, properties={'value': 2}) path01 = Path.new_empty_path().add_node(node0).add_edge( edge01).add_node(node1) path12 = Path.new_empty_path().add_node(node1).add_edge( edge12).add_node(node2) # Test a path equality filter query = "MATCH p1 = (:L1)-[:R1]->(:L1) MATCH p2 = (:L1)-[:R1]->(:L1) WHERE p1 = p2 RETURN p1" expected_results = [[path01], [path12]] query_info = QueryInfo(query=query, description="Test path equality", expected_result=expected_results) self._assert_resultset_and_expected_mutually_included( redis_graph.query(query), query_info) # Test a path inequality filter query = "MATCH p1 = (:L1)-[:R1]->(:L1) MATCH p2 = (:L1)-[:R1]->(:L1) WHERE p1 <> p2 RETURN DISTINCT p1, p2" expected_results = [[path01, path12], [path12, path01]] query_info = QueryInfo(query=query, description="Test path inequality", expected_result=expected_results) self._assert_resultset_and_expected_mutually_included( redis_graph.query(query), query_info)
def retrieve_playthrough_nsteps( gamename, redis=None, # redis server: expected to be a RedisJSON client connection ptid=playthrough_id(), # default playthrough, can optionally specify redisbasekey=REDIS_FTWC_PLAYTHROUGHS, randseed=DEFAULT_PTHRU_SEED): nsteps = _rj.jsonobjlen(f'{redisbasekey}:{gamename}', Path(f".{ptid}")) if redis is None: _rj.close() return nsteps
def test_zero_length_path(self): node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L2") edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1}) redis_graph.add_node(node0) redis_graph.add_node(node1) redis_graph.add_edge(edge01) redis_graph.flush() path01 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1) expected_results=[[path01]] query = "MATCH p=(:L1)-[*0..]->()-[]->(:L2) RETURN p" query_info = QueryInfo(query = query, description="Tests path with zero length variable length paths", \ expected_result = expected_results) self._assert_resultset_and_expected_mutually_included(redis_graph.query(query), query_info)
def test_path(self): redis_graph = Graph('social', self.r) node0 = Node(node_id=0, label="L1") node1 = Node(node_id=1, label="L1") edge01 = Edge(node0, "R1", node1, edge_id=0, properties={'value': 1}) redis_graph.add_node(node0) redis_graph.add_node(node1) redis_graph.add_edge(edge01) redis_graph.flush() path01 = Path.new_empty_path().add_node(node0).add_edge( edge01).add_node(node1) expected_results = [[path01]] query = "MATCH p=(:L1)-[:R1]->(:L1) RETURN p ORDER BY p" result = redis_graph.query(query) self.assertEqual(expected_results, result.result_set) # All done, remove graph. redis_graph.delete()
def retrieve_playthrough_json( gamename, redis=None, # redis server: expected to be a RedisJSON client connection ptid=playthrough_id(), # default playthrough, can optionally specify redisbasekey=REDIS_FTWC_PLAYTHROUGHS, randseed=DEFAULT_PTHRU_SEED): if redis is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: assert isinstance(redis, Client) _rj = redis jsonobj = _rj.jsonget(f'{redisbasekey}:{gamename}', Path('.' + ptid)) if redis is None: _rj.close() step_array = [ ] # convert json dict data (with redundant keys) to an array for convenience for i, step_key in enumerate(list(jsonobj.keys())): assert step_key == format_stepkey( i) # format_stepkey() from twutils.playthroughs step_array.append(jsonobj[step_key]) assert len(step_array) == len(jsonobj.keys()) return step_array
def save_playthrough_to_redis(gamename, gamedir=None, redisbasekey=REDIS_FTWC_PLAYTHROUGHS, randseed=DEFAULT_PTHRU_SEED, goal_type=GOAL_MEAL, do_write=False, redis=None, skip_existing=True): if gamedir is None: if redis: gamedir = get_dir_for_game(redis, gamename) else: gamedir = TW_TRAINING_DIR # best guess, we'll see later if it's there or not if gamename.endswith(".z8") or gamefile.endswith(".ulx"): _gamefile = f"{gamedir}/{gamename}" else: _gamefile = f"{gamedir}/{gamename}.z8" if not os.path.exists(_gamefile): _gamefile = f"{gamedir}/{gamename}.ulx" redis_ops = 0 num_steps = 0 step_array = [ ] # convert json dict data (with redundant keys) to an array for convenience ptid = playthrough_id( objective_name=goal_type, seed=randseed ) # playtrough ID (which of potentially different) for this gamename if not redis: do_write = False if do_write: if not redis.exists(f'{redisbasekey}:{gamename}'): redis_ops += 1 redis.jsonset(f'{redisbasekey}:{gamename}', Path.rootPath(), {ptid: {}}) elif skip_existing: if redis.jsonobjlen(f'{redisbasekey}:{gamename}', Path('.' + ptid)): # if exists and non-empty print(f"SKIPPED EXISTING playthrough {gamename}") return num_steps, redis_ops _dones = [0] _rewards = [0] next_cmds = ['start'] gymenv, _obs, _infos = start_game_for_playthrough(_gamefile) redis_ops, playthru_step_data = save_playthrough_step_info_to_redis( gamename, num_steps, _obs, _rewards, _dones, _infos, next_cmds, redisbasekey=redisbasekey, ptid=ptid, redis=redis, do_write=do_write, redis_ops=redis_ops) step_array.append(playthru_step_data[format_stepkey(num_steps)]) next_cmds = _infos['tw_o_step'] while not _dones[0] and num_steps < MAX_PLAYTHROUGH_STEPS + 1: num_steps += 1 _obs, _rewards, _dones, _infos = step_game_for_playthrough( gymenv, next_cmds) redis_ops, playthru_step_data = save_playthrough_step_info_to_redis( gamename, num_steps, _obs, _rewards, _dones, _infos, next_cmds, redisbasekey=redisbasekey, ptid=ptid, redis=redis, do_write=do_write, redis_ops=redis_ops) step_array.append(playthru_step_data[format_stepkey(num_steps)]) next_cmds = _infos['tw_o_step'] gymenv.close() print( f"----------------- {gamename} playthrough steps: {num_steps} Redis writes {redis_ops} ----------------" ) return num_steps, redis_ops, step_array
def extracted_data_to_redis(game_names=None, xtract_dir=XTRACT_DIR, redis=None): do_write = False if not game_names: game_names = [] itercount = 0 room_count = 0 desc_count = 0 failed_count = 0 max_desc_wlen = 0 max_desc_wlen = 0 max_desc_str = '' redis_ops = 0 for bn in game_names: #[0:10]: if do_write: if not redis.exists(f'{REDIS_EXTRACTED_DATA}:{bn}'): redis_ops += 1 redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}', Path.rootPath(), {'room': {}}) xdir = xtract_dir + '/' + bn itercount += 1 data_files = os.listdir(xdir) # suffixes = ['.room_state', '.desc'] suffix = '.room_state' filtered_files = list( filter(lambda fname: fname.endswith(suffix), data_files)) sorted_files = list( map(lambda fname: fname.split('_', maxsplit=2), filtered_files)) for i, tup in enumerate(sorted_files): tup[1] = int(tup[1]) # convert room numbers from ascii to int tup[2] = tup[2].split('.')[0] # the room name tup.append( filtered_files[i] ) # add the original file name (NOTE: tup is a list, not a tuple) # sort by room number, not alphabetically sorted_files = sorted(sorted_files) # print(sorted_files[0:3]) for (_, i, room, f) in sorted_files: obj_lines = [] room_count += 1 with open(xdir + '/' + f, 'r') as room_state_file: state_str = room_state_file.read() room_name = f.split('.')[0] with open(xdir + '/' + room_name + '.desc', 'r') as desc_file: desc_lines = [line.strip() for line in desc_file] desc_str = ' '.join(desc_lines) if len(desc_str): desc_count += 1 if len(desc_str) > len(max_desc_str): if len(desc_str.split(' ')) > max_desc_wlen: max_desc_wlen = len(desc_str.split(' ')) max_desc_str = desc_str jsonobj = { 'room_name': room, 'room_id': i, 'init_descr': desc_str, 'init_state': state_str } if do_write: redis_ops += 1 redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}', Path(f'.room.{room}'), jsonobj) #print(bn, room.upper(), f) print("Processed", itercount, "games, ", room_count, "rooms.") print(redis_ops, "redis write ops") print("max room description length =", len(max_desc_str), "# words=", max_desc_wlen) assert desc_count == room_count