def read_raw_node_data(pool, root_dir):
""" Read raw node data from root dir. """
_, nodes = lib.read_root_csv(root_dir)
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
return raw_node_data
def main():
parser = argparse.ArgumentParser(
description=
"Reduces raw database dump into prototype tiles, grid, and connections."
)
parser.add_argument('--root_dir', required=True)
parser.add_argument('--output_dir', required=True)
parser.add_argument('--verify_only', action='store_true')
parser.add_argument('--ignored_wires')
args = parser.parse_args()
tiles, nodes = lib.read_root_csv(args.root_dir)
processes = min(multiprocessing.cpu_count(), 10)
print('{} Running {} processes'.format(datetime.datetime.now(), processes))
pool = multiprocessing.Pool(processes=processes)
node_tree_file = os.path.join(args.output_dir, 'node_tree.json')
tileconn_file = os.path.join(args.output_dir, 'tileconn.json')
wire_map_file = os.path.join(args.output_dir, 'wiremap.pickle')
print('{} Reading tilegrid'.format(datetime.datetime.now()))
with open(os.path.join(util.get_db_root(), util.get_part(),
'tilegrid.json')) as f:
grid = json.load(f)
if not args.verify_only:
print('{} Creating tile map'.format(datetime.datetime.now()))
grid2, wire_map = generate_tilegrid(pool, tiles)
# Make sure tilegrid from 005-tilegrid matches tilegrid from
# generate_tilegrid.
db_grid_keys = set(grid.keys())
generated_grid_keys = set(grid2.keys())
assert db_grid_keys == generated_grid_keys, (
db_grid_keys ^ generated_grid_keys)
for tile in db_grid_keys:
for k in grid2[tile]:
assert k in grid[tile], k
assert grid[tile][k] == grid2[tile][k], (
tile, k, grid[tile][k], grid2[tile][k])
with open(wire_map_file, 'wb') as f:
pickle.dump(wire_map, f)
print('{} Reading node tree'.format(datetime.datetime.now()))
with open(node_tree_file) as f:
node_tree = json.load(f)
print('{} Creating tile connections'.format(datetime.datetime.now()))
tileconn, raw_node_data = generate_tileconn(
pool, node_tree, nodes, wire_map, grid)
print('{} Writing tileconn'.format(datetime.datetime.now()))
with open(tileconn_file, 'w') as f:
xjson.pprint(f, tileconn)
else:
with open(wire_map_file, 'rb') as f:
wire_map = pickle.load(f)
print('{} Reading raw_node_data'.format(datetime.datetime.now()))
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
print('{} Reading tileconn'.format(datetime.datetime.now()))
with open(tileconn_file) as f:
tileconn = json.load(f)
wire_nodes_file = os.path.join(args.output_dir, 'wire_nodes.pickle')
if os.path.exists(wire_nodes_file) and args.verify_only:
with open(wire_nodes_file, 'rb') as f:
wire_nodes = pickle.load(f)
else:
print(
"{} Connecting wires to verify tileconn".format(
datetime.datetime.now()))
wire_nodes = connect_wires(grid, tileconn, wire_map)
with open(wire_nodes_file, 'wb') as f:
pickle.dump(wire_nodes, f)
print('{} Verifing tileconn'.format(datetime.datetime.now()))
error_nodes = []
lib.verify_nodes(
[
(node['node'], tuple(wire['wire']
for wire in node['wires']))
for node in raw_node_data
], wire_nodes, error_nodes)
if len(error_nodes) > 0:
error_nodes_file = os.path.join(args.output_dir, 'error_nodes.json')
with open(error_nodes_file, 'w') as f:
xjson.pprint(f, error_nodes)
ignored_wires = []
ignored_wires_file = args.ignored_wires
if os.path.exists(ignored_wires_file):
with open(ignored_wires_file) as f:
ignored_wires = set(l.strip() for l in f)
if not lib.check_errors(error_nodes, ignored_wires):
print(
'{} errors detected, see {} for details.'.format(
len(error_nodes), error_nodes_file))
sys.exit(1)
else:
print(
'{} errors ignored because of {}\nSee {} for details.'.format(
len(error_nodes), ignored_wires_file, error_nodes_file))
def main():
parser = argparse.ArgumentParser(
description="Reduce node names for wire connections.")
parser.add_argument('--root_dir', required=True)
parser.add_argument('--output_dir', required=True)
parser.add_argument('--max_cpu', type=int, default=10)
args = parser.parse_args()
_, nodes = lib.read_root_csv(args.root_dir)
processes = min(multiprocessing.cpu_count(), args.max_cpu)
pool = multiprocessing.Pool(processes=processes)
# Read tile grid and raw node data.
print('{} Reading tilegrid'.format(datetime.datetime.now()))
with open(
os.path.join(util.get_db_root(), util.get_fabric(),
'tilegrid.json')) as f:
grid = Grid(db=None, tilegrid=json.load(f))
raw_node_data = []
with progressbar.ProgressBar(max_value=len(nodes)) as bar:
for idx, node in enumerate(
pool.imap_unordered(
read_json5,
nodes,
chunksize=20,
)):
bar.update(idx)
raw_node_data.append(node)
bar.update(idx + 1)
node_wires = set()
remove_node_wires = set()
specific_node_wires = set()
# Create initial node wire pattern
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
node_tile, node_wire = node['node'].split('/')
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
if node['node'] == wire['wire']:
assert node_tile == wire_tile
assert node_wire == wire_name
gridinfo = grid.gridinfo_at_tilename(node_tile)
node_wires.add((gridinfo.tile_type, wire_name))
print('Initial number of wires that are node drivers: {}'.format(
len(node_wires)))
# Remove exceptional node wire names, create specific_node_wires set,
# which is simply the list of wires that are nodes in the graph.
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if node['node'] == wire['wire']:
assert key in node_wires
else:
if key in node_wires:
specific_node_wires.add(node['node'])
remove_node_wires.add(key)
# Complete the specific_node_wires list after the pruning of the
# node_pattern_wires sets.
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if key in remove_node_wires and node['node'] == wire['wire']:
specific_node_wires.add(node['node'])
node_wires -= remove_node_wires
print('Final number of wires that are node drivers: {}'.format(
len(node_wires)))
print('Number of wires that are node drivers: {}'.format(
len(specific_node_wires)))
# Verify the node wire data.
for node in progressbar.progressbar(raw_node_data):
if len(node['wires']) <= 1:
continue
found_node_wire = False
for wire in node['wires']:
if wire['wire'] in specific_node_wires:
assert wire['wire'] == node['node']
found_node_wire = True
break
if not found_node_wire:
for wire in node['wires']:
wire_tile, wire_name = wire['wire'].split('/')
gridinfo = grid.gridinfo_at_tilename(wire_tile)
key = gridinfo.tile_type, wire_name
if key in node_wires:
assert node['node'] == wire['wire']
else:
assert node['node'] != wire['wire']
# Normalize output.
tile_types = {}
for tile_type, tile_wire in node_wires:
if tile_type not in tile_types:
tile_types[tile_type] = []
tile_types[tile_type].append(tile_wire)
for tile_type in tile_types:
tile_types[tile_type].sort()
out = {
'node_pattern_wires': tile_types,
'specific_node_wires': sorted(specific_node_wires),
}
with open(os.path.join(args.output_dir, 'node_wires.json'), 'w') as f:
json.dump(out, f, indent=2, sort_keys=True)