def _load_json_to_mongo(database: pymongo.database.Database,
collection: str) -> None:
"""Load a MongoDB collection from a JSON file."""
with open(path.join(_TESTDATA_FOLDER, collection + '.json')) as json_file:
json_blob = pyjson5.load(json_file)
database[collection].insert_many(json_blob)
def annotate_speed_model(tile_type, reduced_tile, root_dir):
""" Updates the reduced tile with the correct speed information """
speed_model_indices = get_speed_model_indices(reduced_tile)
tmp_indices_file = os.path.join(root_dir,
'{}_speed_index.tmp'.format(tile_type))
with open(tmp_indices_file, "w") as f:
for index in speed_model_indices:
print(index, file=f)
# Get vivado path
vivado = os.getenv('XRAY_VIVADO')
assert vivado is not None
subprocess.check_call(
"{} -mode batch -source get_speed_model.tcl -tclargs {}".format(
vivado, tmp_indices_file),
shell=True,
stdout=subprocess.DEVNULL)
with open(tmp_indices_file, "r") as f:
speed_model_data = json5.load(f)
for site in reduced_tile['sites']:
annotate_site_pins_speed_model(site['site_pins'], speed_model_data)
annotate_pips_speed_model(reduced_tile['pips'], speed_model_data)
annotate_wires_speed_model(reduced_tile['wires'], speed_model_data)
def read_json5(fname, nodes):
node_lookup = prjxray.lib.NodeLookup()
node_lookup.load_from_nodes(nodes)
#print('{} Reading {} (in pid {})'.format(datetime.datetime.now(), fname, os.getpid()))
with open(fname) as f:
tile = json5.load(f)
#print('{} Done reading {}'.format(datetime.datetime.now(), fname))
def get_site_types():
for site in tile['sites']:
yield get_prototype_site(site)
site_types = tuple(get_site_types())
sites = tuple(get_sites(tile, node_lookup.site_pin_node_to_wires))
pips = get_pips(tile['tile'], tile['pips'])
def inner():
for wire in tile['wires']:
assert wire['wire'].startswith(tile['tile'] + '/')
yield wire['wire'][len(tile['tile']) + 1:]
wires = set(inner())
wires_from_nodes = set(node_lookup.wires_for_tile(tile['tile']))
assert len(wires_from_nodes - wires) == 0, repr((wires, wires_from_nodes))
return fname, tile, site_types, sites, pips, wires
def read_json5(fname, database_file):
node_lookup = prjxray.node_lookup.NodeLookup(database_file)
with open(fname) as f:
tile = json5.load(f)
def get_site_types():
for site in tile['sites']:
yield get_prototype_site(site)
site_types = tuple(get_site_types())
sites = tuple(get_sites(tile, node_lookup.site_pin_node_to_wires))
pips = get_pips(tile['tile'], tile['pips'])
def inner():
for wire in tile['wires']:
assert wire['wire'].startswith(tile['tile'] + '/')
if wire['res'] != '0.000' or wire['cap'] != '0.000':
wire_delay_model = {
'res': wire['res'],
'cap': wire['cap'],
}
else:
wire_delay_model = None
yield wire['wire'][len(tile['tile']) + 1:], wire_delay_model
wires = {k: v for (k, v) in inner()}
wires_from_nodes = set(node_lookup.wires_for_tile(tile['tile']))
assert len(wires_from_nodes - wires.keys()) == 0, repr(
(wires, wires_from_nodes))
return fname, tile, site_types, sites, pips, wires
def load_from_root_csv(self, nodes):
import pyjson5 as json5
import progressbar
for node in progressbar.progressbar(nodes):
with open(node) as f:
node_wires = json5.load(f)
assert node_wires['node'] not in self.nodes
self.nodes[node_wires['node']] = node_wires['wires']
def load(cls, filename: str):
with open(filename, 'r') as f:
data = json.load(f)
assert data['$schema'] == 'vscode://schemas/color-theme'
parent = None
if parent_filename := data.get('include'):
directory = pathlib.Path(os.path.dirname(filename))
parent = cls.load(directory / parent_filename)
def __init__(self, path): with open(path, "rt") as config_file: json_config = pyjson5.load(config_file) #handling secrets if "secrets" in json_config: secrets_path = os.path.join(os.path.dirname(path), json_config["secrets"]) with open(secrets_path, "rt") as secrets_json_file: secrets_config = pyjson5.load(secrets_json_file) for key, value in secrets_config.items(): if key in json_config: json_config[key].update(value) else: json_config[key] = value self.smtp = SmtpConfig(json_config["smtp"]) self.bug_report = BugReportConfig(json_config["bug_report"]) self.parser = ParserConfig(json_config["parser"]) self.www = WwwConfig(json_config["www"]) self.db = DatabaseConfig(json_config["db"]) self.unittest_mode = "DANCEBOOKS_UNITTEST" in os.environ
def __init__(self, path):
with open(path, "rt") as config_file:
json_config = pyjson5.load(config_file)
#handling secrets
if "secrets" in json_config:
secrets_path = os.path.join(os.path.dirname(path),
json_config["secrets"])
with open(secrets_path, "rt") as secrets_json_file:
secrets_config = pyjson5.load(secrets_json_file)
for key, value in secrets_config.items():
if key in json_config:
json_config[key].update(value)
else:
json_config[key] = value
self.smtp = SmtpConfig(json_config["smtp"])
self.bug_report = BugReportConfig(json_config["bug_report"])
self.parser = ParserConfig(json_config["parser"])
self.www = WwwConfig(json_config["www"])
self.db = DatabaseConfig(json_config["db"])
self.unittest_mode = "DANCEBOOKS_UNITTEST" in os.environ
def get_tile_grid_info(fname):
with open(fname, 'r') as f:
tile = json5.load(f)
return {
tile['tile']: {
'type': tile['type'],
'grid_x': tile['x'],
'grid_y': tile['y'],
'sites': dict(
(site['site'], site['type']) for site in tile['sites']),
'wires': set((wire['wire'] for wire in tile['wires']))
},
}
def build_database(self, nodes, tiles):
create_tables(self.conn)
c = self.conn.cursor()
tile_names = []
for tile_type in tiles:
for tile in tiles[tile_type]:
tile_names.append(tile)
tile_pkeys = {}
for tile_file in progressbar.progressbar(tile_names):
# build/specimen_001/tile_DSP_L_X34Y145.json5
root, _ = os.path.splitext(os.path.basename(tile_file))
tile = root[5:]
c.execute("INSERT INTO tile(name) VALUES (?);", (tile, ))
tile_pkeys[tile] = c.lastrowid
nodes_processed = set()
for node in progressbar.progressbar(nodes):
with open(node) as f:
node_wires = json5.load(f)
assert node_wires['node'] not in nodes_processed
nodes_processed.add(node_wires['node'])
c.execute("INSERT INTO node(name) VALUES (?);",
(node_wires['node'], ))
node_pkey = c.lastrowid
for wire in node_wires['wires']:
tile = wire['wire'].split('/')[0]
tile_pkey = tile_pkeys[tile]
c.execute(
"""
INSERT INTO wire(name, tile_pkey, node_pkey) VALUES (?, ?, ?);""",
(wire['wire'], tile_pkey, node_pkey))
self.conn.commit()
c = self.conn.cursor()
c.execute("CREATE INDEX tile_names ON tile(name);")
c.execute("CREATE INDEX node_names ON node(name);")
c.execute("CREATE INDEX wire_node_tile ON wire(node_pkey, tile_pkey);")
c.execute("CREATE INDEX wire_tile ON wire(tile_pkey);")
self.conn.commit()
def load_set(cls, filename: str) -> Dict[str, '_Persona']:
"""Load a set of personas from a JSON file."""
with open(filename) as personas_file:
personas_json = pyjson5.load(personas_file)
personas: Dict[str, _Persona] = {}
for name, blob in personas_json.items():
user = user_pb2.User()
assert proto.parse_from_mongo(blob['user'], user.profile)
if 'featuresEnabled' in blob:
assert proto.parse_from_mongo(blob['featuresEnabled'],
user.features_enabled)
if 'project' in blob:
assert proto.parse_from_mongo(blob['project'],
user.projects.add())
if 'projects' in blob:
for project in blob['projects']:
assert proto.parse_from_mongo(project, user.projects.add())
assert name not in personas
personas[name] = cls(name, user)
return personas
def get_tile_grid_info(fname):
with open(fname, 'r') as f:
tile = json5.load(f)
tile_type = tile['type']
ignored = int(tile['ignored']) != 0
if ignored:
tile_type = 'NULL'
return {
tile['tile']: {
'type': tile_type,
'ignored': ignored,
'grid_x': tile['x'],
'grid_y': tile['y'],
'sites': dict(
(site['site'], site['type']) for site in tile['sites']),
'wires': set((wire['wire'] for wire in tile['wires']))
},
}
def read_json5(fname, database_file):
node_lookup = prjxray.node_lookup.NodeLookup(database_file)
with open(fname) as f:
tile = json5.load(f)
def get_site_types():
for site in tile['sites']:
yield get_prototype_site(site)
site_types = tuple(get_site_types())
sites = tuple(get_sites(tile, node_lookup.site_pin_node_to_wires))
pips = get_pips(tile['tile'], tile['pips'])
def inner():
for wire in tile['wires']:
assert wire['wire'].startswith(tile['tile'] + '/')
yield wire['wire'][len(tile['tile']) + 1:]
wires = set(inner())
wires_from_nodes = set(node_lookup.wires_for_tile(tile['tile']))
assert len(wires_from_nodes - wires) == 0, repr((wires, wires_from_nodes))
return fname, tile, site_types, sites, pips, wires
def config_load(config):
"""Loads the given config file, and puts it in the expected schema.
"""
app_config = pyjson5.load(open(config))
return config_schema(app_config['parsers'])
def read_json5(fname):
with open(fname, 'r') as f:
return json5.load(f)
def main():
simplejson.dump(pyjson5.load(sys.stdin), sys.stdout, indent=2)
def read_json5(fname):
with OpenSafeFile(fname, 'r') as f:
return json5.load(f)
def process_file(src_path, dest_path):
print(src_path, dest_path)
with open(src_path) as infile:
data = pyjson5.load(infile)
with open(dest_path, 'w') as outfile:
json.dump(data, outfile, indent=2)
def _check_config_file(config):
"""For non-deployment editions, we must load config from a json5 file.
This also gets run for deployments, as this function is responsible for
merging multiple configs into the single root config.
This function also contains all schema validation of json5 files, as
the version stored in the observatory image is separate.
"""
global CONFIG_FOLDER, IMAGE_TAG
# Not a deployment -- requires CONFIG_FOLDER
if CONFIG_FOLDER is None:
assert config is not None
CONFIG_FOLDER = config
else:
assert config is None
import pyjson5, schema as s
config_data = pyjson5.load(open(os.path.join(CONFIG_FOLDER,
'config.json5')))
# Before applying schema, merge in child configs
for child_name in os.listdir(CONFIG_FOLDER):
child_path = os.path.join(CONFIG_FOLDER, child_name)
if not os.path.isdir(child_path):
continue
child_config_path = os.path.join(child_path, 'config.json5')
if not os.path.lexists(child_config_path):
continue
child_config = pyjson5.load(open(child_config_path))
# First traversal -- patch keys and values
nodes = [([], child_config)]
while nodes:
path, src = nodes.pop()
for k, v in src.items():
## Rewrite v
# Check executables; amend cwd
if path and (
'file_detail_views' == path[-1]
or 'decision_views' == path[-1]
or 'parsers' == path[-1]
or 'tasks' == path[-1]):
if 'cwd' in v:
v['cwd'] = f'{child_name}/' + v['cwd']
else:
v['cwd'] = child_name
# Docker build stage patch
if path == ['build', 'stages']:
if 'commands' in v:
v['commands'] = [
vv
.replace('{disttarg}', f'{{disttarg}}/{child_name}')
.replace('{dist}', f'{{dist}}/{child_name}')
for vv in v['commands']]
if isinstance(v, dict):
nodes.append((path + [k], v))
# Second traversal -- merge
nodes = [([], config_data, child_config)]
while nodes:
path, dst, src = nodes.pop()
for k, v in src.items():
## Rewrite k
# Docker build stage patch -- rewrites `k`
if path == ['build', 'stages']:
# Adding a build stage. If not 'base' or 'final', then
# prepend config folder name
if k not in ['base', 'final']:
k = f'{child_name}_{k}'
# New entries only for these
if path in [
['pipelines'],
['file_detail_views'],
['decision_views'],
['parsers'],
]:
# Amend these with the child's name, to allow for copying
k = f'{child_name.replace("_", "-")}-{k}'
assert k not in dst, f'Cannot extend {path} {k}; must be new'
## Check for merge type
# Check if new -- if so, assign and be done
if k not in dst:
# Non-existent key; add it to the dictionary
dst[k] = v
continue
# Do merge
if isinstance(v, dict):
if not isinstance(dst[k], dict):
raise ValueError(f'{path} {k} type does not match base config')
# Dictionary merge
nodes.append((path + [k], dst[k], v))
elif isinstance(v, list):
if not isinstance(dst[k], list):
raise ValueError(f'{path} {k} type does not match base config')
# Add to end.
dst[k].extend(v)
else:
raise ValueError(f'May not extend {path} {k}: base config type {dst[k]}')
# Pull in parser-specific schema
import importlib.util
spec = importlib.util.spec_from_file_location('etl_parse',
os.path.join(faw_dir, 'common', 'pdf-etl-parse', 'parse_schema.py'))
etl_parse = importlib.util.module_from_spec(spec)
spec.loader.exec_module(etl_parse)
schema_views = {
s.Optional('decision_views', default={}): s.Or({}, {
str: {
'label': str,
'type': 'program',
'exec': [s.Or(
s.And(str, lambda x: not x.startswith('<')),
s.And(str, lambda x: x in [
'<filesPath>', '<apiInfo>', '<jsonArguments>', '<mongo>', '<outputHtml>',
'<workbenchApiUrl>']),
)],
s.Optional('cwd', default='.'): str,
'execStdin': s.And(str, lambda x: all([
y.group(0) in ['<referenceDecisions>', '<statsbyfile>']
for y in re.finditer('<[^>]*>', x)]),
error="Must be string with any <'s being one of: "
"<referenceDecisions>, <statsbyfile>"),
},
}),
s.Optional('file_detail_views', default={}): s.Or({}, {
str: {
'label': str,
'type': 'program_to_html',
'exec': [str],
s.Optional('cwd', default='.'): str,
s.Optional('outputMimeType', default='text/html'): str,
},
}),
}
# NOTE -- primary schema validation is here, but NOT for submodules such
# as pdf-etl-parse.
sch = s.Schema({
'name': s.And(str, s.Regex(r'^[a-zA-Z0-9-]+$')),
# parsers validated by pdf-etl-parse
'parsers': etl_parse.schema_get(),
s.Optional('parserDefaultTimeout', default=30): s.Or(float, int),
'decision_default': str,
s.Optional('pipelines', default={}): s.Or({}, {
s.And(str, lambda x: '_' not in x and '.' not in x,
error='Must not have underscore or dot'): {
s.Optional('label'): str,
s.Optional('disabled', default=False): s.Or(True, False),
s.Optional('tasks', default={}): s.Or({},
s.And(
{
s.And(str, lambda x: '_' not in x and '.' not in x,
error='Must not have underscore or dot'): {
s.Optional('disabled', default=False): s.Or(True, False),
'version': str,
'exec': [str],
s.Optional('cwd', default='.'): str,
s.Optional('dependsOn', default=[]): [str],
},
},
lambda x: all([d in x for _, task in x.items() for d in task['dependsOn']]),
error="Task `dependsOn` had invalid name",
)),
s.Optional('parsers', default={}): etl_parse.schema_get(),
**schema_views,
},
}),
'build': {
'stages': {
str: {
s.Optional('from'): str,
s.Optional('commands'): [str],
s.Optional('copy_output'): {
str: s.Or(str, bool),
},
},
},
},
**schema_views,
})
try:
config_data = sch.validate(config_data)
except Exception as e:
traceback.print_exc()
sys.stderr.write('\n'.join([str(v) for v in e.autos]) + '\n')
sys.exit(1)
IMAGE_TAG = config_data['name']
return config_data
parser.add_argument('file',
metavar='FILE',
type=str,
help='The file to be verified')
parser.add_argument('--order-desc',
dest='desc',
action='store_true',
help="ID numbers are in descending ordering")
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
with open(args.file, 'r') as f:
data = load(f)
in_order = operator.gt if args.desc else operator.lt
reservations = []
error = False
for i, item in enumerate(data, 1):
if len(item) != 3:
print(f"Invalid record number {i}: {item}", file=sys.stderr)
error = True
else:
first, last, username = item
x = (first, last)
ok = True
order = True
for (num, y, name) in reservations:
