def extract_data_step(self):
"""
Step 4: Extract either VCD or DAT information from run.
"""
# switch later stages based on whether we are outputing vcd or mem files
extract = Step(SourceType.Nothing)
if self.vcd:
def f(_inp, ctx):
f = (Path(ctx["tmpdir"]) / "output.vcd").open("rb")
return (Source(f, SourceType.File), None, 0)
extract.set_func(f, "Read output.vcd.")
else:
def f(_inp, ctx):
# Simulated 91 cycles
r = re.search(r"Simulated (\d+) cycles",
_inp.data.read().decode("ascii"))
data = {
"cycles": int(r.group(1)),
"memories": convert2json(ctx["tmpdir"], "out"),
}
buf = BytesIO(
json.dumps(data, indent=2, sort_keys=True).encode("UTF-8"))
return (Source(buf, SourceType.File), None, 0)
extract.set_func(f, "Convert output memories to json.")
return extract
def _output_dir(self, steps):
# output directory
output = Step(SourceType.Nothing)
def f(_, ctx):
return (Source(ctx["tmpdir_obj"], SourceType.TmpDir), None, 0)
output.set_func(f, "Output synthesis directory.")
steps.append(output)
def cleanup_step(self):
"""
Step 5: Cleanup the temporary directory
"""
cleanup = Step(SourceType.File)
def f(inp, ctx):
ctx["tmpdir_obj"].cleanup()
return (inp, None, 0)
cleanup.set_func(f, "Cleanup tmp directory.")
return cleanup
def _define(self):
# extract
extract = Step(SourceType.Nothing)
def f(inp, _):
res = None
if inp.source_type == SourceType.TmpDir:
res = futil_extract(Path(inp.data.name))
else:
res = futil_extract(Path(inp.data))
return (Source(BytesIO(res.encode("UTF-8")), SourceType.File), None, 0)
extract.set_func(f, "Extract information.")
return [extract]
def _establish_connection(self, steps):
# maybe establish ssh connection
if self.use_ssh:
ssh_connection = Step(SourceType.Nothing)
def f(inp, ctx):
if self.use_ssh:
ssh = self.ssh_client()
ssh.load_system_host_keys()
ssh.connect(self.ssh_host, username=self.ssh_user)
ctx["ssh_client"] = ssh
return (inp, None, 0)
ssh_connection.set_func(f, "Connect to server over SSH")
steps.append(ssh_connection)
def mktmp_step(self):
"""
Step 1: Make a temporary directory.
"""
# Step 1: Make a new temporary directory.
mktmp = Step(SourceType.Nothing)
def f(inp, ctx):
tmpdir = TemporaryDirectory()
ctx["tmpdir"] = tmpdir.name
ctx["tmpdir_obj"] = tmpdir
return (inp, None, 0)
mktmp.set_func(f, "Make temporary directory.")
return mktmp
def _mktmp(self, steps):
# make temporary directory
mktmp = Step(SourceType.Nothing)
def f(inp, ctx):
if self.use_ssh:
_, stdout, _ = ctx["ssh_client"].exec_command("mktemp -d")
tmpdir = stdout.read().decode("ascii").strip()
ctx["tmpdir"] = tmpdir
else:
tmpdir = TemporaryDirectory()
ctx["tmpdir"] = tmpdir.name
ctx["tmpdir_obj"] = tmpdir
return (inp, None, 0)
mktmp.set_func(f, "Make temporary directory.")
steps.append(mktmp)
def _run_vivado(self, steps):
vivado = Step(SourceType.Path)
if self.use_ssh:
def f(inp, ctx):
_, stdout, _ = ctx["ssh_client"].exec_command(
" ".join(
[
f"cd {ctx['tmpdir']}",
"&&",
"vivado -mode batch -source synth.tcl",
]
)
)
for chunk in iter(lambda: stdout.readline(2048), ""):
logging.debug(chunk.strip())
return (inp, None, 0)
ssh_addr = f"{self.ssh_user}@{self.ssh_host}"
vivado.set_func(
f,
" ".join(
[
f"ssh {ssh_addr} 'cd {{ctx[tmpdir]}}",
"&&",
"vivado -mode batch -source synth.tcl'",
]
),
)
else:
vivado.set_cmd(
" ".join(
[
"cd {ctx[tmpdir]}",
"&&",
"vivado -mode batch -source synth.tcl >&2",
]
)
)
steps.append(vivado)
def _run_vivado_hls(self, steps):
vivado_hls = Step(SourceType.Path)
if self.use_ssh:
def f(inp, ctx):
_, stdout, _ = ctx["ssh_client"].exec_command(
f'cd {ctx["tmpdir"]} && vivado_hls -f hls.tcl'
)
for chunk in iter(lambda: stdout.readline(2048), ""):
logging.debug(chunk.strip())
return (inp, None, 0)
ssh_addr = f"{self.ssh_user}@{self.ssh_host}"
vivado_hls.set_func(
f, f'ssh {ssh_addr} cd {{ctx["tmpdir"]}} && vivado_hls -f hls.tcl'
)
else:
vivado_hls.set_cmd(
" ".join(["cd {ctx[tmpdir]}", "&&", "vivado_hls -f hls.tcl >&2"])
)
steps.append(vivado_hls)
def _move_files(self, steps, device_files, src_file):
# copy over files
move = Step(SourceType.Path)
if self.use_ssh:
def f(inp, ctx):
with self.scp_client(ctx["ssh_client"].get_transport()) as scp:
scp.put(device_files, remote_path=ctx["tmpdir"])
scp.put(inp.data,
remote_path=f'{ctx["tmpdir"]}/{src_file}')
return (inp, None, 0)
move.set_func(f, "Copy synth files over SCP.")
else:
move.set_cmd(" ".join([
"cp",
" ".join(device_files),
"{ctx[tmpdir]}",
"&&",
f"cp {{ctx[input_path]}} {{ctx[tmpdir]}}/{src_file}",
]))
steps.append(move)
def _finalize_ssh(self, steps):
if self.use_ssh:
copy = Step(SourceType.Nothing)
def f(inp, ctx):
if self.use_ssh:
tmpdir = TemporaryDirectory()
with self.scp_client(
ctx["ssh_client"].get_transport()) as scp:
scp.get(ctx["tmpdir"],
local_path=f"{tmpdir.name}",
recursive=True)
ctx["old_tmpdir"] = ctx["tmpdir"]
ctx["tmpdir"] = tmpdir.name
ctx["tmpdir_obj"] = tmpdir
return (inp, None, 0)
copy.set_func(f, "Copy files back.")
steps.append(copy)
close_ssh = Step(SourceType.Nothing)
def f(inp, ctx):
if self.use_ssh:
ctx["ssh_client"].exec_command(f'rm -r {ctx["tmpdir"]}')
ctx["ssh_client"].close()
return (inp, None, 0)
close_ssh.set_func(f, "Close SSH")
steps.append(close_ssh)
restructure_tmp = Step(SourceType.Nothing)
restructure_tmp.set_cmd(" ".join([
"mv {ctx[tmpdir]}/tmp.*/* {ctx[tmpdir]}",
"&&",
"rm -r {ctx[tmpdir]}/tmp.*",
]))
steps.append(restructure_tmp)
def json_to_dat_step(self):
"""
Step 2: Transform data from JSON to Dat.
"""
data = Step(SourceType.Path)
data_path = self.config["stages", self.name, "data"]
def f(inp, ctx):
if data_path is None:
with open(inp.data, "r") as verilog_src:
# the verilog expects data, but none has been provided
if "readmemh" in verilog_src.read():
raise errors.MissingDynamicConfiguration(
"verilog.data")
ctx["data_prefix"] = ""
else:
with open(data_path) as f:
convert2dat(ctx["tmpdir"], json.load(f), "dat")
ctx["data_prefix"] = f'DATA={ctx["tmpdir"]}'
return (inp, None, 0)
data.set_func(f, "Convert json data to directory of .dat files.")
return data
def _define(self):
mktmp = Step(SourceType.Nothing)
def f(inp, ctx):
tmpdir = TemporaryDirectory()
ctx['tmpdir'] = tmpdir.name
ctx['tmpdir_obj'] = tmpdir
return (inp, None, 0)
mktmp.set_func(f, "Make temporary directory.")
data = Step(SourceType.Path)
data_path = self.config['stages', self.name, 'data']
def f(inp, ctx):
if data_path is None:
with open(inp.data, 'r') as verilog_src:
# the verilog expects data, but none has been provided
if 'readmemh' in verilog_src.read():
raise errors.MissingDynamicConfiguration(
'verilog.data')
ctx['data_prefix'] = ''
else:
with open(data_path) as f:
convert2dat(ctx['tmpdir'], json.load(f), 'dat')
ctx['data_prefix'] = f'DATA={ctx["tmpdir"]}'
return (inp, None, 0)
data.set_func(f, "Convert json data to directory of .dat files.")
verilator = Step(SourceType.Path)
testbench_files = [
str(
Path(self.config['global', 'futil_directory']) / 'fud' /
'sim' / 'testbench.cpp'),
str(
Path(self.config['global', 'futil_directory']) / 'fud' /
'sim' / 'wrapper.cpp'),
]
verilator.set_cmd(" ".join([
self.cmd,
'-cc',
# Don't trace if we're only looking at memory outputs
'--trace',
'{ctx[input_path]}',
"--exe " + " --exe ".join(testbench_files),
'--top-module main', # TODO: make this use dynamic config
'--Mdir',
'{ctx[tmpdir]}',
'1>&2'
]))
make = Step(SourceType.Nothing)
make.set_cmd("make -j -C {ctx[tmpdir]} -f Vmain.mk Vmain 1>&2")
run = Step(SourceType.Nothing)
run.set_cmd(" ".join([
'{ctx[data_prefix]}', '{ctx[tmpdir]}/Vmain',
'{ctx[tmpdir]}/output.vcd',
str(self.cycle_limit), '--trace' if self.vcd else '', '1>&2'
]))
# switch later stages based on whether we are outputing vcd or mem files
extract = Step(SourceType.Nothing)
if self.vcd:
def f(_inp, ctx):
f = (Path(ctx['tmpdir']) / 'output.vcd').open('rb')
return (Source(f, SourceType.File), None, 0)
extract.set_func(f, "Read output.vcd.")
else:
def f(_inp, ctx):
mem = convert2json(ctx['tmpdir'], 'out')
buf = BytesIO(
json.dumps(mem, indent=2, sort_keys=True).encode('UTF-8'))
return (Source(buf, SourceType.File), None, 0)
extract.set_func(f, "Convert output memories to json.")
cleanup = Step(SourceType.File)
def f(inp, ctx):
ctx['tmpdir_obj'].cleanup()
return (inp, None, 0)
cleanup.set_func(f, "Cleanup tmp directory.")
return [mktmp, data, verilator, make, run, extract, cleanup]