def build_src_project(bindings, jamaicaoutput, targetdir, syscalls, interfaceResolver, debug, classrefs):
"""
Construct the software portion of the project. Copy the C source code for the Jamaica project,
refactoring the functions that are implemented on the FPGA.
Also copies the FPGA interface and build scripts.
bindings:
A map {id -> java method signature} that gives the ID of each hardware method.
Generated from prepare_hls_project.build_from_functions
jamaicaoutput:
Absolute path of the jamaica builder output directory which contains the source C files
targetdir:
Absolute path to place output files
"""
if not os.path.isfile(join(jamaicaoutput, "Main__nc.o")):
raise CaicosError("Cannot find file " + str(join(jamaicaoutput, "Main__nc.o")) +
". Ensure that the application has first be been built by Jamaica Builder.")
mkdir(targetdir)
copy_files(project_path("projectfiles", "juniper_fpga_interface"), join(targetdir, "juniper_fpga_interface"))
copy_files(project_path("projectfiles", "malloc_preload"), join(targetdir, "malloc_preload"))
refactor_src(bindings, jamaicaoutput, join(targetdir, "src"), debug)
if debug:
copy_files(project_path("debug_software"), join(targetdir, "src"))
generate_interrupt_handler(join(targetdir, "src", "caicos_interrupts.c"), syscalls, interfaceResolver, classrefs)
shutil.copy(join(jamaicaoutput, "Main__nc.o"), join(targetdir, "src"))
shutil.copy(project_path("projectfiles", "include", "juniperoperations.h"), join(targetdir, "src"))
shutil.copy(project_path("projectfiles", "scripts", "run.sh"), targetdir)
make_executable([join(targetdir, "run.sh")])
def copy_project_files(
targetdir, jamaicaoutputdir, fpgapartname, filestobuild, reachable_functions, syscalls, interfaceResolver, classrefs
):
"""
Prepare an HLS project. Copies all required files from the local 'projectfiles' dir into targetdir
along with any extra required files.
Args:
targetdir: directory to output to
jamaicaoutputdir: absolute path that contains the output of Jamaica builder
fgpapartname: string of the fpga part name
filestobuild: array of absolute file paths and will be added to the HLS tcl script as source files
reachable_functions: array of FuncDecl nodes that are reachable and require translation
syscalls: map{string->int} names of function calls that should be translated to PCIe system calls -> ID of call
"""
mkdir(targetdir)
copy_files(project_path("projectfiles", "include"), join(targetdir, "include"), [".h"])
copy_files(project_path("projectfiles", "src"), join(targetdir, "src"), [".h", ".c"])
shutil.copy(join(jamaicaoutputdir, "Main__.h"), join(targetdir, "include"))
for f in filestobuild:
if (
not os.path.basename(f) == "fpgaporting.c"
): # We needed fpgaporting to perform reachability analysis, but don't rewrite it
log().info("Adding source file: " + f)
if f.endswith(".c"): # We only parse C files
targetfile = os.path.join(targetdir, "src", os.path.basename(f))
rewrite_source_file(f, targetfile, reachable_functions, syscalls, interfaceResolver, classrefs)
def parse_jamaica_output(filename, includepath = None):
"""
Use pycparser to parse a Jamaica output file.
Because Jamaica's output is pretty complex, cpp is required, and even still
some features have to be removed because it uses GCC extensions unsupported by pycparser.
Returns a pycparser.c_ast or throws ParseError
If includepath is None then the project include files are used for parsing. Else, an absolute
path to alternate includes can be provided.
"""
if '*' in filename:
filename = deglob_file(filename)
cppargs = ['-E',
'-DNDEBUG', #Disable Jamaica debug support
'-U__GNUC__', #Prevents use of __attribute__, will cause an "unsupported compiler" #warning
#'-W#warnings', #And this hides that warning
'-DJAMAICA_NATIVE_TIME_GET_HIGH_RESOLUTION_TIMESTAMP', #These use volatile asm() which is not supported by pycparser
'-DJAMAICA_NATIVE_THREAD_COMPARE_AND_SWAP',
'-D__CAICOS__', #So we can tell if we are being parsed by caicos
r'-I' + project_path("stdlibheaders"), #Override stdlib headers with blank versions (Jamaica builder doesn't use them, but includes them)
]
if includepath == None:
cppargs.append(r'-I' + project_path("projectfiles", "include"))
else:
cppargs.append(r'-I' + str(includepath))
return pycparser.parse_file(filename, use_cpp=True, cpp_path="gcc", cpp_args=cppargs)
def get_files_to_search(sig, jamaicaoutputdir):
"""
When performing flow analysis, the search order of files will hugely affect the execution time
because almost all referenced functions will be in the current file (i.e. Java package), the
FPGA porting layer, or the java.lang package or similar.
This function returns a list containing all of the files in the Jamaica output directory and the
porting layer. However it orders it based on heuristics to try to ensure that resolutions are faster.
"""
filestosearch = []
#First, put the originating file as this is the most likely source
filestosearch.append(c_filename_of_java_method_sig(sig, jamaicaoutputdir))
#Then the FPGA porting layer
filestosearch.append(project_path("projectfiles", "src", "fpgaporting.c"))
#Then the java.lang package
filestosearch.append(deglob_file(os.path.join(jamaicaoutputdir, "PKG_java_lang_V*__.c")))
#Then the other output C files
for f in os.listdir(jamaicaoutputdir):
ffullpath = os.path.join(jamaicaoutputdir, f)
if ffullpath.endswith(".c") and (not ffullpath.endswith("Main__.c")) and not ffullpath in filestosearch:
filestosearch.append(ffullpath)
return filestosearch
def load_config(filename):
"""
Open the provided filename as a caicos config file.
Validates the file, and that all required options are set, using the config_specification structure.
Raises CaicosError if a problem is found.
"""
def make_array_or_append(config, name, val):
if name in config:
config[name].append(val)
else:
config[name] = [val]
if not os.path.exists(filename) and os.path.isfile(filename):
raise CaicosError(str(filename) + " is not a valid file.")
f = open(filename)
config = {"signatures": []}
lno = 1
for line in f.readlines():
line = line.strip()
if not line.startswith("#") and not line == "":
pos = line.find("=")
if pos == -1:
raise CaicosError(
str(filename) + " is not a valid caicos config file. Line " + str(lno) + " does not contain '='."
)
param = line[:pos].strip().lower()
val = line[pos + 1 :].strip()
if param in config_specification and config_specification[param][1] != "":
make_array_or_append(config, config_specification[param][1], val)
else:
if param not in config_specification:
raise CaicosError(
"File "
+ str(filename)
+ " specifies the invalid config parameter '"
+ str(param)
+ "' on line "
+ str(lno)
)
config[param] = val
lno = lno + 1
# Now check all required parameters are defined
for p, spec in config_specification.iteritems():
if spec[0]:
if not p in config:
raise CaicosError("Required parameter '" + str(p) + "' was not defined by config file " + str(filename))
# Now validate parameters
if not os.path.isdir(project_path("dynamic_board_designs", config["targetboard"])):
raise CaicosError("targetboard is set to an unsupported board: " + config["targetboard"])
if not "fpgapart" in config:
config["fpgapart"] = fpgapart[config["targetboard"]]
return config
def build_from_functions(funcs, jamaicaoutputdir, outputdir, additionalsourcefiles, part, notranslatesigs):
"""
Build a Vivado HLS project that contains the hardware for a set of Java functions
Args:
funcs: Iterable of strings of Java signatures that are the methods to include
jamaicaoutputdir: Absolute path of the Jamaica builder output directory (that contains the generated C code)
outputdir: Absolute path of the target directory in which to build the project. Will be created if does not exist.
additionalsourcefiles: Iterable of abs paths for other source files that are required.
part: The FPGA part to target. Passed directly to the Xilinx tools and not checked.
notranslatesigs: Signatures of methods that should not be translated
Returns:
A tuple of (bindings, syscalls, interfaceResolver, classrefs)
bindings = A dictionary of {int -> Java sig} of the hardware methods and their associated call ids.
syscalls = A dictionary of {callname -> callid} of the sys calls that the hardware may make
interfaceResolver = Instance of interfaces.InterfaceResolver which contains information about encountered interface calls
classrefs = Instance of ClassRefs describing the static class references that are used by the hardware
"""
filestobuild = set()
filestobuild.add(project_path("projectfiles", "src", "fpgaporting.c"))
# All functions that should be translated
all_reachable_functions = set()
# Reachable functions that have been excluded by flowanalysis.excluded_functions so need to be handled as a PCIe interrupt
reachable_non_translated = set()
interfaceResolver = trace_from_functions(
funcs, jamaicaoutputdir, additionalsourcefiles, filestobuild, all_reachable_functions, reachable_non_translated
)
classrefs = class_references.enumerate_static_refs(os.path.join(jamaicaoutputdir, "Main__.c"))
log().info("All reachable functions:")
for f in all_reachable_functions:
log().info("\t" + str(f.decl.name) + ": " + str(f.coord.file))
# Build the syscall structure
# TODO: notranslatedecls is not currently used
notranslatedecls = determine_no_translate_fns(all_reachable_functions, notranslatesigs)
callid = 1
syscalls = {}
for sysname in reachable_non_translated:
syscalls[sysname] = callid
callid = callid + 1
if len(syscalls) > 0:
log().info("Generating syscalls for:")
for callname, sysid in syscalls.iteritems():
log().info("\t" + str(sysid) + ": " + str(callname))
copy_project_files(
outputdir, jamaicaoutputdir, part, filestobuild, all_reachable_functions, syscalls, interfaceResolver, classrefs
)
write_toplevel_header(funcs, jamaicaoutputdir, os.path.join(outputdir, "include", "toplevel.h"))
bindings = write_functions_c(funcs, jamaicaoutputdir, os.path.join(outputdir, "src", "functions.c"))
write_hls_script(os.path.join(outputdir, "src"), part, os.path.join(outputdir, "autobuild.tcl"))
return (bindings, syscalls, interfaceResolver, classrefs)
def generate_interrupt_handler(outputfile, syscalls, interfaceResolver, classrefs):
contents = open(project_path("projectfiles", "templates", "caicos_interrupts.c")).read()
template = Template(contents)
with open(outputfile, "w") as outf:
outf.write(template.safe_substitute({
'ADDITIONAL_SYSCALLS': interrupt_handler_syscall_code(syscalls, interfaceResolver),
'ADDITIONAL_CLASSREFS': interrupt_handler_classref_code(classrefs),
}))
def get_funcdecl_of_system_funccall(call):
"""
Given a FuncCall to a Jamaica system call (or just the string of its name), return the corresponding FuncDecl node.
"""
ast = astcache.get(project_path("projectfiles", "include", "jamaica.h"))
fdecs = functions_declared_in_ast(ast)
if isinstance(call, c_ast.FuncCall):
call = call.name.name
for dec in fdecs:
if dec.parent.name == call:
return dec
raise CaicosError("Cannot find the definition of system function: " + str(call))
import apsw
import os.path
import utils
_DB_PATH = utils.project_path('..', '.db')
def exists():
return os.path.isfile(_DB_PATH)
def init():
utils.log('Initializing database')
init_sql_path = utils.project_path('init.sql')
db_connection = connect()
try:
with db_connection, open(init_sql_path) as init_sql_file:
init_sql = init_sql_file.read()
db_connection.cursor().execute(init_sql)
finally:
db_connection.close()
def connect():
return apsw.Connection(_DB_PATH)
# Initialize the database if it doesn't exist when this module is imported.
if not exists():
init()
def test_project_path(self):
path = utils.project_path()
self.assertTrue(type(path) == str)
def test_path_to(self):
project_path = utils.project_path()
data_path = utils.path_to('data')
self.assertEqual(data_path, f'{project_path}/data')
def build_all(config):
"""
Build a JUNIPER project. Format of the config dictionary is described in the docstring for config_specification.
"""
utils.log().setLevel(logging.INFO)
utils.remove_slots_from_classes(pycparser.c_ast)
utils.remove_slots_from_classes(pycparser)
try:
if config.get("cleanoutput", "false").lower() == "true":
if os.path.isdir(config["outputdir"]):
for f in os.listdir(config["outputdir"]):
shutil.rmtree(join(config["outputdir"], f), ignore_errors=True)
mkdir(config["outputdir"])
# Determine output paths
swdir = config.get("swoutputdir", join(config["outputdir"], "software"))
boarddir = config.get("hwoutputdir", join(config["outputdir"], "hardware"))
scriptsdir = config.get("scriptsoutputdir", join(config["outputdir"], "scripts"))
hwdir = join(boarddir, "reconfig", config.get("hlsprojectname", "caicos"))
if "astcache" in config:
astcache.activate_cache(config["astcache"])
# Create board design
log().info("Building board design for " + config["targetboard"] + " in " + str(boarddir) + "...")
utils.copy_files(project_path("dynamic_board_designs", "common"), boarddir)
utils.copy_files(project_path("dynamic_board_designs", config["targetboard"]), boarddir)
# Build hardware project
log().info("Building hardware project in " + str(hwdir) + "...")
if config.get("dev_softwareonly", "false").lower() == "true":
log().warning("dev_softwareonly is set. Generating software only.")
bindings = __getfakebindings(config["signatures"])
else:
bindings, syscalls, interfaceResolver, classrefs = prepare_hls_project.build_from_functions(
config["signatures"],
config.get("jamaicaoutputdir_hw", config["jamaicaoutputdir"]),
hwdir,
config.get("additionalhardwarefiles"),
config["fpgapart"],
config.get("notranslates", []),
)
target = join(config["outputdir"], "push.sh")
shutil.copyfile(project_path("projectfiles", "scripts", "push.sh"), target)
make_executable(
[
target,
join(boarddir, "build_base.sh"),
join(boarddir, "make_reconfig.sh"),
join(boarddir, "base", "build_hls.sh"),
]
)
# Build software project
log().info("Building software project in " + str(swdir) + "...")
prepare_src_project.build_src_project(
bindings,
config["jamaicaoutputdir"],
swdir,
syscalls,
interfaceResolver,
config.get("debug", False),
classrefs,
)
# Output templated Makefile.inc
contents = open(project_path("projectfiles", "templates", "Makefile.inc")).read()
subs = make_options[config["targetboard"]]
subs["SUB_JAMAICATARGET"] = config["jamaicatarget"]
template = Template(contents)
fout = open(join(swdir, "Makefile.inc"), "w")
fout.write(template.safe_substitute(subs))
fout.close()
# Output main makefile
shutil.copyfile(project_path("projectfiles", "templates", "Makefile"), join(swdir, "Makefile"))
# Output scripts folder
mkdir(scriptsdir)
for fn in ["cmd_template.bat", "program.sh", "rescan.sh", "getoffsets.py"]:
shutil.copyfile(project_path("projectfiles", "scripts", fn), join(scriptsdir, fn))
# Output kernel module
copy_files(project_path("system_software", "host_kernel_module"), join(swdir, "host_kernel_module"))
log().info("caicos done.")
except CaicosError, e:
log().error("A critical error was encountered:\n\t" + str(e))