def build_src_project(bindings, jamaicaoutput, targetdir, syscalls, interfaceResolver, debug, classrefs):
"""
Construct the software portion of the project. Copy the C source code for the Jamaica project,
refactoring the functions that are implemented on the FPGA.
Also copies the FPGA interface and build scripts.
bindings:
A map {id -> java method signature} that gives the ID of each hardware method.
Generated from prepare_hls_project.build_from_functions
jamaicaoutput:
Absolute path of the jamaica builder output directory which contains the source C files
targetdir:
Absolute path to place output files
"""
if not os.path.isfile(join(jamaicaoutput, "Main__nc.o")):
raise CaicosError("Cannot find file " + str(join(jamaicaoutput, "Main__nc.o")) +
". Ensure that the application has first be been built by Jamaica Builder.")
mkdir(targetdir)
copy_files(project_path("projectfiles", "juniper_fpga_interface"), join(targetdir, "juniper_fpga_interface"))
copy_files(project_path("projectfiles", "malloc_preload"), join(targetdir, "malloc_preload"))
refactor_src(bindings, jamaicaoutput, join(targetdir, "src"), debug)
if debug:
copy_files(project_path("debug_software"), join(targetdir, "src"))
generate_interrupt_handler(join(targetdir, "src", "caicos_interrupts.c"), syscalls, interfaceResolver, classrefs)
shutil.copy(join(jamaicaoutput, "Main__nc.o"), join(targetdir, "src"))
shutil.copy(project_path("projectfiles", "include", "juniperoperations.h"), join(targetdir, "src"))
shutil.copy(project_path("projectfiles", "scripts", "run.sh"), targetdir)
make_executable([join(targetdir, "run.sh")])
def dsub(job_name,
project,
regions,
machine_type,
image,
script,
tasks,
logging,
preemptible=False,
mount=None,
env=None,
after=None,
dsub_shell_script=None,
submit_jobs=False):
cmd = [
'dsub', '--provider', 'google-v2', '--project', project, '--regions',
regions, '--machine-type', machine_type, '--image', image, '--name',
job_name, '--script', script, '--tasks', tasks, '--logging', logging,
'--disk-size', '100'
]
if preemptible:
cmd += ['--preemptible']
if mount is not None:
cmd += ['--mount', mount]
if env is not None:
if isinstance(env, str):
env = [env]
for e in env:
cmd += ['--env', e]
if after is not None:
cmd += ['--after', after]
logger.info("# " + job_name + " submission cmd:")
formatted_cmd = ' '.join(cmd).replace(' --', ' \\\n--')
logger.info(formatted_cmd)
if dsub_shell_script is not None:
with open(dsub_shell_script, 'w') as f:
f.write('#!/bin/bash\n')
f.write(formatted_cmd + '\n')
make_executable(dsub_shell_script)
if submit_jobs:
if after is None:
return run_command(cmd, stderr=None).strip()
else:
return run_command_bg(cmd)
def build_all(config):
"""
Build a JUNIPER project. Format of the config dictionary is described in the docstring for config_specification.
"""
utils.log().setLevel(logging.INFO)
utils.remove_slots_from_classes(pycparser.c_ast)
utils.remove_slots_from_classes(pycparser)
try:
if config.get("cleanoutput", "false").lower() == "true":
if os.path.isdir(config["outputdir"]):
for f in os.listdir(config["outputdir"]):
shutil.rmtree(join(config["outputdir"], f), ignore_errors=True)
mkdir(config["outputdir"])
# Determine output paths
swdir = config.get("swoutputdir", join(config["outputdir"], "software"))
boarddir = config.get("hwoutputdir", join(config["outputdir"], "hardware"))
scriptsdir = config.get("scriptsoutputdir", join(config["outputdir"], "scripts"))
hwdir = join(boarddir, "reconfig", config.get("hlsprojectname", "caicos"))
if "astcache" in config:
astcache.activate_cache(config["astcache"])
# Create board design
log().info("Building board design for " + config["targetboard"] + " in " + str(boarddir) + "...")
utils.copy_files(project_path("dynamic_board_designs", "common"), boarddir)
utils.copy_files(project_path("dynamic_board_designs", config["targetboard"]), boarddir)
# Build hardware project
log().info("Building hardware project in " + str(hwdir) + "...")
if config.get("dev_softwareonly", "false").lower() == "true":
log().warning("dev_softwareonly is set. Generating software only.")
bindings = __getfakebindings(config["signatures"])
else:
bindings, syscalls, interfaceResolver, classrefs = prepare_hls_project.build_from_functions(
config["signatures"],
config.get("jamaicaoutputdir_hw", config["jamaicaoutputdir"]),
hwdir,
config.get("additionalhardwarefiles"),
config["fpgapart"],
config.get("notranslates", []),
)
target = join(config["outputdir"], "push.sh")
shutil.copyfile(project_path("projectfiles", "scripts", "push.sh"), target)
make_executable(
[
target,
join(boarddir, "build_base.sh"),
join(boarddir, "make_reconfig.sh"),
join(boarddir, "base", "build_hls.sh"),
]
)
# Build software project
log().info("Building software project in " + str(swdir) + "...")
prepare_src_project.build_src_project(
bindings,
config["jamaicaoutputdir"],
swdir,
syscalls,
interfaceResolver,
config.get("debug", False),
classrefs,
)
# Output templated Makefile.inc
contents = open(project_path("projectfiles", "templates", "Makefile.inc")).read()
subs = make_options[config["targetboard"]]
subs["SUB_JAMAICATARGET"] = config["jamaicatarget"]
template = Template(contents)
fout = open(join(swdir, "Makefile.inc"), "w")
fout.write(template.safe_substitute(subs))
fout.close()
# Output main makefile
shutil.copyfile(project_path("projectfiles", "templates", "Makefile"), join(swdir, "Makefile"))
# Output scripts folder
mkdir(scriptsdir)
for fn in ["cmd_template.bat", "program.sh", "rescan.sh", "getoffsets.py"]:
shutil.copyfile(project_path("projectfiles", "scripts", fn), join(scriptsdir, fn))
# Output kernel module
copy_files(project_path("system_software", "host_kernel_module"), join(swdir, "host_kernel_module"))
log().info("caicos done.")
except CaicosError, e:
log().error("A critical error was encountered:\n\t" + str(e))
def install(plugin, url, install_auto=None, install_dir=None):
"""
Installs a plugin to the default plugins directory given an url.
Could have been named 'my_little_dirty_function'.
:param url: Where to fetch the plugin from.
:install_dir: The name of the directory to create in c-lightning's
default plugins directory.
"""
# `lightningd` sets umask to 0 !
os.umask(22)
# We dont support pre-v0.7.2.1 anyway
reply = {"response": "", "format-hint": "simple"}
reply["response"] += " "
reply["response"] += "===== Installation log ======\n\n"
# Check that we have been given a supported url
if url.split("://")[0] not in {"http", "https"}:
reply["response"] += "You did not pass a valid url,"\
" treating as a keyword\n"
search_result = search(plugin, url)
if search_result:
if install_auto:
if len(search_result) > 1:
reply["response"] += "I cannot install the plugin"\
" automatically since I found"\
" two of them. Continuing.\n\n"
else:
return install(plugin, search_result[0]["url_download"])
reply["response"] += "You can install {} by running : ".format(url)
reply["response"] += "`lightning-cli install_plugin {}`\n"\
.format(search_result[0]["url_download"])
if len(search_result) > 1:
reply["response"] += "You can also install it via: "
reply["response"] += ", ".join(r["url_download"]
for r in search_result[1:])
else:
reply["response"] += "No known plugin was found matching {}"\
.format(url)
return reply
# Create the plugin directory..
res_name = urllib.parse.urlparse(url).path.split('/')[-1]
if not install_dir:
install_dir = res_name.split('.')[0]
install_path = os.path.join(plugin.plugins_path, install_dir)
create_dir(install_path)
reply["response"] += "Created {} directory\n".format(install_path)
file_path = os.path.join(install_path, res_name)
if os.path.exists(file_path):
return "Destination {} already exists\n".format(file_path)
# .. Then download it
if "api.github.com" in url and len(res_name.split('.')) == 1:
install_folder_from_github(install_path, url)
elif "github.com" in url:
if "/tree/" not in url:
url += "/tree/master"
*_, owner, repo, _, tree = url.split('/')
api_endpoint = "https://api.github.com"
api_url = "{}/repos/{}/{}/git/trees/{}".format(api_endpoint, owner,
repo, tree)
dl_github_repo(install_path, api_url, url)
else:
urllib.request.urlretrieve(url, file_path)
make_executable(file_path)
reply["response"] += "Downloaded file from {} to {}\n".format(
url, file_path)
# The common case where the plugin is in Python and has dependencies
handle_requirements(install_path)
# The case where the plugin is not written in a scripting language
handle_compilation(install_path)
# Finally get the executable file and make `lightningd` start it
main_file = get_main_file(install_path)
if main_file is not None:
reply["response"] += "Made {} executable\n".format(main_file)
else:
reply["response"] += "Could not find a main file, hence not making"\
" anything executable\n"
return reply
reply["response"] += "\n"
active_plugins = plugin.rpc.plugin_start(os.path.abspath(main_file))
if main_file in [p["name"] for p in active_plugins["plugins"]]:
reply["response"] += "Started {}".format(main_file.split('/')[-1])
else:
reply["response"] += "Timeout while trying to start {}.".format(
main_file)
return reply