def build_src_project(bindings, jamaicaoutput, targetdir, syscalls, interfaceResolver, debug, classrefs):
	"""
	Construct the software portion of the project. Copy the C source code for the Jamaica project, 
	refactoring the functions that are implemented on the FPGA.
	Also copies the FPGA interface and build scripts. 
	
	bindings:
		A map {id -> java method signature} that gives the ID of each hardware method. 
		Generated from prepare_hls_project.build_from_functions
	jamaicaoutput:
		Absolute path of the jamaica builder output directory which contains the source C files
	targetdir:
		Absolute path to place output files
	"""
	if not os.path.isfile(join(jamaicaoutput, "Main__nc.o")):
		raise CaicosError("Cannot find file " + str(join(jamaicaoutput, "Main__nc.o")) + 
						". Ensure that the application has first be been built by Jamaica Builder.")
		
	mkdir(targetdir)
	copy_files(project_path("projectfiles", "juniper_fpga_interface"), join(targetdir, "juniper_fpga_interface"))
	copy_files(project_path("projectfiles", "malloc_preload"), join(targetdir, "malloc_preload"))
	refactor_src(bindings, jamaicaoutput, join(targetdir, "src"), debug)
	if debug:
		copy_files(project_path("debug_software"), join(targetdir, "src"))
	generate_interrupt_handler(join(targetdir, "src", "caicos_interrupts.c"), syscalls, interfaceResolver, classrefs)
	shutil.copy(join(jamaicaoutput, "Main__nc.o"), join(targetdir, "src"))
	shutil.copy(project_path("projectfiles", "include", "juniperoperations.h"), join(targetdir, "src"))
	shutil.copy(project_path("projectfiles", "scripts", "run.sh"), targetdir)
	make_executable([join(targetdir, "run.sh")])
Example #2
0
def dsub(job_name,
         project,
         regions,
         machine_type,
         image,
         script,
         tasks,
         logging,
         preemptible=False,
         mount=None,
         env=None,
         after=None,
         dsub_shell_script=None,
         submit_jobs=False):
    cmd = [
        'dsub', '--provider', 'google-v2', '--project', project, '--regions',
        regions, '--machine-type', machine_type, '--image', image, '--name',
        job_name, '--script', script, '--tasks', tasks, '--logging', logging,
        '--disk-size', '100'
    ]

    if preemptible:
        cmd += ['--preemptible']
    if mount is not None:
        cmd += ['--mount', mount]
    if env is not None:
        if isinstance(env, str):
            env = [env]
        for e in env:
            cmd += ['--env', e]
    if after is not None:
        cmd += ['--after', after]

    logger.info("# " + job_name + " submission cmd:")
    formatted_cmd = ' '.join(cmd).replace(' --', ' \\\n--')
    logger.info(formatted_cmd)

    if dsub_shell_script is not None:
        with open(dsub_shell_script, 'w') as f:
            f.write('#!/bin/bash\n')
            f.write(formatted_cmd + '\n')
        make_executable(dsub_shell_script)

    if submit_jobs:
        if after is None:
            return run_command(cmd, stderr=None).strip()
        else:
            return run_command_bg(cmd)
Example #3
0
def build_all(config):
    """
	Build a JUNIPER project. Format of the config dictionary is described in the docstring for config_specification.
	"""
    utils.log().setLevel(logging.INFO)
    utils.remove_slots_from_classes(pycparser.c_ast)
    utils.remove_slots_from_classes(pycparser)
    try:
        if config.get("cleanoutput", "false").lower() == "true":
            if os.path.isdir(config["outputdir"]):
                for f in os.listdir(config["outputdir"]):
                    shutil.rmtree(join(config["outputdir"], f), ignore_errors=True)

        mkdir(config["outputdir"])

        # Determine output paths
        swdir = config.get("swoutputdir", join(config["outputdir"], "software"))
        boarddir = config.get("hwoutputdir", join(config["outputdir"], "hardware"))
        scriptsdir = config.get("scriptsoutputdir", join(config["outputdir"], "scripts"))
        hwdir = join(boarddir, "reconfig", config.get("hlsprojectname", "caicos"))

        if "astcache" in config:
            astcache.activate_cache(config["astcache"])

            # Create board design
        log().info("Building board design for " + config["targetboard"] + " in " + str(boarddir) + "...")
        utils.copy_files(project_path("dynamic_board_designs", "common"), boarddir)
        utils.copy_files(project_path("dynamic_board_designs", config["targetboard"]), boarddir)

        # Build hardware project
        log().info("Building hardware project in " + str(hwdir) + "...")
        if config.get("dev_softwareonly", "false").lower() == "true":
            log().warning("dev_softwareonly is set. Generating software only.")
            bindings = __getfakebindings(config["signatures"])
        else:
            bindings, syscalls, interfaceResolver, classrefs = prepare_hls_project.build_from_functions(
                config["signatures"],
                config.get("jamaicaoutputdir_hw", config["jamaicaoutputdir"]),
                hwdir,
                config.get("additionalhardwarefiles"),
                config["fpgapart"],
                config.get("notranslates", []),
            )

            target = join(config["outputdir"], "push.sh")
            shutil.copyfile(project_path("projectfiles", "scripts", "push.sh"), target)

            make_executable(
                [
                    target,
                    join(boarddir, "build_base.sh"),
                    join(boarddir, "make_reconfig.sh"),
                    join(boarddir, "base", "build_hls.sh"),
                ]
            )

            # Build software project
        log().info("Building software project in " + str(swdir) + "...")
        prepare_src_project.build_src_project(
            bindings,
            config["jamaicaoutputdir"],
            swdir,
            syscalls,
            interfaceResolver,
            config.get("debug", False),
            classrefs,
        )

        # Output templated Makefile.inc
        contents = open(project_path("projectfiles", "templates", "Makefile.inc")).read()
        subs = make_options[config["targetboard"]]
        subs["SUB_JAMAICATARGET"] = config["jamaicatarget"]
        template = Template(contents)
        fout = open(join(swdir, "Makefile.inc"), "w")
        fout.write(template.safe_substitute(subs))
        fout.close()

        # Output main makefile
        shutil.copyfile(project_path("projectfiles", "templates", "Makefile"), join(swdir, "Makefile"))

        # Output scripts folder
        mkdir(scriptsdir)
        for fn in ["cmd_template.bat", "program.sh", "rescan.sh", "getoffsets.py"]:
            shutil.copyfile(project_path("projectfiles", "scripts", fn), join(scriptsdir, fn))

            # Output kernel module
        copy_files(project_path("system_software", "host_kernel_module"), join(swdir, "host_kernel_module"))

        log().info("caicos done.")

    except CaicosError, e:
        log().error("A critical error was encountered:\n\t" + str(e))
Example #4
0
def install(plugin, url, install_auto=None, install_dir=None):
    """
    Installs a plugin to the default plugins directory given an url.
    Could have been named 'my_little_dirty_function'.

    :param url: Where to fetch the plugin from.
    :install_dir: The name of the directory to create in c-lightning's
                  default plugins directory.
    """
    # `lightningd` sets umask to 0 !
    os.umask(22)
    # We dont support pre-v0.7.2.1 anyway
    reply = {"response": "", "format-hint": "simple"}
    reply["response"] += "                                "
    reply["response"] += "===== Installation log ======\n\n"
    # Check that we have been given a supported url
    if url.split("://")[0] not in {"http", "https"}:
        reply["response"] += "You did not pass a valid url,"\
                             " treating as a keyword\n"
        search_result = search(plugin, url)
        if search_result:
            if install_auto:
                if len(search_result) > 1:
                    reply["response"] += "I cannot install the plugin"\
                                         " automatically since I found"\
                                         " two of them. Continuing.\n\n"
                else:
                    return install(plugin, search_result[0]["url_download"])
            reply["response"] += "You can install {} by running : ".format(url)
            reply["response"] += "`lightning-cli install_plugin {}`\n"\
                                 .format(search_result[0]["url_download"])
            if len(search_result) > 1:
                reply["response"] += "You can also install it via: "
                reply["response"] += ", ".join(r["url_download"]
                                               for r in search_result[1:])
        else:
            reply["response"] += "No known plugin was found matching {}"\
                                 .format(url)
        return reply

    # Create the plugin directory..
    res_name = urllib.parse.urlparse(url).path.split('/')[-1]
    if not install_dir:
        install_dir = res_name.split('.')[0]
    install_path = os.path.join(plugin.plugins_path, install_dir)
    create_dir(install_path)
    reply["response"] += "Created {} directory\n".format(install_path)
    file_path = os.path.join(install_path, res_name)
    if os.path.exists(file_path):
        return "Destination {} already exists\n".format(file_path)

    # .. Then download it
    if "api.github.com" in url and len(res_name.split('.')) == 1:
        install_folder_from_github(install_path, url)
    elif "github.com" in url:
        if "/tree/" not in url:
            url += "/tree/master"
        *_, owner, repo, _, tree = url.split('/')
        api_endpoint = "https://api.github.com"
        api_url = "{}/repos/{}/{}/git/trees/{}".format(api_endpoint, owner,
                                                       repo, tree)
        dl_github_repo(install_path, api_url, url)
    else:
        urllib.request.urlretrieve(url, file_path)
        make_executable(file_path)
    reply["response"] += "Downloaded file from {} to {}\n".format(
        url, file_path)

    # The common case where the plugin is in Python and has dependencies
    handle_requirements(install_path)
    # The case where the plugin is not written in a scripting language
    handle_compilation(install_path)

    # Finally get the executable file and make `lightningd` start it
    main_file = get_main_file(install_path)
    if main_file is not None:
        reply["response"] += "Made {} executable\n".format(main_file)
    else:
        reply["response"] += "Could not find a main file, hence not making"\
                             " anything executable\n"
        return reply
    reply["response"] += "\n"

    active_plugins = plugin.rpc.plugin_start(os.path.abspath(main_file))
    if main_file in [p["name"] for p in active_plugins["plugins"]]:
        reply["response"] += "Started {}".format(main_file.split('/')[-1])
    else:
        reply["response"] += "Timeout while trying to start {}.".format(
            main_file)
    return reply