Esempio n. 1
0
    def _parse_configuration_export(self, it):
        ret = []
        token = it.next()
        if token == lexer.Token.OPEN_PARENTHESIS:

            while True:
                token = it.next()

                if token in [lexer.Token.LITERAL, lexer.Token.VARIABLE]:
                    value = self._token_to_variable(token)
                    token = it.next()
                    if token == lexer.Token.COLON:
                        token = it.next()
                        if token == lexer.Token.VARIABLE:
                            variable = variables.Variable(self.name, token.content, value)
                            ret.append(variable)
                        else:
                            ui.parse_error(token, msg="expected variable")
                    else:
                        ui.parse_error(token, msg="expected colon")
                elif token == lexer.Token.CLOSE_PARENTHESIS:
                    break
                else:
                    ui.parse_error(token)
        else:
            ui.parse_error(token)

        ui.debug("colon list: " + str(ret))
        return ret
Esempio n. 2
0
    def __parse_configuration(self, it):
        configuration = configurations.Configuration()

        # name
        token = it.next()
        if token == lexer.Token.LITERAL:
            configuration.name = token.content
        else:
            ui.parse_error(token)

        while True:
            token = it.next()
            if token == lexer.Token.LITERAL:
                if token.content == "compiler":
                    configuration.compiler = self.__parse_list(it)
                elif token.content == "archiver":
                    configuration.archiver = self.__parse_list(it)
                elif token.content == "application_suffix":
                    configuration.application_suffix = self.__parse_list(it)
                elif token.content == "compiler_flags":
                    configuration.compiler_flags = self.__parse_list(it)
                elif token.content == "linker_flags":
                    configuration.linker_flags = self.__parse_list(it)
                elif token.content == "export":
                    configuration.export = self._parse_configuration_export(it)
                else:
                    ui.parse_error(token)

            elif token == lexer.Token.NEWLINE:
                break
            else:
                ui.parse_error(token)

        ui.debug("configuration parsed:" + str(configuration))
        configurations.add_configuration(configuration)
Esempio n. 3
0
def main(user_args):
    targets = []
    api_keys = get_config_from_file(user_args)
    ui.info_section("\n", ui.darkteal, "Targets")
    user_stdin_target = fetch_emails(args.target_emails)

    if user_stdin_target:
        targets.append(user_stdin_target)
    elif os.path.isfile(user_args.target_emails):
        ui.debug(ui.darkgray, "Reading from file", user_args.target_emails)
        targets.extend(get_emails_from_file(user_args.target_emails))
    else:
        ui.warning("No targets found")


# Launch
    if not user_args.run_local:
        breached_targets = target_factory(targets, api_keys)
    elif user_args.run_local:
        breached_targets = [Target(t) for t in targets]
    if user_args.bc_path:
        breached_targets = breachcomp_check(breached_targets,
                                            user_args.bc_path)
    print_results(breached_targets)
    if user_args.output_file:
        save_results_csv(user_args.output_file, breached_targets)
Esempio n. 4
0
    def _parse_configuration_export(self, it):
        ret = []
        token = it.next()
        if token == lexer.Token.OPEN_PARENTHESIS:

            while True:
                token = it.next()

                if token in [lexer.Token.LITERAL, lexer.Token.VARIABLE]:
                    value = self._token_to_variable(token)
                    token = it.next()
                    if token == lexer.Token.COLON:
                        token = it.next()
                        if token == lexer.Token.VARIABLE:
                            variable = variables.Variable(
                                self.name, token.content, value)
                            ret.append(variable)
                        else:
                            ui.parse_error(token, msg="expected variable")
                    else:
                        ui.parse_error(token, msg="expected colon")
                elif token == lexer.Token.CLOSE_PARENTHESIS:
                    break
                else:
                    ui.parse_error(token)
        else:
            ui.parse_error(token)

        ui.debug("colon list: " + str(ret))
        return ret
Esempio n. 5
0
def _parse_command_line():
    parser = argparse.ArgumentParser(description='Painless buildsystem.')
    parser.add_argument('target',
                        metavar='target',
                        nargs="*",
                        help='targets to be built')
    parser.add_argument('-a',
                        '--all',
                        action="store_true",
                        help='build all targets')
    parser.add_argument('-c',
                        action='store',
                        dest='configuration',
                        default="__default",
                        nargs="?",
                        help='configuration to be used')
    parser.add_argument('-j',
                        action='store',
                        dest='jobs',
                        default="1",
                        nargs="?",
                        help='parallel jobs to be used')
    parser.add_argument('-v',
                        '--verbose',
                        action="store_true",
                        help='show tool invokations')
    args = parser.parse_args()
    ui.debug(str(args))
    return args
Esempio n. 6
0
def run_git(working_path, *cmd, raises=True):
    """ Run git `cmd` in given `working_path`

    If `raises` is True and git return code is non zero, raise
    an exception. Otherwise, return a tuple (returncode, out)

    """
    git_cmd = list(cmd)
    git_cmd.insert(0, "git")
    options = dict()
    if not raises:
        options["stdout"] = subprocess.PIPE
        options["stderr"] = subprocess.STDOUT

    ui.debug(ui.lightgray, working_path, "$", ui.reset, *git_cmd)
    process = subprocess.Popen(git_cmd, cwd=working_path, **options)

    if raises:
        process.wait()
    else:
        out, _ = process.communicate()
        out = out.decode("utf-8")

    returncode = process.returncode
    if raises:
        if returncode != 0:
            raise GitCommandError(working_path, cmd)
    else:
        if out.endswith('\n'):
            out = out.strip('\n')
        ui.debug(ui.lightgray, "[%i]" % returncode, ui.reset, out)
        return returncode, out
Esempio n. 7
0
def storeD():
    '''store the light level in the output file, cropping if necessary'''
    global sensordata, watchdog
    watchdog.reset()  # Reset the watchdog
    light_filetmp = cfg.light_filename + ".tmp"  # Temporary file
    try:
        with open(cfg.light_filename, 'r') as fin:
            data = fin.read().splitlines(
                True)  # Read in existing data if present
    except FileNotFoundError:
        data = []
    if len(data) >= cfg.light_keepreadings:
        data = data[-cfg.light_keepreadings:]


#    data.append(str(int(sum(sensordata)/len(sensordata))) + '\n')                      # Append the new entry
#    print("storing {}".format(int(sum(sensordata)/len(sensordata))))
    if len(sensordata) > 0:
        sensestore = cfg.light_normalise / (sum(sensordata) / len(sensordata))
        data.append(str(sensestore) + '\n')  # Append the new entry
        #        print("storing {}".format(cfg.light_normalise/(sum(sensordata)/len(sensordata))))
        with open(light_filetmp, 'w') as fout:
            fout.writelines(data)  # Store entries in temporary file
        os.rename(light_filetmp, cfg.light_filename
                  )  # Move the temporary file to the non-temporary one
        debug('Storing light sensor average {}'.format(sensestore), 2)
        if cfg.mqtt_engine:
            mqq.put(('lightSense', {'brightness': format(sensestore)}))
    sensordata = []
Esempio n. 8
0
    def __parse_configuration(self, it):
        configuration = configurations.Configuration()

        # name
        token = it.next()
        if token == lexer.Token.LITERAL:
            configuration.name = token.content
        else:
            ui.parse_error(token)

        while True:
            token = it.next()
            if token == lexer.Token.LITERAL:
                if token.content == "compiler": configuration.compiler = self.__parse_list(it)
                elif token.content == "archiver": configuration.archiver = self.__parse_list(it)
                elif token.content == "application_suffix": configuration.application_suffix = self.__parse_list(it)
                elif token.content == "compiler_flags": configuration.compiler_flags = self.__parse_list(it)
                elif token.content == "linker_flags": configuration.linker_flags = self.__parse_list(it)
                elif token.content == "export": configuration.export = self._parse_configuration_export(it)
                else: ui.parse_error(token)

            elif token == lexer.Token.NEWLINE:
                break
            else:
                ui.parse_error(token)

        ui.debug("configuration parsed:" + str(configuration))
        configurations.add_configuration(configuration)
Esempio n. 9
0
    def make_request(self,
                     url,
                     cf=False,
                     meth="GET",
                     timeout=30,
                     redirs=True,
                     data=None,
                     params=None):
        if cf is False:
            try:
                response = requests.request(url=url,
                                            headers=self.headers,
                                            method=meth,
                                            timeout=timeout,
                                            allow_redirects=redirs,
                                            data=data,
                                            params=params)
                ui.debug(ui.purple, "REQUEST", response.url, ui.bold,
                         response.status_code)

            except Exception as ex:
                ui.error("Request could not be made for", self.email, ex)
        else:  # cf is True
            try:
                import cfscrape
                scraper = cfscrape.create_scraper()
                response = scraper.get(url)
            except Exception as ex:
                ui.error("Cloudflare bypass request could not be made for",
                         self.email, ex)

        if response.status_code == 429:
            ui.warning("Reached RATE LIMIT, sleeping", ui.purple, self.email)
            sleep(2.5)
        return response
Esempio n. 10
0
def append(module_name, name, value):
    if name not in modules[module_name]:
        modules[module_name][name] = Variable(module_name, name)

    variable = modules[module_name][name]
    variable.content.append(value)

    ui.debug("setting variable: {!s}".format(variable))
Esempio n. 11
0
    def __parse(self):
        it = iter(self.tokens)

        try:
            if not self.__parse_directive(it):
                ui.parse_error(msg="unknown :(")
        except StopIteration:
            ui.debug("eof")
Esempio n. 12
0
def append(module_name, name, value):
    if name not in modules[module_name]:
        modules[module_name][name] = Variable(module_name, name)

    variable = modules[module_name][name]
    variable.content.append(value)

    ui.debug("setting variable: {!s}".format(variable))
Esempio n. 13
0
    def __parse(self):
        it = iter(self.tokens)

        try:
            if not self.__parse_directive(it):
                ui.parse_error(msg="unknown :(")
        except StopIteration:
            ui.debug("eof")
Esempio n. 14
0
def is_newer_than(prerequisite, target):
    if os.path.isfile(target) and os.path.isfile(prerequisite):
        ret = get_mtime(prerequisite) > get_mtime(target)
        ui.debug("is {} newer than {} = {!s}".format(prerequisite, target, ret))
        return ret
    else:
        ui.debug(target + " doesn't exist, treating like older")
        return True
Esempio n. 15
0
def dbD():
    '''database access thread
    '''
    debug('Database thread initialising')
    #    if cfg.ui_debugmod['db']: uiq.put(('Database thread initialising', 'DEBUG'))
    dbopen()
    while True:
        item = dbq.get()
        dbstorebeat(*item)
Esempio n. 16
0
def is_newer_than(prerequisite, target):
    if os.path.isfile(target):
        ret = get_mtime(prerequisite) > get_mtime(target)
        ui.debug("is {} newer than {} = {!s}".format(prerequisite, target,
                                                     ret))
        return ret
    else:
        ui.debug(target + " doesn't exist, treating like older")
        return True
Esempio n. 17
0
def _parse_command_line():
    parser = argparse.ArgumentParser(description='Painless buildsystem.')
    parser.add_argument('target', metavar='target', nargs="*", help='targets to be built')
    parser.add_argument('-a', '--all',  action="store_true", help='build all targets')
    parser.add_argument('-c', action='store', dest='configuration', default="__default", nargs="?", help='configuration to be used')
    parser.add_argument('-j', action='store', dest='jobs', default="1", nargs="?", help='parallel jobs to be used')
    parser.add_argument('-v', '--verbose',  action="store_true", help='show tool invokations')
    args = parser.parse_args()
    ui.debug(str(args))
    return args
Esempio n. 18
0
    def build(self, configuration):
        evaluated_artefacts = self.common_parameters.artefacts.eval()
        evaluated_prerequisites = self.common_parameters.prerequisites.eval()

        if not evaluated_artefacts or not evaluated_prerequisites:
            ui.warning(("target {} has no artifacts or prerequisites defined, "
                        "this means, whatever is defined in run_before or run_after "
                        "will be always executed")
                       .format(self.common_parameters.name))

        ui.debug("phony build")
Esempio n. 19
0
    def build(self, configuration):
        evaluated_artefacts = self.common_parameters.artefacts.eval()
        evaluated_prerequisites = self.common_parameters.prerequisites.eval()

        if not evaluated_artefacts or not evaluated_prerequisites:
            ui.warning(
                ("target {} has no artifacts or prerequisites defined, "
                 "this means, whatever is defined in run_before or run_after "
                 "will be always executed").format(
                     self.common_parameters.name))

        ui.debug("phony build")
Esempio n. 20
0
 def get_hunterio_private(self, api_key):
     try:
         ui.debug(self.email, "Getting HunterIO private data on domain")
         url = "https://api.hunter.io/v2/domain-search?domain={target}&api_key={key}".format(
             target=self.hostname, key=api_key)
         req = self.make_request(url, cf=True)
         response = req.json()
         for e in response["data"]["emails"]:
             self.hunterio_mails.append(e["value"])
     except Exception as ex:
         ui.warning(ui.yellow, "HunterIO (private API) error:", self.email,
                    ex, url)
Esempio n. 21
0
    def __scan_includes(self, in_filename, include_dirs, compiler_flags):
        ui.debug("scanning includes for " + in_filename)
        try:
            flags = self.__prepare_compiler_flags(include_dirs, compiler_flags)
            out = shell.execute(" ".join(
                [configurations.compiler(), flags, "-M", in_filename]),
                                capture_output=True).split()
        except Exception as e:
            raise Exception("error while building dependency graph for"
                            "{!s}, {!s}".format(in_filename, e))

        return [token for token in out[2:] if not token == "\\"]
Esempio n. 22
0
    def _build_object(self, sem, toolchain, name, object_file,
                       source, include_dirs, compiler_flags):
        try:
            if self.error:
                return

            with sem:
                toolchain.build_object(name, object_file, source, include_dirs, compiler_flags)
        except Exception as e:
            ui.debug("catched during compilation {!s}".format(e))
            self.error_reason = str(e)
            self.error = True
Esempio n. 23
0
 def get_pastebin(self):
     try:
         ui.debug(self.email, "Getting pastebin private data")
         url = "https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=filtered_cse&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&cx=013305635491195529773:0ufpuq-fpt0&q=%22{email}%22&cse_tok=AF14hljK02r_xNhCjU6rz69O3mWFxQgr0w:1534517796222&sort=&googlehost=www.google.com".format(
             email=self.email)
         req = self.make_request(url, cf=False)
         response = req.json()
         for e in response["results"]:
             self.pastebin_urls.append(e["formattedUrl"])
     except Exception as ex:
         ui.warning(ui.yellow, "Google api error for pastebin.com:",
                    self.email, ex, url)
Esempio n. 24
0
 def get_hunterio_public(self):
     try:
         ui.debug(self.email, "Getting HunterIO public data on domain")
         url = "https://api.hunter.io/v2/email-count?domain={}".format(
             self.hostname)
         req = self.make_request(url, cf=True)
         response = req.json()
         if response["data"]["total"] != 0:
             self.related_emails = response["data"]["total"]
     except Exception as ex:
         ui.warning(ui.yellow, "HunterIO (pubic API) error:", self.email,
                    ex)
Esempio n. 25
0
    def _build_object(self, sem, toolchain, name, object_file, source,
                      include_dirs, compiler_flags):
        try:
            if self.error:
                return

            with sem:
                toolchain.build_object(name, object_file, source, include_dirs,
                                       compiler_flags)
        except Exception as e:
            ui.debug("catched during compilation {!s}".format(e))
            self.error_reason = str(e)
            self.error = True
Esempio n. 26
0
def execute(command, capture_output=False):
    out = ''
    try:
        if capture_output:
            out = subprocess.check_output(command, shell=True)
        else:
            subprocess.check_call(command, shell=True)
    except subprocess.CalledProcessError as e:
        raise Exception("command exited with error({}): {}".format(
            str(e.returncode), command))

    ui.debug("command completed: " + command)
    return out
Esempio n. 27
0
def export_special_variables(configuration):
    ui.debug("exporting special variables")

    with ui.ident:
        add_empty("__configuration", "$__null")

        add("__configuration", "$__name", configuration.name)

        for variable in configuration.export:
            add("__configuration", variable.name, variable)

        for module in modules:
            add(module, "$__build", fsutils.build_dir(configuration.name))
Esempio n. 28
0
def export_special_variables(configuration):
    ui.debug("exporting special variables")

    with ui.ident:
        add_empty("__configuration", "$__null")

        add("__configuration", "$__name", configuration.name)

        for variable in configuration.export:
            add("__configuration", variable.name, variable)

        for module in modules:
            add(module, "$__build", fsutils.build_dir(configuration.name))
Esempio n. 29
0
def fullcap(a, b, c):
    '''the capacitor has charged enough to pull the pin high'''
    global timenow, sensordata
    delta = c - timenow  # Stop the counter
    print(delta)
    #    if delta < 0: delta += 4294967295                   # Counter wrapped
    if delta < 0:
        draincap()
        return
    debug('Light sensor read took {} ticks'.format(delta), 3)
    sensordata.append(delta)
    time.sleep(cfg.light_read_interval)  # Wait for the next reading
    draincap()  # Drain the cap to prime the next reading
    return
Esempio n. 30
0
def envD(pig):
    '''environmental data thread (pig = pigpiod handle)'''
    debug('Environmental data thread initialising')

    if cfg.env_frequency - 3 * cfg.env_delay < 0:
        uiq.put(('ERROR: env_frequency too low (must be at least {})'.format(
            3 * cfg.env_delay), 'ERR'))
        return
    while True:
        globs.temperature, globs.humidity = temphumid(pig)
        debug(
            'Read temperature {} and humidity {}'.format(
                globs.temperature, globs.humidity), 2)
        time.sleep(cfg.env_frequency - 3 * cfg.env_delay)
Esempio n. 31
0
    def link_application(self, out_filename, in_filenames, link_with,
                         library_dirs):
        if fsutils.is_any_newer_than(
                in_filenames,
                out_filename) or self.__are_libs_newer_than_target(
                    link_with, out_filename):
            ui.debug("linking application")
            ui.debug("  files: " + str(in_filenames))
            ui.debug("  with libs: " + str(link_with))
            ui.debug("  lib dirs: " + str(library_dirs))

            parameters = " ".join("-L " + lib_dir for lib_dir in library_dirs)

            ui.bigstep("linking", out_filename)
            try:
                shell.execute(" ".join([
                    configurations.compiler(),
                    configurations.linker_flags(), "-o", out_filename,
                    " ".join(in_filenames),
                    self.__prepare_linker_flags(link_with), parameters
                ]))
            except Exception as e:
                ui.fatal("cannot link {}, reason: {!s}".format(
                    out_filename, e))
        else:
            ui.bigstep("up to date", out_filename)
Esempio n. 32
0
    def __try_run(self, cmds):
        root_dir = os.getcwd()
        os.chdir(self.common_parameters.root_path)

        evaluated_artefacts = self.common_parameters.artefacts.eval()
        evaluated_prerequisites = self.common_parameters.prerequisites.eval()

        should_run = True
        if evaluated_prerequisites and evaluated_artefacts:
            should_run = False

            ui.debug("checking prerequisites ({!s}) for making {!s}".format(
                evaluated_prerequisites, evaluated_artefacts))

            for artefact in evaluated_artefacts:
                ui.debug("  " + artefact)
                if fsutils.is_any_newer_than(evaluated_prerequisites,
                                             artefact):
                    ui.debug(("going on because {!s}"
                              "needs to be rebuilt").format(artefact))
                    should_run = True
                    break

        if should_run:
            variables.pollute_environment(self.common_parameters.module_name)

            evaluated_cmds = cmds.eval()

            for cmd in evaluated_cmds:
                ui.debug("running {!s}".format(cmd))
                shell.execute(cmd)

        os.chdir(root_dir)
Esempio n. 33
0
    def __fetch_includes(self, target_name, in_filename, include_dirs, compiler_flags):
        ui.debug("getting includes for " + in_filename)

        with ui.ident:
            cache_file = self.cache_directory(target_name) + in_filename + ".includes"
            includes = None
            if os.path.exists(cache_file) and fsutils.is_newer_than(cache_file, in_filename):
                includes = marshal.load(open(cache_file, "rb"))
            else:
                fsutils.mkdir_recursive(os.path.dirname(cache_file));
                includes = self.__scan_includes(in_filename, include_dirs, compiler_flags)
                marshal.dump(includes, open(cache_file, "wb"))

        return includes
Esempio n. 34
0
def wait_until_something_changes():
    mkdir_recursive(BUILD_ROOT)
    touch_file=os.path.join(BUILD_ROOT, "ci.touch")

    with open(touch_file, "w") as f:
        f.write("don't bother about this file :)")

    while True:
        time.sleep(1)
        for dirpath, _, filenames in os.walk(os.getcwd()):
            for filename in filenames:
                if is_newer_than(os.path.join(dirpath, filename), touch_file):
                    ui.debug("found change: {}".format(filename))
                    return
Esempio n. 35
0
    def __scan_includes(self, in_filename, include_dirs, compiler_flags):
        ui.debug("scanning includes for " + in_filename)
        try:
            flags = self.__prepare_compiler_flags(include_dirs, compiler_flags)
            out = shell.execute(" ".join([configurations.compiler(), flags, "-M",
                                          in_filename]),
                                capture_output=True).split()
        except Exception as e:
            raise Exception("error while building dependency graph for"
                            "{!s}, {!s}".format(in_filename, e))

        def is_system_include(filename):
            return filename.startswith("/usr/include") or filename.startswith("/usr/lib")

        return [token for token in out[2:] if not token == "\\" and not is_system_include(token)]
Esempio n. 36
0
def mqttD():
    '''MQTT broker thread'''
    debug('MQTT thread initialising')
    global mqclient, watchdog, db2x
    mqclient = mqtt.Client()
    mqclient.connect(cfg.mqtt_broker, cfg.mqtt_port, cfg.mqtt_keepalive)
    mqclient.loop_start()
    if cfg.mqtt_telemetry:
        # Set a watchdog for publishing telemetry data
        watchdog = Watchdog(cfg.mqtt_telemetry_interval, mqttTelemetry)
    while True:
        item = mqq.get()
        payload = json.dumps(item[1])
        mqclient.publish('{}/{}'.format(cfg.mqtt_topicbase, item[0]), payload=payload, qos=0, retain=False)
        debug('MQTT publish: {}/{}: {}'.format(cfg.mqtt_topicbase, item[0], item[1]), 3)
Esempio n. 37
0
    def __init__(self, filename):
        assert isinstance(filename, str)

        ui.debug("parsing " + filename)

        with ui.ident:
            self.filename = filename
            self.name = self.__get_module_name(filename)

            self.tokens = lexer.parse(filename)

            self.__parse()

            variables.add(self.name, "$__path", os.path.dirname(filename))

            variables.add_empty(self.name, "$__null")
Esempio n. 38
0
def execute(command, capture_output = False):
    global _counter

    _counter += 1

    ui.debug("running {!s}: {!s}".format(_counter, command))

    out = ''
    try:
        if capture_output:
            out = subprocess.check_output(command, shell=True)
        else:
            subprocess.check_call(command, shell=True)
    except subprocess.CalledProcessError as e:
        raise Exception("command exited with error({}): {}".format(str(e.returncode), command))

    return out
Esempio n. 39
0
    def get_shodan(self, api_key):
        try:
            self.ip = socket.gethostbyname(self.hostname)
        except Exception as ex:
            ui.debug("Could not fetch host IP address", self.hostname, ex)

        if len(api_key) == 0:
            ui.debug(self.email, "Setting default Shodan API KEY")
            api_key = "UNmOjxeFS2mPA3kmzm1sZwC0XjaTTksy"  # From Infoga tool
        try:
            url = "https://api.shodan.io/shodan/host/{target}?key={key}".format(
                target=self.ip, key=api_key)
            response = self.make_request(url)
            data = json.loads(response.text)
            self.rev_ports.extend(data["ports"])
            self.rev_dns.extend(data["hostnames"])
        except Exception as ex:
            ui.warning(ui.yellow, "Shodan error for:", self.email, ex)
Esempio n. 40
0
    def __fetch_includes(self, target_name, in_filename, include_dirs,
                         compiler_flags):
        ui.debug("getting includes for " + in_filename)

        with ui.ident:
            cache_file = self.cache_directory(
                target_name) + in_filename + ".includes"
            includes = None
            if os.path.exists(cache_file) and fsutils.is_newer_than(
                    cache_file, in_filename):
                includes = marshal.load(open(cache_file, "rb"))
            else:
                fsutils.mkdir_recursive(os.path.dirname(cache_file))
                includes = self.__scan_includes(in_filename, include_dirs,
                                                compiler_flags)
                marshal.dump(includes, open(cache_file, "wb"))

        return includes
Esempio n. 41
0
def mqttTelemetry():
    '''publish telemetry data to MQTT broker every mqtt_telemetry_interval seconds'''
    global mqclient, watchdog
    # Reset the watchdog
    watchdog.reset()
    # Sort the telemetry data and clear the global
    stelemetry = sorted(globs.telemetry)
    globs.telemetry = []
    try:
        avgskew = sum(stelemetry)/len(stelemetry)
    except ZeroDivisionError:                               # This will happen when the clock isn't running
        return
    drift = -avgskew*86400/cfg.p_period
    percents = round_to_100([
        100*len([element for element in stelemetry if element < cfg.p_offset-cfg.p_tolerance2])/len(stelemetry),
        100*len([element for element in stelemetry if cfg.p_offset-cfg.p_tolerance2 <= element < cfg.p_offset-cfg.p_tolerance1])/len(stelemetry),
        100*len([element for element in stelemetry if abs(cfg.p_offset-element) <= cfg.p_tolerance1])/len(stelemetry),
        100*len([element for element in stelemetry if cfg.p_offset+cfg.p_tolerance1 < element <= cfg.p_offset+cfg.p_tolerance2])/len(stelemetry),
        100*len([element for element in stelemetry if element > cfg.p_offset+cfg.p_tolerance2])/len(stelemetry)
    ])
    payload = {
            'interval' : cfg.mqtt_telemetry_interval,
            'avgskew' : avgskew,
            'maxskew' : int(stelemetry[-1]),
            'minskew' : int(stelemetry[0]),
            'bad-' : percents[0],
            'warn-' : percents[1],
            'good' : percents[2],
            'warn+' : percents[3],
            'bad+' : percents[4],
            'drift' : round(drift,1),
            '1mdrift' : globs.driftavgs[0],
            '1hdrift' : globs.driftavgs[1],
            '1ddrift' : globs.driftavgs[2],
            '95max' : int(stelemetry[int(len(stelemetry)*0.95)]),
            '95min' : int(stelemetry[int(len(stelemetry)*0.05)]),
            'temperature' : globs.temperature,
            'humidity' : globs.humidity,
    }
    debug('MQTT publish: {}/telemetry: {}'.format(cfg.mqtt_topicbase, payload), 3)
    payload = json.dumps(payload)
    mqclient.publish('{}/telemetry'.format(cfg.mqtt_topicbase), payload=payload, qos=0, retain=False)
Esempio n. 42
0
    def build_objects(self, toolchain):
        object_files = []
        evaluated_sources = self.cxx_parameters.sources.eval()
        evaluated_include_dirs = self.cxx_parameters.include_dirs.eval()
        evaluated_compiler_flags = self.cxx_parameters.compiler_flags.eval()

        ui.debug("building objects from {!s}".format(evaluated_sources))
        ui.push()

        threads = []

        jobs = command_line.args.jobs
        limit_semaphore = threading.Semaphore(int(jobs))
        ui.debug("limiting jobs to {!s}".format(jobs))

        for source in evaluated_sources:
            object_file = toolchain.object_filename(
                self.common_parameters.name, source)
            object_files.append(object_file)

            thread = threading.Thread(
                target=self._build_object,
                args=(limit_semaphore, toolchain, self.common_parameters.name,
                      object_file, source, evaluated_include_dirs,
                      evaluated_compiler_flags))

            threads.append(thread)
            thread.daemon = True
            thread.start()

        assert len(threads) <= jobs

        for t in threads:
            t.join()

        if self.error:
            ui.fatal("failed building {!s}: {!s}".format(
                self.common_parameters.name, self.error_reason))

        ui.pop()

        return object_files
Esempio n. 43
0
def dbopen():
    '''open the database
    '''
    global dbx
    dbx = None
    debug('Opening database', 2)
    #    if cfg.ui_debugmod['db']: uiq.put(('Opening database', 'DEBUG', 2))
    try:
        dbx = sqlite3.connect(cfg.db_file)
        #        dbx.isolation_level = 'EXCLUSIVE'                   # FIXME - this seems not to lock against other processes
        #        dbx.execute('BEGIN EXCLUSIVE')
        sql = "CREATE TABLE IF NOT EXISTS beats ({});".format(globs.sqltable)
        cur = dbx.cursor()
        try:
            cur.execute(sql)
            dbx.commit()
        except Error as dberr:
            uiq.put(('Database table create error: {}'.format(dberr), 'ERR'))
    except Error as dberr:
        uiq.put(('Database open error: {}'.format(dberr), 'ERR'))
    return dbx
Esempio n. 44
0
    def eval(self):
        ui.debug("evaluating {!s}".format(self))

        parts = self.name.split(".")

        if len(parts) == 1:
            self.module = self.module
            self.name = parts[0]
        elif len(parts) == 2:
            self.module = parts[0][1:]  # lose the $
            self.name = "$" + parts[1]

        global modules

        if self.module not in modules:
            ui.parse_error(msg="no such module: " + self.module)

        if self.name not in modules[self.module]:
            ui.fatal("{!s} does not exist".format(self))

        return modules[self.module][self.name].eval()
Esempio n. 45
0
    def __try_run(self, cmds):
        evaluated_cmds = cmds.eval()

        if evaluated_cmds:
            root_dir = os.getcwd()
            os.chdir(self.common_parameters.root_path)

            evaluated_artefacts = self.common_parameters.artefacts.eval()
            evaluated_prerequisites = self.common_parameters.prerequisites.eval()

            should_run = self.__are_explicit_prerequisities_newer(evaluated_artefacts,
                                                                  evaluated_prerequisites)

            if should_run:
                variables.pollute_environment(self.common_parameters.module_name)

                for cmd in evaluated_cmds:
                    ui.debug("running {!s}".format(cmd))
                    shell.execute(cmd)

            os.chdir(root_dir)
Esempio n. 46
0
    def __init__(self, filename):
        assert isinstance(filename, str)

        ui.debug("parsing " + filename)

        with ui.ident:
            self.filename = filename
            self.name = self.__get_module_name(filename)

            self.tokens = lexer.parse(filename)

            self.__parse()

            variables.add(
                self.name,
                "$__path",
                os.path.dirname(filename))

            variables.add_empty(
                self.name,
                "$__null")
Esempio n. 47
0
    def eval(self):
        ui.debug("evaluating {!s}: ".format(self))

        s = self.content

        ret = []

        STATE_READING = 1
        STATE_WAITING_FOR_PARENTHESIS = 2
        STATE_READING_NAME = 3

        variable_name = "$"
        state = STATE_READING

        for c in s:
            if state == STATE_READING:
                if c == "$":
                    state = STATE_WAITING_FOR_PARENTHESIS
                else:
                    ret.append(c)
            elif state == STATE_WAITING_FOR_PARENTHESIS:
                if c == "{":
                    state = STATE_READING_NAME
                else:
                    ui.parse_error(msg="expecting { after $")
            elif state == STATE_READING_NAME:
                if c == "}":
                    ui.debug("variable: " + variable_name)

                    variable = ReferenceToVariable(self.module, variable_name)
                    ret.append(" ".join(variable.eval()))

                    variable_name = "$"
                    state = STATE_READING
                else:
                    variable_name += c
            elif state == STATE_READING_NAME:
                variable_name = variable_name + c

        return ["".join(ret)]
Esempio n. 48
0
    def build_objects(self, toolchain):
        object_files = []
        evaluated_sources = self.cxx_parameters.sources.eval()
        evaluated_include_dirs = self.cxx_parameters.include_dirs.eval()
        evaluated_compiler_flags = self.cxx_parameters.compiler_flags.eval()

        ui.debug("building objects from {!s}".format(evaluated_sources))
        ui.push()

        threads = []

        jobs = command_line.args.jobs
        limit_semaphore = threading.Semaphore(int(jobs))
        ui.debug("limiting jobs to {!s}".format(jobs))

        for source in evaluated_sources:
            object_file = toolchain.object_filename(self.common_parameters.name,
                                                        source)
            object_files.append(object_file)

            thread = threading.Thread(target=self._build_object,
                                      args=(limit_semaphore, toolchain, self.common_parameters.name, object_file,
                                            source, evaluated_include_dirs, evaluated_compiler_flags))

            threads.append(thread)
            thread.daemon = True
            thread.start()

        assert len(threads) <= jobs

        for t in threads:
            t.join()

        if self.error:
            ui.fatal("failed building {!s}: {!s}"
                     .format(self.common_parameters.name, self.error_reason))

        ui.pop()

        return object_files
Esempio n. 49
0
    def build_object(self, target_name, out_filename, in_filename, include_dirs,
                     compiler_flags):

        abs_source = os.path.join(os.getcwd(), in_filename)

        ui.debug("building object " + out_filename)

        with ui.ident:
            prerequisites = self.__fetch_includes(target_name, abs_source,
                                                  include_dirs, compiler_flags)
            prerequisites.append(in_filename)

            ui.debug("appending prerequisites from pake modules: {!s}"
                     .format(fsutils.pake_files))

            prerequisites.extend(fsutils.pake_files)

            ui.debug("prerequisites: {!r}".format(prerequisites))

            if fsutils.is_any_newer_than(prerequisites, out_filename):
                fsutils.mkdir_recursive(os.path.dirname(out_filename));

                cmd = configurations.compiler() + " " + self.__prepare_compiler_flags(include_dirs, compiler_flags) + " -c -o " + out_filename + " " + abs_source
                if command_line.args.verbose:
                    ui.step(configurations.compiler(), cmd)
                else:
                    ui.step(configurations.compiler(), in_filename)

                shell.execute(cmd)
Esempio n. 50
0
    def link_application(self, out_filename, in_filenames, link_with, library_dirs):
        if fsutils.is_any_newer_than(in_filenames, out_filename) or self.__are_libs_newer_than_target(link_with, out_filename):
            ui.debug("linking application")
            ui.debug("  files: " + str(in_filenames))
            ui.debug("  with libs: " + str(link_with))
            ui.debug("  lib dirs: " + str(library_dirs))

            parameters = " ".join("-L " + lib_dir for lib_dir in library_dirs)

            ui.bigstep("linking", out_filename)
            try:
                shell.execute(" ".join([configurations.compiler(),
                                        configurations.linker_flags(),
                                        "-o", out_filename,
                                        " ".join(in_filenames),
                                        self.__prepare_linker_flags(link_with),
                                        parameters]))
            except Exception as e:
                ui.fatal("cannot link {}, reason: {!s}".format(out_filename, e))
        else:
            ui.bigstep("up to date", out_filename)
Esempio n. 51
0
    def __are_explicit_prerequisities_newer(self, artefacts, prerequisites):
        ui.debug("checking prerequisites ({!s}) for making {!s}"
                 .format(prerequisites, artefacts))

        if prerequisites and artefacts:
            for artefact in artefacts:
                ui.debug("  " + artefact)
                if fsutils.is_any_newer_than(prerequisites, artefact):
                    ui.debug(("going on because {!s}"
                              "needs to be rebuilt").format(artefact))
                    return True
            return False
        else:
            return True
Esempio n. 52
0
def pollute_environment(current_module):
    ui.debug("polluting environment")

    with ui.ident:
        for module in modules:
            for (name, variable) in modules[module].iteritems():
                evaluated = variable.eval()
                env_name = module + "_" + name[1:]
                os.environ[env_name] = " ".join(evaluated)
                ui.debug("  " + env_name + ": " + str(evaluated))
                if module == current_module:
                    env_short_name = name[1:]
                    os.environ[env_short_name] = " ".join(evaluated)
                    ui.debug("  " + env_short_name + ": " + str(evaluated))
Esempio n. 53
0
def _build_and_track_single_target(name):
    """ tracking means putting it to special
        container, when this function is called
        with the same target, it will be skipped """
    configuration = configurations.get_selected_configuration()

    fsutils.make_build_dir(configuration.name)

    ui.debug("building {} with configuration {!s}".format(name, configuration))

    with ui.ident:
        if name in _built_targets:
            ui.debug("{} already build, skipping".format(name))
            return
        else:
            _built_targets.append(name)

        if name not in targets:
            ui.fatal("target {} not found".format(name))

        target = targets[name]

        if not target.is_visible(configuration):
            ui.fatal("target {} is not visible in {!s}"
                     .format(name, configuration))

        for dependency in target.common_parameters.depends_on.eval():
            ui.debug("{} depends on {}".format(name, dependency))
            build(dependency)

        toolchain = compiler.Gnu()

        target.before()
        target.build(toolchain)
        target.after()
        target.copy_resources(toolchain)
Esempio n. 54
0
def add(module_name, name, value):
    variable = Variable(module_name, name, value)
    modules[module_name][name] = variable

    ui.debug("adding variable: {!s}".format(variable))
Esempio n. 55
0
def add_empty(module_name, name):
    variable = Variable(name=name)
    modules[module_name][name] = variable

    ui.debug("adding variable: {!s}".format(variable))
Esempio n. 56
0
 def __init__(self, filename):
     self.filename = filename
     buf = FileReader(filename)
     self.tokens = []
     self.__tokenize(buf)
     ui.debug("tokens: " + str(self.tokens))
Esempio n. 57
0
def add_target(target):
    ui.debug("adding target: {!s}".format(target))

    targets[target.common_parameters.name] = target
Esempio n. 58
0
def add_configuration(configuration):
    ui.debug("adding configuration: " + str(configuration))
    configurations[configuration.name] = configuration
Esempio n. 59
0
 def __prepare_include_dirs_parameters(self, include_dirs):
     ret = " ".join("-I" + include_dir for include_dir in include_dirs)
     ui.debug("include parameters: " + ret)
     return ret