def get_mbed_targets_from_yotta_local_module(mbed_classic_name, yotta_targets_path='./yotta_targets'):
    """! Function is parsing local yotta targets to fetch matching mbed device target's name
    @return Function returns list of possible targets or empty list if value not found
    """
    result = []

    if os.path.exists(yotta_targets_path):
        # All local diorectories with yotta targets
        target_dirs = [target_dir_name for target_dir_name in os.listdir(yotta_targets_path) if os.path.isdir(os.path.join(yotta_targets_path, target_dir_name))]

        gt_log("local yotta target search in '%s' for compatible mbed-target '%s'"% (gt_bright(yotta_targets_path), gt_bright(mbed_classic_name.lower().strip())))

        for target_dir in target_dirs:
            path = os.path.join(yotta_targets_path, target_dir, 'target.json')
            try:
                with open(path, 'r') as data_file:
                    target_json_data = json.load(data_file)
                    yotta_target_name = parse_mbed_target_from_target_json(mbed_classic_name, target_json_data)
                    if yotta_target_name:
                        target_dir_name = os.path.join(yotta_targets_path, target_dir)
                        gt_log_tab("inside '%s' found compatible target '%s'"% (gt_bright(target_dir_name), gt_bright(yotta_target_name)))
                        result.append(yotta_target_name)
            except IOError as e:
                gt_log_err(str(e))
    return result
def get_mbed_targets_from_yotta(mbed_classic_name):
    """! Function is using 'yotta search' command to fetch matching mbed device target's name
    @return Function returns list of possible targets or empty list if value not found
    @details Example:
             $ yt search -k mbed-target:k64f target
             frdm-k64f-gcc 0.0.16: Official mbed build target for the mbed frdm-k64f development board.
             frdm-k64f-armcc 0.0.10: Official mbed build target for the mbed frdm-k64f development board, using the armcc toolchain.

             Note: Function prints on console
    """
    result = []
    cmd = ['yotta', '--plain', 'search', '-k', 'mbed-target:%s'% mbed_classic_name.lower().strip(), 'target']
    gt_log("yotta search for mbed-target '%s'"% gt_bright(mbed_classic_name.lower().strip()))
    gt_log_tab("calling yotta: %s"% " ".join(cmd))
    _stdout, _stderr, _ret = run_cli_process(cmd)
    if not _ret:
        for line in _stdout.splitlines():
            yotta_target_name = parse_yotta_search_cmd_output(line)
            if yotta_target_name:
                if yotta_target_name and yotta_target_name not in result:
                    result.append(yotta_target_name)
                    gt_log_tab("found target '%s'" % gt_bright(yotta_target_name))
    else:
        gt_log_err("calling yotta search failed!")
    return result
Exemple #3
0
def get_mbed_targets_from_yotta(mbed_classic_name):
    """! Function is using 'yotta search' command to fetch matching mbed device target's name
    @return Function returns list of possible targets or empty list if value not found
    @details Example:
             $ yt search -k mbed-target:k64f target
             frdm-k64f-gcc 0.0.16: Official mbed build target for the mbed frdm-k64f development board.
             frdm-k64f-armcc 0.0.10: Official mbed build target for the mbed frdm-k64f development board, using the armcc toolchain.

             Note: Function prints on console
    """
    result = []
    cmd = [
        'yotta', '--plain', 'search', '-k',
        'mbed-target:%s' % mbed_classic_name.lower().strip(), 'target'
    ]
    gt_log("yotta search for mbed-target '%s'" %
           gt_bright(mbed_classic_name.lower().strip()))
    gt_log_tab("calling yotta: %s" % " ".join(cmd))
    _stdout, _stderr, _ret = run_cli_process(cmd)
    if not _ret:
        for line in _stdout.splitlines():
            yotta_target_name = parse_yotta_search_cmd_output(line)
            if yotta_target_name:
                result.append(yotta_target_name)
                gt_log_tab("found target '%s'" % gt_bright(yotta_target_name))
    else:
        gt_log_err("calling yotta search failed!")
    return result
def get_mbed_target_from_current_dir():
    """! Function uses yotta target command to check current target
    @return Returns current target or None if target not found (e.g. not yotta package)
    """
    result = None
    cmd = ['yotta', '--plain', 'target']
    gt_log("checking yotta target in current directory")
    gt_log_tab("calling yotta: %s"% " ".join(cmd))
    _stdout, _stderr, _ret = run_cli_process(cmd)
    if not _ret:
        for line in _stdout.splitlines():
            target = parse_yotta_target_cmd_output(line)
            if target:
                result = target
                break
    return result
def get_mbed_target_from_current_dir():
    """! Function uses yotta target command to check current target
    @return Returns current target or None if target not found (e.g. not yotta package)
    """
    result = None
    cmd = ['yotta', 'target']
    gt_log("checking yotta target in current directory")
    gt_log_tab("calling yotta: %s"% " ".join(cmd))
    _stdout, _stderr, _ret = run_cli_process(cmd)
    if not _ret:
        for line in _stdout.splitlines():
            m = re.search(' \d+\.\d+\.\d+$', line)
            if m and len(m.group()):
                result = line.split()[0]
                break
    return result
Exemple #6
0
def get_mbed_target_from_current_dir():
    """! Function uses yotta target command to check current target
    @return Returns current target or None if target not found (e.g. not yotta package)
    """
    result = None
    cmd = ['yotta', '--plain', 'target']
    gt_log("checking yotta target in current directory")
    gt_log_tab("calling yotta: %s" % " ".join(cmd))
    _stdout, _stderr, _ret = run_cli_process(cmd)
    if not _ret:
        for line in _stdout.splitlines():
            target = parse_yotta_target_cmd_output(line)
            if target:
                result = target
                break
    return result
def list_binaries_for_targets(build_dir='./build', verbose_footer=True):
    """! Prints tests in target directories, only if tests exist.
    @details Skips empty / no tests for target directories.
    """
    dir = build_dir
    sub_dirs = [os.path.join(dir, o) for o in os.listdir(dir) if os.path.isdir(os.path.join(dir, o))]
    gt_log("available tests for built targets, location '%s'"% os.path.abspath(build_dir))
    for sub_dir in sub_dirs:
        test_list = load_ctest_testsuite(sub_dir, binary_type='')
        if len(test_list):
            gt_log_tab("target '%s':" % sub_dir.split(os.sep)[-1])
            for test in test_list:
                gt_log_tab("test '%s'"% test)
    if verbose_footer:
        print
        print "Example: execute 'mbedgt -t TARGET_NAME -n TEST_NAME' to run test TEST_NAME for target TARGET_NAME"
Exemple #8
0
def list_binaries_for_targets(build_dir='./build', verbose_footer=True):
    """! Prints tests in target directories, only if tests exist.
    @details Skips empty / no tests for target directories.
    """
    dir = build_dir
    sub_dirs = [
        os.path.join(dir, o) for o in os.listdir(dir)
        if os.path.isdir(os.path.join(dir, o))
    ]
    gt_log("available tests for built targets, location '%s'" %
           os.path.abspath(build_dir))
    for sub_dir in sub_dirs:
        test_list = load_ctest_testsuite(sub_dir, binary_type='')
        if len(test_list):
            gt_log_tab("target '%s':" % sub_dir.split(os.sep)[-1])
            for test in test_list:
                gt_log_tab("test '%s'" % test)
    if verbose_footer:
        print
        print "Example: execute 'mbedgt -t TARGET_NAME -n TEST_NAME' to run test TEST_NAME for target TARGET_NAME"
Exemple #9
0
def run_host_test(image_path,
                  disk,
                  port,
                  duration=10,
                  micro=None,
                  reset=None,
                  reset_tout=None,
                  verbose=False,
                  copy_method=None,
                  program_cycle_s=None,
                  digest_source=None,
                  json_test_cfg=None,
                  max_failed_properties=5,
                  enum_host_tests_path=None,
                  run_app=None):
    """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process.
    @return Tuple with test results, test output and test duration times
    @param image_path Path to binary file for flashing
    @param disk Currently mounted mbed-enabled devices disk (mount point)
    @param port Currently mounted mbed-enabled devices serial port (console)
    @param duration Test case timeout
    @param micro Mbed-nebaled device name
    @param reset Reset type
    @param reset_tout Reset timeout (sec)
    @param verbose Verbose mode flag
    @param copy_method Copy method type (name)
    @param program_cycle_s Wait after flashing delay (sec)
    @param json_test_cfg Additional test configuration file path passed to host tests in JSON format
    @param max_failed_properties After how many unknown properties we will assume test is not ported
    @param enum_host_tests_path Directory where locally defined host tests may reside
    @param run_app Run application mode flag (we run application and grab serial port data)
    @param digest_source if None mbedhtrun will be executed. If 'stdin',
                           stdin will be used via StdInObserver or file (if
                           file name was given as switch option)
    """

    class StdInObserver(Thread):
        """ Process used to read stdin only as console input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False

    class FileObserver(Thread):
        """ process used to read file content as console input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception as e:
                print "ProcessObserver.stop(): %s" % str(e)
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
            # signals to queue job is done
            obs.queue.task_done()
        except Empty:
            c = None
        except:
            raise
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_auto_property_value(property_name, line):
        """! Scans auto detection line from MUT and returns scanned parameter 'property_name'
        @details Host test case has to print additional properties for test to be set up
        @return Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'"% property_name, line) is not None:
            property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed

    if verbose:
        gt_log("selecting test case observer...")
        if digest_source:
            gt_log_tab("selected digest source: %s"% digest_source)

    # Select who will digest test case serial port data
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = ["mbedhtrun",
                '-d', disk,
                '-p', port,
                '-f', '"%s"'% image_path,
                ]

        # Add extra parameters to host_test
        if program_cycle_s is not None:
            cmd += ["-C", str(program_cycle_s)]
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_app is not None:
            cmd += ["--run"]    # -f stores binary name!
        if enum_host_tests_path:
            cmd += ["-e", '"%s"'% enum_host_tests_path]

        if verbose:
            gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd))
            gt_log("mbed-host-test-runner: started")
        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    coverage_start_data = None
    coverage_start_data_list = []   # List of code coverage reports to dump

    result = None
    update_once_flag = {}   # Stores flags checking if some auto-parameter was already set
    unknown_property_count = 0
    total_duration = 20     # This for flashing, reset and other serial port operations
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (total_duration):
        try:
            c = get_char_from_queue(obs)
        except Exception as e:
            output.append('get_char_from_queue(obs): %s'% str(e))
            break
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Check for unknown property prints
                # If there are too many we will stop test execution and assume test is not ported
                if "HOST: Unknown property" in line:
                    unknown_property_count += 1
                    if unknown_property_count >= max_failed_properties:
                        output.append('{{error}}')
                        break

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()
                    total_duration = duration   # After reset we gonna use real test case duration

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    total_duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                if coverage_start_data and "{{coverage_end}}" in line:
                    idx = line.find("{{coverage_end}}")
                    coverage_end_line = line[:idx]  # Coverage payload
                    cov_data = {
                        "path" : coverage_start_data[1],
                        "payload" : coverage_end_line,
                        "encoding" : 'hex'  # hex, base64
                    }
                    coverage_start_data_list.append(cov_data)
                    coverage_start_data = None

                if line.startswith("{{coverage_start"):
                    coverage_start_line = line.strip("{}")  # Remove {{ }} delimiters
                    coverage_start_data = coverage_start_line.split(";")

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c
    else:
        result = TEST_RESULT_TIMEOUT

    if not '{end}' in line:
        output.append('{{end}}')

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()

    end_time = time()
    testcase_duration = end_time - start_time   # Test case duration from reset to {end}

    def pack_hex_payload(path, payload):
        """! Convert a block of hex string data back to binary and return the binary data
        @param path Where to save file with binary payload
        @param payload String with hex encoded ascii data, e.g.: '6164636772...'
        @return bytearray with payload with data
        """
        hex_pairs = map(''.join, zip(*[iter(payload)] * 2)) # ['61', '64', '63', '67', '72', ... ]
        bin_payload = bytearray([int(s, 16) for s in hex_pairs])
        return bin_payload

    def pack_base64_payload(path, payload):
        """! Convert a block of base64 data back to binary and return the binary data
        """
        return None

    # We may have code coverage data to dump from test prints
    if coverage_start_data_list:
        gt_log("storing coverage data artefacts")
        for cov_data in coverage_start_data_list:
            path = os.path.abspath(cov_data['path'])
            payload = cov_data['payload']
            encoding = cov_data['encoding']
            bin_payload = None
            if encoding == 'hex':
                bin_payload = pack_hex_payload(path, payload)
            else:
                bin_payload = pack_base64_payload(path, payload)
            if bin_payload:
                gt_log_tab("storing %d bytes in '%s'"% (len(bin_payload), path))
                with open(path, "wb") as f:
                    f.write(bin_payload)

    if verbose:
        gt_log("mbed-host-test-runner: stopped")
    if not result:
        result = get_test_result(output)
    if verbose:
        gt_log("mbed-host-test-runner: returned '%s'"% result)
    return (result, "".join(output), testcase_duration, duration)
    "mbed-test-serial_interrupt",
    "mbed-test-stl",
    "mbed-test-sleep_timeout",
    "mbed-test-blinky",
    "mbed-test-heap_and_stack",
    "mbed-test-cstring",
]

def get_mbed_target_from_current_dir():
    """! Function uses yotta target command to check current target
    @return Returns current target or None if target not found (e.g. not yotta package)
    """
    result = None
<<<<<<< HEAD
    cmd = ['yotta', '--plain', 'target']
    gt_log("checking yotta target in current directory")
    gt_log_tab("calling yotta: %s"% " ".join(cmd))
=======
    cmd = ['yotta', 'target']
    print "mbedgt: yotta search for existing mbed-target"
<<<<<<< HEAD
>>>>>>> princeofdarkness76/testing
=======
>>>>>>> origin/testing
    _stdout, _stderr, _ret = run_cli_process(cmd)
    if not _ret:
        for line in _stdout.splitlines():
            target = parse_yotta_target_cmd_output(line)
            if target:
                result = target
                break
Exemple #11
0
def run_host_test(image_path,
                  disk,
                  port,
                  duration=10,
                  micro=None,
                  reset=None,
                  reset_tout=None,
                  verbose=False,
                  copy_method=None,
                  program_cycle_s=None,
                  digest_source=None,
                  json_test_cfg=None,
                  max_failed_properties=5,
                  run_app=None):
    """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process.
    @return Tuple with test results, test output and test duration times
    @param image_path Path to binary file for flashing
    @param disk Currently mounted mbed-enabled devices disk (mount point)
    @param port Currently mounted mbed-enabled devices serial port (console)
    @param duration Test case timeout
    @param micro Mbed-nebaled device name
    @param reset Reset type
    @param reset_tout Reset timeout (sec)
    @param verbose Verbose mode flag
    @param copy_method Copy method type (name)
    @param program_cycle_s Wait after flashing delay (sec)
    @param json_test_cfg Additional test configuration file path passed to host tests in JSON format
    @param max_failed_properties After how many unknown properties we will assume test is not ported
    @param run_app Run application mode flag (we run application and grab serial port data)
    @param digest_source if None mbedhtrun will be executed. If 'stdin',
                           stdin will be used via StdInObserver or file (if
                           file name was given as switch option)
    """
    class StdInObserver(Thread):
        """ Process used to read stdin only as console input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False

    class FileObserver(Thread):
        """ process used to read file content as console input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception as e:
                print "ProcessObserver.stop(): %s" % str(e)
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
            # signals to queue job is done
            obs.queue.task_done()
        except Empty:
            c = None
        except:
            raise
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_auto_property_value(property_name, line):
        """! Scans auto detection line from MUT and returns scanned parameter 'property_name'
        @details Host test case has to print additional properties for test to be set up
        @return Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'" % property_name, line) is not None:
            property = re.search(
                "HOST: Property '%s' = '([\w\d _]+)'" % property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed

    if verbose:
        gt_log("selecting test case observer...")
        if digest_source:
            gt_log_tab("selected digest source: %s" % digest_source)

    # Select who will digest test case serial port data
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = [
            "mbedhtrun",
            '-d',
            disk,
            '-p',
            port,
            '-f',
            '"%s"' % image_path,
        ]

        # Add extra parameters to host_test
        if program_cycle_s is not None:
            cmd += ["-C", str(program_cycle_s)]
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_app is not None:
            cmd += ["--run"]  # -f stores binary name!

        if verbose:
            gt_log_tab("calling mbedhtrun: %s" % " ".join(cmd))
            gt_log("mbed-host-test-runner: started")
        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    result = None
    update_once_flag = {
    }  # Stores flags checking if some auto-parameter was already set
    unknown_property_count = 0
    total_duration = 20  # This for flashing, reset and other serial port operations
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (total_duration):
        try:
            c = get_char_from_queue(obs)
        except Exception as e:
            output.append('get_char_from_queue(obs): %s' % str(e))
            break
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Check for unknown property prints
                # If there are too many we will stop test execution and assume test is not ported
                if "HOST: Unknown property" in line:
                    unknown_property_count += 1
                    if unknown_property_count >= max_failed_properties:
                        output.append('{{error}}')
                        break

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()
                    total_duration = duration  # After reset we gonna use real test case duration

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    total_duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c
    else:
        result = TEST_RESULT_TIMEOUT

    if not '{end}' in line:
        output.append('{{end}}')

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()

    end_time = time()
    testcase_duration = end_time - start_time  # Test case duration from reset to {end}

    if verbose:
        gt_log("mbed-host-test-runner: stopped")
    if not result:
        result = get_test_result(output)
    if verbose:
        gt_log("mbed-host-test-runner: returned '%s'" % result)
    return (result, "".join(output), testcase_duration, duration)
Exemple #12
0
def run_host_test(image_path,
                  disk,
                  port,
                  duration=10,
                  micro=None,
                  reset=None,
                  reset_tout=None,
                  verbose=False,
                  copy_method=None,
                  program_cycle_s=None,
                  digest_source=None,
                  json_test_cfg=None,
                  max_failed_properties=5,
                  run_app=None):
    """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process.
    @return Tuple with test results, test output and test duration times
    @param image_path Path to binary file for flashing
    @param disk Currently mounted mbed-enabled devices disk (mount point)
    @param port Currently mounted mbed-enabled devices serial port (console)
    @param duration Test case timeout
    @param micro Mbed-nebaled device name
    @param reset Reset type
    @param reset_tout Reset timeout (sec)
    @param verbose Verbose mode flag
    @param copy_method Copy method type (name)
    @param program_cycle_s Wait after flashing delay (sec)
    @param json_test_cfg Additional test configuration file path passed to host tests in JSON format
    @param max_failed_properties After how many unknown properties we will assume test is not ported
    @param run_app Run application mode flag (we run application and grab serial port data)
    @param digest_source if None mbedhtrun will be executed. If 'stdin',
                           stdin will be used via StdInObserver or file (if
                           file name was given as switch option)
    """

    class StdInObserver(Thread):
        """ Process used to read stdin only as console input from MUT
        """
        def __init__(self):
            Thread.__init__(self)
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = sys.stdin.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    class FileObserver(Thread):
        """ process used to read file content as console input from MUT
        """
        def __init__(self, filename):
            Thread.__init__(self)
            self.filename = filename
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            with open(self.filename) as f:
                while self.active:
                    c = f.read(1)
                    self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    class ProcessObserver(Thread):
        """ Default process used to observe stdout of another process as console input from MUT
        """
        def __init__(self, proc):
            Thread.__init__(self)
            self.proc = proc
            self.queue = Queue()
            self.daemon = True
            self.active = True
            self.start()

        def run(self):
            while self.active:
                c = self.proc.stdout.read(1)
                self.queue.put(c)

        def stop(self):
            self.active = False
            try:
                self.proc.terminate()
            except Exception:
                pass

    def get_char_from_queue(obs):
        """ Get character from queue safe way
        """
        try:
            c = obs.queue.get(block=True, timeout=0.5)
            # signals to queue job is done
            obs.queue.task_done()
        except Empty:
            c = None
        except:
            raise
        return c

    def filter_queue_char(c):
        """ Filters out non ASCII characters from serial port
        """
        if ord(c) not in range(128):
            c = ' '
        return c

    def get_test_result(output):
        """! Parse test 'output' data
        @details If test result not found returns by default TEST_RESULT_TIMEOUT value
        @return Returns found test result
        """
        result = TEST_RESULT_TIMEOUT
        for line in "".join(output).splitlines():
            search_result = RE_DETECT_TESTCASE_RESULT.search(line)
            if search_result and len(search_result.groups()):
                result = TEST_RESULT_MAPPING[search_result.groups(0)[0]]
                break
        return result

    def get_auto_property_value(property_name, line):
        """! Scans auto detection line from MUT and returns scanned parameter 'property_name'
        @details Host test case has to print additional properties for test to be set up
        @return Returns string or None if property search failed
        """
        result = None
        if re.search("HOST: Property '%s'"% property_name, line) is not None:
            property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
            if property is not None and len(property.groups()) == 1:
                result = property.groups()[0]
        return result

    # Detect from where input should be taken, if no --digest switch is specified
    # normal test execution can be performed

    if verbose:
        gt_log("selecting test case observer...")
        if digest_source:
            gt_log_tab("selected digest source: %s"% digest_source)

    # Select who will digest test case serial port data
    if digest_source == 'stdin':
        # When we want to scan stdin for test results
        obs = StdInObserver()
    elif digest_source is not None:
        # When we want to open file to scan for test results
        obs = FileObserver(digest_source)
    else:
        # Command executing CLI for host test supervisor (in detect-mode)
        cmd = ["mbedhtrun",
                '-d', disk,
                '-p', port,
                '-f', '"%s"'% image_path,
                ]

        # Add extra parameters to host_test
        if program_cycle_s is not None:
            cmd += ["-C", str(program_cycle_s)]
        if copy_method is not None:
            cmd += ["-c", copy_method]
        if micro is not None:
            cmd += ["-m", micro]
        if reset is not None:
            cmd += ["-r", reset]
        if reset_tout is not None:
            cmd += ["-R", str(reset_tout)]
        if json_test_cfg is not None:
            cmd += ["--test-cfg", '"%s"' % str(json_test_cfg)]
        if run_app is not None:
            cmd += ["--run"]    # -f stores binary name!

        if verbose:
            gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd))
            gt_log("mbed-host-test-runner: started")
        proc = Popen(cmd, stdout=PIPE)
        obs = ProcessObserver(proc)

    update_once_flag = {}   # Stores flags checking if some auto-parameter was already set
    unknown_property_count = 0
    total_duration = 20     # This for flashing, reset and other serial port operations
    line = ''
    output = []
    start_time = time()
    while (time() - start_time) < (total_duration):
        try:
            c = get_char_from_queue(obs)
        except:
            break
        if c:
            if verbose:
                sys.stdout.write(c)
            c = filter_queue_char(c)
            output.append(c)
            # Give the mbed under test a way to communicate the end of the test
            if c in ['\n', '\r']:

                # Check for unknown property prints
                # If there are too many we will stop test exeuction and assume test is not ported
                if "HOST: Unknown property" in line:
                    unknown_property_count += 1
                    if unknown_property_count >= max_failed_properties:
                        output.append('{{error}}')
                        break

                # Checking for auto-detection information from the test about MUT reset moment
                if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['reset_target'] = True
                    start_time = time()
                    total_duration = duration   # After reset we gonna use real test case duration

                # Checking for auto-detection information from the test about timeout
                auto_timeout_val = get_auto_property_value('timeout', line)
                if 'timeout' not in update_once_flag and auto_timeout_val is not None:
                    # We will update this marker only once to prevent multiple time resets
                    update_once_flag['timeout'] = True
                    total_duration = int(auto_timeout_val)

                # Detect mbed assert:
                if 'mbed assertation failed: ' in line:
                    output.append('{{mbed_assert}}')
                    break

                # Check for test end
                if '{end}' in line:
                    break
                line = ''
            else:
                line += c

    if not '{end}' in line:
        output.append('{{end}}')

    c = get_char_from_queue(obs)

    if c:
        if verbose:
            sys.stdout.write(c)
        c = filter_queue_char(c)
        output.append(c)

    # Stop test process
    obs.stop()

    end_time = time()
    testcase_duration = end_time - start_time   # Test case duration from reset to {end}

    if verbose:
        gt_log("mbed-host-test-runner: stopped")
    result = get_test_result(output)
    return (result, "".join(output), testcase_duration, duration)
Exemple #13
0
def main():
    """ Closure for main_cli() function """

    parser = optparse.OptionParser()

    parser.add_option('-t', '--target',
                    dest='list_of_targets',
                    help='You can specify list of targets you want to build. Use comma to sepatate them')

    parser.add_option('-n', '--test-by-names',
                    dest='test_by_names',
                    help='Runs only test enumerated it this switch. Use comma to separate test case names.')

    parser.add_option("-O", "--only-build",
                    action="store_true",
                    dest="only_build_tests",
                    default=False,
                    help="Only build repository and tests, skips actual test procedures (flashing etc.)")

    parser.add_option("", "--skip-build",
                    action="store_true",
                    dest="skip_yotta_build",
                    default=False,
                    help="Skip calling 'yotta build' on this module")

    copy_methods_str = "Plugin support: " + ', '.join(mbed_host_tests.host_tests_plugins.get_plugin_caps('CopyMethod'))
    parser.add_option("-c", "--copy",
                    dest="copy_method",
                    help="Copy (flash the target) method selector. " + copy_methods_str,
                    metavar="COPY_METHOD")

    parser.add_option('', '--config',
                    dest='verbose_test_configuration_only',
                    default=False,
                    action="store_true",
                    help='Displays connected boards and detected targets and exits.')

    parser.add_option('', '--release',
                    dest='build_to_release',
                    default=False,
                    action="store_true",
                    help='If possible force build in release mode (yotta -r).')

    parser.add_option('', '--debug',
                    dest='build_to_debug',
                    default=False,
                    action="store_true",
                    help='If possible force build in debug mode (yotta -d).')

    parser.add_option('', '--list',
                    dest='list_binaries',
                    default=False,
                    action="store_true",
                    help='List available binaries')

    parser.add_option('', '--lock',
                    dest='lock_by_target',
                    default=False,
                    action="store_true",
                    help='Use simple resource locking mechanism to run multiple application instances')

    parser.add_option('', '--digest',
                    dest='digest_source',
                    help='Redirect input from where test suite should take console input. You can use stdin or file name to get test case console output')

    parser.add_option('', '--test-cfg',
                    dest='json_test_configuration',
                    help='Pass to host test data with host test configuration')

    parser.add_option('', '--run',
                    dest='run_app',
                    help='Flash, reset and dump serial from selected binary application')

    parser.add_option('', '--report-junit',
                    dest='report_junit_file_name',
                    help='You can log test suite results in form of JUnit compliant XML report')

    parser.add_option('', '--report-text',
                    dest='report_text_file_name',
                    help='You can log test suite results to text file')

    parser.add_option('', '--report-json',
                    dest='report_json',
                    default=False,
                    action="store_true",
                    help='Outputs test results in JSON')

    parser.add_option('', '--report-fails',
                    dest='report_fails',
                    default=False,
                    action="store_true",
                    help='Prints console outputs for failed tests')

    parser.add_option('-V', '--verbose-test-result',
                    dest='verbose_test_result_only',
                    default=False,
                    action="store_true",
                    help='Prints test serial output')

    parser.add_option('-v', '--verbose',
                    dest='verbose',
                    default=False,
                    action="store_true",
                    help='Verbose mode (prints some extra information)')

    parser.add_option('', '--version',
                    dest='version',
                    default=False,
                    action="store_true",
                    help='Prints package version and exits')

    parser.description = """This automated test script is used to test mbed SDK 3.0 on mbed-enabled devices with support from yotta build tool"""
    parser.epilog = """Example: mbedgt --target frdm-k64f-gcc"""

    (opts, args) = parser.parse_args()

    cli_ret = 0

    start = time()
    if opts.lock_by_target:
        # We are using Greentea proprietary locking meachnism to lock between platforms and targets
        gt_log("using (experimental) simple locking mechaism")
        gt_log_tab("kettle: %s"% GREENTEA_KETTLE_PATH)
        gt_file_sem, gt_file_sem_name, gt_instance_uuid = greentea_get_app_sem()
        with gt_file_sem:
            greentea_update_kettle(gt_instance_uuid)
            try:
                cli_ret = main_cli(opts, args, gt_instance_uuid)
            except KeyboardInterrupt:
                greentea_clean_kettle(gt_instance_uuid)
                gt_log_err("ctrl+c keyboard interrupt!")
                exit(-2)    # Keyboard interrupt
            except:
                greentea_clean_kettle(gt_instance_uuid)
                gt_log_err("Unexpected error:")
                gt_log_tab(sys.exc_info()[0])
                raise
            greentea_clean_kettle(gt_instance_uuid)
    else:
        # Standard mode of operation
        # Other instance must provide mutually exclusive access control to platforms and targets
        try:
            cli_ret = main_cli(opts, args)
        except KeyboardInterrupt:
            gt_log_err("ctrl+c keyboard interrupt!")
            exit(-2)    # Keyboard interrupt
        except Exception as e:
            gt_log_err("Unexpected error:")
            gt_log_tab(str(e))
            raise

    if not any([opts.list_binaries, opts.version]):
        print "Completed in %.2f sec"% (time() - start)
    exit(cli_ret)
Exemple #14
0
def main_cli(opts, args, gt_instance_uuid=None):
    """! This is main CLI function with all command line parameters
    @details This function also implements CLI workflow depending on CLI parameters inputed
    @return This function doesn't return, it exits to environment with proper success code
    """

    if not MBED_LMTOOLS:
        gt_log_err("error: mbed-ls proprietary module not installed")
        return (-1)

    if not MBED_HOST_TESTS:
        gt_log_err("error: mbed-host-tests proprietary module not installed")
        return (-1)

    # List available test binaries (names, no extension)
    if opts.list_binaries:
        list_binaries_for_targets()
        return (0)

    # Prints version and exits
    if opts.version:
        print_version()
        return (0)

    # Capture alternative test console inputs, used e.g. in 'yotta test command'
    if opts.digest_source:
        host_test_result = run_host_test(image_path=None,
                                         disk=None,
                                         port=None,
                                         digest_source=opts.digest_source,
                                         verbose=opts.verbose_test_result_only)

        single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
        status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1
        return (status)

    # mbed-enabled devices auto-detection procedures
    mbeds = mbed_lstools.create()
    mbeds_list = mbeds.list_mbeds()
    platform_list = mbeds.list_platforms_ext()

    # Option -t <opts.list_of_targets> supersedes yotta target set in current directory
    if opts.list_of_targets is None:
        if opts.verbose:
            gt_log("yotta target not set from command line (specified with -t option)")
        # Trying to use locally set yotta target
        current_target = get_mbed_target_from_current_dir()

        if current_target:
            gt_log("yotta target in current directory is set to '%s'"% gt_bright(current_target))
            # Assuming first target printed by 'yotta search' will be used
            opts.list_of_targets = current_target.split(',')[0]
        else:
            gt_log("yotta target in current directory is not set")
            gt_log_err("yotta target is not specified. Use '%s' or '%s' command to set target"%
            (
                gt_bright('mbedgt -t <target>'),
                gt_bright('yotta target <target>')
            ))
            return (-1)

    gt_log("detecting connected mbed-enabled devices... %s"% ("no devices detected" if not len(mbeds_list) else ""))
    if mbeds_list:
        gt_log("detected %d device%s"% (len(mbeds_list), 's' if len(mbeds_list) != 1 else ''))
    else:
        gt_log("no devices detected")

    list_of_targets = opts.list_of_targets.split(',') if opts.list_of_targets is not None else None

    test_report = {}    # Test report used to export to Junit, HTML etc...

    if opts.list_of_targets is None:
        gt_log("assuming default target as '%s'"% gt_bright(current_target))
        gt_log_tab("reason: no --target switch set")
        list_of_targets = [current_target]

    test_exec_retcode = 0       # Decrement this value each time test case result is not 'OK'
    test_platforms_match = 0    # Count how many tests were actually ran with current settings
    target_platforms_match = 0  # Count how many platforms were actually tested with current settings

    for mut in mbeds_list:
        platform_text = gt_bright(mut['platform_name'])
        serial_text = gt_bright(mut['serial_port'])
        mount_text = gt_bright(mut['mount_point'])
        platform_target_id = gt_bright(mut['target_id'])    # We can use it to do simple resource lock

        if not all([platform_text, serial_text, mount_text]):
            gt_log_err("can't detect all properties of the device!")
            gt_log_tab("detected '%s', console at '%s', mounted at '%s'"% (platform_text, serial_text, mount_text))
            continue

        gt_log_tab("detected '%s', console at '%s', mounted at '%s'"% (platform_text, serial_text, mount_text))

        # Check if mbed classic target name can be translated to yotta target name
        gt_log("scan available targets for '%s' platform..."% gt_bright(mut['platform_name']))
        mut_info = get_mbed_clasic_target_info(mut['platform_name'])

        if mut_info is not None:
            for yotta_target in mut_info['yotta_targets']:
                yotta_target_name = yotta_target['yotta_target']

                if yotta_target_name in list_of_targets:
                    target_platforms_match += 1

                # Configuration print mode:
                if opts.verbose_test_configuration_only:
                    continue

                # Demo mode: --run implementation (already added --run to mbedhtrun)
                # We want to pass file name to mbedhtrun (--run NAME  =>  -f NAME_ and run only one binary
                if opts.run_app and yotta_target_name in list_of_targets:
                    gt_log("running '%s' for '%s'"% (gt_bright(opts.run_app), gt_bright(yotta_target_name)))
                    disk = mut['mount_point']
                    port = mut['serial_port']
                    micro = mut['platform_name']
                    program_cycle_s = mut_info['properties']['program_cycle_s']
                    copy_method = opts.copy_method if opts.copy_method else 'shell'
                    verbose = opts.verbose_test_result_only

                    test_platforms_match += 1
                    host_test_result = run_host_test(opts.run_app,
                                                     disk,
                                                     port,
                                                     micro=micro,
                                                     copy_method=copy_method,
                                                     program_cycle_s=program_cycle_s,
                                                     digest_source=opts.digest_source,
                                                     json_test_cfg=opts.json_test_configuration,
                                                     run_app=opts.run_app,
                                                     verbose=True)

                    single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
                    status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1
                    if single_test_result != TEST_RESULT_OK:
                        test_exec_retcode += 1
                    continue

                # Regression test mode:
                # Building sources for given target and perform normal testing
                if yotta_target_name in list_of_targets:
                    gt_log("using '%s' target, prepare to build"% gt_bright(yotta_target_name))
                    cmd = ['yotta'] # "yotta %s --target=%s,* build"% (yotta_verbose, yotta_target_name)
                    if opts.verbose is not None: cmd.append('-v')
                    cmd.append('--target=%s,*' % yotta_target_name)
                    cmd.append('build')
                    if opts.build_to_release:
                        cmd.append('-r')
                    elif opts.build_to_debug:
                        cmd.append('-d')

                    if not opts.skip_yotta_build:
                        gt_log("building your sources and tests with yotta...")
                        gt_log_tab("calling yotta: %s"% " ".join(cmd))
                        yotta_result, yotta_ret = run_cli_command(cmd, shell=False, verbose=opts.verbose)
                        if yotta_result:
                            gt_log("yotta build for target '%s' was successful"% gt_bright(yotta_target_name))
                        else:
                            gt_log_err("yotta build failed!")
                    else:
                        gt_log("skipping calling yotta (specified with --skip-build option)")
                        yotta_result, yotta_ret = True, 0   # Skip build and assume 'yotta build' was successful

                    # Build phase will be followed by test execution for each target
                    if yotta_result and not opts.only_build_tests:
                        binary_type = mut_info['properties']['binary_type']
                        ctest_test_list = load_ctest_testsuite(os.path.join('.', 'build', yotta_target_name),
                            binary_type=binary_type)

                        test_list = None
                        if opts.test_by_names:
                            test_list = opts.test_by_names.split(',')
                            gt_log("test case filter: %s (specified with -n option)"% ', '.join(["'%s'"% gt_bright(t) for t in test_list]))

                            invalid_test_names = False
                            for test_n in test_list:
                                if test_n not in ctest_test_list:
                                    gt_log_tab("test name '%s' not found in CTestTestFile.cmake (specified with -n option)"% gt_bright(test_n))
                                    invalid_test_names = True
                            if invalid_test_names:
                                gt_log("invalid test case names (specified with -n option)")
                                gt_log_tab("note: test case names are case sensitive")
                                gt_log_tab("note: see list of available test cases below")
                                list_binaries_for_targets(verbose_footer=False)

                        gt_log("running tests for target '%s'" % gt_bright(yotta_target_name))
                        for test_bin, image_path in ctest_test_list.iteritems():
                            test_result = 'SKIPPED'
                            # Skip test not mentioned in -n option
                            if opts.test_by_names:
                                if test_bin not in test_list:
                                    continue

                            if get_mbed_supported_test(test_bin):
                                disk = mut['mount_point']
                                port = mut['serial_port']
                                micro = mut['platform_name']
                                program_cycle_s = mut_info['properties']['program_cycle_s']
                                copy_method = opts.copy_method if opts.copy_method else 'shell'
                                verbose = opts.verbose_test_result_only

                                test_platforms_match += 1
                                gt_log_tab("running host test...")
                                host_test_result = run_host_test(image_path,
                                                                 disk,
                                                                 port,
                                                                 micro=micro,
                                                                 copy_method=copy_method,
                                                                 program_cycle_s=program_cycle_s,
                                                                 digest_source=opts.digest_source,
                                                                 json_test_cfg=opts.json_test_configuration,
                                                                 verbose=verbose)

                                single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
                                test_result = single_test_result
                                if single_test_result != TEST_RESULT_OK:
                                    test_exec_retcode += 1

                                # Update report for optional reporting feature
                                test_name = test_bin.lower()
                                if yotta_target_name not in test_report:
                                    test_report[yotta_target_name] = {}
                                if test_name not in test_report[yotta_target_name]:
                                    test_report[yotta_target_name][test_name] = {}

                                test_report[yotta_target_name][test_name]['single_test_result'] = single_test_result
                                test_report[yotta_target_name][test_name]['single_test_output'] = single_test_output
                                test_report[yotta_target_name][test_name]['elapsed_time'] = single_testduration
                                test_report[yotta_target_name][test_name]['platform_name'] = micro
                                test_report[yotta_target_name][test_name]['copy_method'] = copy_method

                                if single_test_result != 'OK' and not verbose and opts.report_fails:
                                    # In some cases we want to print console to see why test failed
                                    # even if we are not in verbose mode
                                    gt_log_tab("test failed, reporting console output (specified with --report-fails option)")
                                    print
                                    print single_test_output

                                gt_log_tab("test '%s' %s %s in %.2f sec"% (test_bin, '.' * (80 - len(test_bin)), test_result, single_testduration))
                    # We need to stop executing if yotta build fails
                    if not yotta_result:
                        gt_log_err("yotta returned %d"% yotta_ret)
                        test_exec_retcode = -1
                        return (test_exec_retcode)
        else:
            gt_log_err("mbed classic target name '%s' is not in target database"% gt_bright(mut['platform_name']))

    if opts.verbose_test_configuration_only:
        print
        print "Example: execute 'mbedgt --target=TARGET_NAME' to start testing for TARGET_NAME target"
        return (0)

    # This tool is designed to work in CI
    # We want to return success codes based on tool actions,
    # only if testes were executed and all passed we want to
    # return 0 (success)
    if not opts.only_build_tests:
        # Reports (to file)
        if opts.report_junit_file_name:
            junit_report = exporter_junit(test_report)
            with open(opts.report_junit_file_name, 'w') as f:
                f.write(junit_report)
        if opts.report_text_file_name:
            gt_log("exporting to junit '%s'..."% gt_bright(opts.report_text_file_name))
            text_report, text_results = exporter_text(test_report)
            with open(opts.report_text_file_name, 'w') as f:
                f.write(text_report)
        # Reports (to console)
        if opts.report_json:
            # We will not print summary and json report together
            gt_log("json test report:")
            print exporter_json(test_report)
        else:
            # Final summary
            gt_log("test report:")
            text_report, text_results = exporter_text(test_report)
            print text_report
            print
            print "Result: " + text_results

        # This flag guards 'build only' so we expect only yotta errors
        if test_platforms_match == 0:
            # No tests were executed
            gt_log("no target matching tests were found!")
            test_exec_retcode += -10
        if target_platforms_match == 0:
            # No platforms were tested
            gt_log("no target matching platforms were found!")
            test_exec_retcode += -100

    return (test_exec_retcode)