Exemple #1
0
    def __init__(self, project_root):
        # set project root
        self.project_root = project_root

        # test timestamp - for storing results
        self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
        self.epoch = int(time.time())

        # parse cmd  args
        self.cmd_args = self.handle_cmd_args()

        # Get SUT build for use in reporting
        self.test_build = self.cmd_args['build']

        self.reporter = Reporter(project_root, self.test_timestamp)
        self.shishito_support = ShishitoSupport(cmd_args=self.cmd_args,
                                                project_root=self.project_root)
Exemple #2
0
    def __init__(self, user, password, timestamp, epoch, build):
        self.shishito_support = ShishitoSupport()
        self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp
        self.epoch = epoch
        self.build = build

        # project specific config
        self.project_id = self.shishito_support.get_opt('qastats_project_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.result_url = self.qastats_base_url + '/api/v1/results'
        self.project_url = self.shishito_support.get_opt('base_url')
Exemple #3
0
    def __init__(self, user, password, timestamp, build):
        self.shishito_support = ShishitoSupport()
        self.test_rail_instance = self.shishito_support.get_opt(
            'test_rail_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp

        # project specific config
        self.project_id = self.shishito_support.get_opt('test_rail_project_id')
        self.section_id = self.shishito_support.get_opt('test_rail_section_id')
        self.test_plan_id = self.shishito_support.get_opt(
            'test_rail_test_plan_id')
        self.test_plan_name = self.shishito_support.get_opt(
            'test_rail_test_plan_name') or build
        self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'
    def __init__(self, project_root):
        # set project root
        self.project_root = project_root

        # test timestamp - for storing results
        self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")

        # parse cmd  args
        self.cmd_args = self.handle_cmd_args()

        self.reporter = Reporter(project_root, self.test_timestamp)
        self.shishito_support = ShishitoSupport(
            cmd_args=self.cmd_args,
            project_root=self.project_root
        )
Exemple #5
0
    def __init__(self, project_root):
        # set project root
        self.project_root = project_root

        # test timestamp - for storing results
        self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
        self.epoch = int(time.time())

        # parse cmd  args
        self.cmd_args = self.handle_cmd_args()

        # Get SUT build for use in reporting
        self.test_build = self.cmd_args['build']

        self.reporter = Reporter(project_root, self.test_timestamp)
        self.shishito_support = ShishitoSupport(
            cmd_args=self.cmd_args,
            project_root=self.project_root
        )
Exemple #6
0
    def __init__(self, user, password, timestamp):
        self.shishito_support = ShishitoSupport()
        self.test_rail_instance = self.shishito_support.get_opt('test_rail_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp

        # project specific config
        self.project_id = self.shishito_support.get_opt('test_rail_project_id')
        self.section_id = self.shishito_support.get_opt('test_rail_section_id')
        self.test_plan_id = self.shishito_support.get_opt('test_rail_test_plan_id')
        self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'
Exemple #7
0
    def __init__(self, user, password, timestamp, epoch, build):
        self.shishito_support = ShishitoSupport()
        self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp
        self.epoch = epoch
        self.build = build

        # project specific config
        self.project_id = self.shishito_support.get_opt('qastats_project_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.result_url = self.qastats_base_url + '/api/v1/results'
        self.project_url = self.shishito_support.get_opt('base_url')
Exemple #8
0
class ShishitoRunner(object):
    """ Base shishito test runner.

    - runs python selenium tests on customizable configurations (using PyTest)
    - archive the test results in .zip file """

    def __init__(self, project_root):
        # set project root
        self.project_root = project_root

        # test timestamp - for storing results
        self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
        self.epoch = int(time.time())

        # parse cmd  args
        self.cmd_args = self.handle_cmd_args()

        # Get SUT build for use in reporting
        self.test_build = self.cmd_args['build']

        self.reporter = Reporter(project_root, self.test_timestamp)
        self.shishito_support = ShishitoSupport(
            cmd_args=self.cmd_args,
            project_root=self.project_root
        )

    def handle_cmd_args(self):
        """ Retrieve command line arguments passed to the script.

        :return: dict with parsed command line arguments
        """

        parser = argparse.ArgumentParser(description='Selenium Python test runner execution arguments.')

        parser.add_argument('--platform',
                            help='Platform on which run tests.',
                            dest='test_platform')
        parser.add_argument('--environment',
                            help='Environment for which run tests.',
                            dest='test_environment')
        parser.add_argument('--test_directory',
                            help='Directory where to lookup for tests')
        parser.add_argument('--smoke',
                            help='Run only smoke tests',
                            action='store_true')
        parser.add_argument('--browserstack',
                            help='BrowserStack credentials; format: "username:token"')
        parser.add_argument('--saucelabs',
                            help='Saucelabs credentials; format: "username:token"')
        parser.add_argument('--test_rail',
                            help='TestRail Test Management tool credentials; format: "username:password"')
        parser.add_argument('--qastats',
                            help='QAStats Test Management tool credentials; format: "token"')
        parser.add_argument('--node_webkit_chromedriver_path',
                            help='Path to chromedriver located in same directory as node-webkit application')
        parser.add_argument('--app',
                            help='Path to appium application')
        parser.add_argument('--test',
                            help='Run specified test (PyTest string expression)')
        parser.add_argument('--build',
                            help='Specify build number for reporting purposes')
        parser.add_argument('--maxfail',
                            help='stop after x failures')
        args = parser.parse_args()

        # return args dict --> for use in other classes
        return vars(args)

    def run_tests(self):
        """ Execute tests for given platform and environment. Platform and Environment can be passed as command lines
        argument or settings in config file.
        """

        if __name__ == "__main__":
            sys.exit('The runner cannot be executed directly.'
                     ' You need to import it within project specific runner. Session terminated.')

        # cleanup previous results
        self.reporter.cleanup_results()

        # import execution class
        executor_class = self.shishito_support.get_module('platform_execution')
        # executor_class = getattr(import_module(platform_path), 'ControlExecution')
        executor = executor_class(self.shishito_support, self.test_timestamp)

        # run test
        exit_code = executor.run_tests()

        # archive results + generate combined report
        self.reporter.archive_results()
        self.reporter.generate_combined_report()

        # upload results to QAStats test management app
        qastats_credentials = self.shishito_support.get_opt('qastats')
        if qastats_credentials:
            try:
                qas_user, qas_password = qastats_credentials.split(':', 1)
            except (AttributeError, ValueError):
                raise ValueError('QAStats credentials were not specified! Unable to connect to QAStats.')

            qastats = QAStats(qas_user, qas_password, self.test_timestamp, self.epoch, self.test_build)
            qastats.post_results()

        # upload results to TestRail test management app
        test_rail_credentials = self.shishito_support.get_opt('test_rail')
        if test_rail_credentials:
            try:
                tr_user, tr_password = test_rail_credentials.split(':', 1)
            except (AttributeError, ValueError):
                raise ValueError('TestRail credentials were not specified! Unable to connect to TestRail.')

            test_rail = TestRail(tr_user, tr_password, self.test_timestamp, self.test_build)
            test_rail.post_results()

        return exit_code
Exemple #9
0
class TestRail(object):
    """ TestRail object """

    def __init__(self, user, password, timestamp, build):
        self.shishito_support = ShishitoSupport()
        self.test_rail_instance = self.shishito_support.get_opt('test_rail_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp

        # project specific config
        self.project_id = self.shishito_support.get_opt('test_rail_project_id')
        self.section_id = self.shishito_support.get_opt('test_rail_section_id')
        self.test_plan_id = self.shishito_support.get_opt('test_rail_test_plan_id')
        self.test_plan_name = self.shishito_support.get_opt('test_rail_test_plan_name') or build
        self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'

    def post_results(self):
        """ Create test-cases on TestRail, adds a new test run and update results for the run """
        self.create_missing_test_cases()

        if self.test_plan_name:
            test_plan_id = self.add_test_plan()
        else:
            test_plan_id = self.test_plan_id

        test_run = self.add_test_run(test_plan_id)
        self.add_test_results(test_run)

    def tr_get(self, url):
        """ GET request for TestRail API

        :param url: url endpoint snippet
        :return: response JSON
        """
        response = requests.get(self.uri_base + url, auth=(self.user, self.password), headers=self.default_headers)
        #print(self.uri_base + url, response, response.text)
        return response.json()

    def tr_post(self, url, payload):
        """ GET request for TestRail API

        :param url: url endpoint snippet
        :param payload: payload for the POST api call
        :return: response object
        """
        return requests.post(self.uri_base + url, auth=(self.user, self.password), data=json.dumps(payload),
                             headers=self.default_headers)

    def get_all_test_plans(self):
        """ Gets list of all test-plans from certain project

        :return: list of test-plans (names = strings)
        """
        test_plans_list = self.tr_get('get_plans/{}'.format(self.project_id))
        return [{'name': test_plan['name'], 'id': test_plan['id']} for test_plan in test_plans_list]

    def get_all_test_cases(self):
        """ Gets list of all test-cases from certain project

        :return: list of test-cases (names = strings)
        """
        test_case_list = self.tr_get('get_cases/{}&section_id={}&suite_id={}'.format(self.project_id, self.section_id, self.suite_id))
        return [{'title': test_case['title'], 'id': test_case['id']} for test_case in test_case_list]

    def create_test_case(self, title):
        """ Creates a new test case in TestRail

        :param title: Title of the test-case
        :return: response object
        """
        return self.tr_post('add_case/{}'.format(self.section_id), {"title": title})

    def create_missing_test_cases(self):
        """ Creates new test-cases on TestRail for those in test project (those run by pytest).
         Does not create test-cases if already existed on TestRail.

        :return: list of test-cases that could not be created on TestRail (post failure)
        """
        post_errors = []
        test_case_names = [item['title'] for item in self.get_all_test_cases()]
        # Iterate over results for each environment combination
        for result_combination in self.shishito_results:
            # Create TestRail entry for every test-case in combination (if missing)
            for item in result_combination['cases']:
                if item['name'] not in test_case_names:
                    response = self.create_test_case(item['name'])
                    if response.status_code != requests.codes.ok:
                        post_errors.append(item['name'])
                    else:
                        test_case_names.append(item['name'])
        return post_errors

    def add_test_plan(self):
        test_plan_id = 0

        # Check if already exists
        for plan in self.get_all_test_plans():
            if plan['name'] == self.test_plan_name:
                return plan['id']

        result = self.tr_post('add_plan/{}'.format(self.project_id), {"name": self.test_plan_name})

        return json.loads(result.text)['id']

    def add_test_run(self, test_plan_id = None):
        """ Adds new test run under certain test plan into TestRail

        :return: dictionary of TestRail run names & IDs
        """
        test_plan_id = test_plan_id or self.test_plan_id
        runs_created = []
        # Iterate over results for each environment combination
        for result_combination in self.shishito_results:
            run_name = '{} ({})'.format(result_combination['name'][:-4], self.timestamp)
            test_run = {"case_ids": [case['id'] for case in self.get_all_test_cases()]}
            result = self.tr_post('add_plan_entry/{}'.format(test_plan_id),
                                  {"suite_id": self.suite_id, "name": run_name, "runs": [test_run]}).json()
            # lookup test run id
            for run in result['runs']:
                if run['name'] == run_name:
                    runs_created.append({'combination': result_combination['name'], 'id': run['id']})
        return runs_created

    def add_test_results(self, test_runs):
        """ Add test results for specific test run based on parsed xUnit results

        :return: list of test run IDs for which results could not be added (post failure)
        """
        post_errors = []
        run_ids = {r['combination']: r['id'] for r in test_runs}
        # Iterate over results for each environment combination
        for result in self.shishito_results:
            run_id = run_ids.get(result['name'])
            if not run_id:
                continue

            test_results = []
            tr_tests = {t['title']: t['id'] for t in self.tr_get('get_tests/{}'.format(run_id))}
            # Create TestRail entry for every test-case in combination (if missing)
            for xunit_test in result['cases']:
                tr_test_id = tr_tests.get(xunit_test['name'])
                result_id = {'success': 1, 'failure': 5}.get(xunit_test['result'])
                if tr_test_id and result_id:
                    # Add result content into the payload list
                    result = {'test_id': tr_test_id, 'status_id': result_id}
                    if result_id == 5:
                        result['comment'] = xunit_test['failure_message']
                    test_results.append(result)

            response = self.tr_post('add_results/{}'.format(run_id), {'results': test_results})
            if response.status_code != requests.codes.ok:
                post_errors.append(run_id)
        return post_errors
Exemple #10
0
class ShishitoRunner(object):
    """ Base shishito test runner.

    - runs python selenium tests on customizable configurations (using PyTest)
    - archive the test results in .zip file """
    def __init__(self, project_root):
        # set project root
        self.project_root = project_root

        # test timestamp - for storing results
        self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
        self.epoch = int(time.time())

        # parse cmd  args
        self.cmd_args = self.handle_cmd_args()

        # Get SUT build for use in reporting
        self.test_build = self.cmd_args['build']

        self.reporter = Reporter(project_root, self.test_timestamp)
        self.shishito_support = ShishitoSupport(cmd_args=self.cmd_args,
                                                project_root=self.project_root)

    def handle_cmd_args(self):
        """ Retrieve command line arguments passed to the script.

        :return: dict with parsed command line arguments
        """

        parser = argparse.ArgumentParser(
            description='Selenium Python test runner execution arguments.')

        parser.add_argument('--platform',
                            help='Platform on which run tests.',
                            dest='test_platform')
        parser.add_argument('--environment',
                            help='Environment for which run tests.',
                            dest='test_environment')
        parser.add_argument('--test_directory',
                            help='Directory where to lookup for tests')
        parser.add_argument('--smoke',
                            help='Run only smoke tests',
                            action='store_true')
        parser.add_argument(
            '--browserstack',
            help='BrowserStack credentials; format: "username:token"')
        parser.add_argument(
            '--saucelabs',
            help='Saucelabs credentials; format: "username:token"')
        parser.add_argument(
            '--test_rail',
            help=
            'TestRail Test Management tool credentials; format: "username:password"'
        )
        parser.add_argument(
            '--qastats',
            help='QAStats Test Management tool credentials; format: "token"')
        parser.add_argument(
            '--node_webkit_chromedriver_path',
            help=
            'Path to chromedriver located in same directory as node-webkit application'
        )
        parser.add_argument('--app', help='Path to appium application')
        parser.add_argument(
            '--test', help='Run specified test (PyTest string expression)')
        parser.add_argument('--build',
                            help='Specify build number for reporting purposes')
        args = parser.parse_args()

        # return args dict --> for use in other classes
        return vars(args)

    def run_tests(self):
        """ Execute tests for given platform and environment. Platform and Environment can be passed as command lines
        argument or settings in config file.
        """

        if __name__ == "__main__":
            sys.exit(
                'The runner cannot be executed directly.'
                ' You need to import it within project specific runner. Session terminated.'
            )

        # cleanup previous results
        self.reporter.cleanup_results()

        # import execution class
        executor_class = self.shishito_support.get_module('platform_execution')
        # executor_class = getattr(import_module(platform_path), 'ControlExecution')
        executor = executor_class(self.shishito_support, self.test_timestamp)

        # run test
        exit_code = executor.run_tests()

        # archive results + generate combined report
        self.reporter.archive_results()
        self.reporter.generate_combined_report()

        # upload results to QAStats test management app
        qastats_credentials = self.shishito_support.get_opt('qastats')
        if qastats_credentials:
            try:
                qas_user, qas_password = qastats_credentials.split(':', 1)
            except (AttributeError, ValueError):
                raise ValueError(
                    'QAStats credentials were not specified! Unable to connect to QAStats.'
                )

            qastats = QAStats(qas_user, qas_password, self.test_timestamp,
                              self.epoch, self.test_build)
            qastats.post_results()

        # upload results to TestRail test management app
        test_rail_credentials = self.shishito_support.get_opt('test_rail')
        if test_rail_credentials:
            try:
                tr_user, tr_password = test_rail_credentials.split(':', 1)
            except (AttributeError, ValueError):
                raise ValueError(
                    'TestRail credentials were not specified! Unable to connect to TestRail.'
                )

            test_rail = TestRail(tr_user, tr_password, self.test_timestamp,
                                 self.test_build)
            test_rail.post_results()

        return exit_code
Exemple #11
0
class TestRail(object):
    """ TestRail object """
    def __init__(self, user, password, timestamp, build):
        self.shishito_support = ShishitoSupport()
        self.test_rail_instance = self.shishito_support.get_opt(
            'test_rail_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp

        # project specific config
        self.project_id = self.shishito_support.get_opt('test_rail_project_id')
        self.section_id = self.shishito_support.get_opt('test_rail_section_id')
        self.test_plan_id = self.shishito_support.get_opt(
            'test_rail_test_plan_id')
        self.test_plan_name = self.shishito_support.get_opt(
            'test_rail_test_plan_name') or build
        self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'

    def post_results(self):
        """ Create test-cases on TestRail, adds a new test run and update results for the run """
        self.create_missing_test_cases()

        if self.test_plan_name:
            test_plan_id = self.add_test_plan()
        else:
            test_plan_id = self.test_plan_id

        test_run = self.add_test_run(test_plan_id)
        self.add_test_results(test_run)

    def tr_get(self, url):
        """ GET request for TestRail API

        :param url: url endpoint snippet
        :return: response JSON
        """
        response = requests.get(self.uri_base + url,
                                auth=(self.user, self.password),
                                headers=self.default_headers)
        #print(self.uri_base + url, response, response.text)
        return response.json()

    def tr_post(self, url, payload):
        """ GET request for TestRail API

        :param url: url endpoint snippet
        :param payload: payload for the POST api call
        :return: response object
        """
        return requests.post(self.uri_base + url,
                             auth=(self.user, self.password),
                             data=json.dumps(payload),
                             headers=self.default_headers)

    def get_all_test_plans(self):
        """ Gets list of all test-plans from certain project

        :return: list of test-plans (names = strings)
        """
        test_plans_list = self.tr_get('get_plans/{}'.format(self.project_id))
        return [{
            'name': test_plan['name'],
            'id': test_plan['id']
        } for test_plan in test_plans_list]

    def get_all_test_cases(self):
        """ Gets list of all test-cases from certain project

        :return: list of test-cases (names = strings)
        """
        test_case_list = self.tr_get(
            'get_cases/{}&section_id={}&suite_id={}'.format(
                self.project_id, self.section_id, self.suite_id))
        return [{
            'title': test_case['title'],
            'id': test_case['id']
        } for test_case in test_case_list]

    def create_test_case(self, title):
        """ Creates a new test case in TestRail

        :param title: Title of the test-case
        :return: response object
        """
        return self.tr_post('add_case/{}'.format(self.section_id),
                            {"title": title})

    def create_missing_test_cases(self):
        """ Creates new test-cases on TestRail for those in test project (those run by pytest).
         Does not create test-cases if already existed on TestRail.

        :return: list of test-cases that could not be created on TestRail (post failure)
        """
        post_errors = []
        test_case_names = [item['title'] for item in self.get_all_test_cases()]
        # Iterate over results for each environment combination
        for result_combination in self.shishito_results:
            # Create TestRail entry for every test-case in combination (if missing)
            for item in result_combination['cases']:
                if item['name'] not in test_case_names:
                    response = self.create_test_case(item['name'])
                    if response.status_code != requests.codes.ok:
                        post_errors.append(item['name'])
                    else:
                        test_case_names.append(item['name'])
        return post_errors

    def add_test_plan(self):
        test_plan_id = 0

        # Check if already exists
        for plan in self.get_all_test_plans():
            if plan['name'] == self.test_plan_name:
                return plan['id']

        result = self.tr_post('add_plan/{}'.format(self.project_id),
                              {"name": self.test_plan_name})

        return json.loads(result.text)['id']

    def add_test_run(self, test_plan_id=None):
        """ Adds new test run under certain test plan into TestRail

        :return: dictionary of TestRail run names & IDs
        """
        test_plan_id = test_plan_id or self.test_plan_id
        runs_created = []
        # Iterate over results for each environment combination
        for result_combination in self.shishito_results:
            run_name = '{} ({})'.format(result_combination['name'][:-4],
                                        self.timestamp)
            test_run = {
                "case_ids": [case['id'] for case in self.get_all_test_cases()]
            }
            result = self.tr_post('add_plan_entry/{}'.format(test_plan_id), {
                "suite_id": self.suite_id,
                "name": run_name,
                "runs": [test_run]
            }).json()
            # lookup test run id
            for run in result['runs']:
                if run['name'] == run_name:
                    runs_created.append({
                        'combination':
                        result_combination['name'],
                        'id':
                        run['id']
                    })
        return runs_created

    def add_test_results(self, test_runs):
        """ Add test results for specific test run based on parsed xUnit results

        :return: list of test run IDs for which results could not be added (post failure)
        """
        post_errors = []
        run_ids = {r['combination']: r['id'] for r in test_runs}
        # Iterate over results for each environment combination
        for result in self.shishito_results:
            run_id = run_ids.get(result['name'])
            if not run_id:
                continue

            test_results = []
            tr_tests = {
                t['title']: t['id']
                for t in self.tr_get('get_tests/{}'.format(run_id))
            }
            # Create TestRail entry for every test-case in combination (if missing)
            for xunit_test in result['cases']:
                tr_test_id = tr_tests.get(xunit_test['name'])
                result_id = {
                    'success': 1,
                    'failure': 5
                }.get(xunit_test['result'])
                if tr_test_id and result_id:
                    # Add result content into the payload list
                    result = {'test_id': tr_test_id, 'status_id': result_id}
                    if result_id == 5:
                        result['comment'] = xunit_test['failure_message']
                    test_results.append(result)

            response = self.tr_post('add_results/{}'.format(run_id),
                                    {'results': test_results})
            if response.status_code != requests.codes.ok:
                post_errors.append(run_id)
        return post_errors
Exemple #12
0
class QAStats(object):
    """ QAStats object """
    def __init__(self, user, password, timestamp, epoch, build):
        self.shishito_support = ShishitoSupport()
        self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp
        self.epoch = epoch
        self.build = build

        # project specific config
        self.project_id = self.shishito_support.get_opt('qastats_project_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.result_url = self.qastats_base_url + '/api/v1/results'
        self.project_url = self.shishito_support.get_opt('base_url')

    def post_results(self):
        """ Create test-cases on QAStats, adds a new test run and update results for the run
            {
               "project_id": 123,
               "timestamp": 1470133472,
               "build": "773",              // optional
               "environment": "Firefox"     // optional
               "branch": "develop",         // optional
               "git": "ae232a",             // optional
               "results": [
                  { "test": "test_login", "result": "pass" },   // [pass fail err nr]
                  ...
               ],
            }
        """

        for (i, run) in enumerate(self.shishito_results):
            environment = run['name']
            m = re.match('^(.*)\.xml$', environment)
            if m != None: environment = m.group(1)

            payload = {
                'project_id': self.project_id,
                'timestamp': self.epoch + i,
                'environment': environment
            }
            if self.build:
                payload['build'] = self.build
            if 'QA_BRANCH_TO_TEST' in os.environ:
                payload['branch'] = os.environ['QA_BRANCH_TO_TEST']
            if 'QA_GIT_COMMIT' in os.environ:
                payload['git'] = os.environ["QA_GIT_COMMIT"]
            elif 'CIRCLE_REPOSITORY_URL' in os.environ:
                print('github url is known')
                payload['git'] = os.environ['CIRCLE_REPOSITORY_URL']
            if 'CIRCLE_TEST_REPORTS' in os.environ:
                print('result url is known')
                payload['reporturl'] = os.environ.get('CIRCLE_TEST_REPORTS')
            if self.project_url is not None:
                payload['testurl'] = self.project_url
            status_map = {
                'error': 'err',
                'failure': 'fail',
                'success': 'pass',
                'skipped': 'nr'
            }
            results = [{
                'test': t['name'],
                'result': status_map[t['result']]
            } for t in run['cases']]
            payload['results'] = results
            print(payload)
            json_payload = json.dumps(payload)
            r = requests.post(self.result_url,
                              auth=(self.user, self.password),
                              data=json_payload,
                              headers=self.default_headers)

            if r.status_code == requests.codes.ok:
                try:
                    resp = r.json()
                    if 'result' in resp and resp['result'] == 'OK':
                        print("Results uploaded to QAStats")
                        return True
                except (ValueError, AttributeError):
                    pass

            print("Error: uploading tests to QAStats\n\n", json_payload, "\n")
            print("\tStatus-code:\t" + str(r.status_code) + "\n")
            for n, v in r.headers.items():
                print("\t" + n + "\t" + v)
            print("")
            print(r.text)
Exemple #13
0
class QAStats(object):
    """ QAStats object """

    def __init__(self, user, password, timestamp, epoch, build):
        self.shishito_support = ShishitoSupport()
        self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
        self.user = user
        self.password = password
        self.timestamp = timestamp
        self.epoch = epoch
        self.build = build

        # project specific config
        self.project_id = self.shishito_support.get_opt('qastats_project_id')

        # shishito results
        self.reporter = Reporter()
        self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)

        self.default_headers = {'Content-Type': 'application/json'}
        self.result_url = self.qastats_base_url + '/api/v1/results'
        self.project_url = self.shishito_support.get_opt('base_url')

    def post_results(self):
        """ Create test-cases on QAStats, adds a new test run and update results for the run
            {
               "project_id": 123,
               "timestamp": 1470133472,
               "build": "773",              // optional
               "environment": "Firefox"     // optional
               "branch": "develop",         // optional
               "git": "ae232a",             // optional
               "results": [
                  { "test": "test_login", "result": "pass" },   // [pass fail err nr]
                  ...
               ],
            }
        """

        for (i, run) in enumerate(self.shishito_results):
            environment = run['name'];
            m =re.match('^(.*)\.xml$', environment)
            if m != None: environment = m.group(1)

            payload = {
                    'project_id': self.project_id,
                    'timestamp': self.epoch + i,
                    'environment': environment
            }
            if self.build:
                payload['build'] = self.build
            if 'QA_BRANCH_TO_TEST' in os.environ:
                payload['branch'] = os.environ['QA_BRANCH_TO_TEST']
            if 'QA_GIT_COMMIT' in os.environ:
                payload['git'] = os.environ["QA_GIT_COMMIT"]
            elif 'CIRCLE_REPOSITORY_URL' in os.environ:
                print('github url is known')
                payload['git'] = os.environ['CIRCLE_REPOSITORY_URL']
            if 'CIRCLE_TEST_REPORTS' in os.environ:
                print('result url is known')
                payload['reporturl'] = os.environ.get('CIRCLE_TEST_REPORTS')
            if self.project_url is not None:
                payload['testurl']=self.project_url
            status_map = {
                'error': 'err',
                'failure': 'fail',
                'success': 'pass',
                'skipped': 'nr'
            }
            results = [ {'test': t['name'], 'result': status_map[t['result']]} for t in run['cases'] ]
            payload['results'] = results
            print(payload)
            json_payload = json.dumps(payload)
            r = requests.post(self.result_url, auth=(self.user, self.password), data=json_payload,
                                 headers=self.default_headers)

            if r.status_code == requests.codes.ok:
                try:
                    resp = r.json()
                    if 'result' in resp and resp['result'] == 'OK':
                        print("Results uploaded to QAStats")
                        return True
                except (ValueError, AttributeError):
                    pass

            print("Error: uploading tests to QAStats\n\n", json_payload, "\n")
            print("\tStatus-code:\t" + str(r.status_code) + "\n")
            for n, v in r.headers.items():
                print("\t" + n + "\t" + v)
            print("")
            print(r.text)