def is_repo_allowed(repo): """ Check if repo is allowed. A repository name is checked against a list of denied and allowed repos. The 'denied_repo' check takes precendence over 'allowed_repo' check. The list of denied/allowed repos is defined with settings 'denied_repo' and 'allowed_repo'. If the settings are not defined, the repo is not checked against the denied/allowed lists. Both 'denied_repo' and 'allowed_repo' can have multiple values, if any of them matches a substring of the repo, the repo is denied/allowed. Parameters: -repo : repository name """ if repo is None: logger.warning("Repo is not defined") return False denied_message = "Project '%s' is not allowed." denied_repo = Settings().get_setting("denied_repo") allowed_repo = Settings().get_setting("allowed_repo") if denied_repo is not None and \ any(x in repo for x in denied_repo) or \ allowed_repo is not None and \ not any(x in repo for x in allowed_repo): logger.warning(denied_message, repo) return False return True
def analyse(argv): ''' Analyse timestamp file ''' settings = Settings() # load Travis environment variables and save them in settings load_travis_env_vars() # process command line arguments process_argv(argv) # read build data from timestamp CSV file build = Build(TIMESTAMP_FILE) # load build properties from settings build.load_properties_from_settings() # retrieve data from Travis CI API if build.get_property("ci_platform") == "travis": travis_data = TravisData( build.get_property("repo"), build.get_property("build"), ) travis_data.get_build_data() build.set_started_at(travis_data.get_started_at()) # log data if settings.get_setting("mode_native") is True: log_build_native(build) if settings.get_setting("mode_keen") is True: log_build_keen(build)
class Root(object): """Root handler.""" def __init__(self): """ Initialise class. Load config file and set loglevel, define error page handler """ self.settings = Settings() self.settings.load_settings(config_file=constants.CONFIG_FILE) self.settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION) self.file_index = os.path.join(STATIC_DIR, "index.html") cherrypy.config.update({'error_page.404': self.error_page_404}) # get logger self.logger = logger @cherrypy.expose def index(self): """Index page.""" if check_file(self.file_index): return open(self.file_index) else: raise cherrypy.HTTPError(404, "File not found") def error_page_404(self, status, message, traceback, version): """Error Page (404).""" self.logger.error( "Cherrypy %s : Error loading page (%s) : %s\nTraceback : %s", version, status, message, traceback) return "This page doesn't exist, please check usage on " \ "the {} website.".format(SERVICE_WEBSITE_LINK)
def send_build_data(buildjob, detail=None): """ Send build data generated by client to keen.io. Parameters: - buildjob : BuildJob instance - detail : Data storage detail level : 'minimal', 'basic', 'full', 'extended' """ if not isinstance(buildjob, BuildJob): raise TypeError("param buildjob should be a BuildJob instance") data_detail = Settings().get_value_or_setting("data_detail", detail) if is_writable(): logger.info( "Sending client build job data to Keen.io (data detail: %s)", data_detail ) # store build job data add_event("build_jobs", {"job": buildjob.to_dict()}) # store build stages if data_detail in ("full", "extended"): add_events("build_stages", buildjob.stages_to_list())
def add_project_info_dict(payload): """ Add project info to a dictonary. Param payload: dictonary payload """ # check if payload is a dictionary, throws an exception if it isn't check_dict(payload, "payload") payload_as_dict = copy.deepcopy(payload) payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info() if "job" in payload: # override project_name, set to build_job repo if "repo" in payload["job"]: payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \ payload["job"]["repo"] # override timestamp, set to finished_at timestamp if "finished_at" in payload["job"]: payload_as_dict["keen"] = { "timestamp": payload["job"]["finished_at"]["isotimestamp"] } return payload_as_dict
def is_worker_enabled(): """Check if a task queue is configured.""" task_queue = Settings().get_setting("task_queue") # use double not to force boolean evaluation return check_dict(task_queue, key_list=["broker_url", "backend"]) and \ not not task_queue["broker_url"] and \ not not task_queue["backend"]
def generate_trend(argv): """Generate trends from analysed buildtime data.""" settings = Settings() # load settings from config file, env_var and cli parameters if settings.load_settings(argv) is None: return # load Travis CI environment variables env_var.load_all(settings) # run trend_keen() always, # if $KEEN_PROJECT_ID variable is set (checked later), it will be executed if settings.get_setting("mode_native") is True: trend_native() if settings.get_setting("mode_keen") is True: trend_keen()
def __init__(self): """ Initialise class. Load config file and set loglevel, define error page handler """ self.settings = Settings() self.settings.load_settings(config_file=constants.CONFIG_FILE) self.settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION) self.file_index = os.path.join(STATIC_DIR, "index.html") cherrypy.config.update({'error_page.404': self.error_page_404}) # get logger self.logger = logger
def __init__(self): """Constructor.""" self.settings = Settings() self.logger = logger self.file_stats = os.path.join(DASHBOARD_DIR, "stats.html") self.file_stats_service = os.path.join(DASHBOARD_DIR, "stats_service.html")
def create_worker_app(): """Create worker app.""" # load settings settings = Settings() settings.load_settings(config_file=constants.CONFIG_FILE) settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION) if is_worker_enabled(): task_queue = settings.get_setting("task_queue") worker_app = Celery( 'tasks', backend=task_queue["backend"], broker=task_queue["broker_url"] ) # configure worker worker_app.conf.update( CELERY_TASK_SERIALIZER='json', CELERY_ACCEPT_CONTENT=['json'] ) if worker_app is None: logger.error("Error connection to task queue") else: logger.info( "Connected to task queue : %s", task_queue["broker_url"] ) else: worker_app = Celery() logger.warning( "Task queue is not defined," " check README.md to configure task queue" ) return worker_app
def generate_trend(argv): ''' Generate trends from analised buildtime data ''' settings = Settings() # load Travis environment variables and save them in settings load_travis_env_vars() # process command line arguments process_argv(argv) # run trend_keen() always, # if $KEEN_PROJECT_ID variable is set (checked later), it will be executed if settings.get_setting("mode_native") is True: trend_native() if settings.get_setting("mode_keen") is True: trend_keen()
def get_read_key(argv): """Generate a read key for the project and print that key.""" settings = Settings() # load settings from config file, env_var and cli parameters args = settings.load_settings(argv) # exit script if processing cli parameters failed (result = None) if args is None: return # get project name from argument # check if collection is empty if args: settings.set_project_name(args[0]) # generate a read key print(keenio.generate_read_key(settings.get_project_name()))
def setUpClass(self): self.settings = Settings() self.project_name = buildtimetrend.NAME self.project_info = { "version": buildtimetrend.VERSION, "schema_version": buildtimetrend.SCHEMA_VERSION, "project_name": self.project_name}
def test_generate_config_file_ioerror(self): """ Test dashboard.generate_config_file() if creation fails because of unexisting path. """ # set config file path Settings().add_setting("dashboard_configfile", "build/unexisting_path/config_test.js") # generation should return false self.assertFalse(dashboard.generate_config_file("test/repo4"))
def load_properties_from_settings(self): """Load build properties from settings.""" self.load_property_from_settings("build") self.load_property_from_settings("job") self.load_property_from_settings("branch") self.load_property_from_settings("ci_platform") self.load_property_from_settings("build_trigger") self.load_property_from_settings("pull_request") self.load_property_from_settings("result") self.load_property_from_settings("build_matrix") self.add_property("repo", Settings().get_project_name())
def get_repo_data_detail(repo): """ Get level of data detail storage of a repo. A repository name is checked against a list of repository names. If a match is found, the corresponding data detail level is used. Else, the default global setting is returned. Parameters: -repo : repository name """ settings = Settings() if repo is None: logger.warning("Repo is not defined") else: repo_settings = settings.get_setting("repo_data_detail") for repo_substring, setting in repo_settings.items(): if repo_substring in repo: return setting # return default global data_detail setting return settings.get_setting("data_detail")
def multi_build(self, repo, first_build, last_build): """ Schedule processing multiple consecutive builds. All builds from first_build until last_build will be retrieved and processed. The total number of builds to be scheduled is limited by the `multi_import.max_builds` config parameter. Every next scheduled build will be delayed by the `multi_import.delay` config parameter. Parameters: - repo : repo name (fe. buildtimetrend/service) - first_build : first build number to process (int) - last_build : last build number to process (int) """ first_build = int(first_build) last_build = int(last_build) message = "" multi_import = Settings().get_setting("multi_import") max_multi_builds = int(multi_import["max_builds"]) if last_build < first_build: tmp_msg = "Warning : last_build should be equal" \ " or larger than first_build" self.logger.warning(tmp_msg) message += tmp_msg + "\n" last_build = first_build if (last_build - first_build) > max_multi_builds: tmp_msg = "Warning : number of multiple builds is limited to {:d}" self.logger.warning(tmp_msg.format(max_multi_builds)) message += tmp_msg.format(max_multi_builds) + "\n" last_build = first_build + max_multi_builds message += "Request to process build(s) #{first_build:d} to" \ " #{last_build:d} of repo {repo}:\n".format(**locals()) build = first_build delay = 0 while build <= last_build: message += self.schedule_task(repo, build, delay) + "\n" delay += multi_import["delay"] build += 1 return message
def load_travis_env_vars(): ''' Loads Travis CI environment variables and assigns them to the corresponding settings item. ''' if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true": settings = Settings() # set ci_platform setting to "travis" settings.add_setting("ci_platform", "travis") # set settings with TRAVIS values env_var_to_settings("TRAVIS_BUILD_NUMBER", "build") env_var_to_settings("TRAVIS_JOB_NUMBER", "job") env_var_to_settings("TRAVIS_BRANCH", "branch") env_var_to_settings("TRAVIS_REPO_SLUG", "project_name") # convert and set Travis build result if "TRAVIS_TEST_RESULT" in os.environ: # map $TRAVIS_TEST_RESULT to a more readable value settings.add_setting( "result", convert_build_result(os.environ["TRAVIS_TEST_RESULT"]) )
def load_property_from_settings(self, property_name, setting_name=None): """ Load the value of a setting and set it as a build property. Parameters - property_name : name of the build property - setting_name : name of the setting (takes property_name if not set) """ if setting_name is None: setting_name = property_name value = Settings().get_setting(setting_name) if value is not None: self.add_property(property_name, value)
def setUpClass(cls): """Set up test fixture.""" # show full diff in case of assert mismatch cls.maxDiff = None cls.settings = Settings() cls.project_name = buildtimetrend.NAME cls.project_info = { "lib_version": buildtimetrend.VERSION, "schema_version": buildtimetrend.SCHEMA_VERSION, "client": 'None', "client_version": 'None', "project_name": cls.project_name }
def load_all(settings=None): """ Load all Travis CI environment variables. Load Travis CI environment variables and assign their values to the corresponding setting value : - general - build matrix - pull request """ if not isinstance(settings, Settings): settings = Settings() load_general_env_vars(settings) load_build_matrix_env_vars(settings) load_travis_pr_env_vars(settings)
def setUpClass(cls): """Set up test fixture.""" cls.settings = Settings() cls.project_info = cls.settings.get_project_info() cls.maxDiff = None cls.test_api_error = {"message": "test message", "error_code": "123"} # copy Keen.io environment variables if "KEEN_PROJECT_ID" in os.environ: cls.copy_keen_project_id = os.environ["KEEN_PROJECT_ID"] if "KEEN_WRITE_KEY" in os.environ: cls.copy_keen_write_key = os.environ["KEEN_WRITE_KEY"] if "KEEN_READ_KEY" in os.environ: cls.copy_keen_read_key = os.environ["KEEN_READ_KEY"] if "KEEN_MASTER_KEY" in os.environ: cls.copy_keen_master_key = os.environ["KEEN_MASTER_KEY"]
def get_read_key(): ''' Generate a read key for the project and print that key ''' settings = Settings() settings.load_config_file("config.yml") # get project name from argument if len(sys.argv) > 1: settings.set_project_name(sys.argv[1]) # generate a read key print keen_io_generate_read_key(settings.get_project_name())
def check_authorization(repo, auth_header): """ Check if Travis CI notification has a correct Authorization header. This check is enabled if travis_account_token is defined in settings. More information on the Authorization header : http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks Returns true if Authorization header is valid, but also if travis_account_token is not defined. Parameters: - repo : git repo name - auth_header : Travis CI notification Authorization header """ # get Travis account token from Settings token = Settings().get_setting("travis_account_token") # return True if token is not set if token is None: logger.info("Setting travis_account_token is not defined," " Travis CI notification Authorization header" " is not checked.") return True # check if parameters are strings if is_string(repo) and is_string(auth_header) and is_string(token): # generate hash (encode string to bytes first) auth_hash = sha256((repo + token).encode('utf-8')).hexdigest() # compare hash with Authorization header if auth_hash == auth_header: logger.info("Travis CI notification Authorization header" " is correct.") return True else: logger.error("Travis CI notification Authorization header" " is incorrect.") return False else: logger.debug("repo, auth_header and travis_auth_token" " should be strings.") return False
def process_notification_payload(payload): """ Extract repo slug and build number from Travis notification payload. Returns a dictionary with "repo" and "build" information, or an empty dictionary if the payload could not be processed. Deprecated behaviour : Currently the repo and build information are also stored in the "settings" object, but this will be removed in the near future. Parameters: - payload : Travis CI notification payload """ settings = Settings() parameters = {} if payload is None: logger.warning("Travis notification payload is not set") return parameters if not is_string(payload): logger.warning( "Travis notification payload is incorrect :" " string expected, got %s", type(payload)) return parameters json_payload = json.loads(payload) logger.info("Travis Payload : %r.", json_payload) # get repo name from payload if ("repository" in json_payload and "owner_name" in json_payload["repository"] and "name" in json_payload["repository"]): repo = get_repo_slug(json_payload["repository"]["owner_name"], json_payload["repository"]["name"]) logger.info("Build repo : %s", repo) settings.set_project_name(repo) parameters["repo"] = repo # get build number from payload if "number" in json_payload: logger.info("Build number : %s", str(json_payload["number"])) settings.add_setting('build', json_payload['number']) parameters["build"] = json_payload['number'] return parameters
def retrieve_and_store_data(argv): """ Load timing and build data, process and store it. Retrieve timing and build data from Travis CI log, parse it and store the result in Keen.io. Parameters: - argv : command line parameters """ settings = Settings() settings.set_client(CLIENT_NAME, CLIENT_VERSION) # load settings from config file, env_var and cli parameters if settings.load_settings(argv, "config_service.yml") is None: return build = settings.get_setting('build') if build is None: print("Build number is not set, use --build=build_id") return travis_data = TravisData(settings.get_project_name(), build) # retrieve build data using Travis CI API print( "Retrieve build #{:s} data of {:s} from Travis CI".format( build, settings.get_project_name() ) ) travis_data.get_build_data() # process all build jobs travis_data.process_build_jobs() if not keenio.is_writable(): print("Keen IO write key not set, no data was sent") return # send build job data to Keen.io for build_job in travis_data.build_jobs: print("Send build job #{:s} data to Keen.io".format(build_job)) keenio.send_build_data_service(travis_data.build_jobs[build_job])
def test_generate_config_file(self, get_cfg_str_func): """Test dashboard.generate_config_file()""" # set config file path Settings().add_setting("dashboard_configfile", constants.DASHBOARD_TEST_CONFIG_FILE) # check if configfile exists self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE)) # generate config file with empty repo name self.assertRaises(TypeError, dashboard.generate_config_file) # generate config file with empty repo name self.assertTrue(dashboard.generate_config_file(None)) self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE)) # check if mock was called with correct parameters args, kwargs = get_cfg_str_func.call_args self.assertEqual(args, (None, )) self.assertDictEqual(kwargs, {}) # generate config file self.assertTrue(dashboard.generate_config_file("test/repo3")) self.assertTrue(check_file(constants.DASHBOARD_TEST_CONFIG_FILE)) # check if mock was called with correct parameters args, kwargs = get_cfg_str_func.call_args self.assertEqual(args, ("test/repo3", )) self.assertDictEqual(kwargs, {}) # test generated config file contents with open(constants.DASHBOARD_TEST_CONFIG_FILE, 'r') as config_file: self.assertEqual("var config = {'projectName': 'test/repo3'};\n", next(config_file)) self.assertEqual("var keenConfig = {'projectId': '1234abcd'};", next(config_file))
def test_generate_config_file_fails(self): """Test dashboard.generate_config_file() if creation fails""" # set config file path Settings().add_setting("dashboard_configfile", constants.DASHBOARD_TEST_CONFIG_FILE) # check if configfile exists self.assertFalse(check_file(constants.DASHBOARD_TEST_CONFIG_FILE)) # init mock patcher = mock.patch('buildtimetrend.tools.check_file', return_value=False) check_file_func = patcher.start() # generation should return false self.assertFalse(dashboard.generate_config_file("test/repo4")) # check if mock was called with correct parameters args, kwargs = check_file_func.call_args self.assertEqual(args, (constants.DASHBOARD_TEST_CONFIG_FILE, )) self.assertDictEqual(kwargs, {}) patcher.stop()
def setUpClass(cls): """Set up test fixture.""" cls.project_info = Settings().get_project_info() cls.maxDiff = None
def test_load_properties(self): """Test loading properties""" self.build.load_properties_from_settings() self.assertDictEqual( {'duration': 0, "repo": buildtimetrend.NAME}, self.build.get_properties()) settings = Settings() settings.add_setting("ci_platform", "travis") settings.add_setting("build", "123") settings.add_setting("job", "123.1") settings.add_setting("branch", "branch1") settings.add_setting("result", "passed") settings.add_setting("build_trigger", "push") settings.add_setting( "pull_request", { "is_pull_request": False, "title": None, "number": None } ) settings.set_project_name("test/project") self.build.load_properties_from_settings() self.assertDictEqual( { 'duration': 0, 'ci_platform': "travis", 'build': "123", 'job': "123.1", 'branch': "branch1", 'result': "passed", 'build_trigger': "push", 'pull_request': { "is_pull_request": False, "title": None, "number": None}, 'repo': "test/project" }, self.build.get_properties())
def test_load_properties(self): """Test loading properties""" self.build.load_properties_from_settings() self.assertDictEqual({ 'duration': 0, "repo": buildtimetrend.NAME }, self.build.get_properties()) settings = Settings() settings.add_setting("ci_platform", "travis") settings.add_setting("build", "123") settings.add_setting("job", "123.1") settings.add_setting("branch", "branch1") settings.add_setting("result", "passed") settings.add_setting("build_trigger", "push") settings.add_setting("pull_request", { "is_pull_request": False, "title": None, "number": None }) settings.set_project_name("test/project") self.build.load_properties_from_settings() self.assertDictEqual( { 'duration': 0, 'ci_platform': "travis", 'build': "123", 'job': "123.1", 'branch': "branch1", 'result': "passed", 'build_trigger': "push", 'pull_request': { "is_pull_request": False, "title": None, "number": None }, 'repo': "test/project" }, self.build.get_properties())
def setUp(self): """Initialise test environment before each test.""" self.build = BuildJob() # reinitialise settings Settings().__init__()
def test_load_build_matrix_env_vars(self): """Test load_build_matrix_env_vars""" settings = Settings() self.assertEqual(None, settings.get_setting("build_matrix")) # setup Travis env vars if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true": reset_travis_vars = False expected_os = os.environ["TRAVIS_OS_NAME"] copy_python = os.environ["TRAVIS_PYTHON_VERSION"] del os.environ["TRAVIS_PYTHON_VERSION"] else: reset_travis_vars = True os.environ["TRAVIS"] = "true" expected_os = os.environ["TRAVIS_OS_NAME"] = "test_os" # temporarily remove PYTHON VERSION if "TRAVIS_PYTHON_VERSION" in os.environ: reset_python_version = True copy_python = os.environ["TRAVIS_PYTHON_VERSION"] del os.environ["TRAVIS_PYTHON_VERSION"] else: reset_python_version = False # test language and language versions test_languages = [{ 'env_var': 'TRAVIS_DART_VERSION', 'language': 'dart', 'test_value': "1.1" }, { 'env_var': 'TRAVIS_GO_VERSION', 'language': 'go', 'test_value': "1.2" }, { 'env_var': 'TRAVIS_HAXE_VERSION', 'language': 'haxe', 'test_value': "1.3" }, { 'env_var': 'TRAVIS_JDK_VERSION', 'language': 'java', 'test_value': "1.4" }, { 'env_var': 'TRAVIS_JULIA_VERSION', 'language': 'julia', 'test_value': "1.5" }, { 'env_var': 'TRAVIS_NODE_VERSION', 'language': 'javascript', 'test_value': "1.6" }, { 'env_var': 'TRAVIS_OTP_RELEASE', 'language': 'erlang', 'test_value': "1.7" }, { 'env_var': 'TRAVIS_PERL_VERSION', 'language': 'perl', 'test_value': "1.8" }, { 'env_var': 'TRAVIS_PHP_VERSION', 'language': 'php', 'test_value': "1.9" }, { 'env_var': 'TRAVIS_PYTHON_VERSION', 'language': 'python', 'test_value': "1.10" }, { 'env_var': 'TRAVIS_R_VERSION', 'language': 'r', 'test_value': "1.11" }, { 'env_var': 'TRAVIS_RUBY_VERSION', 'language': 'ruby', 'test_value': "1.12" }, { 'env_var': 'TRAVIS_RUST_VERSION', 'language': 'rust', 'test_value': "1.13" }, { 'env_var': 'TRAVIS_SCALA_VERSION', 'language': 'scala', 'test_value': "1.14" }] # test languages for language in test_languages: if language['env_var'] in os.environ: reset_travis_lang_version = False expected_lang_version = os.environ[language['env_var']] else: reset_travis_lang_version = True expected_lang_version = \ os.environ[language['env_var']] = language['test_value'] env_var.load_build_matrix_env_vars(settings) self.assertDictEqual( { 'os': expected_os, 'language': language['language'], 'language_version': expected_lang_version, 'summary': "{0!s} {1!s} {2!s}".format(language['language'], expected_lang_version, expected_os) }, settings.get_setting("build_matrix")) # reset Travis test result if reset_travis_lang_version: del os.environ[language['env_var']] # reset test Travis vars if reset_travis_vars: del os.environ["TRAVIS"] del os.environ["TRAVIS_OS_NAME"] # reset removed python version if reset_python_version: os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
def setUpClass(cls): """Set up test fixture.""" cls.settings = Settings()
def test_load_build_matrix_env_vars_parameters(self): """Test load_travis_env_vars, optional parameters""" # setup Travis env vars if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true": reset_travis_vars = False copy_os = os.environ["TRAVIS_OS_NAME"] else: reset_travis_vars = True os.environ["TRAVIS"] = "true" # temporarily remove OS VERSION if "TRAVIS_OS_NAME" in os.environ: reset_os = True copy_os = os.environ["TRAVIS_OS_NAME"] del os.environ["TRAVIS_OS_NAME"] else: reset_os = False # temporarily remove PYTHON VERSION if "TRAVIS_PYTHON_VERSION" in os.environ: reset_python_version = True copy_python = os.environ["TRAVIS_PYTHON_VERSION"] del os.environ["TRAVIS_PYTHON_VERSION"] else: reset_python_version = False # test optional build matrix parameters test_parameters = [{ 'env_var': 'TRAVIS_XCODE_SDK', 'parameter': 'xcode_sdk', 'test_value': "test_x_sdk" }, { 'env_var': 'TRAVIS_XCODE_SCHEME', 'parameter': 'xcode_scheme', 'test_value': "test_x_scheme" }, { 'env_var': 'TRAVIS_XCODE_PROJECT', 'parameter': 'xcode_project', 'test_value': "test_x_project" }, { 'env_var': 'TRAVIS_XCODE_WORKSPACE', 'parameter': 'xcode_workspace', 'test_value': "test_x_workspace" }, { 'env_var': 'CC', 'parameter': 'compiler', 'test_value': "test_gcc" }, { 'env_var': 'ENV', 'parameter': 'parameters', 'test_value': "test_env" }] # test parameters for parameter in test_parameters: Settings().__init__() settings = Settings() if parameter['env_var'] in os.environ: reset_travis_parameter = False expected_param_value = os.environ[parameter['env_var']] else: reset_travis_parameter = True expected_param_value = os.environ[parameter['env_var']] = \ parameter['test_value'] env_var.load_build_matrix_env_vars(settings) self.assertDictEqual( { parameter["parameter"]: expected_param_value, 'summary': expected_param_value }, settings.get_setting("build_matrix")) # reset Travis parameters if reset_travis_parameter: del os.environ[parameter['env_var']] # reset test Travis vars if reset_travis_vars: del os.environ["TRAVIS"] # reset removed os name if reset_os: os.environ["TRAVIS_OS_NAME"] = copy_os # reset removed python version if reset_python_version: os.environ["TRAVIS_PYTHON_VERSION"] = copy_python
def setUp(): """Initialise test environment before each test.""" # reinit settings singleton Settings().__init__()
def test_load_travis_env_vars(self): """Test load_travis_env_vars""" settings = Settings() self.assertEqual(None, settings.get_setting("ci_platform")) self.assertEqual(None, settings.get_setting("build")) self.assertEqual(None, settings.get_setting("job")) self.assertEqual(None, settings.get_setting("branch")) self.assertEqual(None, settings.get_setting("result")) self.assertEqual(None, settings.get_setting("build_trigger")) self.assertEqual(None, settings.get_setting("pull_request")) self.assertEqual(buildtimetrend.NAME, settings.get_project_name()) # setup Travis env vars if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true": reset_travis_vars = False expected_build = os.environ["TRAVIS_BUILD_NUMBER"] expected_job = os.environ["TRAVIS_JOB_NUMBER"] expected_branch = os.environ["TRAVIS_BRANCH"] expected_project_name = os.environ["TRAVIS_REPO_SLUG"] copy_pull_request = os.environ["TRAVIS_PULL_REQUEST"] else: reset_travis_vars = True os.environ["TRAVIS"] = "true" expected_build = os.environ["TRAVIS_BUILD_NUMBER"] = "123" expected_job = os.environ["TRAVIS_JOB_NUMBER"] = "123.1" expected_branch = os.environ["TRAVIS_BRANCH"] = "branch1" expected_project_name = \ os.environ["TRAVIS_REPO_SLUG"] = "test/project" # setup Travis test result if "TRAVIS_TEST_RESULT" in os.environ: reset_travis_result = False copy_result = os.environ["TRAVIS_TEST_RESULT"] else: reset_travis_result = True os.environ["TRAVIS_TEST_RESULT"] = "0" os.environ["TRAVIS_PULL_REQUEST"] = "false" env_var.load_all(settings) self.assertEqual("travis", settings.get_setting("ci_platform")) self.assertEqual(expected_build, settings.get_setting("build")) self.assertEqual(expected_job, settings.get_setting("job")) self.assertEqual(expected_branch, settings.get_setting("branch")) self.assertEqual(expected_project_name, settings.get_project_name()) self.assertEqual("passed", settings.get_setting("result")) self.assertEqual("push", settings.get_setting("build_trigger")) self.assertDictEqual( { 'is_pull_request': False, 'title': None, 'number': None }, settings.get_setting("pull_request")) os.environ["TRAVIS_TEST_RESULT"] = "1" # build is a pull request expected_pull_request = os.environ["TRAVIS_PULL_REQUEST"] = "123" env_var.load_all() self.assertEqual("failed", settings.get_setting("result")) self.assertEqual("pull_request", settings.get_setting("build_trigger")) self.assertDictEqual( { 'is_pull_request': True, 'title': "unknown", 'number': expected_pull_request }, settings.get_setting("pull_request")) # build is not a pull request os.environ["TRAVIS_PULL_REQUEST"] = "false" env_var.load_all(settings) self.assertEqual("push", settings.get_setting("build_trigger")) self.assertDictEqual( { 'is_pull_request': False, 'title': None, 'number': None }, settings.get_setting("pull_request")) # reset test Travis vars if reset_travis_vars: del os.environ["TRAVIS"] del os.environ["TRAVIS_BUILD_NUMBER"] del os.environ["TRAVIS_JOB_NUMBER"] del os.environ["TRAVIS_BRANCH"] del os.environ["TRAVIS_REPO_SLUG"] del os.environ["TRAVIS_PULL_REQUEST"] else: os.environ["TRAVIS_PULL_REQUEST"] = copy_pull_request # reset Travis test result if reset_travis_result: del os.environ["TRAVIS_TEST_RESULT"] else: os.environ["TRAVIS_TEST_RESULT"] = copy_result
def __init__(self): """Initialise class.""" self.settings = Settings() # get logger self.logger = logger
class TravisParser(object): """ Travis CI build timing and build data parser. Retrieve timing data from Travis CI, parse it and store it in Keen.io. """ def __init__(self): """Initialise class.""" self.settings = Settings() # get logger self.logger = logger @cherrypy.expose def default(self, repo_owner=None, repo_name=None, first_build=None, last_build=None, payload=None): """ Default handler. Visiting this page triggers loading and processing the build log and data of a travis CI build process. If last_build is defined, all builds from first_build until last_build will be retrieved and processed. If only payload is defined, repo and build data will be extracted from the payload. Parameters: - repo_owner : name of the Github repo owner, fe. `buildtimetrend` - repo_name : name of the Github repo, fe. `service` - first_build : first build number to process (int) - last_build : last build number to process (int) - payload : Travis CI notification payload (json) """ cherrypy.response.headers['Content-Type'] = 'text/plain' # reset settings self.settings.set_project_name(None) self.settings.add_setting('build', None) self.logger.debug("Check Travis headers : %r", cherrypy.request.headers) repo = get_repo_slug(repo_owner, repo_name) # load parameters from the Travis notification payload if self.check_travis_notification(): payload_params = process_notification_payload(payload) # assign payload parameters if repo is None and "repo" in payload_params: repo = payload_params["repo"] if first_build is None and last_build is None and \ "build" in payload_params: first_build = payload_params["build"] # check parameter validity, check returns error message # or None if parameters are valid params_valid = validate_travis_request(repo, first_build) if params_valid is not None: self.logger.warning(params_valid) return params_valid if last_build is None: self.logger.warning("Request to process build #%s of repo %s", first_build, repo) # schedule task with 10 second delay to give Travis CI time # to add the finished_at property. (issue #96) return self.schedule_task(repo, first_build, 10) return self.multi_build(repo, first_build, last_build) def multi_build(self, repo, first_build, last_build): """ Schedule processing multiple consecutive builds. All builds from first_build until last_build will be retrieved and processed. The total number of builds to be scheduled is limited by the `multi_import.max_builds` config parameter. Every next scheduled build will be delayed by the `multi_import.delay` config parameter. Parameters: - repo : repo name (fe. buildtimetrend/service) - first_build : first build number to process (int) - last_build : last build number to process (int) """ first_build = int(first_build) last_build = int(last_build) message = "" multi_import = Settings().get_setting("multi_import") max_multi_builds = int(multi_import["max_builds"]) if last_build < first_build: tmp_msg = "Warning : last_build should be equal" \ " or larger than first_build" self.logger.warning(tmp_msg) message += tmp_msg + "\n" last_build = first_build if (last_build - first_build) > max_multi_builds: tmp_msg = "Warning : number of multiple builds is limited to {:d}" self.logger.warning(tmp_msg.format(max_multi_builds)) message += tmp_msg.format(max_multi_builds) + "\n" last_build = first_build + max_multi_builds message += "Request to process build(s) #{first_build:d} to" \ " #{last_build:d} of repo {repo}:\n".format(**locals()) build = first_build delay = 0 while build <= last_build: message += self.schedule_task(repo, build, delay) + "\n" delay += multi_import["delay"] build += 1 return message def schedule_task(self, repo, build, delay=0): """ Schedule task. Parameters: - repo : repo name (fe. buildtimetrend/service) - build : build number to process (int) - delay : delay before task should be started, in seconds """ # process travis build if is_worker_enabled(): task = tasks.process_travis_buildlog.apply_async( (repo, build), countdown=int(delay)) temp_msg = "Task scheduled to process build #{build}" \ " of repo {repo} : {task_id}" self.logger.warning( temp_msg.format(build=build, repo=repo, task_id=task.id)) return temp_msg.format(build=cgi.escape(str(build)), repo=cgi.escape(str(repo)), task_id=cgi.escape(str(task.id))) else: return tasks.process_travis_buildlog(repo, build) def check_travis_notification(self): """ Check Travis CI notification request. Load Authorization and Travis-Repo-Slug headers and check if the Authorization header is correct """ if "Authorization" not in cherrypy.request.headers: self.logger.debug("Authorization header is not set") return True if "Travis-Repo-Slug" not in cherrypy.request.headers: self.logger.debug("Travis-Repo-Slug header is not set") return False return check_authorization( cherrypy.request.headers["Travis-Repo-Slug"], cherrypy.request.headers["Authorization"])
class TestSettings(unittest.TestCase): @classmethod def setUpClass(self): self.settings = Settings() self.project_name = buildtimetrend.NAME self.project_info = { "version": buildtimetrend.VERSION, "schema_version": buildtimetrend.SCHEMA_VERSION, "project_name": self.project_name} def setUp(self): # reinit settings singleton if self.settings is not None: self.settings.__init__() def test_get_project_info(self): self.assertDictEqual(self.project_info, self.settings.get_project_info()) def test_get_set_project_name(self): self.assertEquals(self.project_name, self.settings.get_project_name()) self.settings.set_project_name("test_name") self.assertEquals("test_name", self.settings.get_project_name()) self.settings.set_project_name(None) self.assertEquals(None, self.settings.get_project_name()) self.settings.set_project_name("") self.assertEquals("", self.settings.get_project_name()) def test_get_add_setting(self): # setting is not set yet self.assertEquals(None, self.settings.get_setting("test_name")) self.settings.add_setting("test_name", "test_value") self.assertEquals("test_value", self.settings.get_setting("test_name")) self.settings.add_setting("test_name", None) self.assertEquals(None, self.settings.get_setting("test_name")) self.settings.add_setting("test_name", "") self.assertEquals("", self.settings.get_setting("test_name")) self.settings.add_setting("test_name", 6) self.assertEquals(6, self.settings.get_setting("test_name")) def test_get_setting(self): self.assertEquals(None, self.settings.get_setting("test_name")) self.assertEquals( self.project_name, self.settings.get_setting("project_name")) self.assertDictEqual( { "project_name": self.project_name, "mode_native": False, "mode_keen": True }, self.settings.settings.get_items()) def test_no_config_file(self): # function should return false when file doesn't exist self.assertFalse(self.settings.load_config_file('no_file.yml')) self.assertDictEqual( { "project_name": self.project_name, "mode_native": False, "mode_keen": True }, self.settings.settings.get_items()) self.assertFalse(self.settings.load_config_file('')) self.assertDictEqual( { "project_name": self.project_name, "mode_native": False, "mode_keen": True }, self.settings.settings.get_items()) # function should throw an error when no filename is set self.assertRaises(TypeError, self.settings.load_config_file) def test_load_config_file(self): # checking if Keen.io configuration is not set (yet) self.assertEquals(None, keen.project_id) self.assertEquals(None, keen.write_key) self.assertEquals(None, keen.read_key) # load sample config file self.assertTrue(self.settings.load_config_file(constants.TEST_SAMPLE_CONFIG_FILE)) self.assertDictEqual( {"project_name": "test_project", "mode_native": True, "mode_keen": False, "setting1": "test_value1"}, self.settings.settings.get_items()) # checking if Keen.io configuration is set self.assertEquals("1234", keen.project_id) self.assertEquals("12345678", keen.write_key) self.assertEquals("abcdefg", keen.read_key) self.assertTrue(keen_is_readable()) self.assertTrue(keen_is_writable())
def test_load_properties(self): self.build.load_properties_from_settings() self.assertDictEqual( {'duration': 0, "repo": buildtimetrend.NAME}, self.build.get_properties()) settings = Settings() settings.add_setting("ci_platform", "travis") settings.add_setting("build", "123") settings.add_setting("job", "123.1") settings.add_setting("branch", "branch1") settings.add_setting("result", "passed") settings.set_project_name("test/project") self.build.load_properties_from_settings() self.assertDictEqual( { 'duration': 0, 'ci_platform': "travis", 'build': "123", 'job': "123.1", 'branch': "branch1", 'result': "passed", 'repo': "test/project"}, self.build.get_properties())