def is_repo_allowed(repo):
"""
Check if repo is allowed.
A repository name is checked against a list of denied and allowed repos.
The 'denied_repo' check takes precendence over 'allowed_repo' check.
The list of denied/allowed repos is defined with settings 'denied_repo'
and 'allowed_repo'.
If the settings are not defined,
the repo is not checked against the denied/allowed lists.
Both 'denied_repo' and 'allowed_repo' can have multiple values,
if any of them matches a substring of the repo, the repo is denied/allowed.
Parameters:
-repo : repository name
"""
if repo is None:
logger.warning("Repo is not defined")
return False
denied_message = "Project '%s' is not allowed."
denied_repo = Settings().get_setting("denied_repo")
allowed_repo = Settings().get_setting("allowed_repo")
if denied_repo is not None and \
any(x in repo for x in denied_repo) or \
allowed_repo is not None and \
not any(x in repo for x in allowed_repo):
logger.warning(denied_message, repo)
return False
return True
def create_worker_app():
"""Create worker app."""
# load settings
settings = Settings()
settings.load_settings(config_file=constants.CONFIG_FILE)
settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION)
if is_worker_enabled():
task_queue = settings.get_setting("task_queue")
worker_app = Celery(
'tasks',
backend=task_queue["backend"],
broker=task_queue["broker_url"]
)
# configure worker
worker_app.conf.update(
CELERY_TASK_SERIALIZER='json',
CELERY_ACCEPT_CONTENT=['json']
)
if worker_app is None:
logger.error("Error connection to task queue")
else:
logger.info(
"Connected to task queue : %s", task_queue["broker_url"]
)
else:
worker_app = Celery()
logger.warning(
"Task queue is not defined,"
" check README.md to configure task queue"
)
return worker_app
def is_readable():
"""Check if login keys for Keen IO API are set, to allow reading."""
if has_project_id() and has_read_key():
return True
logger.warning("Keen.io Read Key is not set")
return False
def has_read_key():
"""Check if Keen.io Read key is set."""
if "KEEN_READ_KEY" in os.environ or keen.read_key is not None:
return True
logger.warning("Keen.io Read Key is not set")
return False
def is_writable():
"""Check if login keys for Keen IO API are set, to allow writing."""
if has_project_id() and has_write_key():
return True
logger.warning("Keen.io Write Key is not set")
return False
def has_master_key():
"""Check if Keen.io Master API key is set."""
if "KEEN_MASTER_KEY" in os.environ or keen.master_key is not None:
return True
logger.warning("Keen.io Master API Key is not set")
return False
def has_write_key():
"""Check if Keen.io Write Key is set."""
if "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None:
return True
logger.warning("Keen.io Write Key is not set")
return False
def get_dashboard_keen_config(repo):
"""
Generate the Keen.io settings for the configuration of the dashboard.
The dashboard is Javascript powered HTML file that contains the
graphs generated by Keen.io.
"""
# initialise config settings
keen_config = {}
if not has_project_id() or not has_master_key():
logger.warning("Keen.io related config settings could not be created,"
" keen.project_id and/or keen.master_key"
" are not defined.")
return keen_config
# set keen project ID
if keen.project_id is None:
keen_project_id = os.environ["KEEN_PROJECT_ID"]
else:
keen_project_id = keen.project_id
keen_config['projectId'] = str(keen_project_id)
# generate read key
read_key = generate_read_key(repo)
if read_key is not None:
# convert bytes to string
if isinstance(read_key, bytes):
read_key = read_key.decode('utf-8')
keen_config['readKey'] = str(read_key)
# return keen config settings
return keen_config
def has_project_id():
"""Check if Keen.io project ID is set."""
if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
return True
logger.warning("Keen.io Project ID is not set")
return False
def get_substage_name(self, command):
"""
Resolve Travis CI substage name that corresponds to a cli command.
Parameters:
- command : cli command
"""
if not tools.is_string(command):
return ""
if len(self.current_build_data) > 0 and \
"config" in self.current_build_data:
build_config = self.current_build_data["config"]
else:
logger.warning("Travis CI build config is not set")
return ""
# check if build_config collection is empty
if build_config:
for stage_name, commands in build_config.items():
if tools.is_list(commands) and command in commands:
substage_number = commands.index(command) + 1
substage_name = "{stage}.{substage:d}".format(
stage=stage_name, substage=substage_number)
logger.debug("Substage %s corresponds to '%s'",
substage_name, command)
return substage_name
return ""
def set_build_matrix(self, job_data):
"""
Retrieve build matrix data from job data and store in properties.
Properties :
- language
- language version (if applicable)
- compiler (if applicable)
- operating system
- environment parameters
Parameters:
- job_data : dictionary with Travis CI job data
"""
# check if job config data exists
if 'job' not in job_data or 'config' not in job_data['job']:
logger.warning("Job config data doesn't exist")
return
build_matrix = Collection()
job_config = job_data['job']['config']
if 'language' in job_config:
language = job_config['language']
build_matrix.add_item('language', language)
# set language version
# ('d', 'dart', 'go', 'perl', 'php', 'python', 'rust')
if language in job_config:
if language == 'android':
build_matrix.add_item(
"language_components",
" ".join(job_config[language]["components"]))
else:
build_matrix.add_item('language_version',
str(job_config[language]))
# language specific build matrix parameters
parameters = {
'ghc': 'ghc', # Haskell
'jdk': 'jdk', # Java, Android, Groovy, Ruby, Scala
'lein': 'lein', # Clojure
'mono': 'mono', # C#, F#, Visual Basic
'node_js': 'node_js', # Javascript
'otp_release': 'otp_release', # Erlang
'rvm': 'rvm', # Ruby, Objective-C
'gemfile': 'gemfile', # Ruby, Objective-C
'xcode_sdk': 'xcode_sdk', # Objective-C
'xcode_scheme': 'xcode_scheme', # Objective-C
'compiler': 'compiler', # C, C++
'os': 'os',
'env': 'parameters'
}
for parameter, name in parameters.items():
if parameter in job_config:
build_matrix.add_item(name, str(job_config[parameter]))
self.current_job.add_property("build_matrix",
build_matrix.get_items_with_summary())
def set_finished_at(self, isotimestamp):
"""
Set timestamp when build finished.
Parameters :
- isotimestamp : timestamp in iso format when build started
"""
try:
self.add_property("finished_at", split_isotimestamp(isotimestamp))
except (TypeError, ValueError) as msg:
logger.warning(
"isotimestamp expected when setting finished_at : %s", msg)
def process_notification_payload(payload):
"""
Extract repo slug and build number from Travis notification payload.
Returns a dictionary with "repo" and "build" information,
or an empty dictionary if the payload could not be processed.
Deprecated behaviour : Currently the repo and build information are
also stored in the "settings" object,
but this will be removed in the near future.
Parameters:
- payload : Travis CI notification payload
"""
settings = Settings()
parameters = {}
if payload is None:
logger.warning("Travis notification payload is not set")
return parameters
if not is_string(payload):
logger.warning(
"Travis notification payload is incorrect :"
" string expected, got %s", type(payload))
return parameters
json_payload = json.loads(payload)
logger.info("Travis Payload : %r.", json_payload)
# get repo name from payload
if ("repository" in json_payload
and "owner_name" in json_payload["repository"]
and "name" in json_payload["repository"]):
repo = get_repo_slug(json_payload["repository"]["owner_name"],
json_payload["repository"]["name"])
logger.info("Build repo : %s", repo)
settings.set_project_name(repo)
parameters["repo"] = repo
# get build number from payload
if "number" in json_payload:
logger.info("Build number : %s", str(json_payload["number"]))
settings.add_setting('build', json_payload['number'])
parameters["build"] = json_payload['number']
return parameters
def generate_write_key():
"""Create scoped key for write access to Keen.io database."""
if not has_master_key():
logger.warning("Keen.io Write Key was not created,"
" keen.master_key is not defined.")
return None
master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")
privileges = {
"allowed_operations": ["write"]
}
logger.info("Keen.io Write Key is created")
return scoped_keys.encrypt(master_key, privileges)
def validate_travis_request(repo=None, build=None):
"""
Validate repo and build parameters of travis web request.
Check parameters (repo and build)
Returns error message, None when all parameters are fine.
"""
if repo is None or build is None:
logger.warning("Repo or build number are not set")
return "Repo or build are not set, format : " \
"/travis/<repo_owner>/<repo_name>/<build>"
# check if repo is allowed
if not is_repo_allowed(repo):
return "Project '{}' is not allowed.".format(cgi.escape(repo))
return None
def load_config_ini_file(self, config_file="config.ini"):
"""
Load settings from a config file, using objconfig.
Parameters :
- config_file : name of the config file
"""
spec_filename = "buildtimetrend/config_spec.ini"
configspec = ConfigObj(spec_filename,
interpolation=False,
list_values=False,
_inspec=True)
config = ConfigObj(config_file, configspec=configspec)
if not config.validate(Validator()):
logger.warning("Error validating config file %s", config_file)
return config
def generate_read_key(repo):
"""
Create scoped key for reading only the build-stages related data.
Param repo : github repository slug (fe. buildtimetrend/python-lib)
"""
if not has_master_key():
logger.warning("Keen.io Read Key was not created,"
" keen.master_key is not defined.")
return None
master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")
privileges = {
"allowed_operations": ["read"]
}
if repo is not None:
privileges["filters"] = [get_repo_filter(repo)]
logger.info("Keen.io Read Key is created for %s", repo)
return scoped_keys.encrypt(master_key, privileges)
def process_travis_buildlog(self, repo, build):
"""
Process Travis CI buildlog.
Check parameters, load build data from Travis CI,
process it and send to Keen.io for storage.
"""
repo = str(repo)
build = str(build)
try:
result = check_process_parameters(repo, build)
except Exception as msg:
if self.request.called_directly:
return msg
else:
# When checking if build exists fails, retry later
# Keen.io API might be down
raise self.retry()
if result is not None:
logger.warning(result)
return result
travis_data = TravisData(repo, build)
data_detail = service.get_repo_data_detail(repo)
# retrieve build data using Travis CI API
message = "Retrieving build #%s data of %s from Travis CI"
logger.info(message, build, repo)
ret_msg = message % (cgi.escape(build), cgi.escape(repo))
if not travis_data.get_build_data():
if self.request.called_directly:
ret_msg += "\nError retrieving build data."
return ret_msg
else:
# retry if retrieving build data failed.
raise self.retry()
# process all build jobs and
# send build job data to Keen.io
for build_job in travis_data.process_build_jobs():
build_job_id = build_job.properties.get_items()["job"]
message = "Send %s build job #%s data to Keen.io"
logger.warning(message, repo, build_job_id)
ret_msg += "\n" + message % \
(cgi.escape(repo), cgi.escape(build_job_id))
keenio.send_build_data_service(build_job, data_detail)
# check if collection is empty
if travis_data.build_jobs:
message = "Successfully retrieved build #%s data of %s" \
" from Travis CI and sent to Keen.io"
else:
message = "No data found for build #%s of %s"
logger.warning(message, build, repo)
ret_msg += "\n" + message % (cgi.escape(build), cgi.escape(repo))
return ret_msg
def get_repo_data_detail(repo):
"""
Get level of data detail storage of a repo.
A repository name is checked against a list of repository names.
If a match is found, the corresponding data detail level is used.
Else, the default global setting is returned.
Parameters:
-repo : repository name
"""
settings = Settings()
if repo is None:
logger.warning("Repo is not defined")
else:
repo_settings = settings.get_setting("repo_data_detail")
for repo_substring, setting in repo_settings.items():
if repo_substring in repo:
return setting
# return default global data_detail setting
return settings.get_setting("data_detail")