def get_total_builds(repo=None, interval=None):
"""
Query Keen.io database and retrieve total number of builds.
Parameters :
- repo : repo name (fe. buildtimetrend/service)
- interval : timeframe, possible values : 'week', 'month', 'year',
anything else defaults to 'week'
"""
if repo is None or not is_readable():
return -1
interval_data = check_time_interval(interval)
try:
return keen.count_unique(
"build_jobs",
target_property="job.build",
timeframe=interval_data['timeframe'],
max_age=interval_data['max_age'],
filters=[get_repo_filter(repo)]
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
return -1
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.get_total_builds() : " + str(msg))
return -1
def get_latest_buildtime(repo=None):
"""
Query Keen.io database and retrieve buildtime duration of last build.
Parameters :
- repo : repo name (fe. buildtimetrend/python-lib)
"""
if repo is None or not is_readable():
return -1
try:
result = keen.extraction(
"build_jobs",
property_names="job.duration",
latest=1,
filters=[get_repo_filter(repo)]
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
return -1
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.get_latest_buildtime() : " + str(msg))
return -1
if is_list(result) and len(result) > 0 and \
check_dict(result[0], None, ['job']) and \
check_dict(result[0]['job'], None, ['duration']):
return result[0]['job']['duration']
return -1
def create_worker_app():
"""Create worker app."""
# load settings
settings = Settings()
settings.load_settings(config_file=constants.CONFIG_FILE)
settings.set_client(constants.CLIENT_NAME, constants.CLIENT_VERSION)
if is_worker_enabled():
task_queue = settings.get_setting("task_queue")
worker_app = Celery(
'tasks',
backend=task_queue["backend"],
broker=task_queue["broker_url"]
)
# configure worker
worker_app.conf.update(
CELERY_TASK_SERIALIZER='json',
CELERY_ACCEPT_CONTENT=['json']
)
if worker_app is None:
logger.error("Error connection to task queue")
else:
logger.info(
"Connected to task queue : %s", task_queue["broker_url"]
)
else:
worker_app = Celery()
logger.warning(
"Task queue is not defined,"
" check README.md to configure task queue"
)
return worker_app
def get_items_with_summary(self):
"""Return items collection as dictionary with an summary property."""
items = self.get_items()
# concatenate all properties in a summary field
matrix_params = self.get_key_sorted_items().values()
try:
items["summary"] = " ".join(matrix_params)
except TypeError as msg:
logger.error(
"Error parsing build matrix properties : %s, message : %s",
matrix_params, str(msg)
)
return items
def check_authorization(repo, auth_header):
"""
Check if Travis CI notification has a correct Authorization header.
This check is enabled if travis_account_token is defined in settings.
More information on the Authorization header :
http://docs.travis-ci.com/user/notifications/#Authorization-for-Webhooks
Returns true if Authorization header is valid, but also if
travis_account_token is not defined.
Parameters:
- repo : git repo name
- auth_header : Travis CI notification Authorization header
"""
# get Travis account token from Settings
token = Settings().get_setting("travis_account_token")
# return True if token is not set
if token is None:
logger.info("Setting travis_account_token is not defined,"
" Travis CI notification Authorization header"
" is not checked.")
return True
# check if parameters are strings
if is_string(repo) and is_string(auth_header) and is_string(token):
# generate hash (encode string to bytes first)
auth_hash = sha256((repo + token).encode('utf-8')).hexdigest()
# compare hash with Authorization header
if auth_hash == auth_header:
logger.info("Travis CI notification Authorization header"
" is correct.")
return True
else:
logger.error("Travis CI notification Authorization header"
" is incorrect.")
return False
else:
logger.debug("repo, auth_header and travis_auth_token"
" should be strings.")
return False
def get_build_data(self):
"""
Retrieve Travis CI build data.
Returns true if retrieving data was succesful, false on error.
"""
request = 'repos/{repo}/builds?number={build_id}'.format(
repo=self.repo, build_id=self.build_id)
try:
self.builds_data = self.connector.json_request(request)
except (HTTPError, URLError) as msg:
logger.error("Error getting build data from Travis CI: %s", msg)
return False
# log builds_data
logger.debug("Build #%s data : %s", str(self.build_id),
json.dumps(self.builds_data, sort_keys=True, indent=2))
return True
def has_build_id(repo=None, build_id=None):
"""
Check if build_id exists in Keen.io database.
Parameters :
- repo : repo name (fe. buildtimetrend/python-lib)
- build_id : ID of the build
"""
if repo is None or build_id is None:
logger.error("Repo or build_id is not set")
raise ValueError("Repo or build_id is not set")
if not is_readable():
raise SystemError("Keen.io Project ID or API Read Key is not set")
try:
count = keen.count(
"build_jobs",
filters=[get_repo_filter(repo), {
"property_name": "job.build",
"operator": "eq",
"property_value": str(build_id)}]
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
raise SystemError("Connection to Keen.io API failed")
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.has_build_id : " + str(msg))
raise SystemError(msg)
return count > 0
def get_all_projects():
"""Query Keen.io database and retrieve a list of all projects."""
if not is_readable():
return []
try:
result = keen.select_unique(
"build_jobs",
"buildtime_trend.project_name",
max_age=3600 * 24 # cache for 24 hours
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
return []
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.get_all_projects() : " + str(msg))
return []
if is_list(result):
return result
return []
def validate_task_parameters(repo=None, build=None):
"""
Validate repo and build parameters of process_travis_buildlog().
Check parameters (repo and build)
Returns error message, None when all parameters are fine.
"""
if not keenio.is_writable():
return "Keen IO write key not set, no data was sent"
try:
if has_build_id(repo, build):
template = "Build #{build} of project {repo} " \
"already exists in database"
return template.format(build=cgi.escape(str(build)),
repo=cgi.escape(str(repo)))
except Exception as msg:
# Raise last exception again
logger.error("Error checking if build exists : %s", msg)
raise SystemError("Error checking if build exists.")
return None
def get_days_since_fail(repo=None):
"""
Query Keen.io database and retrieve time since last failed buildjob.
Parameters :
- repo : repo name (fe. buildtimetrend/python-lib)
"""
if repo is None or not is_readable():
return -1
try:
failed_timestamp = keen.maximum(
"build_jobs",
target_property="job.finished_at.timestamp_seconds",
filters=[
get_repo_filter(repo),
{
"operator": "ne",
"property_name": "job.result",
"property_value": "passed"
}
]
)
except requests.ConnectionError:
logger.error("Connection to Keen.io API failed")
return -1
except keen.exceptions.KeenApiError as msg:
logger.error("Error in keenio.get_days_since_fail() : " + str(msg))
return -1
if isinstance(failed_timestamp, (int, float)) and failed_timestamp > 0:
dt_failed = datetime.fromtimestamp(failed_timestamp)
dt_now = datetime.now()
return math.floor((dt_now - dt_failed).total_seconds() / (3600 * 24))
return -1