def process_build_info_and_store(self,
build=None,
job_name=None,
timestamp=None,
build_id=None,
status=None):
"""
Retrieve further details for a build including console output
analysis and store the data into datastore.
"""
# get rid of decimal point 0:18:19.931000 at build duration
duration = str(build.get_duration()).split('.')[0]
console_output = build.get_console()
result = self.process_console_output(console_output)
key_id = "%s-%s" % (job_name, build_id)
builds_stats = BuildsStatisticsModel(id=key_id,
name=job_name,
bid=build_id,
status=status,
# naive datetime (no timezone)
ts=timestamp.replace(tzinfo=None),
duration=duration)
if result:
for item in ("passed", "failed", "skipped", "error"):
setattr(builds_stats, item, result[item])
log.debug("Storing %s ..." % builds_stats)
builds_stats.put()
def test_builds_stats_model_in_memcache(self):
self.jenkins.process_build_info_and_store(build=Build(),
job_name="Selenium_Portal_MTV_development_sandbox",
timestamp=datetime.datetime.now(),
build_id=51,
status="FAILED")
builds = BuildsStatisticsModel.get_builds_data(days_limit=1)
# current time is added, remove from the comparison
del builds["current_time"]
assert memcache.get(MEMCACHE_BUILDS_KEY)[1] == builds
assert (0 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
assert (1 in memcache.get(MEMCACHE_BUILDS_KEY)) is True
assert (2 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
assert (3 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
prev_builds_resp = builds
builds = BuildsStatisticsModel.get_builds_data(days_limit=2)
# current time is added, remove from the comparison
del builds["current_time"]
assert memcache.get(MEMCACHE_BUILDS_KEY)[2] == builds
assert memcache.get(MEMCACHE_BUILDS_KEY)[1] == prev_builds_resp
assert (0 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
assert (1 in memcache.get(MEMCACHE_BUILDS_KEY)) is True
assert (2 in memcache.get(MEMCACHE_BUILDS_KEY)) is True
assert (3 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY).put()
self.jenkins.update_builds_stats()
assert memcache.get(MEMCACHE_BUILDS_KEY) == None
def test_initialization(self):
initialization()
d = ActivitySummaryModel.get_data()
items = ('sent_emails_counter_total',
'stopped_builds_counter',
'sent_emails_counter',
'stopped_builds_counter_total',
'overview_update_counter_total',
'overview_update_counter')
for i in items:
assert d[i] == 0
b = BuildsStatisticsModel.get_builds_data()
assert b["num_builds"] == 0
assert b["builds"] == {}
def test_process_build_info_and_store(self):
self.jenkins.process_build_info_and_store(build=Build(),
job_name="Selenium_Portal_MTV_development_sandbox",
timestamp=datetime.datetime.now(),
build_id=51,
status="FAILED")
builds = BuildsStatisticsModel.get_builds_data()
assert builds["num_builds"]
assert len(builds["builds"]["Selenium_Portal_MTV_development_sandbox"]) == 1
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["status"] == "FAILED"
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["build_id"] == 51
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["duration"] == "0:18:19"
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["failed"] == 1
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["skipped"] == 12
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["passed"] == 20
assert builds["builds"]["Selenium_Portal_MTV_development_sandbox"][0]["error"] == 0
def get_builds_stats(self):
try:
arg = self.request.get("days_limit", 1)
days_limit = int(arg)
except Exception as ex:
self.response.out.write("wrong argument: '%s'" % arg)
return
resp = BuildsStatisticsModel.get_builds_data(days_limit=days_limit)
asm = ActivitySummaryModel.get_data()
if asm:
resp["builds_statistics_model_last_update_at"] = \
asm["builds_statistics_model_last_update_at"]
self.response.headers["Content-Type"] = "application/json"
# seconds (10minutes)
self.response.headers[
"Cache-Control"] = "max-age=600" # , must-revalidate, private"
# self.response.cache_control = 'public'
# self.response.cache_control.max_age = 600
self.response.out.write(json.dumps(resp))
def initialization():
"""
Initialize datastore types.
"""
msg = "Initialization run at %s ..." % get_current_timestamp_str()
log.info(msg)
if ActivitySummaryModel.get_data() is None:
log.debug("ActivitySummaryModel initialization ...")
activity = ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY)
activity.put()
log.debug("Finished ActivitySummaryModel initialization.")
else:
log.debug("ActivitySummaryModel is already initialized.")
if len(BuildsStatisticsModel.query().fetch(keys_only=True)) == 0:
deferred.defer(get_jenkins_instance().builds_stats_init)
log.debug("Finished BuildsStatisticsModel initialization.")
else:
log.debug("BuildStatisticsModel is already initialized.")
return msg
def update_builds_stats(self):
"""
Main task to run over job types and builds from the last one:
retrieve information about a build and store into datastore
if it has not been processed in the previous run of this
routine.
"""
log.info("Start update builds stats at '%s'" % get_current_timestamp_str())
for job_name in self.job_names:
job = self.server.get_job(job_name)
# returns iterator of available build id numbers in
# reverse order, most recent first
bids = job.get_build_ids()
for bid in bids:
log.debug("Retrieving data on %s #%s ..." % (job_name, bid))
build = job.get_build(bid)
status = build.get_status()
if not status:
log.debug("%s #%s has not finished, status: %s, going to "
"another build ..." % (job_name, bid, status))
continue
# this build considered finished now
# check if we have not hit a build which is already stored
key_id = "%s-%s" % (job_name, bid)
if BuildsStatisticsModel.get_by_id(key_id) is not None:
log.debug("%s #%s is already stored, going to the "
"next job type ..." % (job_name, bid))
break
ts = build.get_timestamp()
self.process_build_info_and_store(build=build,
job_name=job_name,
timestamp=ts,
build_id=bid,
status=status)
ActivitySummaryModel.increase_counters(which_counters=["builds_stats_update_counter"])
memcache.set(MEMCACHE_BUILDS_KEY, None)
log.info("Finished update builds stats at '%s'" % get_current_timestamp_str())
def test_get_builds_stats(self):
BuildsStatisticsModel.get_builds_data(days_limit=1)
def test_builds_statistics_model_get_builds_data(self):
initialization()
b = BuildsStatisticsModel.get_builds_data()
assert len(b["builds"]) == 0
assert b["num_builds"] == 0