def stop_running_build(self, job=None, build_id=-1):
"""
Stop the build_id on the project type identified by the job
reference (as returned by the .get_job(job_name) call.
"""
build = job.get_build(build_id)
log.warn("Stopping build %s ..." % build)
stop_call_response = build.stop()
for _ in range(self.stop_build_timeout):
time.sleep(1)
# when using the same build reference to check the current status,
# even when jenkins page reports the build ABORTED, this call upon
# the same reference of build will receive None
# get a fresh build reference after the stop() call
build = job.get_build(build_id)
status = build.get_status()
if status == "ABORTED":
ActivitySummaryModel.increase_counters(which_counters=["stopped_builds_counter"])
break
else:
status = ("status '%s' - after %s seconds timeout, should be 'ABORTED'" %
(status, self.stop_build_timeout))
log.debug("Finished build stop method, result: %s" % status)
return stop_call_response, status
def test_send_activity_summary(self):
# put some data into datastore
ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY).put()
data = ActivitySummaryModel.get_data()
formatted_data = pprint.pformat(data)
send_email(subject="activity summary",
body="activity summary: " + "\n\n" + formatted_data)
ActivitySummaryModel.reset()
def send_activity_summary(self):
msg = "Sending activity summary email at %s ..." % get_current_timestamp_str(
)
logging.info(msg)
formatted_data = pprint.pformat(ActivitySummaryModel.get_data())
send_email(subject="activity summary",
body="activity summary: " + "\n\n" + formatted_data)
ActivitySummaryModel.reset()
logging.info("Finished sending activity summary.")
self.response.out.write(msg)
def check_running_build(self, job_name=None, current_build_id=-1):
"""
Check duration of the build build.
Dictionary resp is updated about actions performed in this method.
"""
resp = {}
job = self.server.get_job(job_name)
build = job.get_build(current_build_id)
# get_timestamp returns this type of data, is in UTC:
# datetime.datetime(2015, 3, 3, 19, 41, 56, tzinfo=<UTC>) (is not JSON serializable)
ts = build.get_timestamp()
resp["start_timestamp"] = get_localized_timestamp_str(ts)
resp["retrieved_at"] = get_current_timestamp_str()
console_url = "%s/job/%s/%s/console" % (self.jenkins_url, job_name, current_build_id)
now = datetime.datetime.utcnow() # there is no timezone info, putting UTC
duration = pytz.utc.localize(now) - ts
duration_str = str(duration).split('.')[0]
resp["duration"] = duration_str
resp["stop_threshold_minutes"] = self.current_build_duration_threshold_hard
resp["email_notification"] = False
if duration.total_seconds() > self.current_build_duration_threshold_hard * 60:
stop_call_response, status = self.stop_running_build(job=job,
build_id=current_build_id)
msg = (("Build '%s' has been running for more than %s minutes.\n"
"duration: %s\nconsole output: %s\nstopping ... current status: %s") %
(build,
self.current_build_duration_threshold_hard,
duration_str,
console_url,
status))
resp["stop_call_response"] = stop_call_response
resp["current_status"] = status
resp["email_notification"] = True
elif duration.total_seconds() > self.current_build_duration_threshold_soft * 60:
msg = (("Build '%s' has been running for more than %s minutes.\n"
"duration: %s\nconsole output: %s\n[soft threshold, no action taken]") %
(build,
self.current_build_duration_threshold_soft,
duration_str,
console_url))
resp["email_notification"] = True
if resp["email_notification"]:
log.warn(msg)
formatted_data = pprint.pformat(resp)
log.debug("build check response:\n%s" % formatted_data)
subject = "long #%s %s" % (current_build_id, job_name)
result = send_email(subject=subject, body=msg + "\n\n" + formatted_data)
if result:
ActivitySummaryModel.increase_counters(which_counters=["sent_emails_counter"])
return resp
def test_update_builds_stats(self):
asm = ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY)
asm.put()
# since the mock classes return empty values, this method
# is tested only partially
self.jenkins.update_builds_stats()
now = datetime.datetime.utcnow()
asm_data = ActivitySummaryModel.get_by_id(ACTIVITY_SUMMARY_MODEL_ID_KEY)
assert asm_data.builds_stats_update_counter_total == 1
assert asm_data.builds_stats_update_counter == 1
time_diff = now - asm_data.builds_statistics_model_last_update_at
assert time_diff.seconds <= 1
assert memcache.get(MEMCACHE_BUILDS_KEY) == None
def test_update_overview_check_running_builds(self):
# put some data into datastore
asm = ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY,
overview_update_counter=72,
sent_emails_counter=4,
stopped_builds_counter=1,
overview_update_counter_total=3448,
sent_emails_counter_total=593,
stopped_builds_counter_total=134)
asm.put()
self.jenkins.update_overview_check_running_builds()
asm = ActivitySummaryModel.get_data()
assert asm["overview_update_counter"] == 73
assert asm["overview_update_counter_total"] == 3449
def test_builds_stats_model_in_memcache(self):
self.jenkins.process_build_info_and_store(build=Build(),
job_name="Selenium_Portal_MTV_development_sandbox",
timestamp=datetime.datetime.now(),
build_id=51,
status="FAILED")
builds = BuildsStatisticsModel.get_builds_data(days_limit=1)
# current time is added, remove from the comparison
del builds["current_time"]
assert memcache.get(MEMCACHE_BUILDS_KEY)[1] == builds
assert (0 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
assert (1 in memcache.get(MEMCACHE_BUILDS_KEY)) is True
assert (2 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
assert (3 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
prev_builds_resp = builds
builds = BuildsStatisticsModel.get_builds_data(days_limit=2)
# current time is added, remove from the comparison
del builds["current_time"]
assert memcache.get(MEMCACHE_BUILDS_KEY)[2] == builds
assert memcache.get(MEMCACHE_BUILDS_KEY)[1] == prev_builds_resp
assert (0 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
assert (1 in memcache.get(MEMCACHE_BUILDS_KEY)) is True
assert (2 in memcache.get(MEMCACHE_BUILDS_KEY)) is True
assert (3 in memcache.get(MEMCACHE_BUILDS_KEY)) is False
ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY).put()
self.jenkins.update_builds_stats()
assert memcache.get(MEMCACHE_BUILDS_KEY) == None
def update_overview_check_running_builds(self):
"""
Combines 2 actions - update overview data (info about currently
running builds and checks them if they are not for too long
in execution. Entry point.
"""
log.info("Start update overview, check builds at '%s'" % get_current_timestamp_str())
data = dict(total_queued_jobs=self.get_total_queued_jobs(),
running_jobs=self.check_running_builds_get_info())
# if there is no date on running jobs, add timestamp, it would be there otherwise
if len(data["running_jobs"]) == 0:
data["retrieved_at"] = get_current_timestamp_str()
OverviewModel.update_overview_data(data)
log.debug("OverviewModel data updated:\n%s" % pprint.pformat(data))
ActivitySummaryModel.increase_counters(which_counters=["overview_update_counter"])
log.info("Finished update overview, check builds at '%s'" % get_current_timestamp_str())
def initialization():
"""
Initialize datastore types.
"""
msg = "Initialization run at %s ..." % get_current_timestamp_str()
log.info(msg)
if ActivitySummaryModel.get_data() is None:
log.debug("ActivitySummaryModel initialization ...")
activity = ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY)
activity.put()
log.debug("Finished ActivitySummaryModel initialization.")
else:
log.debug("ActivitySummaryModel is already initialized.")
if len(BuildsStatisticsModel.query().fetch(keys_only=True)) == 0:
deferred.defer(get_jenkins_instance().builds_stats_init)
log.debug("Finished BuildsStatisticsModel initialization.")
else:
log.debug("BuildStatisticsModel is already initialized.")
return msg
def update_builds_stats(self):
"""
Main task to run over job types and builds from the last one:
retrieve information about a build and store into datastore
if it has not been processed in the previous run of this
routine.
"""
log.info("Start update builds stats at '%s'" % get_current_timestamp_str())
for job_name in self.job_names:
job = self.server.get_job(job_name)
# returns iterator of available build id numbers in
# reverse order, most recent first
bids = job.get_build_ids()
for bid in bids:
log.debug("Retrieving data on %s #%s ..." % (job_name, bid))
build = job.get_build(bid)
status = build.get_status()
if not status:
log.debug("%s #%s has not finished, status: %s, going to "
"another build ..." % (job_name, bid, status))
continue
# this build considered finished now
# check if we have not hit a build which is already stored
key_id = "%s-%s" % (job_name, bid)
if BuildsStatisticsModel.get_by_id(key_id) is not None:
log.debug("%s #%s is already stored, going to the "
"next job type ..." % (job_name, bid))
break
ts = build.get_timestamp()
self.process_build_info_and_store(build=build,
job_name=job_name,
timestamp=ts,
build_id=bid,
status=status)
ActivitySummaryModel.increase_counters(which_counters=["builds_stats_update_counter"])
memcache.set(MEMCACHE_BUILDS_KEY, None)
log.info("Finished update builds stats at '%s'" % get_current_timestamp_str())
def test_initialization(self):
initialization()
d = ActivitySummaryModel.get_data()
items = ('sent_emails_counter_total',
'stopped_builds_counter',
'sent_emails_counter',
'stopped_builds_counter_total',
'overview_update_counter_total',
'overview_update_counter')
for i in items:
assert d[i] == 0
b = BuildsStatisticsModel.get_builds_data()
assert b["num_builds"] == 0
assert b["builds"] == {}
def get_builds_stats(self):
try:
arg = self.request.get("days_limit", 1)
days_limit = int(arg)
except Exception as ex:
self.response.out.write("wrong argument: '%s'" % arg)
return
resp = BuildsStatisticsModel.get_builds_data(days_limit=days_limit)
asm = ActivitySummaryModel.get_data()
if asm:
resp["builds_statistics_model_last_update_at"] = \
asm["builds_statistics_model_last_update_at"]
self.response.headers["Content-Type"] = "application/json"
# seconds (10minutes)
self.response.headers[
"Cache-Control"] = "max-age=600" # , must-revalidate, private"
# self.response.cache_control = 'public'
# self.response.cache_control.max_age = 600
self.response.out.write(json.dumps(resp))
def test_update_builds(self):
asm = ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY)
asm.put()
get_jenkins_instance().update_builds_stats()
def test_update_overview_check_running_builds(self):
ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY).put()
get_jenkins_instance().update_overview_check_running_builds()
def test_get_activity_summary(self):
resp = ActivitySummaryModel.get_data()
json.dumps(resp)
def get_activity_summary(self):
resp = ActivitySummaryModel.get_data()
self.response.headers["Content-Type"] = "application/json"
self.response.out.write(json.dumps(resp))
def test_send_activity_summary_no_data(self):
data = ActivitySummaryModel.get_data()
formatted_data = pprint.pformat(data)
send_email(subject="activity summary",
body="activity summary: " + "\n\n" + formatted_data)
def test_activity_summary_model(self):
asm = ActivitySummaryModel(id=ACTIVITY_SUMMARY_MODEL_ID_KEY,
overview_update_counter=72,
sent_emails_counter=4,
stopped_builds_counter=1,
overview_update_counter_total=3448,
sent_emails_counter_total=593,
stopped_builds_counter_total=134,
builds_stats_update_counter=65,
builds_stats_update_counter_total=165)
asm.put()
ActivitySummaryModel.reset()
asm = ActivitySummaryModel.get_data()
assert asm["overview_update_counter"] == 0
assert asm["sent_emails_counter"] == 0
assert asm["builds_stats_update_counter"] == 0
assert asm["overview_update_counter_total"] == 3448
assert asm["stopped_builds_counter_total"] == 134
assert asm["builds_stats_update_counter_total"] == 165
ActivitySummaryModel.increase_counters(which_counters=["overview_update_counter"])
asm = ActivitySummaryModel.get_data()
assert asm["overview_update_counter"] == 1
assert asm["overview_update_counter_total"] == 3449
ActivitySummaryModel.increase_counters(which_counters=["sent_emails_counter",
"stopped_builds_counter",
"builds_stats_update_counter"])
asm = ActivitySummaryModel.get_data()
assert asm["sent_emails_counter"] == 1
assert asm["sent_emails_counter_total"] == 594
assert asm["stopped_builds_counter_total"] == 135
assert asm["stopped_builds_counter"] == 1
assert asm["builds_stats_update_counter_total"] == 166
assert asm["builds_stats_update_counter"] == 1