def test_add_summary_merge_with_existing(self):
with ioutil.TempDir() as td:
sb = sandbox.Sandbox(os.path.join(td.path, 'buildscripts.trunk.dev'))
sb.layout()
es = get_sample_eval_summary()
db = Dashboard(sb.get_report_root())
start_time = es.get_start_time()
start_text = str(es)
fldr = os.path.join(sb.get_report_root(), 'zufa')
os.makedirs(fldr)
fname = os.path.join(fldr, 'results.txt')
with open(fname, 'w') as f:
for i in range(100):
# Make it look like the results file has 100 entries, spaced 4
# hrs apart, stretching back 400 hrs. At 168 hrs per week, this
# should look like about 2.5 weeks of history.
es.start_time -= (4 * 3600)
f.write(str(es) + '\n')
es.start_time = start_time
db.add_summary(es)
with open(fname, 'r') as f:
lines = [l.strip() for l in f.readlines()]
self.assertEqual(start_text, lines[0])
self.assertEqual(85, len(lines))
ld = dateutils.format_standard_date_with_tz_offset(dateutils.parse_standard_date_with_tz_offset('2011-08-27 14:51:40.490000-0600'))
self.assertEqual('sadm.trunk.136.20,OFFICIAL,zufa,TEST,,' + ld + ',11.08 0.44 9.18,linux_x86-64,Linux,64,2.6.35.13-92.fc14', lines[1])
ld = dateutils.format_standard_date_with_tz_offset(dateutils.parse_standard_date_with_tz_offset('2011-08-13 18:51:40.490000-0600'))
self.assertEqual('sadm.trunk.136.20,OFFICIAL,zufa,TEST,,' + ld + ',11.08 0.44 9.18,linux_x86-64,Linux,64,2.6.35.13-92.fc14', lines[84])
def __str__(self):
# Could use config parser, but I want these keys to always appear in same
# order and I don't need any of cp's other features, so I'm doing this raw...
return '''purpose=%s
start date=%s
last status=%s
last status date=%s
pid=%s
''' % (self.purpose,
dateutils.format_standard_date_with_tz_offset(
self.start_date), self.last_status,
dateutils.format_standard_date_with_tz_offset(
self.last_status_date), self.pid)
def __str__(self):
# Could use config parser, but I want these keys to always appear in same
# order and I don't need any of cp's other features, so I'm doing this raw...
return """purpose=%s
start date=%s
last status=%s
last status date=%s
pid=%s
""" % (
self.purpose,
dateutils.format_standard_date_with_tz_offset(self.start_date),
self.last_status,
dateutils.format_standard_date_with_tz_offset(self.last_status_date),
self.pid,
)
def get_build_id(self):
"""
Return a BuildID (named tuple with .component, .branch, .code_revno,
and .test_revno) that can be used to uniquely identify this sandbox's
state (in terms of what code and tests it contains) for build
reproducibility.
Build IDs are used to tag built artifacts and to coordinate test results
across platforms. It consists of the name of the top-level component in
the sandbox, its branch, plus the revno of the code and test aspects of
that component. The format of build ids is documented on our build site
(https:// ... /working-with-code) at #TODO Kim refer to correct doc site
/overview-and-concepts/version-tags.
Build ids are less useful when the top component aspect is built instead
of code (as with test-only sandboxes), or when the sandbox is
experimental and contains checkouts.
"""
top = self.get_top_component()
aspects = self.get_component_aspects(top)
if component.CODE_ASPECT_NAME in aspects:
crev = int(vcs.revno(self.get_component_path(top, component.CODE_ASPECT_NAME)))
else:
crev = 0
if component.TEST_ASPECT_NAME in aspects:
trev = int(vcs.revno(self.get_component_path(top, component.TEST_ASPECT_NAME)))
else:
trev = 0
date = dateutils.format_standard_date_with_tz_offset(time.time()).split(".")[0].strip()
guid = str(uuid.uuid1())
return build_id.BuildID(top, self.get_branch(), crev, trev, guid, date)
def _show_property(prop_getter, label=False):
value = prop_getter()
if value is None:
if _is_boolean(prop_getter):
value = False
elif hasattr(value, "extend"):
value = _serialize_list(value)
elif hasattr(value, "keys"):
print("%s: " % _get_property_name(prop_getter).ljust(28), end="")
indent = ""
for key, val in value.iteritems():
if hasattr(val, "extend"):
val = _serialize_list(val)
print(indent + "%s: %s" % (key, val))
if not indent:
indent = " ".rjust(30)
return
if prop_getter.__name__.endswith("date"):
try:
value = dateutils.format_standard_date_with_tz_offset(value)
except:
pass
else:
value = str(value)
if label:
print("%s: %s" % (_get_property_name(prop_getter).ljust(28), value))
else:
print(value)
def update_status_log(self):
if self.linked_to_vcs():
os.system('bzr up "%s"' % self._root)
add_needed = False
self._data = None
self._load_if_needed()
status = self.get_status()
result_name = enum_to_str(EvalResult, status.result)
status_line = '%s,%s,%s' % (
dateutils.format_standard_date_with_tz_offset(
status.when), result_name, ' '.join(status.reasons))
log_path = os.path.join(self._root, STATUS_LOG)
if os.path.isfile(log_path):
with open(log_path, 'r') as f:
first_line = f.readline().strip()
if result_name not in first_line:
lines = [status_line]
else:
lines = []
lines.append(first_line)
lines.extend([l.strip() for l in f.readlines()])
if len(lines) > 1000:
lines = lines[0:1000]
else:
if not os.path.isdir(self._root):
os.makedirs(self._root)
else:
add_needed = self.linked_to_vcs()
lines = [status_line]
with open(log_path, 'w') as f:
for l in lines:
f.write(l + '\r\n')
if add_needed:
os.system('bzr add "%s"' % self._root)
def _show_property(prop_getter, label=False):
value = prop_getter()
if value is None:
if _is_boolean(prop_getter):
value = False
elif hasattr(value, 'extend'):
value = _serialize_list(value)
elif hasattr(value, 'keys'):
print('%s: ' % _get_property_name(prop_getter).ljust(28), end='')
indent = ''
for key, val in value.iteritems():
if hasattr(val, 'extend'):
val = _serialize_list(val)
print(indent + '%s: %s' % (key, val))
if not indent:
indent = ' '.rjust(30)
return
if prop_getter.__name__.endswith('date'):
try:
value = dateutils.format_standard_date_with_tz_offset(value)
except:
pass
else:
value = str(value)
if label:
print('%s: %s' % (_get_property_name(prop_getter).ljust(28), value))
else:
print(value)
def update_status_log(self):
if self.linked_to_vcs():
os.system('bzr up "%s"' % self._root)
add_needed = False
self._data = None
self._load_if_needed()
status = self.get_status()
result_name = enum_to_str(EvalResult, status.result)
status_line = '%s,%s,%s' % (
dateutils.format_standard_date_with_tz_offset(status.when),
result_name,
' '.join(status.reasons))
log_path = os.path.join(self._root, STATUS_LOG)
if os.path.isfile(log_path):
with open(log_path, 'r') as f:
first_line = f.readline().strip()
if result_name not in first_line:
lines = [status_line]
else:
lines = []
lines.append(first_line)
lines.extend([l.strip() for l in f.readlines()])
if len(lines) > 1000:
lines = lines[0:1000]
else:
if not os.path.isdir(self._root):
os.makedirs(self._root)
else:
add_needed = self.linked_to_vcs()
lines = [status_line]
with open(log_path, 'w') as f:
for l in lines:
f.write(l + '\r\n')
if add_needed:
os.system('bzr add "%s"' % self._root)
def test_format_and_parse_tandard_date_with_tz_offset(self):
when = round(time.time(), 3)
x = dateutils.format_standard_date_with_tz_offset(when)
if not re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d+)?\s*[-+]\d\d:?\d\d', x):
self.fail('Expected standard format; got "%s" instead.' % x)
when2 = round(dateutils.parse_standard_date_with_tz_offset(x), 3)
if when != when2:
self.fail('Expected round trip to be lossless. Instead, started with %d and ended with %d (diff = %d)'
% (int(when), when2, when2-int(when)))
def add_manifest(sb, folder_to_publish):
try:
files = walktree('', folder_to_publish)
fp = open(os.path.join(folder_to_publish, MANIFEST), 'w')
fp.write('Last published on %s\n' % dateutils.format_standard_date_with_tz_offset(time.time()))
fp.write('\n'.join(files))
fp.close()
except:
return 1
return 0
def _load_host(self, h):
summaries = []
path = os.path.join(self._root, h, RESULTS_FILE)
if self.debug: print('Loading data for %s from %s.' % (h, path))
with open(path, 'r') as f:
lines = [l.strip() for l in f.readlines()]
cutoff = self._get_cutoff_date()
if self.debug:
print('Cutoff date = %.2f (%s)' %
(cutoff,
dateutils.format_standard_date_with_tz_offset(cutoff)))
code_revno = 0
n = 0
for l in lines:
if not l:
continue
try:
n += 1
es = parse_eval_summary_line(l)
if es.get_start_time() < cutoff:
if self.debug:
print('Passed cutoff date after %d lines (line=%s)' %
(n, l))
break
if not self.get_load_all():
# We only want to load build results for the most recent
# revno; for a given machine, anything other than its most
# recent results don't impact our analysis of status.
if es.build_id.code_revno < code_revno:
if self.debug:
print(
'Passed revno of %d after %d lines (line=%s)' %
(code_revno, n, l))
break
else:
code_revno = es.build_id.code_revno
summaries.append(es)
except:
if self.debug: print('Unable to parse line %s' % l)
pass
if self.debug: print('Loaded %d summaries.' % n)
if summaries:
for es in summaries:
if es.build_id not in self._build_groups:
self._build_groups[es.build_id] = []
self._build_groups[es.build_id].append(es)
self._data[h] = summaries
rno = summaries[0].build_id.code_revno
if not rno in self._hosts_by_revno:
self._hosts_by_revno[rno] = []
self._hosts_by_revno[rno].append(h)
style = summaries[0].style
if not style in self._hosts_by_style:
self._hosts_by_style[style] = []
self._hosts_by_style[style].append(h)
def __str__(self):
fr = self.failure_reason
if not fr:
fr = ''
return '%s.%s.%s.%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s' % (
self.build_id.component, self.build_id.branch,
self.build_id.code_revno, self.build_id.test_revno, self.style,
self.host, enum_to_str(EvalPhase, self.final_phase), fr,
dateutils.format_standard_date_with_tz_offset(self.start_time),
' '.join([str(round(x, 2)) for x in self.durations
]), self.tpv, self.os, self.bitness, self.version)
def test_format_and_parse_tandard_date_with_tz_offset(self):
when = round(time.time(), 3)
x = dateutils.format_standard_date_with_tz_offset(when)
if not re.match(
r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d+)?\s*[-+]\d\d:?\d\d',
x):
self.fail('Expected standard format; got "%s" instead.' % x)
when2 = round(dateutils.parse_standard_date_with_tz_offset(x), 3)
if when != when2:
self.fail(
'Expected round trip to be lossless. Instead, started with %d and ended with %d (diff = %d)'
% (int(when), when2, when2 - int(when)))
def _load_host(self, h):
summaries = []
path = os.path.join(self._root, h, RESULTS_FILE)
if self.debug: print('Loading data for %s from %s.' % (h, path))
with open(path, 'r') as f:
lines = [l.strip() for l in f.readlines()]
cutoff = self._get_cutoff_date()
if self.debug: print('Cutoff date = %.2f (%s)' % (cutoff, dateutils.format_standard_date_with_tz_offset(cutoff)))
code_revno = 0
n = 0
for l in lines:
if not l:
continue
try:
n += 1
es = parse_eval_summary_line(l)
if es.get_start_time() < cutoff:
if self.debug: print('Passed cutoff date after %d lines (line=%s)' % (n, l))
break
if not self.get_load_all():
# We only want to load build results for the most recent
# revno; for a given machine, anything other than its most
# recent results don't impact our analysis of status.
if es.build_id.code_revno < code_revno:
if self.debug: print('Passed revno of %d after %d lines (line=%s)' % (code_revno, n, l))
break
else:
code_revno = es.build_id.code_revno
summaries.append(es)
except:
if self.debug: print('Unable to parse line %s' % l)
pass
if self.debug: print('Loaded %d summaries.' % n)
if summaries:
for es in summaries:
if es.build_id not in self._build_groups:
self._build_groups[es.build_id] = []
self._build_groups[es.build_id].append(es)
self._data[h] = summaries
rno = summaries[0].build_id.code_revno
if not rno in self._hosts_by_revno:
self._hosts_by_revno[rno] = []
self._hosts_by_revno[rno].append(h)
style = summaries[0].style
if not style in self._hosts_by_style:
self._hosts_by_style[style] = []
self._hosts_by_style[style].append(h)
def __str__(self):
fr = self.failure_reason
if not fr:
fr = ''
return '%s.%s.%s.%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s' % (
self.build_id.component,
self.build_id.branch,
self.build_id.code_revno,
self.build_id.test_revno,
self.style,
self.host,
enum_to_str(EvalPhase, self.final_phase),
fr,
dateutils.format_standard_date_with_tz_offset(self.start_time),
' '.join([str(round(x, 2)) for x in self.durations]),
self.tpv,
self.os,
self.bitness,
self.version
)
def get_build_id(self):
'''
Return a BuildID (named tuple with .component, .branch, .code_revno,
and .test_revno) that can be used to uniquely identify this sandbox's
state (in terms of what code and tests it contains) for build
reproducibility.
Build IDs are used to tag built artifacts and to coordinate test results
across platforms. It consists of the name of the top-level component in
the sandbox, its branch, plus the revno of the code and test aspects of
that component. The format of build ids is documented on our build site
(https:// ... /working-with-code) at #TODO Kim refer to correct doc site
/overview-and-concepts/version-tags.
Build ids are less useful when the top component aspect is built instead
of code (as with test-only sandboxes), or when the sandbox is
experimental and contains checkouts.
'''
top = self.get_top_component()
aspects = self.get_component_aspects(top)
if component.CODE_ASPECT_NAME in aspects:
crev = int(
vcs.revno(
self.get_component_path(top, component.CODE_ASPECT_NAME)))
else:
crev = 0
if component.TEST_ASPECT_NAME in aspects:
trev = int(
vcs.revno(
self.get_component_path(top, component.TEST_ASPECT_NAME)))
else:
trev = 0
date = dateutils.format_standard_date_with_tz_offset(
time.time()).split('.')[0].strip()
guid = str(uuid.uuid1())
return build_id.BuildID(top, self.get_branch(), crev, trev, guid, date)
def _set_date_conf(self, section, key, value):
if value is not None:
value = dateutils.format_standard_date_with_tz_offset(value)
self._set_conf(section, key, value)
def _set_date_conf(self, section, key, value):
if value is not None:
value = dateutils.format_standard_date_with_tz_offset(value)
self._set_conf(section, key, value)
