def test_add_summary_merge_with_existing(self): with ioutil.TempDir() as td: sb = sandbox.Sandbox(os.path.join(td.path, 'buildscripts.trunk.dev')) sb.layout() es = get_sample_eval_summary() db = Dashboard(sb.get_report_root()) start_time = es.get_start_time() start_text = str(es) fldr = os.path.join(sb.get_report_root(), 'zufa') os.makedirs(fldr) fname = os.path.join(fldr, 'results.txt') with open(fname, 'w') as f: for i in range(100): # Make it look like the results file has 100 entries, spaced 4 # hrs apart, stretching back 400 hrs. At 168 hrs per week, this # should look like about 2.5 weeks of history. es.start_time -= (4 * 3600) f.write(str(es) + '\n') es.start_time = start_time db.add_summary(es) with open(fname, 'r') as f: lines = [l.strip() for l in f.readlines()] self.assertEqual(start_text, lines[0]) self.assertEqual(85, len(lines)) ld = dateutils.format_standard_date_with_tz_offset(dateutils.parse_standard_date_with_tz_offset('2011-08-27 14:51:40.490000-0600')) self.assertEqual('sadm.trunk.136.20,OFFICIAL,zufa,TEST,,' + ld + ',11.08 0.44 9.18,linux_x86-64,Linux,64,2.6.35.13-92.fc14', lines[1]) ld = dateutils.format_standard_date_with_tz_offset(dateutils.parse_standard_date_with_tz_offset('2011-08-13 18:51:40.490000-0600')) self.assertEqual('sadm.trunk.136.20,OFFICIAL,zufa,TEST,,' + ld + ',11.08 0.44 9.18,linux_x86-64,Linux,64,2.6.35.13-92.fc14', lines[84])
def __str__(self): # Could use config parser, but I want these keys to always appear in same # order and I don't need any of cp's other features, so I'm doing this raw... return '''purpose=%s start date=%s last status=%s last status date=%s pid=%s ''' % (self.purpose, dateutils.format_standard_date_with_tz_offset( self.start_date), self.last_status, dateutils.format_standard_date_with_tz_offset( self.last_status_date), self.pid)
def __str__(self): # Could use config parser, but I want these keys to always appear in same # order and I don't need any of cp's other features, so I'm doing this raw... return """purpose=%s start date=%s last status=%s last status date=%s pid=%s """ % ( self.purpose, dateutils.format_standard_date_with_tz_offset(self.start_date), self.last_status, dateutils.format_standard_date_with_tz_offset(self.last_status_date), self.pid, )
def get_build_id(self): """ Return a BuildID (named tuple with .component, .branch, .code_revno, and .test_revno) that can be used to uniquely identify this sandbox's state (in terms of what code and tests it contains) for build reproducibility. Build IDs are used to tag built artifacts and to coordinate test results across platforms. It consists of the name of the top-level component in the sandbox, its branch, plus the revno of the code and test aspects of that component. The format of build ids is documented on our build site (https:// ... /working-with-code) at #TODO Kim refer to correct doc site /overview-and-concepts/version-tags. Build ids are less useful when the top component aspect is built instead of code (as with test-only sandboxes), or when the sandbox is experimental and contains checkouts. """ top = self.get_top_component() aspects = self.get_component_aspects(top) if component.CODE_ASPECT_NAME in aspects: crev = int(vcs.revno(self.get_component_path(top, component.CODE_ASPECT_NAME))) else: crev = 0 if component.TEST_ASPECT_NAME in aspects: trev = int(vcs.revno(self.get_component_path(top, component.TEST_ASPECT_NAME))) else: trev = 0 date = dateutils.format_standard_date_with_tz_offset(time.time()).split(".")[0].strip() guid = str(uuid.uuid1()) return build_id.BuildID(top, self.get_branch(), crev, trev, guid, date)
def _show_property(prop_getter, label=False): value = prop_getter() if value is None: if _is_boolean(prop_getter): value = False elif hasattr(value, "extend"): value = _serialize_list(value) elif hasattr(value, "keys"): print("%s: " % _get_property_name(prop_getter).ljust(28), end="") indent = "" for key, val in value.iteritems(): if hasattr(val, "extend"): val = _serialize_list(val) print(indent + "%s: %s" % (key, val)) if not indent: indent = " ".rjust(30) return if prop_getter.__name__.endswith("date"): try: value = dateutils.format_standard_date_with_tz_offset(value) except: pass else: value = str(value) if label: print("%s: %s" % (_get_property_name(prop_getter).ljust(28), value)) else: print(value)
def update_status_log(self): if self.linked_to_vcs(): os.system('bzr up "%s"' % self._root) add_needed = False self._data = None self._load_if_needed() status = self.get_status() result_name = enum_to_str(EvalResult, status.result) status_line = '%s,%s,%s' % ( dateutils.format_standard_date_with_tz_offset( status.when), result_name, ' '.join(status.reasons)) log_path = os.path.join(self._root, STATUS_LOG) if os.path.isfile(log_path): with open(log_path, 'r') as f: first_line = f.readline().strip() if result_name not in first_line: lines = [status_line] else: lines = [] lines.append(first_line) lines.extend([l.strip() for l in f.readlines()]) if len(lines) > 1000: lines = lines[0:1000] else: if not os.path.isdir(self._root): os.makedirs(self._root) else: add_needed = self.linked_to_vcs() lines = [status_line] with open(log_path, 'w') as f: for l in lines: f.write(l + '\r\n') if add_needed: os.system('bzr add "%s"' % self._root)
def _show_property(prop_getter, label=False): value = prop_getter() if value is None: if _is_boolean(prop_getter): value = False elif hasattr(value, 'extend'): value = _serialize_list(value) elif hasattr(value, 'keys'): print('%s: ' % _get_property_name(prop_getter).ljust(28), end='') indent = '' for key, val in value.iteritems(): if hasattr(val, 'extend'): val = _serialize_list(val) print(indent + '%s: %s' % (key, val)) if not indent: indent = ' '.rjust(30) return if prop_getter.__name__.endswith('date'): try: value = dateutils.format_standard_date_with_tz_offset(value) except: pass else: value = str(value) if label: print('%s: %s' % (_get_property_name(prop_getter).ljust(28), value)) else: print(value)
def update_status_log(self): if self.linked_to_vcs(): os.system('bzr up "%s"' % self._root) add_needed = False self._data = None self._load_if_needed() status = self.get_status() result_name = enum_to_str(EvalResult, status.result) status_line = '%s,%s,%s' % ( dateutils.format_standard_date_with_tz_offset(status.when), result_name, ' '.join(status.reasons)) log_path = os.path.join(self._root, STATUS_LOG) if os.path.isfile(log_path): with open(log_path, 'r') as f: first_line = f.readline().strip() if result_name not in first_line: lines = [status_line] else: lines = [] lines.append(first_line) lines.extend([l.strip() for l in f.readlines()]) if len(lines) > 1000: lines = lines[0:1000] else: if not os.path.isdir(self._root): os.makedirs(self._root) else: add_needed = self.linked_to_vcs() lines = [status_line] with open(log_path, 'w') as f: for l in lines: f.write(l + '\r\n') if add_needed: os.system('bzr add "%s"' % self._root)
def test_format_and_parse_tandard_date_with_tz_offset(self): when = round(time.time(), 3) x = dateutils.format_standard_date_with_tz_offset(when) if not re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d+)?\s*[-+]\d\d:?\d\d', x): self.fail('Expected standard format; got "%s" instead.' % x) when2 = round(dateutils.parse_standard_date_with_tz_offset(x), 3) if when != when2: self.fail('Expected round trip to be lossless. Instead, started with %d and ended with %d (diff = %d)' % (int(when), when2, when2-int(when)))
def add_manifest(sb, folder_to_publish): try: files = walktree('', folder_to_publish) fp = open(os.path.join(folder_to_publish, MANIFEST), 'w') fp.write('Last published on %s\n' % dateutils.format_standard_date_with_tz_offset(time.time())) fp.write('\n'.join(files)) fp.close() except: return 1 return 0
def _load_host(self, h): summaries = [] path = os.path.join(self._root, h, RESULTS_FILE) if self.debug: print('Loading data for %s from %s.' % (h, path)) with open(path, 'r') as f: lines = [l.strip() for l in f.readlines()] cutoff = self._get_cutoff_date() if self.debug: print('Cutoff date = %.2f (%s)' % (cutoff, dateutils.format_standard_date_with_tz_offset(cutoff))) code_revno = 0 n = 0 for l in lines: if not l: continue try: n += 1 es = parse_eval_summary_line(l) if es.get_start_time() < cutoff: if self.debug: print('Passed cutoff date after %d lines (line=%s)' % (n, l)) break if not self.get_load_all(): # We only want to load build results for the most recent # revno; for a given machine, anything other than its most # recent results don't impact our analysis of status. if es.build_id.code_revno < code_revno: if self.debug: print( 'Passed revno of %d after %d lines (line=%s)' % (code_revno, n, l)) break else: code_revno = es.build_id.code_revno summaries.append(es) except: if self.debug: print('Unable to parse line %s' % l) pass if self.debug: print('Loaded %d summaries.' % n) if summaries: for es in summaries: if es.build_id not in self._build_groups: self._build_groups[es.build_id] = [] self._build_groups[es.build_id].append(es) self._data[h] = summaries rno = summaries[0].build_id.code_revno if not rno in self._hosts_by_revno: self._hosts_by_revno[rno] = [] self._hosts_by_revno[rno].append(h) style = summaries[0].style if not style in self._hosts_by_style: self._hosts_by_style[style] = [] self._hosts_by_style[style].append(h)
def __str__(self): fr = self.failure_reason if not fr: fr = '' return '%s.%s.%s.%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s' % ( self.build_id.component, self.build_id.branch, self.build_id.code_revno, self.build_id.test_revno, self.style, self.host, enum_to_str(EvalPhase, self.final_phase), fr, dateutils.format_standard_date_with_tz_offset(self.start_time), ' '.join([str(round(x, 2)) for x in self.durations ]), self.tpv, self.os, self.bitness, self.version)
def test_format_and_parse_tandard_date_with_tz_offset(self): when = round(time.time(), 3) x = dateutils.format_standard_date_with_tz_offset(when) if not re.match( r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d+)?\s*[-+]\d\d:?\d\d', x): self.fail('Expected standard format; got "%s" instead.' % x) when2 = round(dateutils.parse_standard_date_with_tz_offset(x), 3) if when != when2: self.fail( 'Expected round trip to be lossless. Instead, started with %d and ended with %d (diff = %d)' % (int(when), when2, when2 - int(when)))
def _load_host(self, h): summaries = [] path = os.path.join(self._root, h, RESULTS_FILE) if self.debug: print('Loading data for %s from %s.' % (h, path)) with open(path, 'r') as f: lines = [l.strip() for l in f.readlines()] cutoff = self._get_cutoff_date() if self.debug: print('Cutoff date = %.2f (%s)' % (cutoff, dateutils.format_standard_date_with_tz_offset(cutoff))) code_revno = 0 n = 0 for l in lines: if not l: continue try: n += 1 es = parse_eval_summary_line(l) if es.get_start_time() < cutoff: if self.debug: print('Passed cutoff date after %d lines (line=%s)' % (n, l)) break if not self.get_load_all(): # We only want to load build results for the most recent # revno; for a given machine, anything other than its most # recent results don't impact our analysis of status. if es.build_id.code_revno < code_revno: if self.debug: print('Passed revno of %d after %d lines (line=%s)' % (code_revno, n, l)) break else: code_revno = es.build_id.code_revno summaries.append(es) except: if self.debug: print('Unable to parse line %s' % l) pass if self.debug: print('Loaded %d summaries.' % n) if summaries: for es in summaries: if es.build_id not in self._build_groups: self._build_groups[es.build_id] = [] self._build_groups[es.build_id].append(es) self._data[h] = summaries rno = summaries[0].build_id.code_revno if not rno in self._hosts_by_revno: self._hosts_by_revno[rno] = [] self._hosts_by_revno[rno].append(h) style = summaries[0].style if not style in self._hosts_by_style: self._hosts_by_style[style] = [] self._hosts_by_style[style].append(h)
def __str__(self): fr = self.failure_reason if not fr: fr = '' return '%s.%s.%s.%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s' % ( self.build_id.component, self.build_id.branch, self.build_id.code_revno, self.build_id.test_revno, self.style, self.host, enum_to_str(EvalPhase, self.final_phase), fr, dateutils.format_standard_date_with_tz_offset(self.start_time), ' '.join([str(round(x, 2)) for x in self.durations]), self.tpv, self.os, self.bitness, self.version )
def get_build_id(self): ''' Return a BuildID (named tuple with .component, .branch, .code_revno, and .test_revno) that can be used to uniquely identify this sandbox's state (in terms of what code and tests it contains) for build reproducibility. Build IDs are used to tag built artifacts and to coordinate test results across platforms. It consists of the name of the top-level component in the sandbox, its branch, plus the revno of the code and test aspects of that component. The format of build ids is documented on our build site (https:// ... /working-with-code) at #TODO Kim refer to correct doc site /overview-and-concepts/version-tags. Build ids are less useful when the top component aspect is built instead of code (as with test-only sandboxes), or when the sandbox is experimental and contains checkouts. ''' top = self.get_top_component() aspects = self.get_component_aspects(top) if component.CODE_ASPECT_NAME in aspects: crev = int( vcs.revno( self.get_component_path(top, component.CODE_ASPECT_NAME))) else: crev = 0 if component.TEST_ASPECT_NAME in aspects: trev = int( vcs.revno( self.get_component_path(top, component.TEST_ASPECT_NAME))) else: trev = 0 date = dateutils.format_standard_date_with_tz_offset( time.time()).split('.')[0].strip() guid = str(uuid.uuid1()) return build_id.BuildID(top, self.get_branch(), crev, trev, guid, date)
def _set_date_conf(self, section, key, value): if value is not None: value = dateutils.format_standard_date_with_tz_offset(value) self._set_conf(section, key, value)