def long_message(self): out = StringIO() out.write(self._long_message if self._long_message else '') if (self.module, self.name) in ChildError.build_errors: # The error happened in some external executed process. Show # the build log with errors highlighted. if self.build_log: errors, warnings = parse_log_events(self.build_log) nerr = len(errors) if nerr > 0: if nerr == 1: out.write("\n1 error found in build log:\n") else: out.write("\n%d errors found in build log:\n" % nerr) out.write(make_log_context(errors)) else: # The error happened in in the Python code, so try to show # some context from the Package itself. if self.context: out.write('\n') out.write('\n'.join(self.context)) out.write('\n') if out.getvalue(): out.write('\n') if self.build_log: out.write('See build log for details:\n') out.write(' %s' % self.build_log) return out.getvalue()
def long_message(self): out = StringIO() out.write(self._long_message if self._long_message else '') if (self.module, self.name) in ChildError.build_errors: # The error happened in some external executed process. Show # the build log with errors or warnings highlighted. if self.build_log and os.path.exists(self.build_log): errors, warnings = parse_log_events(self.build_log) nerr = len(errors) nwar = len(warnings) if nerr > 0: # If errors are found, only display errors out.write("\n%s found in build log:\n" % plural(nerr, 'error')) out.write(make_log_context(errors)) elif nwar > 0: # If no errors are found but warnings are, display warnings out.write("\n%s found in build log:\n" % plural(nwar, 'warning')) out.write(make_log_context(warnings)) else: # The error happened in in the Python code, so try to show # some context from the Package itself. if self.context: out.write('\n') out.write('\n'.join(self.context)) out.write('\n') if out.getvalue(): out.write('\n') if self.build_log and os.path.exists(self.build_log): out.write('See build log for details:\n') out.write(' %s\n' % self.build_log) return out.getvalue()
def long_message(self): out = StringIO() out.write(self._long_message if self._long_message else '') if (self.module, self.name) in ChildError.build_errors: # The error happened in some external executed process. Show # the build log with errors or warnings highlighted. if self.build_log and os.path.exists(self.build_log): errors, warnings = parse_log_events(self.build_log) nerr = len(errors) nwar = len(warnings) if nerr > 0: # If errors are found, only display errors out.write( "\n%s found in build log:\n" % plural(nerr, 'error')) out.write(make_log_context(errors)) elif nwar > 0: # If no errors are found but warnings are, display warnings out.write( "\n%s found in build log:\n" % plural(nwar, 'warning')) out.write(make_log_context(warnings)) else: # The error happened in in the Python code, so try to show # some context from the Package itself. if self.context: out.write('\n') out.write('\n'.join(self.context)) out.write('\n') if out.getvalue(): out.write('\n') if self.build_log and os.path.exists(self.build_log): out.write('See build log for details:\n') out.write(' %s' % self.build_log) return out.getvalue()
def write_log_summary(out, log_type, log, last=None): errors, warnings = parse_log_events(log) nerr = len(errors) nwar = len(warnings) if nerr > 0: if last and nerr > last: errors = errors[-last:] nerr = last # If errors are found, only display errors out.write("\n%s found in %s log:\n" % (plural(nerr, 'error'), log_type)) out.write(make_log_context(errors)) elif nwar > 0: if last and nwar > last: warnings = warnings[-last:] nwar = last # If no errors are found but warnings are, display warnings out.write("\n%s found in %s log:\n" % (plural(nwar, 'warning'), log_type)) out.write(make_log_context(warnings))
def log_parse(parser, args): input = args.file if args.file == '-': input = sys.stdin errors, warnings = parse_log_events( input, args.context, args.jobs, args.profile) if args.profile: return types = [s.strip() for s in args.show.split(',')] for e in types: if e not in event_types: tty.die('Invalid event type: %s' % e) events = [] if 'errors' in types: events.extend(errors) print('%d errors' % len(errors)) if 'warnings' in types: events.extend(warnings) print('%d warnings' % len(warnings)) print(make_log_context(events, args.width))
def log_parse(parser, args): input = args.file if args.file == '-': input = sys.stdin errors, warnings = parse_log_events( input, args.context, args.jobs, args.profile) if args.profile: return types = [s.strip() for s in args.show.split(',')] for e in types: if e not in event_types: tty.die('Invalid event type: %s' % e) events = [] if 'errors' in types: events.extend(errors) print('%d errors' % len(errors)) if 'warnings' in types: events.extend(warnings) print('%d warnings' % len(warnings)) print(make_log_context(events, args.width))
def build_report(self, filename, report_data): self.initialize_report(filename, report_data) for phase in cdash_phases: report_data[phase] = {} report_data[phase]['loglines'] = [] report_data[phase]['status'] = 0 report_data[phase]['endtime'] = self.endtime # Track the phases we perform so we know what reports to create. phases_encountered = [] total_duration = 0 # Parse output phase-by-phase. phase_regexp = re.compile(r"Executing phase: '(.*)'") cdash_phase = '' for spec in report_data['specs']: if 'time' in spec: total_duration += int(spec['time']) for package in spec['packages']: if 'stdout' in package: current_phase = '' cdash_phase = '' for line in package['stdout'].splitlines(): match = None if line.find("Executing phase: '") != -1: match = phase_regexp.search(line) if match: current_phase = match.group(1) if current_phase not in map_phases_to_cdash: current_phase = '' continue cdash_phase = \ map_phases_to_cdash[current_phase] if cdash_phase not in phases_encountered: phases_encountered.append(cdash_phase) report_data[cdash_phase]['loglines'].append( text_type("{0} output for {1}:".format( cdash_phase, package['name']))) elif cdash_phase: report_data[cdash_phase]['loglines'].append( xml.sax.saxutils.escape(line)) phases_encountered.append('update') # Move the build phase to the front of the list if it occurred. # This supports older versions of CDash that expect this phase # to be reported before all others. if "build" in phases_encountered: build_pos = phases_encountered.index("build") phases_encountered.insert(0, phases_encountered.pop(build_pos)) self.starttime = self.endtime - total_duration for phase in phases_encountered: report_data[phase]['starttime'] = self.starttime report_data[phase]['log'] = \ '\n'.join(report_data[phase]['loglines']) errors, warnings = parse_log_events(report_data[phase]['loglines']) nerrors = len(errors) if phase == 'configure' and nerrors > 0: report_data[phase]['status'] = 1 if phase == 'build': # Convert log output from ASCII to Unicode and escape for XML. def clean_log_event(event): event = vars(event) event['text'] = xml.sax.saxutils.escape(event['text']) event['pre_context'] = xml.sax.saxutils.escape('\n'.join( event['pre_context'])) event['post_context'] = xml.sax.saxutils.escape('\n'.join( event['post_context'])) # source_file and source_line_no are either strings or # the tuple (None,). Distinguish between these two cases. if event['source_file'][0] is None: event['source_file'] = '' event['source_line_no'] = '' else: event['source_file'] = xml.sax.saxutils.escape( event['source_file']) return event report_data[phase]['errors'] = [] report_data[phase]['warnings'] = [] for error in errors: report_data[phase]['errors'].append(clean_log_event(error)) for warning in warnings: report_data[phase]['warnings'].append( clean_log_event(warning)) if phase == 'update': report_data[phase]['revision'] = self.revision # Write the report. report_name = phase.capitalize() + ".xml" phase_report = os.path.join(filename, report_name) with codecs.open(phase_report, 'w', 'utf-8') as f: env = spack.tengine.make_environment() if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. site_template = os.path.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = os.path.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report) self.print_cdash_link()
def test_report_for_package(self, directory_name, package, duration): if 'stdout' not in package: # Skip reporting on packages that did not generate any output. return self.current_package_name = package['name'] self.buildname = "{0} - {1}".format( self.base_buildname, package['name']) report_data = self.initialize_report(directory_name) for phase in ('test', 'update'): report_data[phase] = {} report_data[phase]['loglines'] = [] report_data[phase]['status'] = 0 report_data[phase]['endtime'] = self.endtime # Track the phases we perform so we know what reports to create. # We always report the update step because this is how we tell CDash # what revision of Spack we are using. phases_encountered = ['test', 'update'] # Generate a report for this package. # The first line just says "Testing package name-hash" report_data['test']['loglines'].append( text_type("{0} output for {1}:".format( 'test', package['name']))) for line in package['stdout'].splitlines()[1:]: report_data['test']['loglines'].append( xml.sax.saxutils.escape(line)) self.starttime = self.endtime - duration for phase in phases_encountered: report_data[phase]['starttime'] = self.starttime report_data[phase]['log'] = \ '\n'.join(report_data[phase]['loglines']) errors, warnings = parse_log_events(report_data[phase]['loglines']) # Cap the number of errors and warnings at 50 each. errors = errors[0:49] warnings = warnings[0:49] if phase == 'test': # Convert log output from ASCII to Unicode and escape for XML. def clean_log_event(event): event = vars(event) event['text'] = xml.sax.saxutils.escape(event['text']) event['pre_context'] = xml.sax.saxutils.escape( '\n'.join(event['pre_context'])) event['post_context'] = xml.sax.saxutils.escape( '\n'.join(event['post_context'])) # source_file and source_line_no are either strings or # the tuple (None,). Distinguish between these two cases. if event['source_file'][0] is None: event['source_file'] = '' event['source_line_no'] = '' else: event['source_file'] = xml.sax.saxutils.escape( event['source_file']) return event # Convert errors to warnings if the package reported success. if package['result'] == 'success': warnings = errors + warnings errors = [] report_data[phase]['errors'] = [] report_data[phase]['warnings'] = [] for error in errors: report_data[phase]['errors'].append(clean_log_event(error)) for warning in warnings: report_data[phase]['warnings'].append( clean_log_event(warning)) if phase == 'update': report_data[phase]['revision'] = self.revision # Write the report. report_name = phase.capitalize() + ".xml" report_file_name = package['name'] + "_" + report_name phase_report = os.path.join(directory_name, report_file_name) with codecs.open(phase_report, 'w', 'utf-8') as f: env = spack.tengine.make_environment() if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. site_template = os.path.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = os.path.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report)
def build_report_for_package(self, directory_name, package, duration): if 'stdout' not in package: # Skip reporting on packages that did not generate any output. return self.current_package_name = package['name'] if self.multiple_packages: self.buildname = "{0} - {1}".format( self.base_buildname, package['name']) else: self.buildname = self.base_buildname report_data = self.initialize_report(directory_name) for phase in cdash_phases: report_data[phase] = {} report_data[phase]['loglines'] = [] report_data[phase]['status'] = 0 report_data[phase]['endtime'] = self.endtime # Track the phases we perform so we know what reports to create. # We always report the update step because this is how we tell CDash # what revision of Spack we are using. phases_encountered = ['update'] # Generate a report for this package. current_phase = '' cdash_phase = '' for line in package['stdout'].splitlines(): match = None if line.find("Executing phase: '") != -1: match = self.phase_regexp.search(line) if match: current_phase = match.group(1) if current_phase not in map_phases_to_cdash: current_phase = '' continue cdash_phase = \ map_phases_to_cdash[current_phase] if cdash_phase not in phases_encountered: phases_encountered.append(cdash_phase) report_data[cdash_phase]['loglines'].append( text_type("{0} output for {1}:".format( cdash_phase, package['name']))) elif cdash_phase: report_data[cdash_phase]['loglines'].append( xml.sax.saxutils.escape(line)) # Move the build phase to the front of the list if it occurred. # This supports older versions of CDash that expect this phase # to be reported before all others. if "build" in phases_encountered: build_pos = phases_encountered.index("build") phases_encountered.insert(0, phases_encountered.pop(build_pos)) self.starttime = self.endtime - duration for phase in phases_encountered: report_data[phase]['starttime'] = self.starttime report_data[phase]['log'] = \ '\n'.join(report_data[phase]['loglines']) errors, warnings = parse_log_events(report_data[phase]['loglines']) # Cap the number of errors and warnings at 50 each. errors = errors[:50] warnings = warnings[:50] nerrors = len(errors) if phase == 'configure' and nerrors > 0: report_data[phase]['status'] = 1 self.success = False if phase == 'build': # Convert log output from ASCII to Unicode and escape for XML. def clean_log_event(event): event = vars(event) event['text'] = xml.sax.saxutils.escape(event['text']) event['pre_context'] = xml.sax.saxutils.escape( '\n'.join(event['pre_context'])) event['post_context'] = xml.sax.saxutils.escape( '\n'.join(event['post_context'])) # source_file and source_line_no are either strings or # the tuple (None,). Distinguish between these two cases. if event['source_file'][0] is None: event['source_file'] = '' event['source_line_no'] = '' else: event['source_file'] = xml.sax.saxutils.escape( event['source_file']) return event # Convert errors to warnings if the package reported success. if package['result'] == 'success': warnings = errors + warnings errors = [] else: self.success = False report_data[phase]['errors'] = [] report_data[phase]['warnings'] = [] for error in errors: report_data[phase]['errors'].append(clean_log_event(error)) for warning in warnings: report_data[phase]['warnings'].append( clean_log_event(warning)) if phase == 'update': report_data[phase]['revision'] = self.revision # Write the report. report_name = phase.capitalize() + ".xml" if self.multiple_packages: report_file_name = package['name'] + "_" + report_name else: report_file_name = report_name phase_report = os.path.join(directory_name, report_file_name) with codecs.open(phase_report, 'w', 'utf-8') as f: env = spack.tengine.make_environment() if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. site_template = os.path.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = os.path.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report)
def build_report(self, filename, report_data): self.initialize_report(filename, report_data) for phase in cdash_phases: report_data[phase] = {} report_data[phase]['log'] = "" report_data[phase]['status'] = 0 report_data[phase]['starttime'] = self.starttime report_data[phase]['endtime'] = self.starttime # Track the phases we perform so we know what reports to create. phases_encountered = [] # Parse output phase-by-phase. phase_regexp = re.compile(r"Executing phase: '(.*)'") for spec in report_data['specs']: for package in spec['packages']: if 'stdout' in package: current_phase = '' for line in package['stdout'].splitlines(): match = phase_regexp.search(line) if match: current_phase = match.group(1) if current_phase not in map_phases_to_cdash: current_phase = '' continue beginning_of_phase = True else: if beginning_of_phase: cdash_phase = \ map_phases_to_cdash[current_phase] if cdash_phase not in phases_encountered: phases_encountered.append(cdash_phase) report_data[cdash_phase]['log'] += \ text_type("{0} output for {1}:\n".format( cdash_phase, package['name'])) beginning_of_phase = False report_data[cdash_phase]['log'] += \ xml.sax.saxutils.escape(line) + "\n" for phase in phases_encountered: errors, warnings = parse_log_events( report_data[phase]['log'].splitlines()) nerrors = len(errors) if phase == 'configure' and nerrors > 0: report_data[phase]['status'] = 1 if phase == 'build': # Convert log output from ASCII to Unicode and escape for XML. def clean_log_event(event): event = vars(event) event['text'] = xml.sax.saxutils.escape(event['text']) event['pre_context'] = xml.sax.saxutils.escape('\n'.join( event['pre_context'])) event['post_context'] = xml.sax.saxutils.escape('\n'.join( event['post_context'])) # source_file and source_line_no are either strings or # the tuple (None,). Distinguish between these two cases. if event['source_file'][0] is None: event['source_file'] = '' event['source_line_no'] = '' else: event['source_file'] = xml.sax.saxutils.escape( event['source_file']) return event report_data[phase]['errors'] = [] report_data[phase]['warnings'] = [] for error in errors: report_data[phase]['errors'].append(clean_log_event(error)) for warning in warnings: report_data[phase]['warnings'].append( clean_log_event(warning)) # Write the report. report_name = phase.capitalize() + ".xml" phase_report = os.path.join(filename, report_name) with codecs.open(phase_report, 'w', 'utf-8') as f: env = spack.tengine.make_environment() site_template = os.path.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = os.path.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report)
def build_report(self, filename, report_data): self.initialize_report(filename, report_data) for phase in cdash_phases: report_data[phase] = {} report_data[phase]['log'] = "" report_data[phase]['status'] = 0 report_data[phase]['starttime'] = self.starttime report_data[phase]['endtime'] = self.starttime # Track the phases we perform so we know what reports to create. phases_encountered = [] # Parse output phase-by-phase. phase_regexp = re.compile(r"Executing phase: '(.*)'") for spec in report_data['specs']: for package in spec['packages']: if 'stdout' in package: current_phase = '' for line in package['stdout'].splitlines(): match = phase_regexp.search(line) if match: current_phase = match.group(1) if current_phase not in map_phases_to_cdash: current_phase = '' continue beginning_of_phase = True else: if beginning_of_phase: cdash_phase = \ map_phases_to_cdash[current_phase] if cdash_phase not in phases_encountered: phases_encountered.append(cdash_phase) report_data[cdash_phase]['log'] += \ text_type("{0} output for {1}:\n".format( cdash_phase, package['name'])) beginning_of_phase = False report_data[cdash_phase]['log'] += \ xml.sax.saxutils.escape(line) + "\n" for phase in phases_encountered: errors, warnings = parse_log_events( report_data[phase]['log'].splitlines()) nerrors = len(errors) if phase == 'configure' and nerrors > 0: report_data[phase]['status'] = 1 if phase == 'build': # Convert log output from ASCII to Unicode and escape for XML. def clean_log_event(event): event = vars(event) event['text'] = xml.sax.saxutils.escape(event['text']) event['pre_context'] = xml.sax.saxutils.escape( '\n'.join(event['pre_context'])) event['post_context'] = xml.sax.saxutils.escape( '\n'.join(event['post_context'])) # source_file and source_line_no are either strings or # the tuple (None,). Distinguish between these two cases. if event['source_file'][0] is None: event['source_file'] = '' event['source_line_no'] = '' else: event['source_file'] = xml.sax.saxutils.escape( event['source_file']) return event report_data[phase]['errors'] = [] report_data[phase]['warnings'] = [] for error in errors: report_data[phase]['errors'].append(clean_log_event(error)) for warning in warnings: report_data[phase]['warnings'].append( clean_log_event(warning)) # Write the report. report_name = phase.capitalize() + ".xml" phase_report = os.path.join(filename, report_name) with codecs.open(phase_report, 'w', 'utf-8') as f: env = spack.tengine.make_environment() site_template = os.path.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) phase_template = os.path.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report)