def long_message(self):
out = StringIO()
out.write(self._long_message if self._long_message else '')
if (self.module, self.name) in ChildError.build_errors:
# The error happened in some external executed process. Show
# the build log with errors highlighted.
if self.build_log:
errors, warnings = parse_log_events(self.build_log)
nerr = len(errors)
if nerr > 0:
if nerr == 1:
out.write("\n1 error found in build log:\n")
else:
out.write("\n%d errors found in build log:\n" % nerr)
out.write(make_log_context(errors))
else:
# The error happened in in the Python code, so try to show
# some context from the Package itself.
if self.context:
out.write('\n')
out.write('\n'.join(self.context))
out.write('\n')
if out.getvalue():
out.write('\n')
if self.build_log:
out.write('See build log for details:\n')
out.write(' %s' % self.build_log)
return out.getvalue()
def long_message(self):
out = StringIO()
out.write(self._long_message if self._long_message else '')
if (self.module, self.name) in ChildError.build_errors:
# The error happened in some external executed process. Show
# the build log with errors or warnings highlighted.
if self.build_log and os.path.exists(self.build_log):
errors, warnings = parse_log_events(self.build_log)
nerr = len(errors)
nwar = len(warnings)
if nerr > 0:
# If errors are found, only display errors
out.write("\n%s found in build log:\n" %
plural(nerr, 'error'))
out.write(make_log_context(errors))
elif nwar > 0:
# If no errors are found but warnings are, display warnings
out.write("\n%s found in build log:\n" %
plural(nwar, 'warning'))
out.write(make_log_context(warnings))
else:
# The error happened in in the Python code, so try to show
# some context from the Package itself.
if self.context:
out.write('\n')
out.write('\n'.join(self.context))
out.write('\n')
if out.getvalue():
out.write('\n')
if self.build_log and os.path.exists(self.build_log):
out.write('See build log for details:\n')
out.write(' %s\n' % self.build_log)
return out.getvalue()
def long_message(self):
out = StringIO()
out.write(self._long_message if self._long_message else '')
if (self.module, self.name) in ChildError.build_errors:
# The error happened in some external executed process. Show
# the build log with errors or warnings highlighted.
if self.build_log and os.path.exists(self.build_log):
errors, warnings = parse_log_events(self.build_log)
nerr = len(errors)
nwar = len(warnings)
if nerr > 0:
# If errors are found, only display errors
out.write(
"\n%s found in build log:\n" % plural(nerr, 'error'))
out.write(make_log_context(errors))
elif nwar > 0:
# If no errors are found but warnings are, display warnings
out.write(
"\n%s found in build log:\n" % plural(nwar, 'warning'))
out.write(make_log_context(warnings))
else:
# The error happened in in the Python code, so try to show
# some context from the Package itself.
if self.context:
out.write('\n')
out.write('\n'.join(self.context))
out.write('\n')
if out.getvalue():
out.write('\n')
if self.build_log and os.path.exists(self.build_log):
out.write('See build log for details:\n')
out.write(' %s' % self.build_log)
return out.getvalue()
def write_log_summary(out, log_type, log, last=None):
errors, warnings = parse_log_events(log)
nerr = len(errors)
nwar = len(warnings)
if nerr > 0:
if last and nerr > last:
errors = errors[-last:]
nerr = last
# If errors are found, only display errors
out.write("\n%s found in %s log:\n" %
(plural(nerr, 'error'), log_type))
out.write(make_log_context(errors))
elif nwar > 0:
if last and nwar > last:
warnings = warnings[-last:]
nwar = last
# If no errors are found but warnings are, display warnings
out.write("\n%s found in %s log:\n" %
(plural(nwar, 'warning'), log_type))
out.write(make_log_context(warnings))
def log_parse(parser, args):
input = args.file
if args.file == '-':
input = sys.stdin
errors, warnings = parse_log_events(
input, args.context, args.jobs, args.profile)
if args.profile:
return
types = [s.strip() for s in args.show.split(',')]
for e in types:
if e not in event_types:
tty.die('Invalid event type: %s' % e)
events = []
if 'errors' in types:
events.extend(errors)
print('%d errors' % len(errors))
if 'warnings' in types:
events.extend(warnings)
print('%d warnings' % len(warnings))
print(make_log_context(events, args.width))
def log_parse(parser, args):
input = args.file
if args.file == '-':
input = sys.stdin
errors, warnings = parse_log_events(
input, args.context, args.jobs, args.profile)
if args.profile:
return
types = [s.strip() for s in args.show.split(',')]
for e in types:
if e not in event_types:
tty.die('Invalid event type: %s' % e)
events = []
if 'errors' in types:
events.extend(errors)
print('%d errors' % len(errors))
if 'warnings' in types:
events.extend(warnings)
print('%d warnings' % len(warnings))
print(make_log_context(events, args.width))
def build_report(self, filename, report_data):
self.initialize_report(filename, report_data)
for phase in cdash_phases:
report_data[phase] = {}
report_data[phase]['loglines'] = []
report_data[phase]['status'] = 0
report_data[phase]['endtime'] = self.endtime
# Track the phases we perform so we know what reports to create.
phases_encountered = []
total_duration = 0
# Parse output phase-by-phase.
phase_regexp = re.compile(r"Executing phase: '(.*)'")
cdash_phase = ''
for spec in report_data['specs']:
if 'time' in spec:
total_duration += int(spec['time'])
for package in spec['packages']:
if 'stdout' in package:
current_phase = ''
cdash_phase = ''
for line in package['stdout'].splitlines():
match = None
if line.find("Executing phase: '") != -1:
match = phase_regexp.search(line)
if match:
current_phase = match.group(1)
if current_phase not in map_phases_to_cdash:
current_phase = ''
continue
cdash_phase = \
map_phases_to_cdash[current_phase]
if cdash_phase not in phases_encountered:
phases_encountered.append(cdash_phase)
report_data[cdash_phase]['loglines'].append(
text_type("{0} output for {1}:".format(
cdash_phase, package['name'])))
elif cdash_phase:
report_data[cdash_phase]['loglines'].append(
xml.sax.saxutils.escape(line))
phases_encountered.append('update')
# Move the build phase to the front of the list if it occurred.
# This supports older versions of CDash that expect this phase
# to be reported before all others.
if "build" in phases_encountered:
build_pos = phases_encountered.index("build")
phases_encountered.insert(0, phases_encountered.pop(build_pos))
self.starttime = self.endtime - total_duration
for phase in phases_encountered:
report_data[phase]['starttime'] = self.starttime
report_data[phase]['log'] = \
'\n'.join(report_data[phase]['loglines'])
errors, warnings = parse_log_events(report_data[phase]['loglines'])
nerrors = len(errors)
if phase == 'configure' and nerrors > 0:
report_data[phase]['status'] = 1
if phase == 'build':
# Convert log output from ASCII to Unicode and escape for XML.
def clean_log_event(event):
event = vars(event)
event['text'] = xml.sax.saxutils.escape(event['text'])
event['pre_context'] = xml.sax.saxutils.escape('\n'.join(
event['pre_context']))
event['post_context'] = xml.sax.saxutils.escape('\n'.join(
event['post_context']))
# source_file and source_line_no are either strings or
# the tuple (None,). Distinguish between these two cases.
if event['source_file'][0] is None:
event['source_file'] = ''
event['source_line_no'] = ''
else:
event['source_file'] = xml.sax.saxutils.escape(
event['source_file'])
return event
report_data[phase]['errors'] = []
report_data[phase]['warnings'] = []
for error in errors:
report_data[phase]['errors'].append(clean_log_event(error))
for warning in warnings:
report_data[phase]['warnings'].append(
clean_log_event(warning))
if phase == 'update':
report_data[phase]['revision'] = self.revision
# Write the report.
report_name = phase.capitalize() + ".xml"
phase_report = os.path.join(filename, report_name)
with codecs.open(phase_report, 'w', 'utf-8') as f:
env = spack.tengine.make_environment()
if phase != 'update':
# Update.xml stores site information differently
# than the rest of the CTest XML files.
site_template = os.path.join(self.template_dir, 'Site.xml')
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = os.path.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
self.upload(phase_report)
self.print_cdash_link()
def test_report_for_package(self, directory_name, package, duration):
if 'stdout' not in package:
# Skip reporting on packages that did not generate any output.
return
self.current_package_name = package['name']
self.buildname = "{0} - {1}".format(
self.base_buildname, package['name'])
report_data = self.initialize_report(directory_name)
for phase in ('test', 'update'):
report_data[phase] = {}
report_data[phase]['loglines'] = []
report_data[phase]['status'] = 0
report_data[phase]['endtime'] = self.endtime
# Track the phases we perform so we know what reports to create.
# We always report the update step because this is how we tell CDash
# what revision of Spack we are using.
phases_encountered = ['test', 'update']
# Generate a report for this package.
# The first line just says "Testing package name-hash"
report_data['test']['loglines'].append(
text_type("{0} output for {1}:".format(
'test', package['name'])))
for line in package['stdout'].splitlines()[1:]:
report_data['test']['loglines'].append(
xml.sax.saxutils.escape(line))
self.starttime = self.endtime - duration
for phase in phases_encountered:
report_data[phase]['starttime'] = self.starttime
report_data[phase]['log'] = \
'\n'.join(report_data[phase]['loglines'])
errors, warnings = parse_log_events(report_data[phase]['loglines'])
# Cap the number of errors and warnings at 50 each.
errors = errors[0:49]
warnings = warnings[0:49]
if phase == 'test':
# Convert log output from ASCII to Unicode and escape for XML.
def clean_log_event(event):
event = vars(event)
event['text'] = xml.sax.saxutils.escape(event['text'])
event['pre_context'] = xml.sax.saxutils.escape(
'\n'.join(event['pre_context']))
event['post_context'] = xml.sax.saxutils.escape(
'\n'.join(event['post_context']))
# source_file and source_line_no are either strings or
# the tuple (None,). Distinguish between these two cases.
if event['source_file'][0] is None:
event['source_file'] = ''
event['source_line_no'] = ''
else:
event['source_file'] = xml.sax.saxutils.escape(
event['source_file'])
return event
# Convert errors to warnings if the package reported success.
if package['result'] == 'success':
warnings = errors + warnings
errors = []
report_data[phase]['errors'] = []
report_data[phase]['warnings'] = []
for error in errors:
report_data[phase]['errors'].append(clean_log_event(error))
for warning in warnings:
report_data[phase]['warnings'].append(
clean_log_event(warning))
if phase == 'update':
report_data[phase]['revision'] = self.revision
# Write the report.
report_name = phase.capitalize() + ".xml"
report_file_name = package['name'] + "_" + report_name
phase_report = os.path.join(directory_name, report_file_name)
with codecs.open(phase_report, 'w', 'utf-8') as f:
env = spack.tengine.make_environment()
if phase != 'update':
# Update.xml stores site information differently
# than the rest of the CTest XML files.
site_template = os.path.join(self.template_dir, 'Site.xml')
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = os.path.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
self.upload(phase_report)
def build_report_for_package(self, directory_name, package, duration):
if 'stdout' not in package:
# Skip reporting on packages that did not generate any output.
return
self.current_package_name = package['name']
if self.multiple_packages:
self.buildname = "{0} - {1}".format(
self.base_buildname, package['name'])
else:
self.buildname = self.base_buildname
report_data = self.initialize_report(directory_name)
for phase in cdash_phases:
report_data[phase] = {}
report_data[phase]['loglines'] = []
report_data[phase]['status'] = 0
report_data[phase]['endtime'] = self.endtime
# Track the phases we perform so we know what reports to create.
# We always report the update step because this is how we tell CDash
# what revision of Spack we are using.
phases_encountered = ['update']
# Generate a report for this package.
current_phase = ''
cdash_phase = ''
for line in package['stdout'].splitlines():
match = None
if line.find("Executing phase: '") != -1:
match = self.phase_regexp.search(line)
if match:
current_phase = match.group(1)
if current_phase not in map_phases_to_cdash:
current_phase = ''
continue
cdash_phase = \
map_phases_to_cdash[current_phase]
if cdash_phase not in phases_encountered:
phases_encountered.append(cdash_phase)
report_data[cdash_phase]['loglines'].append(
text_type("{0} output for {1}:".format(
cdash_phase, package['name'])))
elif cdash_phase:
report_data[cdash_phase]['loglines'].append(
xml.sax.saxutils.escape(line))
# Move the build phase to the front of the list if it occurred.
# This supports older versions of CDash that expect this phase
# to be reported before all others.
if "build" in phases_encountered:
build_pos = phases_encountered.index("build")
phases_encountered.insert(0, phases_encountered.pop(build_pos))
self.starttime = self.endtime - duration
for phase in phases_encountered:
report_data[phase]['starttime'] = self.starttime
report_data[phase]['log'] = \
'\n'.join(report_data[phase]['loglines'])
errors, warnings = parse_log_events(report_data[phase]['loglines'])
# Cap the number of errors and warnings at 50 each.
errors = errors[:50]
warnings = warnings[:50]
nerrors = len(errors)
if phase == 'configure' and nerrors > 0:
report_data[phase]['status'] = 1
self.success = False
if phase == 'build':
# Convert log output from ASCII to Unicode and escape for XML.
def clean_log_event(event):
event = vars(event)
event['text'] = xml.sax.saxutils.escape(event['text'])
event['pre_context'] = xml.sax.saxutils.escape(
'\n'.join(event['pre_context']))
event['post_context'] = xml.sax.saxutils.escape(
'\n'.join(event['post_context']))
# source_file and source_line_no are either strings or
# the tuple (None,). Distinguish between these two cases.
if event['source_file'][0] is None:
event['source_file'] = ''
event['source_line_no'] = ''
else:
event['source_file'] = xml.sax.saxutils.escape(
event['source_file'])
return event
# Convert errors to warnings if the package reported success.
if package['result'] == 'success':
warnings = errors + warnings
errors = []
else:
self.success = False
report_data[phase]['errors'] = []
report_data[phase]['warnings'] = []
for error in errors:
report_data[phase]['errors'].append(clean_log_event(error))
for warning in warnings:
report_data[phase]['warnings'].append(
clean_log_event(warning))
if phase == 'update':
report_data[phase]['revision'] = self.revision
# Write the report.
report_name = phase.capitalize() + ".xml"
if self.multiple_packages:
report_file_name = package['name'] + "_" + report_name
else:
report_file_name = report_name
phase_report = os.path.join(directory_name, report_file_name)
with codecs.open(phase_report, 'w', 'utf-8') as f:
env = spack.tengine.make_environment()
if phase != 'update':
# Update.xml stores site information differently
# than the rest of the CTest XML files.
site_template = os.path.join(self.template_dir, 'Site.xml')
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = os.path.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
self.upload(phase_report)
def build_report(self, filename, report_data):
self.initialize_report(filename, report_data)
for phase in cdash_phases:
report_data[phase] = {}
report_data[phase]['log'] = ""
report_data[phase]['status'] = 0
report_data[phase]['starttime'] = self.starttime
report_data[phase]['endtime'] = self.starttime
# Track the phases we perform so we know what reports to create.
phases_encountered = []
# Parse output phase-by-phase.
phase_regexp = re.compile(r"Executing phase: '(.*)'")
for spec in report_data['specs']:
for package in spec['packages']:
if 'stdout' in package:
current_phase = ''
for line in package['stdout'].splitlines():
match = phase_regexp.search(line)
if match:
current_phase = match.group(1)
if current_phase not in map_phases_to_cdash:
current_phase = ''
continue
beginning_of_phase = True
else:
if beginning_of_phase:
cdash_phase = \
map_phases_to_cdash[current_phase]
if cdash_phase not in phases_encountered:
phases_encountered.append(cdash_phase)
report_data[cdash_phase]['log'] += \
text_type("{0} output for {1}:\n".format(
cdash_phase, package['name']))
beginning_of_phase = False
report_data[cdash_phase]['log'] += \
xml.sax.saxutils.escape(line) + "\n"
for phase in phases_encountered:
errors, warnings = parse_log_events(
report_data[phase]['log'].splitlines())
nerrors = len(errors)
if phase == 'configure' and nerrors > 0:
report_data[phase]['status'] = 1
if phase == 'build':
# Convert log output from ASCII to Unicode and escape for XML.
def clean_log_event(event):
event = vars(event)
event['text'] = xml.sax.saxutils.escape(event['text'])
event['pre_context'] = xml.sax.saxutils.escape('\n'.join(
event['pre_context']))
event['post_context'] = xml.sax.saxutils.escape('\n'.join(
event['post_context']))
# source_file and source_line_no are either strings or
# the tuple (None,). Distinguish between these two cases.
if event['source_file'][0] is None:
event['source_file'] = ''
event['source_line_no'] = ''
else:
event['source_file'] = xml.sax.saxutils.escape(
event['source_file'])
return event
report_data[phase]['errors'] = []
report_data[phase]['warnings'] = []
for error in errors:
report_data[phase]['errors'].append(clean_log_event(error))
for warning in warnings:
report_data[phase]['warnings'].append(
clean_log_event(warning))
# Write the report.
report_name = phase.capitalize() + ".xml"
phase_report = os.path.join(filename, report_name)
with codecs.open(phase_report, 'w', 'utf-8') as f:
env = spack.tengine.make_environment()
site_template = os.path.join(self.template_dir, 'Site.xml')
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = os.path.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
self.upload(phase_report)
def build_report(self, filename, report_data):
self.initialize_report(filename, report_data)
for phase in cdash_phases:
report_data[phase] = {}
report_data[phase]['log'] = ""
report_data[phase]['status'] = 0
report_data[phase]['starttime'] = self.starttime
report_data[phase]['endtime'] = self.starttime
# Track the phases we perform so we know what reports to create.
phases_encountered = []
# Parse output phase-by-phase.
phase_regexp = re.compile(r"Executing phase: '(.*)'")
for spec in report_data['specs']:
for package in spec['packages']:
if 'stdout' in package:
current_phase = ''
for line in package['stdout'].splitlines():
match = phase_regexp.search(line)
if match:
current_phase = match.group(1)
if current_phase not in map_phases_to_cdash:
current_phase = ''
continue
beginning_of_phase = True
else:
if beginning_of_phase:
cdash_phase = \
map_phases_to_cdash[current_phase]
if cdash_phase not in phases_encountered:
phases_encountered.append(cdash_phase)
report_data[cdash_phase]['log'] += \
text_type("{0} output for {1}:\n".format(
cdash_phase, package['name']))
beginning_of_phase = False
report_data[cdash_phase]['log'] += \
xml.sax.saxutils.escape(line) + "\n"
for phase in phases_encountered:
errors, warnings = parse_log_events(
report_data[phase]['log'].splitlines())
nerrors = len(errors)
if phase == 'configure' and nerrors > 0:
report_data[phase]['status'] = 1
if phase == 'build':
# Convert log output from ASCII to Unicode and escape for XML.
def clean_log_event(event):
event = vars(event)
event['text'] = xml.sax.saxutils.escape(event['text'])
event['pre_context'] = xml.sax.saxutils.escape(
'\n'.join(event['pre_context']))
event['post_context'] = xml.sax.saxutils.escape(
'\n'.join(event['post_context']))
# source_file and source_line_no are either strings or
# the tuple (None,). Distinguish between these two cases.
if event['source_file'][0] is None:
event['source_file'] = ''
event['source_line_no'] = ''
else:
event['source_file'] = xml.sax.saxutils.escape(
event['source_file'])
return event
report_data[phase]['errors'] = []
report_data[phase]['warnings'] = []
for error in errors:
report_data[phase]['errors'].append(clean_log_event(error))
for warning in warnings:
report_data[phase]['warnings'].append(
clean_log_event(warning))
# Write the report.
report_name = phase.capitalize() + ".xml"
phase_report = os.path.join(filename, report_name)
with codecs.open(phase_report, 'w', 'utf-8') as f:
env = spack.tengine.make_environment()
site_template = os.path.join(self.template_dir, 'Site.xml')
t = env.get_template(site_template)
f.write(t.render(report_data))
phase_template = os.path.join(self.template_dir, report_name)
t = env.get_template(phase_template)
f.write(t.render(report_data))
self.upload(phase_report)
