def command_started(self, package, cmd, location):
if package not in self.__command_log:
raise RuntimeError("Command started received for package '{0}' before package job started: '{1}' in '{2}'"
.format(package, cmd.pretty, location))
msg = clr("[{package}] ==> '{cmd.cmd_str}' in '{location}'").format(**locals())
self.__command_log[package].start_command(cmd, msg)
if not self.quiet and self.interleave:
wide_log(msg)
def command_started(self, package, cmd, location):
if package not in self.__command_log:
raise RuntimeError(
"Command started received for package '{0}' before package job started: '{1}' in '{2}'"
.format(package, cmd.pretty, location))
msg = clr("[{package}] ==> '{cmd.cmd_str}' in '{location}'").format(
**locals())
self.__command_log[package].start_command(cmd, msg)
if not self.quiet and self.interleave:
wide_log(msg)
def print_items_in_columns(items_in, number_of_columns):
number_of_items_in_line = 0
line_template = "{}" * number_of_columns
line_items = []
items = list(items_in)
while items:
line_items.append(items.pop(0))
number_of_items_in_line += 1
if number_of_items_in_line == number_of_columns:
wide_log(line_template.format(*line_items))
line_items = []
number_of_items_in_line = 0
if line_items:
wide_log(("{}" * len(line_items)).format(*line_items))
def command_finished(self, package, cmd, location, retcode):
if package not in self.__command_log:
raise RuntimeError(
"Command finished received for package '{0}' before package job started: '{1}' in '{2}' returned '{3}'"
.format(package, cmd.pretty, location, retcode))
if self.__command_log[package].current_cmd is None:
raise RuntimeError(
"Command finished received for package '{0}' before command started: '{1}' in '{2}' returned '{3}'"
.format(package, cmd.pretty, location, retcode))
msg = clr("[{package}] <== '{cmd.cmd_str}' finished with return code '{retcode}'").format(**locals())
self.__command_log[package].finish_command(msg)
if not self.quiet and not self.interleave:
self.__command_log[package].print_last_command_log()
elif not self.quiet:
wide_log(msg)
def command_finished(self, package, cmd, location, retcode):
if package not in self.__command_log:
raise RuntimeError(
"Command finished received for package '{0}' before package job started: '{1}' in '{2}' returned '{3}'"
.format(package, cmd.pretty, location, retcode))
if self.__command_log[package].current_cmd is None:
raise RuntimeError(
"Command finished received for package '{0}' before command started: '{1}' in '{2}' returned '{3}'"
.format(package, cmd.pretty, location, retcode))
msg = clr(
"[{package}] <== '{cmd.cmd_str}' finished with return code '{retcode}'"
).format(**locals())
self.__command_log[package].finish_command(msg)
if not self.quiet and not self.interleave:
self.__command_log[package].print_last_command_log()
elif not self.quiet:
wide_log(msg)
def command_log(self, package, msg):
if package not in self.__command_log:
raise RuntimeError("Command log received for package '{0}' before package job started: '{1}'"
.format(package, msg))
if self.__command_log[package].current_cmd is None:
raise RuntimeError("Command log received for package '{0}' before command started: '{1}'"
.format(package, msg))
self.__command_log[package].append(msg)
if not self.color:
msg = remove_ansi_escape(msg)
if not self.quiet and self.interleave:
msg = msg.rstrip(ansi('reset'))
msg = msg.rstrip()
if self.interleave and self.prefix_output:
wide_log(clr("[{package}] {msg}").format(**locals()))
else:
wide_log(msg)
def handle_make_arguments(input_make_args,
force_single_threaded_when_running_tests=False):
"""Special handling for make arguments.
If force_single_threaded_when_running_tests is True, jobs flags are
replaced with -j1, because tests cannot handle parallelization.
If no job flags are present and there are none in the MAKEFLAGS environment
variable, then make flags are set to the cpu_count, e.g. -j4 -l4.
:param input_make_args: list of make arguments to be handled
:type input_make_args: list
:param force_single_threaded_when_running_tests: self explanatory
:type force_single_threaded_when_running_tests: bool
:returns: copied list of make arguments, potentially with some modifications
:rtype: list
"""
make_args = list(input_make_args)
if force_single_threaded_when_running_tests:
# force single threaded execution when running test since rostest does not support multiple parallel runs
run_tests = [a for a in make_args if a.startswith('run_tests')]
if run_tests:
wide_log('Forcing "-j1" for running unit tests.')
make_args.append('-j1')
# If no -j/--jobs/-l/--load-average flags are in make_args
if not extract_jobs_flags(' '.join(make_args)):
# If -j/--jobs/-l/--load-average are in MAKEFLAGS
if 'MAKEFLAGS' in os.environ and extract_jobs_flags(
os.environ['MAKEFLAGS']):
# Do not extend make arguments, let MAKEFLAGS set things
pass
else:
# Else extend the make_arguments to include some jobs flags
# Use the number of CPU cores
try:
jobs = cpu_count()
make_args.append('-j{0}'.format(jobs))
make_args.append('-l{0}'.format(jobs))
except NotImplementedError:
# If the number of cores cannot be determined, do not extend args
pass
return make_args
def handle_make_arguments(
input_make_args,
force_single_threaded_when_running_tests=False):
"""Special handling for make arguments.
If force_single_threaded_when_running_tests is True, jobs flags are
replaced with -j1, because tests cannot handle parallelization.
If no job flags are present and there are none in the MAKEFLAGS environment
variable, then make flags are set to the cpu_count, e.g. -j4 -l4.
:param input_make_args: list of make arguments to be handled
:type input_make_args: list
:param force_single_threaded_when_running_tests: self explanatory
:type force_single_threaded_when_running_tests: bool
:returns: copied list of make arguments, potentially with some modifications
:rtype: list
"""
make_args = list(input_make_args)
# Get the values for the jobs flags which may be in the make args
jobs_dict = extract_jobs_flags_values(' '.join(make_args))
jobs_args = extract_jobs_flags(' '.join(make_args))
if len(jobs_args) > 0:
# Remove jobs flags from cli args if they're present
make_args = re.sub(' '.join(jobs_args), '', ' '.join(make_args)).split()
if force_single_threaded_when_running_tests:
# force single threaded execution when running test since rostest does not support multiple parallel runs
run_tests = [a for a in make_args if a.startswith('run_tests')]
if run_tests:
wide_log('Forcing "-j1" for running unit tests.')
jobs_dict['jobs'] = 1
if job_server.gnu_make_enabled():
make_args.extend(job_server.gnu_make_args())
else:
if 'jobs' in jobs_dict:
make_args.append('-j{0}'.format(jobs_dict['jobs']))
if 'load-average' in jobs_dict:
make_args.append('-l{0}'.format(jobs_dict['load-average']))
return make_args
def print_items_in_columns(items, n_cols):
"""Print items in columns
:param items: list of tuples (identifier, template) where the template takes `jid` as a parameter
:param n_cols: number of columns
"""
# Format all the items
formatted_items = [t.format(jid=j) for j, t in items]
# Compute the number of rows
n_items = len(items)
if n_items <= n_cols:
n_cols = 1
n_rows = int(math.ceil(n_items / float(n_cols)))
# Print each row
for r in range(n_rows):
wide_log(''.join(formatted_items[(r * n_cols):((r + 1) * n_cols)]))
def print_items_in_columns(items, n_cols):
"""Print items in columns
:param items: list of tuples (identifier, template) where the template takes `jid` as a parameter
:param n_cols: number of columns
"""
# Format all the items
formatted_items = [t.format(jid=j) for j, t in items]
# Compute the number of rows
n_items = len(items)
if n_items <= n_cols:
n_cols = 1
n_rows = int(math.ceil(n_items / float(n_cols)))
# Print each row
for r in range(n_rows):
wide_log(''.join(formatted_items[(r * n_cols):((r + 1) * n_cols)]))
def command_log(self, package, msg):
if package not in self.__command_log:
raise RuntimeError(
"Command log received for package '{0}' before package job started: '{1}'"
.format(package, msg))
if self.__command_log[package].current_cmd is None:
raise RuntimeError(
"Command log received for package '{0}' before command started: '{1}'"
.format(package, msg))
self.__command_log[package].append(msg)
if not self.color:
msg = remove_ansi_escape(msg)
if not self.quiet and self.interleave:
msg = msg.rstrip(ansi('reset'))
msg = msg.rstrip()
if self.interleave and self.prefix_output:
wide_log(clr("[{package}] {msg}").format(**locals()))
else:
wide_log(msg)
def handle_make_arguments(input_make_args,
force_single_threaded_when_running_tests=False):
"""Special handling for make arguments.
If force_single_threaded_when_running_tests is True, jobs flags are
replaced with -j1, because tests cannot handle parallelization.
If no job flags are present and there are none in the MAKEFLAGS environment
variable, then make flags are set to the cpu_count, e.g. -j4 -l4.
:param input_make_args: list of make arguments to be handled
:type input_make_args: list
:param force_single_threaded_when_running_tests: self explanatory
:type force_single_threaded_when_running_tests: bool
:returns: copied list of make arguments, potentially with some modifications
:rtype: list
"""
make_args = list(input_make_args)
# Get the values for the jobs flags which may be in the make args
jobs_dict = extract_jobs_flags_values(' '.join(make_args))
jobs_args = extract_jobs_flags(' '.join(make_args))
if len(jobs_args) > 0:
# Remove jobs flags from cli args if they're present
make_args = re.sub(' '.join(jobs_args), '',
' '.join(make_args)).split()
if force_single_threaded_when_running_tests:
# force single threaded execution when running test since rostest does not support multiple parallel runs
run_tests = [a for a in make_args if a.startswith('run_tests')]
if run_tests:
wide_log('Forcing "-j1" for running unit tests.')
jobs_dict['jobs'] = 1
if job_server.gnu_make_enabled():
make_args.extend(job_server.gnu_make_args())
else:
if 'jobs' in jobs_dict:
make_args.append('-j{0}'.format(jobs_dict['jobs']))
if 'load-average' in jobs_dict:
make_args.append('-l{0}'.format(jobs_dict['load-average']))
return make_args
def handle_make_arguments(input_make_args, force_single_threaded_when_running_tests=False):
"""Special handling for make arguments.
If force_single_threaded_when_running_tests is True, jobs flags are
replaced with -j1, because tests cannot handle parallelization.
If no job flags are present and there are none in the MAKEFLAGS environment
variable, then make flags are set to the cpu_count, e.g. -j4 -l4.
:param input_make_args: list of make arguments to be handled
:type input_make_args: list
:param force_single_threaded_when_running_tests: self explanatory
:type force_single_threaded_when_running_tests: bool
:returns: copied list of make arguments, potentially with some modifications
:rtype: list
"""
make_args = list(input_make_args)
if force_single_threaded_when_running_tests:
# force single threaded execution when running test since rostest does not support multiple parallel runs
run_tests = [a for a in make_args if a.startswith('run_tests')]
if run_tests:
wide_log('Forcing "-j1" for running unit tests.')
make_args.append('-j1')
# If no -j/--jobs/-l/--load-average flags are in make_args
if not extract_jobs_flags(' '.join(make_args)):
# If -j/--jobs/-l/--load-average are in MAKEFLAGS
if 'MAKEFLAGS' in os.environ and extract_jobs_flags(os.environ['MAKEFLAGS']):
# Do not extend make arguments, let MAKEFLAGS set things
pass
else:
# Else extend the make_arguments to include some jobs flags
# Use the number of CPU cores
try:
jobs = cpu_count()
make_args.append('-j{0}'.format(jobs))
make_args.append('-l{0}'.format(jobs))
except NotImplementedError:
# If the number of cores cannot be determined, do not extend args
pass
return make_args
def print_last_command_log(self):
wide_log("")
command_output = ""
with open(self.log_path, 'r') as f:
line_number = 0
for line in f:
if line_number is not None and line_number != self.last_command_line:
line_number += 1
continue
line_number = None
command_output += line
if not self.color:
wide_log(remove_ansi_escape(command_output), end='')
else:
wide_log(command_output, end='')
wide_log("")
def print_last_command_log(self):
wide_log("")
command_output = ""
with open(self.log_path, 'r') as f:
line_number = 0
for line in f:
if line_number is not None and line_number != self.last_command_line:
line_number += 1
continue
line_number = None
command_output += line
if not self.color:
wide_log(remove_ansi_escape(command_output), end='')
else:
wide_log(command_output, end='')
wide_log("")
def print_error_summary(errors, no_notify, log_dir):
wide_log(clr("[build] There were '" + str(len(errors)) + "' @!@{rf}errors@|:"))
if not no_notify:
notify("Build Failed", "there were {0} errors".format(len(errors)))
for error in errors:
if error.event_type == 'exit':
wide_log("""\
Executor '{exec_id}' had an unhandled exception while processing package '{package}':
{data[exc]}""".format(exec_id=error.executor_id + 1, **error.__dict__))
else:
wide_log(clr("""
@{rf}Failed@| to build package '@{cf}{package}@|' because the following command:
@!@{kf}# Command to reproduce:@|
cd {location} && {cmd.cmd_str}; cd -
@!@{kf}# Path to log:@|
cat {log_dir}
@{rf}Exited@| with return code: @!{retcode}@|""").format(package=error.package,
log_dir=os.path.join(log_dir, error.package + '.log'),
**error.data))
def determine_packages_to_be_built(packages, context, workspace_packages):
"""Returns list of packages which should be built, and those package's deps.
:param packages: list of packages to be built, if None all packages are built
:type packages: list
:param context: Workspace context
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:returns: tuple of packages to be built and those package's deps
:rtype: tuple
"""
start = time.time()
# If there are no packages raise
if not workspace_packages:
log("[build] No packages were found in the source space '{0}'".format(
context.source_space_abs))
else:
wide_log("[build] Found '{0}' packages in {1}.".format(
len(workspace_packages), format_time_delta(time.time() - start)))
# Order the packages by topology
ordered_packages = topological_order_packages(workspace_packages)
# Set the packages in the workspace for the context
context.packages = ordered_packages
# Determin the packages which should be built
packages_to_be_built = []
packages_to_be_built_deps = []
# Determine the packages to be built
if packages:
# First assert all of the packages given are in the workspace
workspace_package_names = dict([(pkg.name, (path, pkg))
for path, pkg in ordered_packages])
for package in packages:
if package not in workspace_package_names:
sys.exit("[build] Given package '{0}' is not in the workspace".
format(package))
# If metapackage, include run depends which are in the workspace
package_obj = workspace_package_names[package][1]
if 'metapackage' in [e.tagname for e in package_obj.exports]:
for rdep in package_obj.run_depends:
if rdep.name in workspace_package_names:
packages.append(rdep.name)
# Limit the packages to be built to just the provided packages
for pkg_path, package in ordered_packages:
if package.name in packages:
packages_to_be_built.append((pkg_path, package))
# Get the recursive dependencies for each of these packages
pkg_build_deps = get_cached_recursive_build_depends_in_workspace(
package, ordered_packages)
packages_to_be_built_deps.extend(pkg_build_deps)
pkg_run_deps = get_cached_recursive_run_depends_in_workspace(
package, ordered_packages)
packages_to_be_built_deps.extend(pkg_run_deps)
else:
# Only use whitelist when no other packages are specified
if len(context.whitelist) > 0:
packages_to_be_built = [
p for p in ordered_packages if (p[1].name in context.whitelist)
]
else:
packages_to_be_built = ordered_packages
# Filter packages with blacklist
if len(context.blacklist) > 0:
packages_to_be_built = [
(path, pkg) for path, pkg in packages_to_be_built
if (pkg.name not in context.blacklist or pkg.name in packages)
]
packages_to_be_built_deps = [
(path, pkg) for path, pkg in packages_to_be_built_deps
if (pkg.name not in context.blacklist or pkg.name in packages)
]
ordered_packages = ordered_packages
return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def job_failed(self, package, time):
self.__command_log[package].close()
del self.__command_log[package]
msg = clr("Failed <== {package:<") + str(
self.max_package_name_length) + clr("} [ {time} ]")
wide_log(msg.format(**locals()))
def run(self):
queued_jobs = []
active_jobs = []
completed_jobs = {}
failed_jobs = []
warned_jobs = []
cumulative_times = dict()
start_times = dict()
active_stages = dict()
start_time = self.pre_start_time or time.time()
last_update_time = time.time()
# If the status rate is too low, just disable it
if self.active_status_rate < 1E-3:
self.show_active_status = False
else:
update_duration = 1.0 / self.active_status_rate
# Disable the wide log padding if the status is disabled
if not self.show_active_status:
disable_wide_log()
while True:
# Check if we should stop
if not self.keep_running:
wide_log(clr('[{}] An internal error occurred!').format(self.label))
return
# Write a continuously-updated status line
if self.show_active_status:
# Try to get an event from the queue (non-blocking)
try:
event = self.event_queue.get(False)
except Empty:
# Determine if the status should be shown based on the desired
# status rate
elapsed_time = time.time() - last_update_time
show_status_now = elapsed_time > update_duration
if show_status_now:
# Print live status (overwrites last line)
status_line = clr('[{} {} s] [{}/{} complete] [{}/{} jobs] [{} queued]').format(
self.label,
format_time_delta_short(time.time() - start_time),
len(completed_jobs),
len(self.jobs),
job_server.running_jobs(),
job_server.max_jobs(),
len(queued_jobs) + len(active_jobs) - len(active_stages)
)
# Show failed jobs
if len(failed_jobs) > 0:
status_line += clr(' [@!@{rf}{}@| @{rf}failed@|]').format(len(failed_jobs))
# Check load / mem
if not job_server.load_ok():
status_line += clr(' [@!@{rf}High Load@|]')
if not job_server.mem_ok():
status_line += clr(' [@!@{rf}Low Memory@|]')
# Add active jobs
if len(active_jobs) == 0:
status_line += clr(' @/@!@{kf}Waiting for jobs...@|')
else:
active_labels = []
for j, (s, t, p) in active_stages.items():
d = format_time_delta_short(cumulative_times[j] + time.time() - t)
if p == '':
active_labels.append(clr('[{}:{} - {}]').format(j, s, d))
else:
active_labels.append(clr('[{}:{} ({}%) - {}]').format(j, s, p, d))
status_line += ' ' + ' '.join(active_labels)
# Print the status line
# wide_log(status_line)
wide_log(status_line, rhs='', end='\r')
sys.stdout.flush()
# Store this update time
last_update_time = time.time()
else:
time.sleep(max(0.0, min(update_duration - elapsed_time, 0.01)))
# Only continue when no event was received
continue
else:
# Try to get an event from the queue (blocking)
try:
event = self.event_queue.get(True)
except Empty:
break
# A `None` event is a signal to terminate
if event is None:
break
# Handle the received events
eid = event.event_id
if 'JOB_STATUS' == eid:
queued_jobs = event.data['queued']
active_jobs = event.data['active']
completed_jobs = event.data['completed']
# Check if all jobs have finished in some way
if all([len(event.data[t]) == 0 for t in ['pending', 'queued', 'active']]):
break
elif 'STARTED_JOB' == eid:
cumulative_times[event.data['job_id']] = 0.0
wide_log(clr('Starting >>> {:<{}}').format(
event.data['job_id'],
self.max_jid_length))
elif 'FINISHED_JOB' == eid:
duration = format_time_delta(cumulative_times[event.data['job_id']])
if event.data['succeeded']:
wide_log(clr('Finished <<< {:<{}} [ {} ]').format(
event.data['job_id'],
self.max_jid_length,
duration))
else:
failed_jobs.append(event.data['job_id'])
wide_log(clr('Failed <<< {:<{}} [ {} ]').format(
event.data['job_id'],
self.max_jid_length,
duration))
elif 'ABANDONED_JOB' == eid:
# Create a human-readable reason string
if 'DEP_FAILED' == event.data['reason']:
direct = event.data['dep_job_id'] == event.data['direct_dep_job_id']
if direct:
reason = clr('Depends on failed job {}').format(event.data['dep_job_id'])
else:
reason = clr('Depends on failed job {} via {}').format(
event.data['dep_job_id'],
event.data['direct_dep_job_id'])
elif 'PEER_FAILED' == event.data['reason']:
reason = clr('Unrelated job failed')
elif 'MISSING_DEPS' == event.data['reason']:
reason = clr('Depends on unknown jobs: {}').format(
', '.join([clr('@!{}@|').format(jid) for jid in event.data['dep_ids']]))
wide_log(clr('Abandoned <<< {:<{}} [ {} ]').format(
event.data['job_id'],
self.max_jid_length,
reason))
elif 'STARTED_STAGE' == eid:
active_stages[event.data['job_id']] = [event.data['stage_label'], event.time, '']
start_times[event.data['job_id']] = event.time
if self.show_stage_events:
wide_log(clr('Starting >> {}:{}').format(
event.data['job_id'],
event.data['stage_label']))
elif 'STAGE_PROGRESS' == eid:
active_stages[event.data['job_id']][2] = event.data['percent']
elif 'SUBPROCESS' == eid:
if self.show_stage_events:
wide_log(clr('Subprocess > {}:{} `{}`').format(
event.data['job_id'],
event.data['stage_label'],
event.data['stage_repro']))
elif 'FINISHED_STAGE' == eid:
# Get the stage duration
duration = event.time - start_times[event.data['job_id']]
cumulative_times[event.data['job_id']] += duration
# This is no longer the active stage for this job
del active_stages[event.data['job_id']]
header_border = None
header_border_file = sys.stdout
header_title = None
header_title_file = sys.stdout
lines = []
footer_title = None
footer_title_file = sys.stdout
footer_border = None
footer_border_file = sys.stdout
# Generate headers / borders for output
if event.data['succeeded']:
footer_title = clr(
'Finished << {}:{}').format(
event.data['job_id'],
event.data['stage_label'])
if len(event.data['stderr']) > 0:
# Mark that this job warned about something
if event.data['job_id'] not in warned_jobs:
warned_jobs.append(event.data['job_id'])
# Output contains warnings
header_border = clr('@!@{yf}' + '_' * (terminal_width() - 1) + '@|')
header_border_file = sys.stderr
header_title = clr(
'Warnings << {}:{} {}').format(
event.data['job_id'],
event.data['stage_label'],
event.data['logfile_filename'])
header_title_file = sys.stderr
footer_border = clr('@{yf}' + '.' * (terminal_width() - 1) + '@|')
footer_border_file = sys.stderr
else:
# Normal output, no warnings
header_title = clr(
'Output << {}:{} {}').format(
event.data['job_id'],
event.data['stage_label'],
event.data['logfile_filename'])
# Don't print footer title
if not self.show_stage_events:
footer_title = None
else:
# Output contains errors
header_border = clr('@!@{rf}' + '_' * (terminal_width() - 1) + '@|')
header_border_file = sys.stderr
header_title = clr(
'Errors << {}:{} {}').format(
event.data['job_id'],
event.data['stage_label'],
event.data['logfile_filename'])
header_title_file = sys.stderr
footer_border = clr('@{rf}' + '.' * (terminal_width() - 1) + '@|')
footer_border_file = sys.stderr
footer_title = clr(
'Failed << {}:{:<{}} [ Exited with code {} ]').format(
event.data['job_id'],
event.data['stage_label'],
max(0, self.max_jid_length - len(event.data['job_id'])),
event.data['retcode'])
footer_title_file = sys.stderr
lines_target = sys.stdout
if self.show_buffered_stdout:
if len(event.data['interleaved']) > 0:
lines = [
l
for l in event.data['interleaved'].splitlines(True)
if (self.show_compact_io is False or len(l.strip()) > 0)
]
else:
header_border = None
header_title = None
footer_border = None
elif self.show_buffered_stderr:
if len(event.data['stderr']) > 0:
lines = [
l
for l in event.data['stderr'].splitlines(True)
if (self.show_compact_io is False or len(l.strip()) > 0)
]
lines_target = sys.stderr
else:
header_border = None
header_title = None
footer_border = None
if len(lines) > 0:
if self.show_repro_cmd:
if event.data['repro'] is not None:
lines.append(clr('@!@{kf}{}@|\n').format(event.data['repro']))
# Print the output
if header_border:
wide_log(header_border, file=header_border_file)
if header_title:
wide_log(header_title, file=header_title_file)
if len(lines) > 0:
wide_log(''.join(lines), end='\r', file=lines_target)
if footer_border:
wide_log(footer_border, file=footer_border_file)
if footer_title:
wide_log(footer_title, file=footer_title_file)
elif 'STDERR' == eid:
if self.show_live_stderr and len(event.data['data']) > 0:
wide_log(self.format_interleaved_lines(event.data), end='\r', file=sys.stderr)
elif 'STDOUT' == eid:
if self.show_live_stdout and len(event.data['data']) > 0:
wide_log(self.format_interleaved_lines(event.data), end='\r')
elif 'MESSAGE' == eid:
wide_log(event.data['msg'])
# Print the full summary
if self.show_full_summary:
self.print_exec_summary(completed_jobs, warned_jobs, failed_jobs)
# Print a compact summary
if self.show_summary or self.show_full_summary:
self.print_compact_summary(completed_jobs, warned_jobs, failed_jobs)
# Print final runtime
wide_log(clr('[{}] Runtime: {} total.').format(
self.label,
format_time_delta(time.time() - start_time)))
def test_workspace(context,
packages=None,
tests=None,
list_tests=False,
start_with=None,
n_jobs=None,
force_color=False,
quiet=False,
interleave_output=False,
no_status=False,
limit_status_rate=10.0,
no_notify=False,
summarize_build=None):
pre_start_time = time.time()
# Get our list of packages based on what's in the source space and our
# command line switches.
packages_to_test = get_packages_to_test(context, packages)
if len(packages_to_test) == 0:
log(fmt('[test] No tests in the available packages.'))
# Get the full list of tests available in those packages, as configured.
packages_tests = get_packages_tests(context, packages_to_test)
print packages_tests
if list_tests:
# Don't build or run, just list available targets.
log(fmt('[test] Tests available in workspace packages:'))
for package, tests in sorted(packages_tests):
log(fmt('[test] * %s' % package.name))
for test in sorted(tests):
log(fmt('[test] - %s' % test))
return 0
else:
jobs = []
# Construct jobs for running tests.
for package, package_tests in packages_tests:
jobs.append(create_package_job(context, package, package_tests))
package_names = [p[0].name for p in packages_tests]
jobs.append(create_results_check_job(context, package_names))
# Queue for communicating status.
event_queue = Queue()
try:
# Spin up status output thread.
status_thread = ConsoleStatusController(
'test', ['package', 'packages'],
jobs,
n_jobs, [pkg.name for _, pkg in context.packages],
[p for p in context.whitelist], [p for p in context.blacklist],
event_queue,
show_notifications=not no_notify,
show_active_status=not no_status,
show_buffered_stdout=not quiet and not interleave_output,
show_buffered_stderr=not interleave_output,
show_live_stdout=interleave_output,
show_live_stderr=interleave_output,
show_stage_events=not quiet,
show_full_summary=(summarize_build is True),
pre_start_time=pre_start_time,
active_status_rate=limit_status_rate)
status_thread.start()
# Block while running N jobs asynchronously
try:
all_succeeded = run_until_complete(
execute_jobs('test',
jobs,
None,
event_queue,
context.log_space_abs,
max_toplevel_jobs=n_jobs))
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
return 0 if all_succeeded else 1
except KeyboardInterrupt:
wide_log("[test] Interrupted by user!")
event_queue.put(None)
return 130 # EOWNERDEAD
def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs):
"""
Print verbose execution summary.
"""
# Calculate the longest jid
max_jid_len = max([len(jid) for jid in self.available_jobs])
templates = {
'successful': clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len),
'warned': clr(" [ @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len),
'failed': clr(" [ @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len),
'ignored': clr(" [ @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len),
'abandoned': clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len),
}
# Calculate the maximum _printed_ length for each template
max_column_len = max([
len(remove_ansi_escape(t.format(jid=("?" * max_jid_len))))
for t in templates.values()
])
# Calculate the number of columns
number_of_columns = (terminal_width() / max_column_len) or 1
# Construct different categories of jobs (jid -> output template)
successfuls = {}
warneds = {}
faileds = {}
ignoreds = {}
abandoneds = {}
non_whitelisted = {}
blacklisted = {}
# Give each package an output template to use
for jid in self.available_jobs:
if jid in self.blacklisted_jobs:
blacklisted[jid] = templates['ignored']
elif jid not in self.jobs:
ignoreds[jid] = templates['ignored']
elif len(self.whitelisted_jobs) > 0 and jid not in self.whitelisted_jobs:
non_whitelisted[jid] = templates['ignored']
elif jid in completed_jobs:
if jid in failed_jobs:
faileds[jid] = templates['failed']
elif jid in warned_jobs:
warneds[jid] = templates['warned']
else:
successfuls[jid] = templates['successful']
else:
abandoneds[jid] = templates['abandoned']
# Combine successfuls and ignoreds, sort by key
if len(successfuls) + len(ignoreds) > 0:
wide_log("")
wide_log(clr("[{}] Successful {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(
sorted(successfuls.items() + ignoreds.items()),
number_of_columns)
else:
wide_log("")
wide_log(clr("[{}] No {} succeeded.").format(self.label, self.jobs_label))
wide_log("")
# Print out whitelisted jobs
if len(non_whitelisted) > 0:
wide_log("")
wide_log(clr("[{}] Non-whitelisted {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(non_whitelisted.items()), number_of_columns)
# Print out blacklisted jobs
if len(blacklisted) > 0:
wide_log("")
wide_log(clr("[{}] Blacklisted {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(blacklisted.items()), number_of_columns)
# Print out jobs that failed
if len(faileds) > 0:
wide_log("")
wide_log(clr("[{}] Failed {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(faileds.items()), number_of_columns)
# Print out jobs that were abandoned
if len(abandoneds) > 0:
wide_log("")
wide_log(clr("[{}] Abandoned {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(abandoneds.items()), number_of_columns)
wide_log("")
def print_compact_summary(self, completed_jobs, warned_jobs, failed_jobs):
"""Print a compact build summary."""
notification_title = ""
notification_msg = []
# Print error summary
if len(completed_jobs) == len(self.jobs) and all(completed_jobs.items()) and len(failed_jobs) == 0:
notification_title = "{} Succeeded".format(self.label.capitalize())
notification_msg.append("All {} {} succeeded!".format(len(self.jobs), self.jobs_label))
wide_log(clr('[{}] Summary: All {} {} succeeded!').format(
self.label,
len(self.jobs),
self.jobs_label))
else:
notification_msg.append("{} of {} {} succeeded.".format(
len([succeeded for jid, succeeded in completed_jobs.items() if succeeded]),
len(self.jobs), self.jobs_label))
wide_log(clr('[{}] Summary: {} of {} {} succeeded.').format(
self.label,
len([succeeded for jid, succeeded in completed_jobs.items() if succeeded]),
len(self.jobs),
self.jobs_label))
# Display number of ignored jobs (jobs which shouldn't have been built)
all_ignored_jobs = [j for j in self.available_jobs if j not in self.jobs]
if len(all_ignored_jobs) == 0:
wide_log(clr('[{}] Ignored: None.').format(
self.label))
else:
notification_msg.append("{} {} were skipped.".format(len(all_ignored_jobs), self.jobs_label))
wide_log(clr('[{}] Ignored: {} {} were skipped or are blacklisted.').format(
self.label,
len(all_ignored_jobs),
self.jobs_label))
# Display number of jobs which produced warnings
if len(warned_jobs) == 0:
wide_log(clr('[{}] Warnings: None.').format(
self.label))
else:
notification_title = "{} Succeeded with Warnings".format(self.label.capitalize())
notification_msg.append("{} {} succeeded with warnings.".format(len(warned_jobs), self.jobs_label))
wide_log(clr('[{}] Warnings: {} {} succeeded with warnings.').format(
self.label,
len(warned_jobs),
self.jobs_label))
# Display number of abandoned jobs
all_abandoned_jobs = [j for j in self.jobs if j not in completed_jobs]
if len(all_abandoned_jobs) == 0:
wide_log(clr('[{}] Abandoned: No {} were abandoned.').format(
self.label,
self.jobs_label))
else:
notification_title = "{} Incomplete".format(self.label.capitalize())
notification_msg.append("{} {} were abandoned.".format(len(all_abandoned_jobs), self.jobs_label))
wide_log(clr('[{}] Abandoned: {} {} were abandoned.').format(
self.label,
len(all_abandoned_jobs),
self.jobs_label))
# Display number of failed jobs
if len(failed_jobs) == 0:
wide_log(clr('[{}] Failed: No {} failed.').format(
self.label,
self.jobs_label))
else:
notification_title = "{} Failed".format(self.label.capitalize())
notification_msg.append("{} {} failed.".format(len(failed_jobs), self.jobs_label))
wide_log(clr('[{}] Failed: {} {} failed.').format(
self.label,
len(failed_jobs),
self.jobs_label))
if self.show_notifications:
if len(failed_jobs) != 0:
notify(notification_title, "\n".join(notification_msg), icon_image='catkin_icon_red.png')
elif len(warned_jobs) != 0:
notify(notification_title, "\n".join(notification_msg), icon_image='catkin_icon_yellow.png')
else:
notify(notification_title, "\n".join(notification_msg))
def job_started(self, package):
self.__command_log[package] = FileBackedLogCache(
package, self.log_dir, self.color)
wide_log(clr("Starting ==> {package}").format(**locals()))
def job_started(self, package):
self.__command_log[package] = FileBackedLogCache(package, self.log_dir, self.color)
wide_log(clr("Starting ==> {package}").format(**locals()))
def test_workspace(
context,
packages=None,
n_jobs=None,
quiet=False,
interleave_output=False,
no_status=False,
limit_status_rate=10.0,
no_notify=False,
continue_on_failure=False,
summarize_build=False,
catkin_test_target='run_tests',
cmake_test_target='test',
):
"""Tests a catkin workspace
:param context: context in which to test the catkin workspace
:type context: :py:class:`catkin_tools.context.Context`
:param packages: list of packages to test
:type packages: list
:param n_jobs: number of parallel package test jobs
:type n_jobs: int
:param quiet: suppresses verbose build or test information
:type quiet: bool
:param interleave_output: prints the output of commands as they are received
:type interleave_output: bool
:param no_status: suppresses the bottom status line
:type no_status: bool
:param limit_status_rate: rate to which status updates are limited; the default 0, places no limit.
:type limit_status_rate: float
:param no_notify: suppresses system notifications
:type no_notify: bool
:param continue_on_failure: do not stop testing other packages on error
:type continue_on_failure: bool
:param summarize_build: summarizes the build at the end
:type summarize_build: bool
:param catkin_test_target: make target for tests in catkin packages
:type catkin_test_target: str
:param cmake_test_target: make target for tests in cmake packages
:type cmake_test_target: str
"""
pre_start_time = time.time()
# Assert that the limit_status_rate is valid
if limit_status_rate < 0:
sys.exit(
"[test] @!@{rf}Error:@| The value of --limit-status-rate must be greater than or equal to zero."
)
# Get all the packages in the context source space
# Suppress warnings since this is a utility function
try:
workspace_packages = find_packages(context.source_space_abs,
exclude_subspaces=True,
warnings=[])
except InvalidPackage as ex:
sys.exit(
clr("@{rf}Error:@| The file {} is an invalid package.xml file."
" See below for details:\n\n{}").format(
ex.package_path, ex.msg))
# Get all build type plugins
test_job_creators = {
ep.name: ep.load()['create_test_job']
for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs')
}
# It's a problem if there aren't any build types available
if len(test_job_creators) == 0:
sys.exit(
'Error: No build types available. Please check your catkin_tools installation.'
)
# Get list of packages to test
ordered_packages = topological_order_packages(workspace_packages)
# Check if topological_order_packages determined any circular dependencies, if so print an error and fail.
# If this is the case, the last entry of ordered packages is a tuple that starts with nil.
if ordered_packages and ordered_packages[-1][0] is None:
guilty_packages = ", ".join(ordered_packages[-1][1:])
sys.exit(
"[test] Circular dependency detected in the following packages: {}"
.format(guilty_packages))
workspace_packages = dict([(pkg.name, (path, pkg))
for path, pkg in ordered_packages])
packages_to_test = []
if packages:
for package in packages:
if package not in workspace_packages:
# Try whether package is a pattern and matches
glob_packages = expand_glob_package(package,
workspace_packages)
if len(glob_packages) > 0:
packages.extend(glob_packages)
else:
sys.exit(
"[test] Given packages '{}' is not in the workspace "
"and pattern does not match any package".format(
package))
for pkg_path, package in ordered_packages:
if package.name in packages:
packages_to_test.append((pkg_path, package))
else:
# Only use buildlist when no other packages are specified
if len(context.buildlist) > 0:
# Expand glob patterns in buildlist
buildlist = []
for buildlisted_package in context.buildlist:
buildlist.extend(
expand_glob_package(buildlisted_package,
workspace_packages))
packages_to_test = [
p for p in ordered_packages if (p[1].name in buildlist)
]
else:
packages_to_test = ordered_packages
# Filter packages on skiplist
if len(context.skiplist) > 0:
# Expand glob patterns in skiplist
skiplist = []
for skiplisted_package in context.skiplist:
skiplist.extend(
expand_glob_package(skiplisted_package, workspace_packages))
# Apply skiplist to packages and dependencies
packages_to_test = [
(path, pkg) for path, pkg in packages_to_test
if (pkg.name not in skiplist or pkg.name in packages)
]
# Check if all packages to test are already built
built_packages = set([
pkg.name
for (path, pkg) in find_packages(context.package_metadata_path(),
warnings=[]).items()
])
packages_to_test_names = set(pkg.name for path, pkg in packages_to_test)
if not built_packages.issuperset(packages_to_test_names):
wide_log(
clr("@{rf}Error: Packages have to be built before they can be tested.@|"
))
wide_log(clr("The following requested packages are not built yet:"))
for package_name in packages_to_test_names.difference(built_packages):
wide_log(' - ' + package_name)
sys.exit(1)
# Construct jobs
jobs = []
for pkg_path, pkg in packages_to_test:
# Determine the job parameters
test_job_kwargs = dict(context=context,
package=pkg,
package_path=pkg_path,
verbose=not quiet)
# Create the job based on the build type
build_type = pkg.get_build_type()
if build_type == 'catkin':
test_job_kwargs['test_target'] = catkin_test_target
elif build_type == 'cmake':
test_job_kwargs['test_target'] = cmake_test_target
if build_type in test_job_creators:
jobs.append(test_job_creators[build_type](**test_job_kwargs))
# Queue for communicating status
event_queue = Queue()
# Initialize job server
job_server.initialize(
max_jobs=n_jobs,
max_load=None,
gnu_make_enabled=context.use_internal_make_jobserver,
)
try:
# Spin up status output thread
status_thread = ConsoleStatusController(
'test',
['package', 'packages'],
jobs,
n_jobs,
[pkg.name for path, pkg in packages_to_test],
[p for p in context.buildlist],
[p for p in context.skiplist],
event_queue,
show_notifications=not no_notify,
show_active_status=not no_status,
show_buffered_stdout=not interleave_output,
show_buffered_stderr=not interleave_output,
show_live_stdout=interleave_output,
show_live_stderr=interleave_output,
show_full_summary=summarize_build,
show_stage_events=not quiet,
pre_start_time=pre_start_time,
active_status_rate=limit_status_rate,
)
status_thread.start()
locks = {}
# Block while running N jobs asynchronously
try:
all_succeeded = run_until_complete(
execute_jobs('test',
jobs,
locks,
event_queue,
context.log_space_abs,
max_toplevel_jobs=n_jobs,
continue_on_failure=continue_on_failure,
continue_without_deps=False))
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
if all_succeeded:
return 0
else:
return 1
except KeyboardInterrupt:
wide_log("[test] Interrupted by user!")
event_queue.put(None)
return 130
def build_isolated_workspace(context,
packages=None,
start_with=None,
no_deps=False,
unbuilt=False,
n_jobs=None,
force_cmake=False,
pre_clean=False,
force_color=False,
quiet=False,
interleave_output=False,
no_status=False,
limit_status_rate=10.0,
lock_install=False,
no_notify=False,
continue_on_failure=False,
summarize_build=None,
relaxed_constraints=False,
influx_url=None,
influx_db=None):
"""Builds a catkin workspace in isolation
This function will find all of the packages in the source space, start some
executors, feed them packages to build based on dependencies and topological
ordering, and then monitor the output of the executors, handling loggings of
the builds, starting builds, failing builds, and finishing builds of
packages, and handling the shutdown of the executors when appropriate.
:param context: context in which to build the catkin workspace
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:param packages: list of packages to build, by default their dependencies will also be built
:type packages: list
:param start_with: package to start with, skipping all packages which proceed it in the topological order
:type start_with: str
:param no_deps: If True, the dependencies of packages will not be built first
:type no_deps: bool
:param n_jobs: number of parallel package build n_jobs
:type n_jobs: int
:param force_cmake: forces invocation of CMake if True, default is False
:type force_cmake: bool
:param force_color: forces colored output even if terminal does not support it
:type force_color: bool
:param quiet: suppresses the output of commands unless there is an error
:type quiet: bool
:param interleave_output: prints the output of commands as they are received
:type interleave_output: bool
:param no_status: disables status bar
:type no_status: bool
:param limit_status_rate: rate to which status updates are limited; the default 0, places no limit.
:type limit_status_rate: float
:param lock_install: causes executors to synchronize on access of install commands
:type lock_install: bool
:param no_notify: suppresses system notifications
:type no_notify: bool
:param continue_on_failure: do not stop building other jobs on error
:type continue_on_failure: bool
:param summarize_build: if True summarizes the build at the end, if None and continue_on_failure is True and the
the build fails, then the build will be summarized, but if False it never will be summarized.
:type summarize_build: bool
:param relaxed_constraints If true, do not use exec_deps for topological ordering
:type relaxed_constraints bool
:param influx_url Url to access an InfluxDB instance
:type influx_url string Url of the form user:password@host:port
:param influx_db Database name in InfluxDB
:type influx_db string
:raises: SystemExit if buildspace is a file or no packages were found in the source space
or if the provided options are invalid
"""
pre_start_time = time.time()
# Assert that the limit_status_rate is valid
if limit_status_rate < 0:
sys.exit(
"[build] @!@{rf}Error:@| The value of --status-rate must be greater than or equal to zero."
)
# Declare a buildspace marker describing the build config for error checking
buildspace_marker_data = {
'workspace': context.workspace,
'profile': context.profile,
'install': context.install,
'install_space': context.install_space_abs,
'devel_space': context.devel_space_abs,
'source_space': context.source_space_abs
}
# Check build config
if os.path.exists(
os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)):
with open(os.path.join(
context.build_space_abs,
BUILDSPACE_MARKER_FILE)) as buildspace_marker_file:
existing_buildspace_marker_data = yaml.safe_load(
buildspace_marker_file)
misconfig_lines = ''
for (k, v) in existing_buildspace_marker_data.items():
new_v = buildspace_marker_data.get(k, None)
if new_v != v:
misconfig_lines += (
'\n - %s: %s (stored) is not %s (commanded)' %
(k, v, new_v))
if len(misconfig_lines) > 0:
sys.exit(
clr("\n@{rf}Error:@| Attempting to build a catkin workspace using build space: "
"\"%s\" but that build space's most recent configuration "
"differs from the commanded one in ways which will cause "
"problems. Fix the following options or use @{yf}`catkin "
"clean -b`@| to remove the build space: %s" %
(context.build_space_abs, misconfig_lines)))
# Summarize the context
summary_notes = []
if force_cmake:
summary_notes += [
clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.")
]
log(context.summary(summary_notes))
# Make sure there is a build folder and it is not a file
if os.path.exists(context.build_space_abs):
if os.path.isfile(context.build_space_abs):
sys.exit(
clr("[build] @{rf}Error:@| Build space '{0}' exists but is a file and not a folder."
.format(context.build_space_abs)))
# If it dosen't exist, create it
else:
log("[build] Creating build space: '{0}'".format(
context.build_space_abs))
os.makedirs(context.build_space_abs)
# Write the current build config for config error checking
with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE),
'w') as buildspace_marker_file:
buildspace_marker_file.write(
yaml.dump(buildspace_marker_data, default_flow_style=False))
# Get all the packages in the context source space
# Suppress warnings since this is a utility function
workspace_packages = find_packages(context.source_space_abs,
exclude_subspaces=True,
warnings=[])
# Get packages which have not been built yet
built_packages, unbuilt_pkgs = get_built_unbuilt_packages(
context, workspace_packages)
# Handle unbuilt packages
if unbuilt:
# Check if there are any unbuilt
if len(unbuilt_pkgs) > 0:
# Add the unbuilt packages
packages.extend(list(unbuilt_pkgs))
else:
log("[build] No unbuilt packages to be built.")
return
# If no_deps is given, ensure packages to build are provided
if no_deps and packages is None:
log(
clr("[build] @!@{rf}Error:@| With no_deps, you must specify packages to build."
))
return
# Find list of packages in the workspace
packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built(
packages, context, workspace_packages)
if not no_deps:
# Extend packages to be built to include their deps
packages_to_be_built.extend(packages_to_be_built_deps)
# Also re-sort
try:
packages_to_be_built = topological_order_packages(
dict(packages_to_be_built))
except AttributeError:
log(
clr("[build] @!@{rf}Error:@| The workspace packages have a circular "
"dependency, and cannot be built. Please run `catkin list "
"--deps` to determine the problematic package(s)."))
return
# Check the number of packages to be built
if len(packages_to_be_built) == 0:
log(clr('[build] No packages to be built.'))
# Assert start_with package is in the workspace
verify_start_with_option(start_with, packages, all_packages,
packages_to_be_built)
# Populate .catkin file if we're not installing
# NOTE: This is done to avoid the Catkin CMake code from doing it,
# which isn't parallel-safe. Catkin CMake only modifies this file if
# it's package source path isn't found.
if not context.install:
dot_catkin_file_path = os.path.join(context.devel_space_abs, '.catkin')
# If the file exists, get the current paths
if os.path.exists(dot_catkin_file_path):
dot_catkin_paths = open(dot_catkin_file_path,
'r').read().split(';')
else:
dot_catkin_paths = []
# Update the list with the new packages (in topological order)
packages_to_be_built_paths = [
os.path.join(context.source_space_abs, path)
for path, pkg in packages_to_be_built
]
new_dot_catkin_paths = [
os.path.join(context.source_space_abs, path) for path in [
os.path.join(context.source_space_abs, path)
for path, pkg in all_packages
] if path in dot_catkin_paths or path in packages_to_be_built_paths
]
# Write the new file if it's different, otherwise, leave it alone
if dot_catkin_paths == new_dot_catkin_paths:
wide_log("[build] Package table is up to date.")
else:
wide_log("[build] Updating package table.")
open(dot_catkin_file_path,
'w').write(';'.join(new_dot_catkin_paths))
# Remove packages before start_with
if start_with is not None:
for path, pkg in list(packages_to_be_built):
if pkg.name != start_with:
wide_log(
clr("@!@{pf}Skipping@| @{gf}---@| @{cf}{}@|").format(
pkg.name))
packages_to_be_built.pop(0)
else:
break
# Get the names of all packages to be built
packages_to_be_built_names = [p.name for _, p in packages_to_be_built]
packages_to_be_built_deps_names = [
p.name for _, p in packages_to_be_built_deps
]
# Generate prebuild and prebuild clean jobs, if necessary
prebuild_jobs = {}
setup_util_present = os.path.exists(
os.path.join(context.devel_space_abs, '_setup_util.py'))
catkin_present = 'catkin' in (packages_to_be_built_names +
packages_to_be_built_deps_names)
catkin_built = 'catkin' in built_packages
prebuild_built = 'catkin_tools_prebuild' in built_packages
# Handle the prebuild jobs if the develspace is linked
prebuild_pkg_deps = []
if context.link_devel:
prebuild_pkg = None
# Construct a dictionary to lookup catkin package by name
pkg_dict = dict([(pkg.name, (pth, pkg)) for pth, pkg in all_packages])
if setup_util_present:
# Setup util is already there, determine if it needs to be
# regenerated
if catkin_built:
if catkin_present:
prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin']
elif prebuild_built:
if catkin_present:
# TODO: Clean prebuild package
ct_prebuild_pkg_path = get_prebuild_package(
context.build_space_abs, context.devel_space_abs,
force_cmake)
ct_prebuild_pkg = parse_package(ct_prebuild_pkg_path)
prebuild_jobs[
'caktin_tools_prebuild'] = create_catkin_clean_job(
context,
ct_prebuild_pkg,
ct_prebuild_pkg_path,
dependencies=[],
dry_run=False,
clean_build=True,
clean_devel=True,
clean_install=True)
# TODO: Build catkin package
prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin']
prebuild_pkg_deps.append('catkin_tools_prebuild')
else:
# How did these get here??
log("Warning: devel space setup files have an unknown origin.")
else:
# Setup util needs to be generated
if catkin_built or prebuild_built:
log("Warning: generated devel space setup files have been deleted."
)
if catkin_present:
# Build catkin package
prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin']
else:
# Generate and buildexplicit prebuild package
prebuild_pkg_path = get_prebuild_package(
context.build_space_abs, context.devel_space_abs,
force_cmake)
prebuild_pkg = parse_package(prebuild_pkg_path)
if prebuild_pkg is not None:
# Create the prebuild job
prebuild_job = create_catkin_build_job(
context,
prebuild_pkg,
prebuild_pkg_path,
build_dependencies=prebuild_pkg_deps,
run_dependencies=[],
force_cmake=force_cmake,
pre_clean=pre_clean,
prebuild=True)
# Add the prebuld job
prebuild_jobs[prebuild_job.jid] = prebuild_job
# Remove prebuild jobs from normal job list
for prebuild_jid, prebuild_job in prebuild_jobs.items():
if prebuild_jid in packages_to_be_built_names:
packages_to_be_built_names.remove(prebuild_jid)
# Initial jobs list is just the prebuild jobs
jobs = [] + list(prebuild_jobs.values())
# Get all build type plugins
build_job_creators = {
ep.name: ep.load()['create_build_job']
for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs')
}
# It's a problem if there aren't any build types available
if len(build_job_creators) == 0:
sys.exit(
'Error: No build types available. Please check your catkin_tools installation.'
)
# Construct jobs
for pkg_path, pkg in all_packages:
if pkg.name not in packages_to_be_built_names:
continue
# Ignore metapackages
if 'metapackage' in [e.tagname for e in pkg.exports]:
continue
# Get actual execution deps
build_deps = [
p.name for _, p in get_cached_recursive_build_depends_in_workspace(
pkg, packages_to_be_built) if p.name not in prebuild_jobs
]
build_for_run_deps = [
p.name for _, p in get_cached_recursive_run_depends_in_workspace(
pkg, packages_to_be_built) if p.name not in prebuild_jobs
]
# All jobs depend on the prebuild jobs if they're defined
if not no_deps:
if relaxed_constraints:
build_for_run_deps = [
p.name for _, p in
get_recursive_build_depends_for_run_depends_in_workspace(
[pkg], packages_to_be_built)
if p.name not in prebuild_jobs
]
else:
# revert to interpreting all dependencies as build dependencies
build_deps = list(set(build_deps + build_for_run_deps))
build_for_run_deps = []
for j in prebuild_jobs.values():
build_deps.append(j.jid)
# Determine the job parameters
build_job_kwargs = dict(context=context,
package=pkg,
package_path=pkg_path,
build_dependencies=build_deps,
run_dependencies=build_for_run_deps,
force_cmake=force_cmake,
pre_clean=pre_clean)
# Create the job based on the build type
build_type = get_build_type(pkg)
if build_type in build_job_creators:
jobs.append(build_job_creators[build_type](**build_job_kwargs))
else:
wide_log(
clr("[build] @!@{yf}Warning:@| Skipping package `{}` because it "
"has an unsupported package build type: `{}`").format(
pkg.name, build_type))
wide_log(clr("[build] Note: Available build types:"))
for bt_name in build_job_creators.keys():
wide_log(clr("[build] - `{}`".format(bt_name)))
# Queue for communicating status
event_queue = Queue()
status_queue = Queue()
monitoring_queue = Queue()
class ForwardingQueue(threading.Thread):
def __init__(self, queues):
super(ForwardingQueue, self).__init__()
self.keep_running = True
self.queues = queues
def run(self):
while self.keep_running:
event = event_queue.get(True)
for queue in self.queues:
queue.put(event)
if event is None:
break
queue_thread = ForwardingQueue([status_queue, monitoring_queue])
threads = [queue_thread]
try:
# Spin up status output thread
status_thread = ConsoleStatusController(
'build', ['package', 'packages'],
jobs,
n_jobs, [pkg.name for _, pkg in context.packages],
[p for p in context.whitelist], [p for p in context.blacklist],
status_queue,
show_notifications=not no_notify,
show_active_status=not no_status,
show_buffered_stdout=not quiet and not interleave_output,
show_buffered_stderr=not interleave_output,
show_live_stdout=interleave_output,
show_live_stderr=interleave_output,
show_stage_events=not quiet,
show_full_summary=(summarize_build is True),
pre_start_time=pre_start_time,
active_status_rate=limit_status_rate)
threads.append(status_thread)
if influx_db is not None:
if not have_influx_db:
sys.exit(
"[build] @!@{rf}Error:@| InfluxDB monitoring is not possible, cannot import influxdb"
)
match = re.match('^(.+):(.+)@(.+):(.+)$', influx_url)
if not match:
sys.exit(
"[build] @!@{rf}Error:@| The value of --influx has to be of the form username:password@host:port"
)
username, password, host, port = match.groups()
influxdb_thread = InfluxDBStatusController(monitoring_queue,
influx_db, host, port,
username, password)
threads.append(influxdb_thread)
for thread in threads:
thread.start()
# Initialize locks
locks = {
'installspace': asyncio.Lock() if lock_install else FakeLock()
}
# Block while running N jobs asynchronously
try:
all_succeeded = run_until_complete(
execute_jobs('build',
jobs,
locks,
event_queue,
context.log_space_abs,
max_toplevel_jobs=n_jobs,
continue_on_failure=continue_on_failure,
continue_without_deps=False,
relaxed_constraints=relaxed_constraints))
except Exception:
all_succeeded = False
for thread in threads:
thread.keep_running = False
for thread in threads:
thread.join(1.0)
wide_log(str(traceback.format_exc()))
event_queue.put(None)
for thread in threads:
thread.join(1.0)
# Warn user about new packages
now_built_packages, now_unbuilt_pkgs = get_built_unbuilt_packages(
context, workspace_packages)
new_pkgs = [p for p in unbuilt_pkgs if p not in now_unbuilt_pkgs]
if len(new_pkgs) > 0:
log(
clr("[build] @/@!Note:@| @/Workspace packages have changed, "
"please re-source setup files to use them.@|"))
if all_succeeded:
# Create isolated devel setup if necessary
if context.isolate_devel:
if not context.install:
_create_unmerged_devel_setup(context, now_unbuilt_pkgs)
else:
_create_unmerged_devel_setup_for_install(context)
return 0
else:
return 1
except KeyboardInterrupt:
wide_log("[build] Interrupted by user!")
event_queue.put(None)
return 130 # EOWNERDEAD return code is not part of the errno module.
def print_build_summary(context, packages_to_be_built, completed_packages, failed_packages):
# Calculate the longest package name
max_name_len = max([len(pkg.name) for _, pkg in context.packages])
def get_template(template_name, column_width):
templates = {
'successful': " @!@{gf}Successful@| @{cf}{package:<" + str(column_width) + "}@|",
'failed': " @!@{rf}Failed@| @{cf}{package:<" + str(column_width) + "}@|",
'not_built': " @!@{kf}Not built@| @{cf}{package:<" + str(column_width) + "}@|",
}
return templates[template_name]
# Setup templates for comparison
successful_template = get_template('successful', max_name_len)
failed_template = get_template('failed', max_name_len)
not_built_template = get_template('not_built', max_name_len)
# Calculate the maximum _printed_ length for each template
faux_package_name = ("x" * max_name_len)
templates = [
remove_ansi_escape(clr(successful_template).format(package=faux_package_name)),
remove_ansi_escape(clr(failed_template).format(package=faux_package_name)),
remove_ansi_escape(clr(not_built_template).format(package=faux_package_name)),
]
# Calculate the longest column using the longest template
max_column_len = max([len(template) for template in templates])
# Calculate the number of columns
number_of_columns = (terminal_width() / max_column_len) or 1
successfuls = {}
faileds = {}
not_builts = {}
non_whitelisted = {}
blacklisted = {}
for (_, pkg) in context.packages:
if pkg.name in context.blacklist:
blacklisted[pkg.name] = clr(not_built_template).format(package=pkg.name)
elif len(context.whitelist) > 0 and pkg.name not in context.whitelist:
non_whitelisted[pkg.name] = clr(not_built_template).format(package=pkg.name)
elif pkg.name in completed_packages:
successfuls[pkg.name] = clr(successful_template).format(package=pkg.name)
else:
if pkg.name in failed_packages:
faileds[pkg.name] = clr(failed_template).format(package=pkg.name)
else:
not_builts[pkg.name] = clr(not_built_template).format(package=pkg.name)
# Combine successfuls and not_builts, sort by key, only take values
wide_log("")
wide_log("Build summary:")
combined = dict(successfuls)
combined.update(not_builts)
non_failed = [v for k, v in sorted(combined.items(), key=operator.itemgetter(0))]
print_items_in_columns(non_failed, number_of_columns)
# Print out whitelisted packages
if len(non_whitelisted) > 0:
wide_log("")
wide_log("Non-Whitelisted Packages:")
non_whitelisted_list = [v for k, v in sorted(non_whitelisted.items(), key=operator.itemgetter(0))]
print_items_in_columns(non_whitelisted_list, number_of_columns)
# Print out blacklisted packages
if len(blacklisted) > 0:
wide_log("")
wide_log("Blacklisted Packages:")
blacklisted_list = [v for k, v in sorted(blacklisted.items(), key=operator.itemgetter(0))]
print_items_in_columns(blacklisted_list, number_of_columns)
# Faileds only, sort by key, only take values
failed = [v for k, v in sorted(faileds.items(), key=operator.itemgetter(0))]
if len(failed) > 0:
wide_log("")
wide_log("Failed packages:")
print_items_in_columns(failed, number_of_columns)
else:
wide_log("")
wide_log("All packages built successfully.")
wide_log("")
wide_log(clr("[build] @!@{gf}Successfully@| built '@!@{cf}{0}@|' packages, "
"@!@{rf}failed@| to build '@!@{cf}{1}@|' packages, "
"and @!@{kf}did not try to build@| '@!@{cf}{2}@|' packages.").format(
len(successfuls), len(faileds), len(not_builts)
))
def build_isolated_workspace(
context,
packages=None,
start_with=None,
no_deps=False,
unbuilt=False,
n_jobs=None,
force_cmake=False,
pre_clean=False,
force_color=False,
quiet=False,
interleave_output=False,
no_status=False,
limit_status_rate=10.0,
lock_install=False,
no_notify=False,
continue_on_failure=False,
summarize_build=None,
):
"""Builds a catkin workspace in isolation
This function will find all of the packages in the source space, start some
executors, feed them packages to build based on dependencies and topological
ordering, and then monitor the output of the executors, handling loggings of
the builds, starting builds, failing builds, and finishing builds of
packages, and handling the shutdown of the executors when appropriate.
:param context: context in which to build the catkin workspace
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:param packages: list of packages to build, by default their dependencies will also be built
:type packages: list
:param start_with: package to start with, skipping all packages which proceed it in the topological order
:type start_with: str
:param no_deps: If True, the dependencies of packages will not be built first
:type no_deps: bool
:param n_jobs: number of parallel package build n_jobs
:type n_jobs: int
:param force_cmake: forces invocation of CMake if True, default is False
:type force_cmake: bool
:param force_color: forces colored output even if terminal does not support it
:type force_color: bool
:param quiet: suppresses the output of commands unless there is an error
:type quiet: bool
:param interleave_output: prints the output of commands as they are received
:type interleave_output: bool
:param no_status: disables status bar
:type no_status: bool
:param limit_status_rate: rate to which status updates are limited; the default 0, places no limit.
:type limit_status_rate: float
:param lock_install: causes executors to synchronize on access of install commands
:type lock_install: bool
:param no_notify: suppresses system notifications
:type no_notify: bool
:param continue_on_failure: do not stop building other jobs on error
:type continue_on_failure: bool
:param summarize_build: if True summarizes the build at the end, if None and continue_on_failure is True and the
the build fails, then the build will be summarized, but if False it never will be summarized.
:type summarize_build: bool
:raises: SystemExit if buildspace is a file or no packages were found in the source space
or if the provided options are invalid
"""
pre_start_time = time.time()
# Assert that the limit_status_rate is valid
if limit_status_rate < 0:
sys.exit("[build] @!@{rf}Error:@| The value of --status-rate must be greater than or equal to zero.")
# Declare a buildspace marker describing the build config for error checking
buildspace_marker_data = {
'workspace': context.workspace,
'profile': context.profile,
'install': context.install,
'install_space': context.install_space_abs,
'devel_space': context.devel_space_abs,
'source_space': context.source_space_abs}
# Check build config
if os.path.exists(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)):
with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)) as buildspace_marker_file:
existing_buildspace_marker_data = yaml.load(buildspace_marker_file)
misconfig_lines = ''
for (k, v) in existing_buildspace_marker_data.items():
new_v = buildspace_marker_data.get(k, None)
if new_v != v:
misconfig_lines += (
'\n - %s: %s (stored) is not %s (commanded)' %
(k, v, new_v))
if len(misconfig_lines) > 0:
sys.exit(clr(
"\n@{rf}Error:@| Attempting to build a catkin workspace using build space: "
"\"%s\" but that build space's most recent configuration "
"differs from the commanded one in ways which will cause "
"problems. Fix the following options or use @{yf}`catkin "
"clean -b`@| to remove the build space: %s" %
(context.build_space_abs, misconfig_lines)))
# Summarize the context
summary_notes = []
if force_cmake:
summary_notes += [clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.")]
log(context.summary(summary_notes))
# Make sure there is a build folder and it is not a file
if os.path.exists(context.build_space_abs):
if os.path.isfile(context.build_space_abs):
sys.exit(clr(
"[build] @{rf}Error:@| Build space '{0}' exists but is a file and not a folder."
.format(context.build_space_abs)))
# If it dosen't exist, create it
else:
log("[build] Creating build space: '{0}'".format(context.build_space_abs))
os.makedirs(context.build_space_abs)
# Write the current build config for config error checking
with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE), 'w') as buildspace_marker_file:
buildspace_marker_file.write(yaml.dump(buildspace_marker_data, default_flow_style=False))
# Get all the packages in the context source space
# Suppress warnings since this is a utility function
workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[])
# Get packages which have not been built yet
unbuilt_pkgs = get_unbuilt_packages(context, workspace_packages)
# Handle unbuilt packages
if unbuilt:
# Check if there are any unbuilt
if len(unbuilt_pkgs) > 0:
# Add the unbuilt packages
packages.extend(list(unbuilt_pkgs))
else:
log("[build] No unbuilt packages to be built.")
return
# If no_deps is given, ensure packages to build are provided
if no_deps and packages is None:
log(clr("[build] @!@{rf}Error:@| With no_deps, you must specify packages to build."))
return
# Find list of packages in the workspace
packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built(
packages, context, workspace_packages)
if not no_deps:
# Extend packages to be built to include their deps
packages_to_be_built.extend(packages_to_be_built_deps)
# Also re-sort
try:
packages_to_be_built = topological_order_packages(dict(packages_to_be_built))
except AttributeError:
log(clr("[build] @!@{rf}Error:@| The workspace packages have a circular "
"dependency, and cannot be built. Please run `catkin list "
"--deps` to determine the problematic package(s)."))
return
# Check the number of packages to be built
if len(packages_to_be_built) == 0:
log(clr('[build] No packages to be built.'))
return
# Assert start_with package is in the workspace
verify_start_with_option(
start_with,
packages,
all_packages,
packages_to_be_built + packages_to_be_built_deps)
# Populate .catkin file if we're not installing
# NOTE: This is done to avoid the Catkin CMake code from doing it,
# which isn't parallel-safe. Catkin CMake only modifies this file if
# it's package source path isn't found.
if not context.install:
dot_catkin_file_path = os.path.join(context.devel_space_abs, '.catkin')
# If the file exists, get the current paths
if os.path.exists(dot_catkin_file_path):
dot_catkin_paths = open(dot_catkin_file_path, 'r').read().split(';')
else:
dot_catkin_paths = []
# Update the list with the new packages (in topological order)
packages_to_be_built_paths = [
os.path.join(context.source_space_abs, path)
for path, pkg in packages_to_be_built
]
new_dot_catkin_paths = [
os.path.join(context.source_space_abs, path)
for path in [os.path.join(context.source_space_abs, path) for path, pkg in all_packages]
if path in dot_catkin_paths or path in packages_to_be_built_paths
]
# Write the new file if it's different, otherwise, leave it alone
if dot_catkin_paths == new_dot_catkin_paths:
wide_log("[build] Package table is up to date.")
else:
wide_log("[build] Updating package table.")
open(dot_catkin_file_path, 'w').write(';'.join(new_dot_catkin_paths))
# Remove packages before start_with
if start_with is not None:
for path, pkg in list(packages_to_be_built):
if pkg.name != start_with:
wide_log(clr("@!@{pf}Skipping@| @{gf}---@| @{cf}{}@|").format(pkg.name))
packages_to_be_built.pop(0)
else:
break
# Get the names of all packages to be built
packages_to_be_built_names = [p.name for _, p in packages_to_be_built]
packages_to_be_built_deps_names = [p.name for _, p in packages_to_be_built_deps]
# Generate prebuild jobs, if necessary
prebuild_jobs = {}
setup_util_exists = os.path.exists(os.path.join(context.devel_space_abs, '_setup_util.py'))
if context.link_devel and (not setup_util_exists or (force_cmake and len(packages) == 0)):
wide_log('[build] Preparing linked develspace...')
pkg_dict = dict([(pkg.name, (pth, pkg)) for pth, pkg in all_packages])
if 'catkin' in packages_to_be_built_names + packages_to_be_built_deps_names:
# Use catkin as the prebuild package
prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin']
else:
# Generate explicit prebuild package
prebuild_pkg_path = generate_prebuild_package(context.build_space_abs, context.devel_space_abs, force_cmake)
prebuild_pkg = parse_package(prebuild_pkg_path)
# Create the prebuild job
prebuild_job = create_catkin_build_job(
context,
prebuild_pkg,
prebuild_pkg_path,
dependencies=[],
force_cmake=force_cmake,
pre_clean=pre_clean,
prebuild=True)
# Add the prebuld job
prebuild_jobs[prebuild_job.jid] = prebuild_job
# Remove prebuild jobs from normal job list
for prebuild_jid, prebuild_job in prebuild_jobs.items():
if prebuild_jid in packages_to_be_built_names:
packages_to_be_built_names.remove(prebuild_jid)
# Initial jobs list is just the prebuild jobs
jobs = [] + list(prebuild_jobs.values())
# Get all build type plugins
build_job_creators = {
ep.name: ep.load()['create_build_job']
for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs')
}
# It's a problem if there aren't any build types available
if len(build_job_creators) == 0:
sys.exit('Error: No build types availalbe. Please check your catkin_tools installation.')
# Construct jobs
for pkg_path, pkg in all_packages:
if pkg.name not in packages_to_be_built_names:
continue
# Ignore metapackages
if 'metapackage' in [e.tagname for e in pkg.exports]:
continue
# Get actual execution deps
deps = [
p.name for _, p
in get_cached_recursive_build_depends_in_workspace(pkg, packages_to_be_built)
if p.name not in prebuild_jobs
]
# All jobs depend on the prebuild job if it's defined
for j in prebuild_jobs.values():
deps.append(j.jid)
# Determine the job parameters
build_job_kwargs = dict(
context=context,
package=pkg,
package_path=pkg_path,
dependencies=deps,
force_cmake=force_cmake,
pre_clean=pre_clean)
# Create the job based on the build type
build_type = get_build_type(pkg)
if build_type in build_job_creators:
jobs.append(build_job_creators[build_type](**build_job_kwargs))
else:
wide_log(clr(
"[build] @!@{yf}Warning:@| Skipping package `{}` because it "
"has an unsupported package build type: `{}`"
).format(pkg.name, build_type))
wide_log(clr("[build] Note: Available build types:"))
for bt_name in build_job_creators.keys():
wide_log(clr("[build] - `{}`".format(bt_name)))
# Queue for communicating status
event_queue = Queue()
try:
# Spin up status output thread
status_thread = ConsoleStatusController(
'build',
['package', 'packages'],
jobs,
n_jobs,
[pkg.name for _, pkg in context.packages],
[p for p in context.whitelist],
[p for p in context.blacklist],
event_queue,
show_notifications=not no_notify,
show_active_status=not no_status,
show_buffered_stdout=not quiet and not interleave_output,
show_buffered_stderr=not interleave_output,
show_live_stdout=interleave_output,
show_live_stderr=interleave_output,
show_stage_events=not quiet,
show_full_summary=(summarize_build is True),
pre_start_time=pre_start_time,
active_status_rate=limit_status_rate)
status_thread.start()
# Initialize locks
locks = {
'installspace': asyncio.Lock() if lock_install else FakeLock()
}
# Block while running N jobs asynchronously
try:
all_succeeded = run_until_complete(execute_jobs(
'build',
jobs,
locks,
event_queue,
os.path.join(context.build_space_abs, '_logs'),
max_toplevel_jobs=n_jobs,
continue_on_failure=continue_on_failure,
continue_without_deps=False))
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
# Warn user about new packages
now_unbuilt_pkgs = get_unbuilt_packages(context, workspace_packages)
new_pkgs = [p for p in unbuilt_pkgs if p not in now_unbuilt_pkgs]
if len(new_pkgs) > 0:
log(clr("[build] @/@!Note:@| @/Workspace packages have changed, "
"please re-source setup files to use them.@|"))
if all_succeeded:
# Create isolated devel setup if necessary
if context.isolate_devel:
if not context.install:
_create_unmerged_devel_setup(context, now_unbuilt_pkgs)
else:
_create_unmerged_devel_setup_for_install(context)
return 0
else:
return 1
except KeyboardInterrupt:
wide_log("[build] Interrupted by user!")
event_queue.put(None)
def print_compact_summary(self, completed_jobs, warned_jobs, failed_jobs):
"""Print a compact build summary."""
notification_title = ""
notification_msg = []
# Print error summary
if len(completed_jobs) == len(self.jobs) and all(
completed_jobs.items()) and len(failed_jobs) == 0:
notification_title = "{} Succeeded".format(self.label.capitalize())
notification_msg.append("All {} {} succeeded!".format(
len(self.jobs), self.jobs_label))
wide_log(
clr('[{}] Summary: All {} {} succeeded!').format(
self.label, len(self.jobs), self.jobs_label))
else:
notification_msg.append("{} of {} {} succeeded.".format(
len([
succeeded for jid, succeeded in completed_jobs.items()
if succeeded
]), len(self.jobs), self.jobs_label))
wide_log(
clr('[{}] Summary: {} of {} {} succeeded.').format(
self.label,
len([
succeeded for jid, succeeded in completed_jobs.items()
if succeeded
]), len(self.jobs), self.jobs_label))
# Display number of ignored jobs (jobs which shouldn't have been built)
all_ignored_jobs = [
j for j in self.available_jobs if j not in self.jobs
]
if len(all_ignored_jobs) == 0:
wide_log(clr('[{}] Ignored: None.').format(self.label))
else:
notification_msg.append("{} {} were skipped.".format(
len(all_ignored_jobs), self.jobs_label))
wide_log(
clr('[{}] Ignored: {} {} were skipped or are blacklisted.').
format(self.label, len(all_ignored_jobs), self.jobs_label))
# Display number of jobs which produced warnings
if len(warned_jobs) == 0:
wide_log(clr('[{}] Warnings: None.').format(self.label))
else:
notification_title = "{} Succeeded with Warnings".format(
self.label.capitalize())
notification_msg.append("{} {} succeeded with warnings.".format(
len(warned_jobs), self.jobs_label))
wide_log(
clr('[{}] Warnings: {} {} succeeded with warnings.').format(
self.label, len(warned_jobs), self.jobs_label))
# Display number of abandoned jobs
all_abandoned_jobs = [j for j in self.jobs if j not in completed_jobs]
if len(all_abandoned_jobs) == 0:
wide_log(
clr('[{}] Abandoned: No {} were abandoned.').format(
self.label, self.jobs_label))
else:
notification_title = "{} Incomplete".format(
self.label.capitalize())
notification_msg.append("{} {} were abandoned.".format(
len(all_abandoned_jobs), self.jobs_label))
wide_log(
clr('[{}] Abandoned: {} {} were abandoned.').format(
self.label, len(all_abandoned_jobs), self.jobs_label))
# Display number of failed jobs
if len(failed_jobs) == 0:
wide_log(
clr('[{}] Failed: No {} failed.').format(
self.label, self.jobs_label))
else:
notification_title = "{} Failed".format(self.label.capitalize())
notification_msg.append("{} {} failed.".format(
len(failed_jobs), self.jobs_label))
wide_log(
clr('[{}] Failed: {} {} failed.').format(
self.label, len(failed_jobs), self.jobs_label))
if self.show_notifications:
if len(failed_jobs) != 0:
notify(notification_title,
"\n".join(notification_msg),
icon_image='catkin_icon_red.png')
elif len(warned_jobs) != 0:
notify(notification_title,
"\n".join(notification_msg),
icon_image='catkin_icon_yellow.png')
else:
notify(notification_title, "\n".join(notification_msg))
def job_finished(self, package, time):
self.__command_log[package].close()
del self.__command_log[package]
msg = clr("Finished <== {package:<") + str(self.max_package_name_length) + clr("} [ {time} ]")
wide_log(msg.format(**locals()))
def build_isolated_workspace(
context,
packages=None,
start_with=None,
no_deps=False,
jobs=None,
force_cmake=False,
force_color=False,
quiet=False,
interleave_output=False,
no_status=False,
lock_install=False,
no_notify=False
):
"""Builds a catkin workspace in isolation
This function will find all of the packages in the source space, start some
executors, feed them packages to build based on dependencies and topological
ordering, and then monitor the output of the executors, handling loggings of
the builds, starting builds, failing builds, and finishing builds of
packages, and handling the shutdown of the executors when appropriate.
:param context: context in which to build the catkin workspace
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:param packages: list of packages to build, by default their dependencies will also be built
:type packages: list
:param start_with: package to start with, skipping all packages which proceed it in the topological order
:type start_with: str
:param no_deps: If True, the dependencies of packages will not be built first
:type no_deps: bool
:param jobs: number of parallel package build jobs
:type jobs: int
:param force_cmake: forces invocation of CMake if True, default is False
:type force_cmake: bool
:param force_color: forces colored output even if terminal does not support it
:type force_color: bool
:param quiet: suppresses the output of commands unless there is an error
:type quiet: bool
:param interleave_output: prints the output of commands as they are received
:type interleave_output: bool
:param no_status: disables status bar
:type no_status: bool
:param lock_install: causes executors to synchronize on access of install commands
:type lock_install: bool
:param no_notify: suppresses system notifications
:type no_notify: bool
:raises: SystemExit if buildspace is a file or no packages were found in the source space
or if the provided options are invalid
"""
# If no_deps is given, ensure packages to build are provided
if no_deps and packages is None:
sys.exit("With --no-deps, you must specify packages to build.")
# Make sure there is a build folder and it is not a file
if os.path.exists(context.build_space_abs):
if os.path.isfile(context.build_space_abs):
sys.exit(clr(
"@{rf}Error:@| Build space '{0}' exists but is a file and not a folder."
.format(context.build_space_abs)))
# If it dosen't exist, create it
else:
log("Creating build space directory, '{0}'".format(context.build_space_abs))
os.makedirs(context.build_space_abs)
# Check for catkin_make droppings
if context.corrupted_by_catkin_make():
sys.exit(
clr("@{rf}Error:@| Build space `{0}` exists but appears to have previously been "
"created by the `catkin_make` or `catkin_make_isolated` tool. "
"Please choose a different directory to use with `catkin build` "
"or clean the build space.".format(context.build_space_abs)))
# Declare a buildspace marker describing the build config for error checking
buildspace_marker_data = {
'workspace': context.workspace,
'profile': context.profile,
'install': context.install,
'install_space': context.install_space_abs,
'devel_space': context.devel_space_abs,
'source_space': context.source_space_abs}
# Check build config
if os.path.exists(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)):
with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)) as buildspace_marker_file:
existing_buildspace_marker_data = yaml.load(buildspace_marker_file)
misconfig_lines = ''
for (k, v) in existing_buildspace_marker_data.items():
new_v = buildspace_marker_data.get(k, None)
if new_v != v:
misconfig_lines += (
'\n - %s: %s (stored) is not %s (commanded)' %
(k, v, new_v))
if len(misconfig_lines) > 0:
sys.exit(clr(
"\n@{rf}Error:@| Attempting to build a catkin workspace using build space: "
"\"%s\" but that build space's most recent configuration "
"differs from the commanded one in ways which will cause "
"problems. Fix the following options or use @{yf}`catkin "
"clean -b`@| to remove the build space: %s" %
(context.build_space_abs, misconfig_lines)))
# Write the current build config for config error checking
with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE), 'w') as buildspace_marker_file:
buildspace_marker_file.write(yaml.dump(buildspace_marker_data, default_flow_style=False))
# Summarize the context
summary_notes = []
if force_cmake:
summary_notes += [clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.")]
log(context.summary(summary_notes))
# Find list of packages in the workspace
packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built(packages, context)
completed_packages = []
if no_deps:
# Consider deps as "completed"
completed_packages.extend(packages_to_be_built_deps)
else:
# Extend packages to be built to include their deps
packages_to_be_built.extend(packages_to_be_built_deps)
# Also resort
packages_to_be_built = topological_order_packages(dict(packages_to_be_built))
max_package_name_length = max([len(pkg.name) for pth, pkg in packages_to_be_built])
# Assert start_with package is in the workspace
verify_start_with_option(start_with, packages, all_packages, packages_to_be_built + packages_to_be_built_deps)
# Setup pool of executors
executors = {}
# The communication queue can have ExecutorEvent's or str's passed into it from the executors
comm_queue = Queue()
# The job queue has Jobs put into it
job_queue = Queue()
# Lock for install space
install_lock = Lock() if lock_install else FakeLock()
# Determine the number of executors
try:
if jobs:
jobs = int(jobs)
if jobs < 1:
sys.exit("Specified number of jobs '{0}' is not positive.".format(jobs))
except ValueError:
sys.exit("Specified number of jobs '{0}' is no integer.".format(jobs))
try:
jobs = cpu_count() if jobs is None else jobs
except NotImplementedError:
log('Failed to determine the cpu_count, falling back to 1 jobs as the default.')
jobs = 1 if jobs is None else jobs
# If only one set of jobs, turn on interleaving to get more responsive feedback
if jobs == 1:
# TODO: make the system more intelligent so that it can automatically switch to streaming output
# when only one job is building, even if multiple jobs could be building
quiet = False
interleave_output = True
# Start the executors
for x in range(jobs):
e = Executor(x, context, comm_queue, job_queue, install_lock)
executors[x] = e
e.start()
# Variables for tracking running jobs and built/building packages
start = time.time()
total_packages = len(packages_to_be_built)
package_count = 0
running_jobs = {}
log_dir = os.path.join(context.build_space_abs, 'build_logs')
color = True
if not force_color and not is_tty(sys.stdout):
color = True
out = OutputController(log_dir, quiet, interleave_output, color, max_package_name_length, prefix_output=(jobs > 1))
if no_status:
disable_wide_log()
# Prime the job_queue
ready_packages = []
if start_with is None:
ready_packages = get_ready_packages(packages_to_be_built, running_jobs, completed_packages)
while start_with is not None:
ready_packages.extend(get_ready_packages(packages_to_be_built, running_jobs, completed_packages))
while ready_packages:
pth, pkg = ready_packages.pop(0)
if pkg.name != start_with:
completed_packages.append(pkg.name)
package_count += 1
wide_log("[build] Skipping package '{0}'".format(pkg.name))
else:
ready_packages.insert(0, (pth, pkg))
start_with = None
break
running_jobs = queue_ready_packages(ready_packages, running_jobs, job_queue, context, force_cmake)
assert running_jobs
error_state = False
errors = []
def set_error_state(error_state):
if error_state:
return
# Set the error state to prevent new jobs
error_state = True
# Empty the job queue
while not job_queue.empty():
job_queue.get()
# Kill the executors by sending a None to the job queue for each of them
for x in range(jobs):
job_queue.put(None)
# While any executors are running, process executor events
while executors:
try:
# Try to get an event from the communications queue
try:
event = comm_queue.get(True, 0.1)
except Empty:
# timeout occured, create null event to pass through checks
event = ExecutorEvent(None, None, None, None)
if event.event_type == 'job_started':
package_count += 1
running_jobs[event.package]['package_number'] = package_count
running_jobs[event.package]['start_time'] = time.time()
out.job_started(event.package)
if event.event_type == 'command_started':
out.command_started(event.package, event.data['cmd'], event.data['location'])
if event.event_type == 'command_log':
out.command_log(event.package, event.data['message'])
if event.event_type == 'command_failed':
out.command_failed(event.package, event.data['cmd'], event.data['location'], event.data['retcode'])
# Add to list of errors
errors.append(event)
# Remove the command from the running jobs
del running_jobs[event.package]
# If it hasn't already been done, stop the executors
set_error_state(error_state)
if event.event_type == 'command_finished':
out.command_finished(event.package, event.data['cmd'], event.data['location'], event.data['retcode'])
if event.event_type == 'job_finished':
completed_packages.append(event.package)
run_time = format_time_delta(time.time() - running_jobs[event.package]['start_time'])
out.job_finished(event.package, run_time)
del running_jobs[event.package]
# If shutting down, do not add new packages
if error_state:
continue
# Calculate new packages
if not no_status:
wide_log('[build] Calculating new jobs...', end='\r')
sys.stdout.flush()
ready_packages = get_ready_packages(packages_to_be_built, running_jobs, completed_packages)
running_jobs = queue_ready_packages(ready_packages, running_jobs, job_queue, context, force_cmake)
# Make sure there are jobs to be/being processed, otherwise kill the executors
if not running_jobs:
# Kill the executors by sending a None to the job queue for each of them
for x in range(jobs):
job_queue.put(None)
# If an executor exit event, join it and remove it from the executors list
if event.event_type == 'exit':
# If an executor has an exception, set the error state
if event.data['reason'] == 'exception':
set_error_state(error_state)
errors.append(event)
# Join and remove it
executors[event.executor_id].join()
del executors[event.executor_id]
if not no_status:
# Update the status bar on the screen
executing_jobs = []
for name, value in running_jobs.items():
number, start_time = value['package_number'], value['start_time']
if number is None or start_time is None:
continue
executing_jobs.append({
'number': number,
'name': name,
'run_time': format_time_delta_short(time.time() - start_time)
})
msg = clr("[build - {run_time}] ").format(run_time=format_time_delta_short(time.time() - start))
# If errors post those
if errors:
for error in errors:
msg += clr("[!{package}] ").format(package=error.package)
# Print them in order of started number
for job_msg_args in sorted(executing_jobs, key=lambda args: args['number']):
msg += clr("[{name} - {run_time}] ").format(**job_msg_args)
msg_rhs = clr("[{0}/{1} Active | {2}/{3} Completed]").format(
len(executing_jobs),
len(executors),
len(packages) if no_deps else len(completed_packages),
total_packages
)
# Update title bar
sys.stdout.write("\x1b]2;[build] {0}/{1}\x07".format(
len(packages) if no_deps else len(completed_packages),
total_packages
))
# Update status bar
wide_log(msg, rhs=msg_rhs, end='\r')
sys.stdout.flush()
except KeyboardInterrupt:
wide_log("[build] User interrupted, stopping.")
set_error_state(error_state)
# All executors have shutdown
sys.stdout.write("\x1b]2;\x07")
if not errors:
if context.isolate_devel:
if not context.install:
_create_unmerged_devel_setup(context)
else:
_create_unmerged_devel_setup_for_install(context)
wide_log("[build] Finished.")
if not no_notify:
notify("Build Finished", "{0} packages built".format(total_packages))
return 0
else:
wide_log(clr("[build] There were @!@{rf}errors@|:"))
if not no_notify:
notify("Build Failed", "there were {0} errors".format(len(errors)))
for error in errors:
if error.event_type == 'exit':
wide_log("""Executor '{exec_id}' had an unhandle exception while processing package '{package}':
{data[exc]}
""".format(exec_id=error.executor_id + 1, **error.__dict__))
else:
wide_log(clr("""
@{rf}Failed@| to build package '@{cf}{package}@|' because the following command:
@!@{kf}# Command run in directory: @|{location}
{cmd.cmd_str}
@{rf}Exited@| with return code: @!{retcode}@|""").format(package=error.package, **error.data))
sys.exit(1)
def clean_profile(opts, profile):
# Load the context
ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False)
if not ctx:
if not opts.workspace:
log(
"[clean] Error: The current or desired workspace could not be "
"determined. Please run `catkin clean` from within a catkin "
"workspace or specify the workspace explicitly with the "
"`--workspace` option.")
else:
log(
"[clean] Error: Could not clean workspace \"%s\" because it "
"either does not exist or it has no catkin_tools metadata." %
opts.workspace)
return False
profile = ctx.profile
# Check if the user wants to do something explicit
actions = [
'build', 'devel', 'install', 'logs',
'packages', 'orphans',
'deinit', 'setup_files']
logs_exists = os.path.exists(ctx.log_space_abs)
build_exists = os.path.exists(ctx.build_space_abs)
devel_exists = os.path.exists(ctx.devel_space_abs)
install_path = (
os.path.join(ctx.destdir, ctx.install_space_abs.lstrip(os.sep))
if ctx.destdir
else ctx.install_space_abs)
install_exists = os.path.exists(install_path)
# Default is to clean all products for this profile
no_specific_action = not any([
v for (k, v) in vars(opts).items()
if k in actions])
clean_all = opts.deinit or no_specific_action
# Initialize action options
if clean_all:
opts.logs = opts.build = opts.devel = opts.install = True
# Make sure the user intends to clena everything
spaces_to_clean = (opts.logs or opts.build or opts.devel or opts.install)
spaces_to_clean_msgs = []
if spaces_to_clean and not (opts.yes or opts.dry_run):
if opts.logs and logs_exists:
spaces_to_clean_msgs.append(clr("[clean] Log Space: @{yf}{}").format(ctx.log_space_abs))
if opts.build and build_exists:
spaces_to_clean_msgs.append(clr("[clean] Build Space: @{yf}{}").format(ctx.build_space_abs))
if opts.devel and devel_exists:
spaces_to_clean_msgs.append(clr("[clean] Devel Space: @{yf}{}").format(ctx.devel_space_abs))
if opts.install and install_exists:
spaces_to_clean_msgs.append(clr("[clean] Install Space: @{yf}{}").format(install_path))
if len(spaces_to_clean_msgs) == 0 and not opts.deinit:
log("[clean] Nothing to be cleaned for profile: `{}`".format(profile))
return True
if len(spaces_to_clean_msgs) > 0:
log("")
log(clr("[clean] @!@{yf}Warning:@| This will completely remove the "
"following directories. (Use `--yes` to skip this check)"))
for msg in spaces_to_clean_msgs:
log(msg)
try:
yes = yes_no_loop(
"\n[clean] Are you sure you want to completely remove the directories listed above?")
if not yes:
log(clr("[clean] Not removing any workspace directories for"
" this profile."))
return True
except KeyboardInterrupt:
log("\n[clean] No actions performed.")
sys.exit(0)
# Initialize flag to be used on the next invocation
needs_force = False
try:
# Remove all installspace files
if opts.install and install_exists:
log("[clean] Removing installspace: %s" % install_path)
if not opts.dry_run:
safe_rmtree(install_path, ctx.workspace, opts.force)
# Remove all develspace files
if opts.devel:
if devel_exists:
log("[clean] Removing develspace: %s" % ctx.devel_space_abs)
if not opts.dry_run:
safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force)
# Clear the cached metadata from the last build run
_, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build')
if os.path.exists(build_metadata_file):
os.unlink(build_metadata_file)
# Clear the cached packages data, if it exists
packages_metadata_path = ctx.package_metadata_path()
if os.path.exists(packages_metadata_path):
safe_rmtree(packages_metadata_path, ctx.workspace, opts.force)
# Remove all buildspace files
if opts.build and build_exists:
log("[clean] Removing buildspace: %s" % ctx.build_space_abs)
if not opts.dry_run:
safe_rmtree(ctx.build_space_abs, ctx.workspace, opts.force)
# Setup file removal
if opts.setup_files:
if devel_exists:
log("[clean] Removing setup files from develspace: %s" % ctx.devel_space_abs)
opts.packages.append('catkin')
opts.packages.append('catkin_tools_prebuild')
else:
log("[clean] No develspace exists, no setup files to clean.")
# Clean log files
if opts.logs and logs_exists:
log("[clean] Removing log space: {}".format(ctx.log_space_abs))
if not opts.dry_run:
safe_rmtree(ctx.log_space_abs, ctx.workspace, opts.force)
# Find orphaned packages
if ctx.link_devel and not any([opts.build, opts.devel]):
if opts.orphans:
if os.path.exists(ctx.build_space_abs):
log("[clean] Determining orphaned packages...")
# Get all existing packages in source space and the
# Suppress warnings since this is looking for packages which no longer exist
found_source_packages = [
pkg.name for (path, pkg) in
find_packages(ctx.source_space_abs, warnings=[]).items()]
built_packages = [
pkg.name for (path, pkg) in
find_packages(ctx.package_metadata_path(), warnings=[]).items()]
# Look for orphaned products in the build space
orphans = [p for p in built_packages
if (p not in found_source_packages and p !=
'catkin_tools_prebuild')]
if len(orphans) > 0:
opts.packages.extend(list(orphans))
else:
log("[clean] No orphans in the workspace.")
else:
log("[clean] No buildspace exists, no potential for orphans.")
# Remove specific packages
if len(opts.packages) > 0:
try:
# Clean the packages
needs_force = clean_packages(
ctx,
opts.packages,
opts.dependents,
opts.verbose,
opts.dry_run)
except KeyboardInterrupt:
wide_log("[build] User interrupted!")
return False
elif opts.orphans or len(opts.packages) > 0:
log("[clean] Error: Individual packages can only be cleaned from "
"workspaces with symbolically-linked develspaces (`catkin "
"config --link-devel`).")
except:
log("[clean] Failed to clean profile `{}`".format(profile))
needs_force = True
raise
finally:
if needs_force:
log(clr(
"[clean] @/@!Note:@| @/Parts of the workspace have been cleaned which will "
"necessitate re-configuring CMake on the next build.@|"))
update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': True})
return True
def clean_profile(opts, profile):
# Load the context
ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False)
if not ctx:
if not opts.workspace:
log(clr("[clean] @!@{rf}Error:@| The current or desired workspace could not be "
"determined. Please run `catkin clean` from within a catkin "
"workspace or specify the workspace explicitly with the "
"`--workspace` option."))
else:
log(clr("[clean] @!@{rf}Error:@| Could not clean workspace \"%s\" because it "
"either does not exist or it has no catkin_tools metadata." %
opts.workspace))
return False
profile = ctx.profile
# Check if the user wants to do something explicit
actions = ['spaces', 'packages', 'clean_this', 'orphans', 'deinit', 'setup_files']
paths = {} # noqa
paths_exists = {} # noqa
paths['install'] = (
os.path.join(ctx.destdir, ctx.install_space_abs.lstrip(os.sep))
if ctx.destdir
else ctx.install_space_abs)
paths_exists['install'] = os.path.exists(paths['install']) and os.path.isdir(paths['install'])
for space in Context.SPACES.keys():
if space in paths:
continue
paths[space] = getattr(ctx, '{}_space_abs'.format(space))
paths_exists[space] = getattr(ctx, '{}_space_exists'.format(space))()
# Default is to clean all products for this profile
no_specific_action = not any([
v for (k, v) in vars(opts).items()
if k in actions])
clean_all = opts.deinit or no_specific_action
# Initialize action options
if clean_all:
opts.spaces = [k for k in Context.SPACES.keys() if k != 'source']
# Make sure the user intends to clean everything
spaces_to_clean_msgs = []
if opts.spaces and not (opts.yes or opts.dry_run):
for space in opts.spaces:
if getattr(ctx, '{}_space_exists'.format(space))():
space_name = Context.SPACES[space]['space']
space_abs = getattr(ctx, '{}_space_abs'.format(space))
spaces_to_clean_msgs.append(clr("[clean] {:14} @{yf}{}").format(space_name + ':', space_abs))
if len(spaces_to_clean_msgs) == 0 and not opts.deinit:
log("[clean] Nothing to be cleaned for profile: `{}`".format(profile))
return True
if len(spaces_to_clean_msgs) > 0:
log("")
log(clr("[clean] @!@{yf}Warning:@| This will completely remove the "
"following directories. (Use `--yes` to skip this check)"))
for msg in spaces_to_clean_msgs:
log(msg)
try:
yes = yes_no_loop(
"\n[clean] Are you sure you want to completely remove the directories listed above?")
if not yes:
log(clr("[clean] Not removing any workspace directories for"
" this profile."))
return True
except KeyboardInterrupt:
log("\n[clean] No actions performed.")
sys.exit(0)
# Initialize flag to be used on the next invocation
needs_force = False
try:
for space in opts.spaces:
if space == 'devel':
# Remove all develspace files
if paths_exists['devel']:
log("[clean] Removing {}: {}".format(Context.SPACES['devel']['space'], ctx.devel_space_abs))
if not opts.dry_run:
safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force)
# Clear the cached metadata from the last build run
_, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build')
if os.path.exists(build_metadata_file):
os.unlink(build_metadata_file)
# Clear the cached packages data, if it exists
packages_metadata_path = ctx.package_metadata_path()
if os.path.exists(packages_metadata_path):
safe_rmtree(packages_metadata_path, ctx.workspace, opts.force)
else:
if paths_exists[space]:
space_name = Context.SPACES[space]['space']
space_path = paths[space]
log("[clean] Removing {}: {}".format(space_name, space_path))
if not opts.dry_run:
safe_rmtree(space_path, ctx.workspace, opts.force)
# Setup file removal
if opts.setup_files:
if paths_exists['devel']:
log("[clean] Removing setup files from {}: {}".format(Context.SPACES['devel']['space'], paths['devel']))
opts.packages.append('catkin')
opts.packages.append('catkin_tools_prebuild')
else:
log("[clean] No {} exists, no setup files to clean.".format(Context.SPACES['devel']['space']))
# Find orphaned packages
if ctx.link_devel or ctx.isolate_devel and not ('devel' in opts.spaces or 'build' in opts.spaces):
if opts.orphans:
if os.path.exists(ctx.build_space_abs):
log("[clean] Determining orphaned packages...")
# Get all existing packages in source space and the
# Suppress warnings since this is looking for packages which no longer exist
found_source_packages = [
pkg.name for (path, pkg) in
find_packages(ctx.source_space_abs, warnings=[]).items()]
built_packages = [
pkg.name for (path, pkg) in
find_packages(ctx.package_metadata_path(), warnings=[]).items()]
# Look for orphaned products in the build space
orphans = [p for p in built_packages
if (p not in found_source_packages and p !=
'catkin_tools_prebuild')]
if len(orphans) > 0:
opts.packages.extend(list(orphans))
else:
log("[clean] No orphans in the workspace.")
else:
log("[clean] No {} exists, no potential for orphans.".format(Context.SPACES['build']['space']))
# Remove specific packages
if len(opts.packages) > 0 or opts.clean_this:
# Determine the enclosing package
try:
ws_path = find_enclosing_workspace(getcwd())
# Suppress warnings since this won't necessarily find all packages
# in the workspace (it stops when it finds one package), and
# relying on it for warnings could mislead people.
this_package = find_enclosing_package(
search_start_path=getcwd(),
ws_path=ws_path,
warnings=[])
except InvalidPackage as ex:
sys.exit(clr("[clean] @!@{rf}Error:@| The file {} is an invalid package.xml file."
" See below for details:\n\n{}").format(ex.package_path, ex.msg))
# Handle context-based package cleaning
if opts.clean_this:
if this_package:
opts.packages += [this_package]
else:
sys.exit(
clr("[clean] @!@{rf}Error:@| In order to use --this, the current directory"
" must be part of a catkin package."))
try:
# Clean the packages
needs_force = clean_packages(
ctx,
opts.packages,
opts.dependents,
opts.verbose,
opts.dry_run)
except KeyboardInterrupt:
wide_log("[clean] User interrupted!")
return False
elif opts.orphans or len(opts.packages) > 0 or opts.clean_this:
log(clr("[clean] @!@{rf}Error:@| Individual packages cannot be cleaned from "
"workspaces with merged develspaces, use a symbolically-linked "
"or isolated develspace instead."))
except: # noqa: E722
# Silencing E722 here since we immediately re-raise the exception.
log("[clean] Failed to clean profile `{}`".format(profile))
needs_force = True
raise
finally:
if needs_force:
log(clr(
"[clean] @/@!Note:@| @/Parts of the workspace have been cleaned which will "
"necessitate re-configuring CMake on the next build.@|"))
update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': True})
return True
def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs):
"""
Print verbose execution summary.
"""
# Calculate the longest jid
max_jid_len = max([len(jid) for jid in self.available_jobs])
templates = {
'successful':
clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len),
'warned':
clr(" [ @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len),
'failed':
clr(" [ @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len),
'ignored':
clr(" [ @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len),
'abandoned':
clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len),
}
# Calculate the maximum _printed_ length for each template
max_column_len = max([
len(remove_ansi_escape(t.format(jid=("?" * max_jid_len))))
for t in templates.values()
])
# Calculate the number of columns
number_of_columns = int((terminal_width() / max_column_len) or 1)
# Construct different categories of jobs (jid -> output template)
successfuls = {}
warneds = {}
faileds = {}
ignoreds = {}
abandoneds = {}
non_whitelisted = {}
blacklisted = {}
# Give each package an output template to use
for jid in self.available_jobs:
if jid in self.blacklisted_jobs:
blacklisted[jid] = templates['ignored']
elif jid not in self.jobs:
ignoreds[jid] = templates['ignored']
elif len(self.whitelisted_jobs
) > 0 and jid not in self.whitelisted_jobs:
non_whitelisted[jid] = templates['ignored']
elif jid in completed_jobs:
if jid in failed_jobs:
faileds[jid] = templates['failed']
elif jid in warned_jobs:
warneds[jid] = templates['warned']
else:
successfuls[jid] = templates['successful']
else:
abandoneds[jid] = templates['abandoned']
# Combine successfuls and ignoreds, sort by key
if len(successfuls) + len(ignoreds) > 0:
wide_log("")
wide_log(
clr("[{}] Successful {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(
sorted(itertools.chain(successfuls.items(), ignoreds.items())),
number_of_columns)
else:
wide_log("")
wide_log(
clr("[{}] No {} succeeded.").format(self.label,
self.jobs_label))
wide_log("")
# Print out whitelisted jobs
if len(non_whitelisted) > 0:
wide_log("")
wide_log(
clr("[{}] Non-whitelisted {}:").format(self.label,
self.jobs_label))
wide_log("")
print_items_in_columns(sorted(non_whitelisted.items()),
number_of_columns)
# Print out blacklisted jobs
if len(blacklisted) > 0:
wide_log("")
wide_log(
clr("[{}] Blacklisted {}:").format(self.label,
self.jobs_label))
wide_log("")
print_items_in_columns(sorted(blacklisted.items()),
number_of_columns)
# Print out jobs that failed
if len(faileds) > 0:
wide_log("")
wide_log(
clr("[{}] Failed {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(faileds.items()), number_of_columns)
# Print out jobs that were abandoned
if len(abandoneds) > 0:
wide_log("")
wide_log(
clr("[{}] Abandoned {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(abandoneds.items()),
number_of_columns)
wide_log("")
def build_isolated_workspace(context,
packages=None,
start_with=None,
no_deps=False,
jobs=None,
force_cmake=False,
force_color=False,
quiet=False,
interleave_output=False,
no_status=False,
lock_install=False,
no_notify=False):
"""Builds a catkin workspace in isolation
This function will find all of the packages in the source space, start some
executors, feed them packages to build based on dependencies and topological
ordering, and then monitor the output of the executors, handling loggings of
the builds, starting builds, failing builds, and finishing builds of
packages, and handling the shutdown of the executors when appropriate.
:param context: context in which to build the catkin workspace
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:param packages: list of packages to build, by default their dependencies will also be built
:type packages: list
:param start_with: package to start with, skipping all packages which proceed it in the topological order
:type start_with: str
:param no_deps: If True, the dependencies of packages will not be built first
:type no_deps: bool
:param jobs: number of parallel package build jobs
:type jobs: int
:param force_cmake: forces invocation of CMake if True, default is False
:type force_cmake: bool
:param force_color: forces colored output even if terminal does not support it
:type force_color: bool
:param quiet: suppresses the output of commands unless there is an error
:type quiet: bool
:param interleave_output: prints the output of commands as they are received
:type interleave_output: bool
:param no_status: disables status bar
:type no_status: bool
:param lock_install: causes executors to synchronize on access of install commands
:type lock_install: bool
:param no_notify: suppresses system notifications
:type no_notify: bool
:raises: SystemExit if buildspace is a file or no packages were found in the source space
or if the provided options are invalid
"""
# If no_deps is given, ensure packages to build are provided
if no_deps and packages is None:
sys.exit("With --no-deps, you must specify packages to build.")
# Make sure there is a build folder and it is not a file
if os.path.exists(context.build_space_abs):
if os.path.isfile(context.build_space_abs):
sys.exit(
clr("@{rf}Error:@| Build space '{0}' exists but is a file and not a folder."
.format(context.build_space_abs)))
# If it dosen't exist, create it
else:
log("Creating build space directory, '{0}'".format(
context.build_space_abs))
os.makedirs(context.build_space_abs)
# Check for catkin_make droppings
if context.corrupted_by_catkin_make():
sys.exit(
clr("@{rf}Error:@| Build space `{0}` exists but appears to have previously been "
"created by the `catkin_make` or `catkin_make_isolated` tool. "
"Please choose a different directory to use with `catkin build` "
"or clean the build space.".format(context.build_space_abs)))
# Declare a buildspace marker describing the build config for error checking
buildspace_marker_data = {
'workspace': context.workspace,
'profile': context.profile,
'install': context.install,
'install_space': context.install_space_abs,
'devel_space': context.devel_space_abs,
'source_space': context.source_space_abs
}
# Check build config
if os.path.exists(
os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)):
with open(os.path.join(
context.build_space_abs,
BUILDSPACE_MARKER_FILE)) as buildspace_marker_file:
existing_buildspace_marker_data = yaml.load(buildspace_marker_file)
misconfig_lines = ''
for (k, v) in existing_buildspace_marker_data.items():
new_v = buildspace_marker_data.get(k, None)
if new_v != v:
misconfig_lines += (
'\n - %s: %s (stored) is not %s (commanded)' %
(k, v, new_v))
if len(misconfig_lines) > 0:
sys.exit(
clr("\n@{rf}Error:@| Attempting to build a catkin workspace using build space: "
"\"%s\" but that build space's most recent configuration "
"differs from the commanded one in ways which will cause "
"problems. Fix the following options or use @{yf}`catkin "
"clean -b`@| to remove the build space: %s" %
(context.build_space_abs, misconfig_lines)))
# Write the current build config for config error checking
with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE),
'w') as buildspace_marker_file:
buildspace_marker_file.write(
yaml.dump(buildspace_marker_data, default_flow_style=False))
# Summarize the context
summary_notes = []
if force_cmake:
summary_notes += [
clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.")
]
log(context.summary(summary_notes))
# Find list of packages in the workspace
packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built(
packages, context)
completed_packages = []
if no_deps:
# Consider deps as "completed"
completed_packages.extend(packages_to_be_built_deps)
else:
# Extend packages to be built to include their deps
packages_to_be_built.extend(packages_to_be_built_deps)
# Also resort
packages_to_be_built = topological_order_packages(
dict(packages_to_be_built))
max_package_name_length = max(
[len(pkg.name) for pth, pkg in packages_to_be_built])
# Assert start_with package is in the workspace
verify_start_with_option(start_with, packages, all_packages,
packages_to_be_built + packages_to_be_built_deps)
# Setup pool of executors
executors = {}
# The communication queue can have ExecutorEvent's or str's passed into it from the executors
comm_queue = Queue()
# The job queue has Jobs put into it
job_queue = Queue()
# Lock for install space
install_lock = Lock() if lock_install else FakeLock()
# Determine the number of executors
try:
if jobs:
jobs = int(jobs)
if jobs < 1:
sys.exit(
"Specified number of jobs '{0}' is not positive.".format(
jobs))
except ValueError:
sys.exit("Specified number of jobs '{0}' is no integer.".format(jobs))
try:
jobs = cpu_count() if jobs is None else jobs
except NotImplementedError:
log('Failed to determine the cpu_count, falling back to 1 jobs as the default.'
)
jobs = 1 if jobs is None else jobs
# If only one set of jobs, turn on interleaving to get more responsive feedback
if jobs == 1:
# TODO: make the system more intelligent so that it can automatically switch to streaming output
# when only one job is building, even if multiple jobs could be building
quiet = False
interleave_output = True
# Start the executors
for x in range(jobs):
e = Executor(x, context, comm_queue, job_queue, install_lock)
executors[x] = e
e.start()
try: # Finally close out now running executors
# Variables for tracking running jobs and built/building packages
start = time.time()
total_packages = len(packages_to_be_built)
package_count = 0
running_jobs = {}
log_dir = os.path.join(context.build_space_abs, 'build_logs')
color = True
if not force_color and not is_tty(sys.stdout):
color = True
out = OutputController(log_dir,
quiet,
interleave_output,
color,
max_package_name_length,
prefix_output=(jobs > 1))
if no_status:
disable_wide_log()
# Prime the job_queue
ready_packages = []
if start_with is None:
ready_packages = get_ready_packages(packages_to_be_built,
running_jobs,
completed_packages)
while start_with is not None:
ready_packages.extend(
get_ready_packages(packages_to_be_built, running_jobs,
completed_packages))
while ready_packages:
pth, pkg = ready_packages.pop(0)
if pkg.name != start_with:
completed_packages.append(pkg.name)
package_count += 1
wide_log("[build] Skipping package '{0}'".format(pkg.name))
else:
ready_packages.insert(0, (pth, pkg))
start_with = None
break
running_jobs = queue_ready_packages(ready_packages, running_jobs,
job_queue, context, force_cmake)
assert running_jobs
error_state = False
errors = []
def set_error_state(error_state):
if error_state:
return
# Set the error state to prevent new jobs
error_state = True
# Empty the job queue
while not job_queue.empty():
job_queue.get()
# Kill the executors by sending a None to the job queue for each of them
for x in range(jobs):
job_queue.put(None)
# While any executors are running, process executor events
while executors:
try:
# Try to get an event from the communications queue
try:
event = comm_queue.get(True, 0.1)
except Empty:
# timeout occured, create null event to pass through checks
event = ExecutorEvent(None, None, None, None)
if event.event_type == 'job_started':
package_count += 1
running_jobs[
event.package]['package_number'] = package_count
running_jobs[event.package]['start_time'] = time.time()
out.job_started(event.package)
if event.event_type == 'command_started':
out.command_started(event.package, event.data['cmd'],
event.data['location'])
if event.event_type == 'command_log':
out.command_log(event.package, event.data['message'])
if event.event_type == 'command_failed':
out.command_failed(event.package, event.data['cmd'],
event.data['location'],
event.data['retcode'])
# Add to list of errors
errors.append(event)
# Remove the command from the running jobs
del running_jobs[event.package]
# If it hasn't already been done, stop the executors
set_error_state(error_state)
if event.event_type == 'command_finished':
out.command_finished(event.package, event.data['cmd'],
event.data['location'],
event.data['retcode'])
if event.event_type == 'job_finished':
completed_packages.append(event.package)
run_time = format_time_delta(
time.time() -
running_jobs[event.package]['start_time'])
out.job_finished(event.package, run_time)
del running_jobs[event.package]
# If shutting down, do not add new packages
if error_state:
continue
# Calculate new packages
if not no_status:
wide_log('[build] Calculating new jobs...', end='\r')
sys.stdout.flush()
ready_packages = get_ready_packages(
packages_to_be_built, running_jobs, completed_packages)
running_jobs = queue_ready_packages(
ready_packages, running_jobs, job_queue, context,
force_cmake)
# Make sure there are jobs to be/being processed, otherwise kill the executors
if not running_jobs:
# Kill the executors by sending a None to the job queue for each of them
for x in range(jobs):
job_queue.put(None)
# If an executor exit event, join it and remove it from the executors list
if event.event_type == 'exit':
# If an executor has an exception, set the error state
if event.data['reason'] == 'exception':
set_error_state(error_state)
errors.append(event)
# Join and remove it
executors[event.executor_id].join()
del executors[event.executor_id]
if not no_status:
# Update the status bar on the screen
executing_jobs = []
for name, value in running_jobs.items():
number, job, start_time = value[
'package_number'], value['job'], value[
'start_time']
if number is None or start_time is None:
continue
executing_jobs.append({
'number':
number,
'name':
name,
'run_time':
format_time_delta_short(time.time() - start_time)
})
msg = clr("[build - {run_time}] ").format(
run_time=format_time_delta_short(time.time() - start))
# If errors post those
if errors:
for error in errors:
msg += clr("[!{package}] ").format(
package=error.package)
# Print them in order of started number
for job_msg_args in sorted(
executing_jobs, key=lambda args: args['number']):
msg += clr("[{name} - {run_time}] ").format(
**job_msg_args)
msg_rhs = clr(
"[{0}/{1} Active | {2}/{3} Completed]").format(
len(executing_jobs), len(executors),
len(packages) if no_deps else
len(completed_packages), total_packages)
# Update title bar
sys.stdout.write("\x1b]2;[build] {0}/{1}\x07".format(
len(packages) if no_deps else len(completed_packages),
total_packages))
# Update status bar
wide_log(msg, rhs=msg_rhs, end='\r')
sys.stdout.flush()
except KeyboardInterrupt:
wide_log("[build] User interrupted, stopping.")
set_error_state(error_state)
# All executors have shutdown
sys.stdout.write("\x1b]2;\x07")
if not errors:
if context.isolate_devel:
if not context.install:
_create_unmerged_devel_setup(context)
else:
_create_unmerged_devel_setup_for_install(context)
wide_log("[build] Finished.")
if not no_notify:
notify("Build Finished",
"{0} packages built".format(total_packages))
return 0
else:
wide_log(clr("[build] There were @!@{rf}errors@|:"))
if not no_notify:
notify("Build Failed",
"there were {0} errors".format(len(errors)))
for error in errors:
if error.event_type == 'exit':
wide_log(
"""Executor '{exec_id}' had an unhandle exception while processing package '{package}':
{data[exc]}
""".format(exec_id=error.executor_id + 1, **error.__dict__))
else:
wide_log(
clr("""
@{rf}Failed@| to build package '@{cf}{package}@|' because the following command:
@!@{kf}# Command run in directory: @|{location}
{cmd.cmd_str}
@{rf}Exited@| with return code: @!{retcode}@|""").format(
package=error.package, **error.data))
sys.exit(1)
finally:
# Ensure executors go down
for x in range(jobs):
job_queue.put(None)
def run(self):
queued_jobs = []
active_jobs = []
completed_jobs = {}
failed_jobs = []
warned_jobs = []
cumulative_times = dict()
start_times = dict()
active_stages = dict()
start_time = self.pre_start_time or time.time()
last_update_time = time.time()
# If the status rate is too low, just disable it
if self.active_status_rate < 1E-3:
self.show_active_status = False
else:
update_duration = 1.0 / self.active_status_rate
# Disable the wide log padding if the status is disabled
if not self.show_active_status:
disable_wide_log()
while True:
# Check if we should stop
if not self.keep_running:
wide_log(
clr('[{}] An internal error occurred!').format(self.label))
return
# Write a continuously-updated status line
if self.show_active_status:
# Try to get an event from the queue (non-blocking)
try:
event = self.event_queue.get(False)
except Empty:
# Determine if the status should be shown based on the desired
# status rate
elapsed_time = time.time() - last_update_time
show_status_now = elapsed_time > update_duration
if show_status_now:
# Print live status (overwrites last line)
status_line = clr(
'[{} {} s] [{}/{} complete] [{}/{} jobs] [{} queued]'
).format(
self.label,
format_time_delta_short(time.time() - start_time),
len(completed_jobs), len(self.jobs),
job_server.running_jobs(), job_server.max_jobs(),
len(queued_jobs) + len(active_jobs) -
len(active_stages))
# Show failed jobs
if len(failed_jobs) > 0:
status_line += clr(
' [@!@{rf}{}@| @{rf}failed@|]').format(
len(failed_jobs))
# Check load / mem
if not job_server.load_ok():
status_line += clr(' [@!@{rf}High Load@|]')
if not job_server.mem_ok():
status_line += clr(' [@!@{rf}Low Memory@|]')
# Add active jobs
if len(active_jobs) == 0:
status_line += clr(
' @/@!@{kf}Waiting for jobs...@|')
else:
active_labels = []
for j, (s, t, p) in active_stages.items():
d = format_time_delta_short(
cumulative_times[j] + time.time() - t)
if p == '':
active_labels.append(
clr('[{}:{} - {}]').format(j, s, d))
else:
active_labels.append(
clr('[{}:{} ({}%) - {}]').format(
j, s, p, d))
status_line += ' ' + ' '.join(active_labels)
# Print the status line
# wide_log(status_line)
wide_log(status_line, rhs='', end='\r')
sys.stdout.flush()
# Store this update time
last_update_time = time.time()
else:
time.sleep(
max(0.0, min(update_duration - elapsed_time,
0.01)))
# Only continue when no event was received
continue
else:
# Try to get an event from the queue (blocking)
try:
event = self.event_queue.get(True)
except Empty:
break
# A `None` event is a signal to terminate
if event is None:
break
# Handle the received events
eid = event.event_id
if 'JOB_STATUS' == eid:
queued_jobs = event.data['queued']
active_jobs = event.data['active']
completed_jobs = event.data['completed']
# Check if all jobs have finished in some way
if all([
len(event.data[t]) == 0
for t in ['pending', 'queued', 'active']
]):
break
elif 'STARTED_JOB' == eid:
cumulative_times[event.data['job_id']] = 0.0
wide_log(
clr('Starting >>> {:<{}}').format(event.data['job_id'],
self.max_jid_length))
elif 'FINISHED_JOB' == eid:
duration = format_time_delta(
cumulative_times[event.data['job_id']])
if event.data['succeeded']:
wide_log(
clr('Finished <<< {:<{}} [ {} ]').format(
event.data['job_id'], self.max_jid_length,
duration))
else:
failed_jobs.append(event.data['job_id'])
wide_log(
clr('Failed <<< {:<{}} [ {} ]').format(
event.data['job_id'], self.max_jid_length,
duration))
elif 'ABANDONED_JOB' == eid:
# Create a human-readable reason string
if 'DEP_FAILED' == event.data['reason']:
direct = event.data['dep_job_id'] == event.data[
'direct_dep_job_id']
if direct:
reason = clr('Depends on failed job {}').format(
event.data['dep_job_id'])
else:
reason = clr('Depends on failed job {} via {}').format(
event.data['dep_job_id'],
event.data['direct_dep_job_id'])
elif 'PEER_FAILED' == event.data['reason']:
reason = clr('Unrelated job failed')
elif 'MISSING_DEPS' == event.data['reason']:
reason = clr('Depends on unknown jobs: {}').format(
', '.join([
clr('@!{}@|').format(jid)
for jid in event.data['dep_ids']
]))
wide_log(
clr('Abandoned <<< {:<{}} [ {} ]').format(
event.data['job_id'], self.max_jid_length, reason))
elif 'STARTED_STAGE' == eid:
active_stages[event.data['job_id']] = [
event.data['stage_label'], event.time, ''
]
start_times[event.data['job_id']] = event.time
if self.show_stage_events:
wide_log(
clr('Starting >> {}:{}').format(
event.data['job_id'], event.data['stage_label']))
elif 'STAGE_PROGRESS' == eid:
active_stages[event.data['job_id']][2] = event.data['percent']
elif 'SUBPROCESS' == eid:
if self.show_stage_events:
wide_log(
clr('Subprocess > {}:{} `{}`').format(
event.data['job_id'], event.data['stage_label'],
event.data['stage_repro']))
elif 'FINISHED_STAGE' == eid:
# Get the stage duration
duration = event.time - start_times[event.data['job_id']]
cumulative_times[event.data['job_id']] += duration
# This is no longer the active stage for this job
del active_stages[event.data['job_id']]
header_border = None
header_border_file = sys.stdout
header_title = None
header_title_file = sys.stdout
lines = []
footer_title = None
footer_title_file = sys.stdout
footer_border = None
footer_border_file = sys.stdout
# Generate headers / borders for output
if event.data['succeeded']:
footer_title = clr('Finished << {}:{}').format(
event.data['job_id'], event.data['stage_label'])
if len(event.data['stderr']) > 0:
# Mark that this job warned about something
if event.data['job_id'] not in warned_jobs:
warned_jobs.append(event.data['job_id'])
# Output contains warnings
header_border = clr('@!@{yf}' + '_' *
(terminal_width() - 1) + '@|')
header_border_file = sys.stderr
header_title = clr('Warnings << {}:{} {}').format(
event.data['job_id'], event.data['stage_label'],
event.data['logfile_filename'])
header_title_file = sys.stderr
footer_border = clr('@{yf}' + '.' *
(terminal_width() - 1) + '@|')
footer_border_file = sys.stderr
else:
# Normal output, no warnings
header_title = clr('Output << {}:{} {}').format(
event.data['job_id'], event.data['stage_label'],
event.data['logfile_filename'])
# Don't print footer title
if not self.show_stage_events:
footer_title = None
else:
# Output contains errors
header_border = clr('@!@{rf}' + '_' *
(terminal_width() - 1) + '@|')
header_border_file = sys.stderr
header_title = clr('Errors << {}:{} {}').format(
event.data['job_id'], event.data['stage_label'],
event.data['logfile_filename'])
header_title_file = sys.stderr
footer_border = clr('@{rf}' + '.' *
(terminal_width() - 1) + '@|')
footer_border_file = sys.stderr
footer_title = clr(
'Failed << {}:{:<{}} [ Exited with code {} ]').format(
event.data['job_id'], event.data['stage_label'],
max(
0, self.max_jid_length -
len(event.data['job_id'])),
event.data['retcode'])
footer_title_file = sys.stderr
lines_target = sys.stdout
if self.show_buffered_stdout:
if len(event.data['interleaved']) > 0:
lines = [
line for line in
event.data['interleaved'].splitlines(True)
if (self.show_compact_io is False
or len(line.strip()) > 0)
]
else:
header_border = None
header_title = None
footer_border = None
elif self.show_buffered_stderr:
if len(event.data['stderr']) > 0:
lines = [
line
for line in event.data['stderr'].splitlines(True)
if (self.show_compact_io is False
or len(line.strip()) > 0)
]
lines_target = sys.stderr
else:
header_border = None
header_title = None
footer_border = None
if len(lines) > 0:
if self.show_repro_cmd:
if event.data['repro'] is not None:
lines.append(
clr('@!@{kf}{}@|\n').format(
event.data['repro']))
# Print the output
if header_border:
wide_log(header_border, file=header_border_file)
if header_title:
wide_log(header_title, file=header_title_file)
if len(lines) > 0:
wide_log(''.join(lines), end='\r', file=lines_target)
if footer_border:
wide_log(footer_border, file=footer_border_file)
if footer_title:
wide_log(footer_title, file=footer_title_file)
elif 'STDERR' == eid:
if self.show_live_stderr and len(event.data['data']) > 0:
wide_log(self.format_interleaved_lines(event.data),
end='\r',
file=sys.stderr)
elif 'STDOUT' == eid:
if self.show_live_stdout and len(event.data['data']) > 0:
wide_log(self.format_interleaved_lines(event.data),
end='\r')
elif 'MESSAGE' == eid:
wide_log(event.data['msg'])
# Print the full summary
if self.show_full_summary:
self.print_exec_summary(completed_jobs, warned_jobs, failed_jobs)
# Print a compact summary
if self.show_summary or self.show_full_summary:
self.print_compact_summary(completed_jobs, warned_jobs,
failed_jobs)
# Print final runtime
wide_log(
clr('[{}] Runtime: {} total.').format(
self.label, format_time_delta(time.time() - start_time)))
def determine_packages_to_be_built(packages, context, workspace_packages):
"""Returns list of packages which should be built, and those package's deps.
:param packages: list of packages to be built, if None all packages are built
:type packages: list
:param context: Workspace context
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:returns: tuple of packages to be built and those package's deps
:rtype: tuple
"""
start = time.time()
# If there are no packages raise
if not workspace_packages:
log("[build] No packages were found in the source space '{0}'".format(
context.source_space_abs))
else:
wide_log("[build] Found '{0}' packages in {1}.".format(
len(workspace_packages), format_time_delta(time.time() - start)))
# Order the packages by topology
ordered_packages = topological_order_packages(workspace_packages)
# Set the packages in the workspace for the context
context.packages = ordered_packages
# Determine the packages which should be built
packages_to_be_built = []
packages_to_be_built_deps = []
# Check if topological_order_packages determined any circular dependencies, if so print an error and fail.
# If this is the case, the last entry of ordered packages is a tuple that starts with nil.
if ordered_packages and ordered_packages[-1][0] is None:
guilty_packages = ", ".join(ordered_packages[-1][1:])
sys.exit(
"[build] Circular dependency detected in the following packages: {}"
.format(guilty_packages))
workspace_package_names = dict([(pkg.name, (path, pkg))
for path, pkg in ordered_packages])
# Determine the packages to be built
if packages:
# First assert all of the packages given are in the workspace
for package in packages:
if package not in workspace_package_names:
# Try whether package is a pattern and matches
glob_packages = expand_glob_package(package,
workspace_package_names)
if len(glob_packages) > 0:
packages.extend(glob_packages)
continue
else:
sys.exit(
"[build] Given package '{0}' is not in the workspace "
"and pattern does not match any package".format(
package))
# If metapackage, include run depends which are in the workspace
package_obj = workspace_package_names[package][1]
if 'metapackage' in [e.tagname for e in package_obj.exports]:
for rdep in package_obj.run_depends:
if rdep.name in workspace_package_names:
packages.append(rdep.name)
# Limit the packages to be built to just the provided packages
for pkg_path, package in ordered_packages:
if package.name in packages:
packages_to_be_built.append((pkg_path, package))
# Get the recursive dependencies for each of these packages
pkg_deps = get_cached_recursive_build_depends_in_workspace(
package, ordered_packages)
packages_to_be_built_deps.extend(pkg_deps)
else:
# Only use whitelist when no other packages are specified
if len(context.whitelist) > 0:
# Expand glob patterns in whitelist
whitelist = []
for whitelisted_package in context.whitelist:
whitelist.extend(
expand_glob_package(whitelisted_package,
workspace_package_names))
packages_to_be_built = [
p for p in ordered_packages if (p[1].name in whitelist)
]
else:
packages_to_be_built = ordered_packages
# Filter packages with blacklist
if len(context.blacklist) > 0:
# Expand glob patterns in blacklist
blacklist = []
for blacklisted_package in context.blacklist:
blacklist.extend(
expand_glob_package(blacklisted_package,
workspace_package_names))
# Apply blacklist to packages and dependencies
packages_to_be_built = [
(path, pkg) for path, pkg in packages_to_be_built
if (pkg.name not in blacklist or pkg.name in packages)
]
packages_to_be_built_deps = [
(path, pkg) for path, pkg in packages_to_be_built_deps
if (pkg.name not in blacklist or pkg.name in packages)
]
return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def determine_packages_to_be_built(packages, context, workspace_packages):
"""Returns list of packages which should be built, and those package's deps.
:param packages: list of packages to be built, if None all packages are built
:type packages: list
:param context: Workspace context
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:returns: tuple of packages to be built and those package's deps
:rtype: tuple
"""
start = time.time()
# If there are no packages raise
if not workspace_packages:
sys.exit("[build] No packages were found in the source space '{0}'".format(context.source_space_abs))
wide_log("[build] Found '{0}' packages in {1}."
.format(len(workspace_packages), format_time_delta(time.time() - start)))
# Order the packages by topology
ordered_packages = topological_order_packages(workspace_packages)
# Set the packages in the workspace for the context
context.packages = ordered_packages
# Determin the packages which should be built
packages_to_be_built = []
packages_to_be_built_deps = []
# Determine the packages to be built
if packages:
# First assert all of the packages given are in the workspace
workspace_package_names = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages])
for package in packages:
if package not in workspace_package_names:
sys.exit("[build] Given package '{0}' is not in the workspace".format(package))
# If metapackage, include run depends which are in the workspace
package_obj = workspace_package_names[package][1]
if 'metapackage' in [e.tagname for e in package_obj.exports]:
for rdep in package_obj.run_depends:
if rdep.name in workspace_package_names:
packages.append(rdep.name)
# Limit the packages to be built to just the provided packages
for pkg_path, package in ordered_packages:
if package.name in packages:
packages_to_be_built.append((pkg_path, package))
# Get the recursive dependencies for each of these packages
pkg_deps = get_cached_recursive_build_depends_in_workspace(package, ordered_packages)
packages_to_be_built_deps.extend(pkg_deps)
else:
# Only use whitelist when no other packages are specified
if len(context.whitelist) > 0:
packages_to_be_built = [p for p in ordered_packages if (p[1].name in context.whitelist)]
else:
packages_to_be_built = ordered_packages
# Filter packages with blacklist
if len(context.blacklist) > 0:
packages_to_be_built = [
(path, pkg) for path, pkg in packages_to_be_built
if (pkg.name not in context.blacklist or pkg.name in packages)]
packages_to_be_built_deps = [
(path, pkg) for path, pkg in packages_to_be_built_deps
if (pkg.name not in context.blacklist or pkg.name in packages)]
ordered_packages = ordered_packages
return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def document_workspace(
context,
packages=None,
start_with=None,
no_deps=False,
n_jobs=None,
force_color=False,
quiet=False,
interleave_output=False,
no_status=False,
limit_status_rate=10.0,
no_notify=False,
continue_on_failure=False,
summarize_build=None
):
pre_start_time = time.time()
# Get all the packages in the context source space
# Suppress warnings since this is a utility function
workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[])
# If no_deps is given, ensure packages to build are provided
if no_deps and packages is None:
log(fmt("[document] @!@{rf}Error:@| With no_deps, you must specify packages to build."))
return
# Find list of packages in the workspace
packages_to_be_documented, packages_to_be_documented_deps, all_packages = determine_packages_to_be_built(
packages, context, workspace_packages)
if not no_deps:
# Extend packages to be documented to include their deps
packages_to_be_documented.extend(packages_to_be_documented_deps)
# Also re-sort
try:
packages_to_be_documented = topological_order_packages(dict(packages_to_be_documented))
except AttributeError:
log(fmt("[document] @!@{rf}Error:@| The workspace packages have a circular "
"dependency, and cannot be documented. Please run `catkin list "
"--deps` to determine the problematic package(s)."))
return
# Check the number of packages to be documented
if len(packages_to_be_documented) == 0:
log(fmt('[document] No packages to be documented.'))
# Assert start_with package is in the workspace
verify_start_with_option(
start_with,
packages,
all_packages,
packages_to_be_documented + packages_to_be_documented_deps)
# Remove packages before start_with
if start_with is not None:
for path, pkg in list(packages_to_be_documented):
if pkg.name != start_with:
wide_log(fmt("@!@{pf}Skipping@| @{gf}---@| @{cf}{}@|").format(pkg.name))
packages_to_be_documented.pop(0)
else:
break
# Get the names of all packages to be built
packages_to_be_documented_names = [p.name for _, p in packages_to_be_documented]
packages_to_be_documeted_deps_names = [p.name for _, p in packages_to_be_documented_deps]
jobs = []
# Construct jobs
for pkg_path, pkg in all_packages:
if pkg.name not in packages_to_be_documented_names:
continue
# Get actual execution deps
deps = [
p.name for _, p
in get_cached_recursive_build_depends_in_workspace(pkg, packages_to_be_documented)
]
jobs.append(create_package_job(context, pkg, pkg_path, deps))
# Special job for post-job summary sphinx step.
jobs.append(create_summary_job(context, package_names=packages_to_be_documented_names))
# Queue for communicating status
event_queue = Queue()
try:
# Spin up status output thread
status_thread = ConsoleStatusController(
'document',
['package', 'packages'],
jobs,
n_jobs,
[pkg.name for _, pkg in context.packages],
[p for p in context.whitelist],
[p for p in context.blacklist],
event_queue,
show_notifications=not no_notify,
show_active_status=not no_status,
show_buffered_stdout=not quiet and not interleave_output,
show_buffered_stderr=not interleave_output,
show_live_stdout=interleave_output,
show_live_stderr=interleave_output,
show_stage_events=not quiet,
show_full_summary=(summarize_build is True),
pre_start_time=pre_start_time,
active_status_rate=limit_status_rate)
status_thread.start()
# Block while running N jobs asynchronously
try:
all_succeeded = run_until_complete(execute_jobs(
'document',
jobs,
None,
event_queue,
context.log_space_abs,
max_toplevel_jobs=n_jobs,
continue_on_failure=continue_on_failure,
continue_without_deps=False))
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
return 0 if all_succeeded else 1
except KeyboardInterrupt:
wide_log("[document] Interrupted by user!")
event_queue.put(None)
return 130 # EOWNERDEAD
def clean_packages(context, names_of_packages_to_be_cleaned, clean_dependents,
verbose, dry_run):
pre_start_time = time.time()
# Update the names of packages to be cleaned with dependents
packages_to_be_cleaned = determine_packages_to_be_cleaned(
context, clean_dependents, names_of_packages_to_be_cleaned)
# print(packages_to_be_cleaned)
# for path, pkg in packages_to_be_cleaned:
# if os.path.exists(os.path.join(context.build_space_abs, pkg.name)):
# print("[clean] Cleaning package: %s" % pkg.name)
# Construct jobs
jobs = []
for path, pkg in packages_to_be_cleaned:
# Get all build type plugins
clean_job_creators = {
ep.name: ep.load()['create_clean_job']
for ep in pkg_resources.iter_entry_points(
group='catkin_tools.jobs')
}
# It's a problem if there aren't any build types available
if len(clean_job_creators) == 0:
sys.exit(
'Error: No build types availalbe. Please check your catkin_tools installation.'
)
# Determine the job parameters
clean_job_kwargs = dict(
context=context,
package=pkg,
package_path=path,
dependencies=[], # Unused because clean jobs are not parallelized
dry_run=dry_run,
clean_build=True,
clean_devel=True,
clean_install=True)
# Create the job based on the build type
build_type = get_build_type(pkg)
if build_type in clean_job_creators:
jobs.append(clean_job_creators[build_type](**clean_job_kwargs))
if len(jobs) == 0:
print(
"[clean] There are no products from the given packages to clean.")
return False
# Queue for communicating status
event_queue = Queue()
# Spin up status output thread
status_thread = ConsoleStatusController(
'clean', ['package', 'packages'],
jobs,
1, [pkg.name for _, pkg in context.packages],
[p for p in context.whitelist], [p for p in context.blacklist],
event_queue,
show_notifications=False,
show_active_status=False,
show_buffered_stdout=verbose or False,
show_buffered_stderr=True,
show_live_stdout=False,
show_live_stderr=False,
show_stage_events=False,
show_full_summary=False,
pre_start_time=pre_start_time,
active_status_rate=10.0)
status_thread.start()
# Initialize locks (none need to be configured here)
locks = {}
# Block while running N jobs asynchronously
try:
ej = execute_jobs('clean',
jobs,
locks,
event_queue,
context.log_space_abs,
max_toplevel_jobs=1,
continue_on_failure=True,
continue_without_deps=False)
all_succeeded = run_until_complete(ej)
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
return all_succeeded
def clean_packages(
context,
names_of_packages_to_be_cleaned,
clean_dependents,
verbose,
dry_run):
pre_start_time = time.time()
# Update the names of packages to be cleaned with dependents
packages_to_be_cleaned = determine_packages_to_be_cleaned(
context,
clean_dependents,
names_of_packages_to_be_cleaned)
# print(packages_to_be_cleaned)
# for path, pkg in packages_to_be_cleaned:
# if os.path.exists(os.path.join(context.build_space_abs, pkg.name)):
# print("[clean] Cleaning package: %s" % pkg.name)
# Construct jobs
jobs = []
for path, pkg in packages_to_be_cleaned:
# Get all build type plugins
clean_job_creators = {
ep.name: ep.load()['create_clean_job']
for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs')
}
# It's a problem if there aren't any build types available
if len(clean_job_creators) == 0:
sys.exit('Error: No build types availalbe. Please check your catkin_tools installation.')
# Determine the job parameters
clean_job_kwargs = dict(
context=context,
package=pkg,
package_path=path,
dependencies=[], # Unused because clean jobs are not parallelized
dry_run=dry_run,
clean_build=True,
clean_devel=True,
clean_install=True)
# Create the job based on the build type
build_type = get_build_type(pkg)
if build_type in clean_job_creators:
jobs.append(clean_job_creators[build_type](**clean_job_kwargs))
if len(jobs) == 0:
print("[clean] There are no products from the given packages to clean.")
return False
# Queue for communicating status
event_queue = Queue()
# Spin up status output thread
status_thread = ConsoleStatusController(
'clean',
['package', 'packages'],
jobs,
1,
[pkg.name for _, pkg in context.packages],
[p for p in context.whitelist],
[p for p in context.blacklist],
event_queue,
show_notifications=False,
show_active_status=False,
show_buffered_stdout=verbose or False,
show_buffered_stderr=True,
show_live_stdout=False,
show_live_stderr=False,
show_stage_events=False,
show_full_summary=False,
pre_start_time=pre_start_time,
active_status_rate=10.0)
status_thread.start()
# Initialize locks (none need to be configured here)
locks = {
}
# Block while running N jobs asynchronously
try:
ej = execute_jobs(
'clean',
jobs,
locks,
event_queue,
context.log_space_abs,
max_toplevel_jobs=1,
continue_on_failure=True,
continue_without_deps=False)
all_succeeded = run_until_complete(ej)
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
return all_succeeded
