def test_log_path():
# use log base path
log_base_path = '/some/path'.replace('/', os.sep)
set_default_log_path(base_path=log_base_path)
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use log base path if environment variable is not set
log_base_path_env_var = 'TEST_COLCON_LOG_BASE_PATH'
set_default_log_path(base_path=log_base_path,
env_var=log_base_path_env_var)
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use explicitly passed log base path even if environment variable is set
with EnvironmentContext(**{log_base_path_env_var: '/not/used'}):
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use environment variable when set and no base path passed
log_base_path = '/other/path'.replace('/', os.sep)
set_default_log_path(base_path=None, env_var=log_base_path_env_var)
with EnvironmentContext(**{log_base_path_env_var: log_base_path}):
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use specific subdirectory
subdirectory = 'sub'
set_default_log_path(base_path=log_base_path,
env_var=log_base_path_env_var,
subdirectory=subdirectory)
assert isinstance(get_log_path(), Path)
assert get_log_path() == Path(log_base_path) / subdirectory
def test_log_path():
# use log base path
log_base_path = '/some/path'.replace('/', os.sep)
set_default_log_path(base_path=log_base_path)
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use log base path if environment variable is not set
log_base_path_env_var = 'TEST_COLCON_LOG_BASE_PATH'
set_default_log_path(
base_path=log_base_path, env_var=log_base_path_env_var)
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use environment variable when set
log_base_path = '/other/path'.replace('/', os.sep)
with EnvironmentContext(**{log_base_path_env_var: log_base_path}):
assert isinstance(get_log_path(), Path)
assert str(get_log_path().parent) == log_base_path
# use specific subdirectory
subdirectory = 'sub'
with patch('colcon_core.location.logger.info') as info:
set_default_log_path(
base_path=log_base_path, env_var=log_base_path_env_var,
subdirectory=subdirectory)
assert isinstance(get_log_path(), Path)
assert get_log_path() == Path(log_base_path) / subdirectory
info.assert_called_once_with(
"Using log path '{log_base_path}/{subdirectory}'"
.format_map(locals()).replace('/', os.sep))
def get_log_directory(job):
"""
Get the log directory for a specific job.
:param job: The job
:rtype: Path
"""
return get_log_path() / job.identifier
def _init_log(self):
# only create log once
if self._path is not None:
return
create_log_path(self.context.args.verb_name)
self._path = get_log_path() / EventLogEventHandler.FILENAME
with self._path.open(mode='w'):
pass
def _init_log(self):
# only create log once
if self._file_handle is not None:
return True
log_path = get_log_path()
if log_path is None:
return False
create_log_path(self.context.args.verb_name)
path = log_path / EventLogEventHandler.FILENAME
self._file_handle = path.open(mode='w')
return True
def _handle(self, event):
job = event[1]
log_f = get_log_path() / job.identifier / STDOUT_STDERR_LOG_FILENAME
if not log_f.exists():
return
self._log_parser.set_package(job.identifier)
with open(log_f, 'r') as in_file:
for line in in_file:
self._log_parser.add_line(line)
with open('sanitizer_report.csv', 'w') as report_csv_f_out:
report_csv_f_out.write(self._log_parser.csv)
with open('sanitizer_report.xml', 'w') as report_xml_f_out:
report_xml_f_out.write(self._log_parser.xml)
def _handle(self, event) -> None:
"""Handle JobEnded event and parse the test log file."""
job = event[1] # type: JobEnded
self._log_parser.set_package(job.identifier)
try:
log_f = get_log_path(
) / job.identifier / STDOUT_STDERR_LOG_FILENAME
with open(log_f, 'r') as in_file:
for line in in_file:
self._log_parser.parse_line(line)
except IOError:
logger.info('Could not open stdout_stderr.log file')
with open('sanitizer_report.csv', 'w') as report_csv_f_out:
report_csv_f_out.write(self._log_parser.get_csv())
with open('test_results.xml', 'w') as report_xml_f_out:
report_xml_f_out.write(self._log_parser.get_xml())
def _init_logs(self, job):
global all_log_filenames
# only create logs once per task
if job in self._jobs:
return True
log_path = get_log_path()
if log_path is None:
return False
self._jobs.add(job)
create_log_path(self.context.args.verb_name)
base_path = get_log_directory(job)
os.makedirs(str(base_path), exist_ok=True)
for filename in all_log_filenames:
path = base_path / filename
self._file_handles[path] = path.open(mode='wb')
return True
def _main(*, command_name, argv):
global colcon_logger
# default log level
colcon_logger.setLevel(logging.WARNING)
set_logger_level_from_env(
colcon_logger, '{command_name}_LOG_LEVEL'.format_map(locals()).upper())
colcon_logger.debug(
'Command line arguments: {argv}'
.format(argv=argv if argv is not None else sys.argv))
# set default locations for config files
set_default_config_path(
path=(
Path('~') / '.{command_name}'.format_map(locals())).expanduser(),
env_var='{command_name}_HOME'.format_map(locals()).upper())
parser = create_parser('colcon_core.environment_variable')
verb_extensions = get_verb_extensions()
# add subparsers for all verb extensions but without arguments for now
subparser = create_subparser(
parser, command_name, verb_extensions, attribute='verb_name')
verb_parsers = add_parsers_without_arguments(
parser, subparser, verb_extensions, attribute='verb_name')
with SuppressUsageOutput([parser] + list(verb_parsers.values())):
known_args, _ = parser.parse_known_args(args=argv)
# add the arguments for the requested verb
if known_args.verb_name:
add_parser_arguments(known_args.verb_parser, known_args.verb_extension)
args = parser.parse_args(args=argv)
context = CommandContext(command_name=command_name, args=args)
if args.log_level:
colcon_logger.setLevel(args.log_level)
colcon_logger.debug(
'Parsed command line arguments: {args}'.format_map(locals()))
# error: no verb provided
if args.verb_name is None:
print(parser.format_usage())
return 'Error: No verb provided'
# set default locations for log files
now = datetime.datetime.now()
now_str = str(now)[:-7].replace(' ', '_').replace(':', '-')
set_default_log_path(
base_path=args.log_base,
env_var='{command_name}_LOG_PATH'.format_map(locals()).upper(),
subdirectory='{args.verb_name}_{now_str}'.format_map(locals()))
# add a file handler writing all levels if logging isn't disabled
log_path = get_log_path()
if log_path is not None:
create_log_path(args.verb_name)
handler = add_file_handler(
colcon_logger, log_path / 'logger_all.log')
# write previous log messages to the file handler
log_record = colcon_logger.makeRecord(
colcon_logger.name, logging.DEBUG, __file__, 0,
'Command line arguments: {argv}'
.format(argv=argv if argv is not None else sys.argv),
None, None)
handler.handle(log_record)
log_record = colcon_logger.makeRecord(
colcon_logger.name, logging.DEBUG, __file__, 0,
'Parsed command line arguments: {args}'.format_map(locals()),
None, None)
handler.handle(log_record)
# invoke verb
return verb_main(context, colcon_logger)
def main(*, command_name='colcon', argv=None):
"""
Execute the main logic of the command.
The overview of the process:
* Configure logging level based on an environment variable
* Configure the configuration path
* Create the argument parser
* Document all environment variables
* Decorate the parsers with additional functionality
* Add the available verbs and their arguments
* Configure logging level based on an arguments
* Create an invocation specific log directory
* Invoke the logic of the selected verb (if applicable)
:param str command_name: The name of the command invoked
:param list argv: The list of arguments
:returns: The return code
"""
global colcon_logger
# default log level
colcon_logger.setLevel(logging.WARN)
set_logger_level_from_env(
colcon_logger, '{command_name}_LOG_LEVEL'.format_map(locals()).upper())
colcon_logger.debug('Command line arguments: {argv}'.format(
argv=argv if argv is not None else sys.argv))
# set default locations for config files
set_default_config_path(
path=(Path('~') / '.{command_name}'.format_map(locals())).expanduser(),
env_var='{command_name}_HOME'.format_map(locals()).upper())
parser = create_parser('colcon_core.environment_variable')
verb_extensions = get_verb_extensions()
# add subparsers for all verb extensions but without arguments for now
subparser = create_subparser(parser,
command_name,
verb_extensions,
attribute='verb_name')
verb_parsers = add_parsers_without_arguments(parser,
subparser,
verb_extensions,
attribute='verb_name')
with SuppressUsageOutput([parser] + list(verb_parsers.values())):
known_args, _ = parser.parse_known_args(args=argv)
# add the arguments for the requested verb
if known_args.verb_name:
add_parser_arguments(known_args.verb_parser, known_args.verb_extension)
args = parser.parse_args(args=argv)
context = CommandContext(command_name=command_name, args=args)
if args.log_level:
colcon_logger.setLevel(args.log_level)
colcon_logger.debug('Parsed command line arguments: {args}'.format_map(
locals()))
# error: no verb provided
if args.verb_name is None:
print(parser.format_usage())
return 'Error: No verb provided'
# set default locations for log files
now = datetime.datetime.now()
now_str = str(now)[:-7].replace(' ', '_').replace(':', '-')
set_default_log_path(
base_path=args.log_base,
env_var='{command_name}_LOG_PATH'.format_map(locals()).upper(),
subdirectory='{args.verb_name}_{now_str}'.format_map(locals()))
# add a file handler writing all levels
create_log_path(args.verb_name)
handler = add_file_handler(colcon_logger,
get_log_path() / 'logger_all.log')
# write previous log messages to the file handler
log_record = colcon_logger.makeRecord(
colcon_logger.name, logging.DEBUG, __file__, 0,
'Command line arguments: {argv}'.format(
argv=argv if argv is not None else sys.argv), None, None)
handler.handle(log_record)
log_record = colcon_logger.makeRecord(
colcon_logger.name, logging.DEBUG, __file__, 0,
'Parsed command line arguments: {args}'.format_map(locals()), None,
None)
handler.handle(log_record)
# invoke verb
return verb_main(context, colcon_logger)
def __call__(self, event): # noqa: D102
global all_log_filenames
data = event[0]
job = event[1]
if isinstance(data, JobStarted):
self._start_times[job] = time.monotonic()
if isinstance(data, JobEnded):
# Skip if the log path is /dev/null
if get_log_path() is None:
return
base_path = get_log_directory(job)
for filename in all_log_filenames:
path = base_path / filename
if path in self._file_handles:
self._file_handles[path].close()
return
filenames = copy.copy(all_log_filenames)
if not isinstance(data, Command):
filenames.remove(COMMAND_LOG_FILENAME)
if not isinstance(data, StdoutLine):
filenames.remove(STDOUT_LOG_FILENAME)
if not isinstance(data, StderrLine):
filenames.remove(STDERR_LOG_FILENAME)
if (not isinstance(data, StdoutLine)
and not isinstance(data, StderrLine)):
filenames.remove(STDOUT_STDERR_LOG_FILENAME)
if len(filenames) <= 1:
# skip if event is neither of the known events
return
if not self._init_logs(job):
return
if isinstance(data, Command):
line = data.to_string() + '\n'
else:
line = data.line
if not isinstance(line, bytes):
# use the same encoding as the default for the opened file
line = line.encode(encoding=locale.getpreferredencoding(False))
base_path = get_log_directory(job)
for filename in filenames:
h = self._file_handles[base_path / filename]
if filename == ALL_STREAMS_LOG_FILENAME:
# prefix line with relative time
relative_time = time.monotonic() - self._start_times[job]
# use the same encoding as the default for the opened file
prefix = ('[%.3fs] ' % relative_time).encode(
encoding=locale.getpreferredencoding(False))
h.write(prefix)
h.write(line)
try:
h.flush()
except OSError as e:
if e.errno == errno.ENOSPC:
# for known error code suppress the stacktrace
# and only show the exception message
raise RuntimeError(
str(e) + ' [{h.name}]'.format_map(locals()))
raise
