def test_log_path(): # use log base path log_base_path = '/some/path'.replace('/', os.sep) set_default_log_path(base_path=log_base_path) assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use log base path if environment variable is not set log_base_path_env_var = 'TEST_COLCON_LOG_BASE_PATH' set_default_log_path(base_path=log_base_path, env_var=log_base_path_env_var) assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use explicitly passed log base path even if environment variable is set with EnvironmentContext(**{log_base_path_env_var: '/not/used'}): assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use environment variable when set and no base path passed log_base_path = '/other/path'.replace('/', os.sep) set_default_log_path(base_path=None, env_var=log_base_path_env_var) with EnvironmentContext(**{log_base_path_env_var: log_base_path}): assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use specific subdirectory subdirectory = 'sub' set_default_log_path(base_path=log_base_path, env_var=log_base_path_env_var, subdirectory=subdirectory) assert isinstance(get_log_path(), Path) assert get_log_path() == Path(log_base_path) / subdirectory
def test_log_path(): # use log base path log_base_path = '/some/path'.replace('/', os.sep) set_default_log_path(base_path=log_base_path) assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use log base path if environment variable is not set log_base_path_env_var = 'TEST_COLCON_LOG_BASE_PATH' set_default_log_path( base_path=log_base_path, env_var=log_base_path_env_var) assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use environment variable when set log_base_path = '/other/path'.replace('/', os.sep) with EnvironmentContext(**{log_base_path_env_var: log_base_path}): assert isinstance(get_log_path(), Path) assert str(get_log_path().parent) == log_base_path # use specific subdirectory subdirectory = 'sub' with patch('colcon_core.location.logger.info') as info: set_default_log_path( base_path=log_base_path, env_var=log_base_path_env_var, subdirectory=subdirectory) assert isinstance(get_log_path(), Path) assert get_log_path() == Path(log_base_path) / subdirectory info.assert_called_once_with( "Using log path '{log_base_path}/{subdirectory}'" .format_map(locals()).replace('/', os.sep))
def get_log_directory(job): """ Get the log directory for a specific job. :param job: The job :rtype: Path """ return get_log_path() / job.identifier
def _init_log(self): # only create log once if self._path is not None: return create_log_path(self.context.args.verb_name) self._path = get_log_path() / EventLogEventHandler.FILENAME with self._path.open(mode='w'): pass
def _init_log(self): # only create log once if self._file_handle is not None: return True log_path = get_log_path() if log_path is None: return False create_log_path(self.context.args.verb_name) path = log_path / EventLogEventHandler.FILENAME self._file_handle = path.open(mode='w') return True
def _handle(self, event): job = event[1] log_f = get_log_path() / job.identifier / STDOUT_STDERR_LOG_FILENAME if not log_f.exists(): return self._log_parser.set_package(job.identifier) with open(log_f, 'r') as in_file: for line in in_file: self._log_parser.add_line(line) with open('sanitizer_report.csv', 'w') as report_csv_f_out: report_csv_f_out.write(self._log_parser.csv) with open('sanitizer_report.xml', 'w') as report_xml_f_out: report_xml_f_out.write(self._log_parser.xml)
def _handle(self, event) -> None: """Handle JobEnded event and parse the test log file.""" job = event[1] # type: JobEnded self._log_parser.set_package(job.identifier) try: log_f = get_log_path( ) / job.identifier / STDOUT_STDERR_LOG_FILENAME with open(log_f, 'r') as in_file: for line in in_file: self._log_parser.parse_line(line) except IOError: logger.info('Could not open stdout_stderr.log file') with open('sanitizer_report.csv', 'w') as report_csv_f_out: report_csv_f_out.write(self._log_parser.get_csv()) with open('test_results.xml', 'w') as report_xml_f_out: report_xml_f_out.write(self._log_parser.get_xml())
def _init_logs(self, job): global all_log_filenames # only create logs once per task if job in self._jobs: return True log_path = get_log_path() if log_path is None: return False self._jobs.add(job) create_log_path(self.context.args.verb_name) base_path = get_log_directory(job) os.makedirs(str(base_path), exist_ok=True) for filename in all_log_filenames: path = base_path / filename self._file_handles[path] = path.open(mode='wb') return True
def _main(*, command_name, argv): global colcon_logger # default log level colcon_logger.setLevel(logging.WARNING) set_logger_level_from_env( colcon_logger, '{command_name}_LOG_LEVEL'.format_map(locals()).upper()) colcon_logger.debug( 'Command line arguments: {argv}' .format(argv=argv if argv is not None else sys.argv)) # set default locations for config files set_default_config_path( path=( Path('~') / '.{command_name}'.format_map(locals())).expanduser(), env_var='{command_name}_HOME'.format_map(locals()).upper()) parser = create_parser('colcon_core.environment_variable') verb_extensions = get_verb_extensions() # add subparsers for all verb extensions but without arguments for now subparser = create_subparser( parser, command_name, verb_extensions, attribute='verb_name') verb_parsers = add_parsers_without_arguments( parser, subparser, verb_extensions, attribute='verb_name') with SuppressUsageOutput([parser] + list(verb_parsers.values())): known_args, _ = parser.parse_known_args(args=argv) # add the arguments for the requested verb if known_args.verb_name: add_parser_arguments(known_args.verb_parser, known_args.verb_extension) args = parser.parse_args(args=argv) context = CommandContext(command_name=command_name, args=args) if args.log_level: colcon_logger.setLevel(args.log_level) colcon_logger.debug( 'Parsed command line arguments: {args}'.format_map(locals())) # error: no verb provided if args.verb_name is None: print(parser.format_usage()) return 'Error: No verb provided' # set default locations for log files now = datetime.datetime.now() now_str = str(now)[:-7].replace(' ', '_').replace(':', '-') set_default_log_path( base_path=args.log_base, env_var='{command_name}_LOG_PATH'.format_map(locals()).upper(), subdirectory='{args.verb_name}_{now_str}'.format_map(locals())) # add a file handler writing all levels if logging isn't disabled log_path = get_log_path() if log_path is not None: create_log_path(args.verb_name) handler = add_file_handler( colcon_logger, log_path / 'logger_all.log') # write previous log messages to the file handler log_record = colcon_logger.makeRecord( colcon_logger.name, logging.DEBUG, __file__, 0, 'Command line arguments: {argv}' .format(argv=argv if argv is not None else sys.argv), None, None) handler.handle(log_record) log_record = colcon_logger.makeRecord( colcon_logger.name, logging.DEBUG, __file__, 0, 'Parsed command line arguments: {args}'.format_map(locals()), None, None) handler.handle(log_record) # invoke verb return verb_main(context, colcon_logger)
def main(*, command_name='colcon', argv=None): """ Execute the main logic of the command. The overview of the process: * Configure logging level based on an environment variable * Configure the configuration path * Create the argument parser * Document all environment variables * Decorate the parsers with additional functionality * Add the available verbs and their arguments * Configure logging level based on an arguments * Create an invocation specific log directory * Invoke the logic of the selected verb (if applicable) :param str command_name: The name of the command invoked :param list argv: The list of arguments :returns: The return code """ global colcon_logger # default log level colcon_logger.setLevel(logging.WARN) set_logger_level_from_env( colcon_logger, '{command_name}_LOG_LEVEL'.format_map(locals()).upper()) colcon_logger.debug('Command line arguments: {argv}'.format( argv=argv if argv is not None else sys.argv)) # set default locations for config files set_default_config_path( path=(Path('~') / '.{command_name}'.format_map(locals())).expanduser(), env_var='{command_name}_HOME'.format_map(locals()).upper()) parser = create_parser('colcon_core.environment_variable') verb_extensions = get_verb_extensions() # add subparsers for all verb extensions but without arguments for now subparser = create_subparser(parser, command_name, verb_extensions, attribute='verb_name') verb_parsers = add_parsers_without_arguments(parser, subparser, verb_extensions, attribute='verb_name') with SuppressUsageOutput([parser] + list(verb_parsers.values())): known_args, _ = parser.parse_known_args(args=argv) # add the arguments for the requested verb if known_args.verb_name: add_parser_arguments(known_args.verb_parser, known_args.verb_extension) args = parser.parse_args(args=argv) context = CommandContext(command_name=command_name, args=args) if args.log_level: colcon_logger.setLevel(args.log_level) colcon_logger.debug('Parsed command line arguments: {args}'.format_map( locals())) # error: no verb provided if args.verb_name is None: print(parser.format_usage()) return 'Error: No verb provided' # set default locations for log files now = datetime.datetime.now() now_str = str(now)[:-7].replace(' ', '_').replace(':', '-') set_default_log_path( base_path=args.log_base, env_var='{command_name}_LOG_PATH'.format_map(locals()).upper(), subdirectory='{args.verb_name}_{now_str}'.format_map(locals())) # add a file handler writing all levels create_log_path(args.verb_name) handler = add_file_handler(colcon_logger, get_log_path() / 'logger_all.log') # write previous log messages to the file handler log_record = colcon_logger.makeRecord( colcon_logger.name, logging.DEBUG, __file__, 0, 'Command line arguments: {argv}'.format( argv=argv if argv is not None else sys.argv), None, None) handler.handle(log_record) log_record = colcon_logger.makeRecord( colcon_logger.name, logging.DEBUG, __file__, 0, 'Parsed command line arguments: {args}'.format_map(locals()), None, None) handler.handle(log_record) # invoke verb return verb_main(context, colcon_logger)
def __call__(self, event): # noqa: D102 global all_log_filenames data = event[0] job = event[1] if isinstance(data, JobStarted): self._start_times[job] = time.monotonic() if isinstance(data, JobEnded): # Skip if the log path is /dev/null if get_log_path() is None: return base_path = get_log_directory(job) for filename in all_log_filenames: path = base_path / filename if path in self._file_handles: self._file_handles[path].close() return filenames = copy.copy(all_log_filenames) if not isinstance(data, Command): filenames.remove(COMMAND_LOG_FILENAME) if not isinstance(data, StdoutLine): filenames.remove(STDOUT_LOG_FILENAME) if not isinstance(data, StderrLine): filenames.remove(STDERR_LOG_FILENAME) if (not isinstance(data, StdoutLine) and not isinstance(data, StderrLine)): filenames.remove(STDOUT_STDERR_LOG_FILENAME) if len(filenames) <= 1: # skip if event is neither of the known events return if not self._init_logs(job): return if isinstance(data, Command): line = data.to_string() + '\n' else: line = data.line if not isinstance(line, bytes): # use the same encoding as the default for the opened file line = line.encode(encoding=locale.getpreferredencoding(False)) base_path = get_log_directory(job) for filename in filenames: h = self._file_handles[base_path / filename] if filename == ALL_STREAMS_LOG_FILENAME: # prefix line with relative time relative_time = time.monotonic() - self._start_times[job] # use the same encoding as the default for the opened file prefix = ('[%.3fs] ' % relative_time).encode( encoding=locale.getpreferredencoding(False)) h.write(prefix) h.write(line) try: h.flush() except OSError as e: if e.errno == errno.ENOSPC: # for known error code suppress the stacktrace # and only show the exception message raise RuntimeError( str(e) + ' [{h.name}]'.format_map(locals())) raise