def _do_call_do_repository(repository, command): """Run the command's _do_repository on the given repository. This will track the invocation and its outcome. """ # pylint: disable=protected-access logging.info("%s processing %s", command.name, repository.name) try: metric_labels = command.determine_metric_labels() metric_labels["repository"] = repository.name result = command.metrics.track_and_time_call( "RunRepositoryCommand", metric_labels, command.metrics.default_determine_outcome_labels, command._do_repository_wrapper, repository, ) logging.info("%s finished %s", command.name, repository.name) return result except Exception as ex: maybe_log_exception( "{command} on repo={repo}".format(command=command.name, repo=repository.name), ex, ) raise
def _do_call_do_repository(repository, command): """Run the command's _do_repository on the given repository. This will track the invocation and its outcome. """ # pylint: disable=protected-access def determine_outcome_labels(): """Add a repository label in addition to whatever the command wanted.""" result = command.determine_outcome_metric_labels() result['repository'] = repository.name return result logging.info('%s processing %s', command.name, repository.name) try: tracking_labels = command.determine_tracking_metric_labels() tracking_labels['repository'] = repository.name result = command.metrics.instrument_track_and_outcome( 'RunRepositoryCommand', 'Command invocations scoped to one repository', tracking_labels, determine_outcome_labels, command._do_repository_wrapper, repository) logging.info('%s finished %s', command.name, repository.name) return result except Exception as ex: maybe_log_exception( '{command} on repo={repo}'.format(command=command.name, repo=repository.name), ex) raise
def load_bom_from_url(self, url): """Returns the bom specification dict from a gcs url.""" logging.debug('Loading %s', url) try: text = check_subprocess('gsutil cat ' + url) return yaml.load(text) except Exception as ex: self.__bad_files[self.url_to_bom_name(url)] = exception_to_message(ex) maybe_log_exception('load_from_from_url', ex, action_msg='Skipping %s' % url) return None
def __call__(self): logging.debug('Running command=%s...', self.name) try: metric_labels = self.determine_metric_labels() result = self.metrics.track_and_time_call( 'RunCommand', metric_labels, self.metrics.default_determine_outcome_labels, self._do_command) logging.debug('Finished command=%s', self.name) return result except Exception as ex: maybe_log_exception(self.name, ex) raise
def wrapped_main(): """Run main and dump outstanding threads when done.""" # pylint: disable=broad-except try: retcode = main() except Exception as ex: sys.stdout.flush() maybe_log_exception('main()', ex, action_msg='Terminating') logging.error("FAILED") retcode = -1 dump_threads() return retcode
def wrapped_main(): """Run main and dump outstanding threads when done.""" # pylint: disable=broad-except try: retcode = main() except Exception as ex: sys.stdout.flush() maybe_log_exception('main()', ex, action_msg='Terminating') logging.error("FAILED") retcode = -1 dump_threads() return retcode
def __call__(self): logging.debug('Running command=%s...', self.name) try: tracking_labels = self.determine_tracking_metric_labels() result = self.metrics.instrument_track_and_outcome( 'RunCommand', 'Command Invocations', tracking_labels, self.determine_outcome_metric_labels, self._do_command) logging.debug('Finished command=%s', self.name) return result except Exception as ex: maybe_log_exception(self.name, ex) raise
def __build_from_live_server(self, repository): """Implements CommandProcessor interface.""" docs_url_path = SWAGGER_URL_PATHS[repository.name] env = dict(os.environ) port = unused_port() env['SERVER_PORT'] = str(port) base_url = 'http://localhost:' + str(port) gate_logfile = self.get_logfile_path(repository.name + '-apidocs-server') logging.info( 'Starting up prototype %s so we can extract docs from it.' ' We will log this instance to %s', repository.name, gate_logfile) boot_run_cmd = './gradlew' # default will run ensure_dir_exists(os.path.dirname(gate_logfile)) gate_logstream = open(gate_logfile, 'w') process = start_subprocess(boot_run_cmd, stream=gate_logstream, stdout=gate_logstream, cwd=repository.git_dir, env=env) max_wait_secs = self.options.max_wait_secs_startup # pylint: disable=broad-except try: logging.info('Waiting up to %s secs for %s to be ready on port %d', max_wait_secs, repository.name, port) self.wait_for_url(base_url + '/health', max_wait_secs) json_path = os.path.join(self.get_output_dir(), 'docs.json') self.__generate_json_from_url(repository, base_url + '/' + docs_url_path, json_path) self.build_swagger_docs(repository, json_path) finally: try: gate_logstream.flush() gate_logstream.write( '\n' + log_timestring() + ' ***** buildtool is killing subprocess *****\n') logging.info( 'Killing %s subprocess %s now that we are done with it', repository.name, process.pid) process.kill() wait_subprocess(process) gate_logstream.close() except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping {name} subprocess {pid}.' .format(name=repository.name, pid=process.pid))
def ingest_bom(self, line): """Function to ingest a single bom into the result map.""" bom = self.load_bom_from_url(line) if not bom: return try: if bom['version'] + '.yml' != line[line.rfind('/') + 1:]: message = 'BOM version "%s" != filename "%s"' % (bom['version'], line) self.__bad_files[self.url_to_bom_name(line.strip())] = message logging.warning(message) raise_and_log_error(UnexpectedError(message)) self.analyze_bom(bom) except Exception as ex: self.__bad_files[self.url_to_bom_name(line.strip())] = ex.message maybe_log_exception('analyze_bom', ex, action_msg='Skipping %s' % line)
def __build_from_live_server(self, repository): """Implements CommandProcessor interface.""" docs_url_path = SWAGGER_URL_PATHS[repository.name] env = dict(os.environ) port = unused_port() env['SERVER_PORT'] = str(port) base_url = 'http://localhost:' + str(port) gate_logfile = self.get_logfile_path(repository.name + '-apidocs-server') logging.info('Starting up prototype %s so we can extract docs from it.' ' We will log this instance to %s', repository.name, gate_logfile) boot_run_cmd = './gradlew' # default will run ensure_dir_exists(os.path.dirname(gate_logfile)) gate_logstream = open(gate_logfile, 'w') process = start_subprocess( boot_run_cmd, stream=gate_logstream, stdout=gate_logstream, cwd=repository.git_dir, env=env) max_wait_secs = self.options.max_wait_secs_startup # pylint: disable=broad-except try: logging.info('Waiting up to %s secs for %s to be ready on port %d', max_wait_secs, repository.name, port) self.wait_for_url(base_url + '/health', max_wait_secs) json_path = os.path.join(self.get_output_dir(), 'docs.json') self.__generate_json_from_url( repository, base_url + '/' + docs_url_path, json_path) self.build_swagger_docs(repository, json_path) finally: try: gate_logstream.flush() gate_logstream.write( '\n' + log_timestring() + ' ***** buildtool is killing subprocess *****\n') logging.info('Killing %s subprocess %s now that we are done with it', repository.name, process.pid) process.kill() wait_subprocess(process) gate_logstream.close() except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping {name} subprocess {pid}.' .format(name=repository.name, pid=process.pid))
def _do_command(self): """Wraps base method so we can start/stop redis if needed.""" self._ensure_templates_directory() redis_name = 'redis-server' start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0 logging.debug('redis-server %s running.', 'IS NOT' if start_redis else 'IS') try: if start_redis: check_subprocess('sudo service %s start' % redis_name) super(BuildApiDocsCommand, self)._do_command() finally: if start_redis: # pylint: disable=broad-except try: check_subprocess('sudo service %s stop' % redis_name) except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping temporary ' + redis_name)
def _do_command(self): """Wraps base method so we can start/stop redis if needed.""" self._ensure_templates_directory() redis_name = 'redis-server' start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0 logging.debug('redis-server %s running.', 'IS NOT' if start_redis else 'IS') try: if start_redis: check_subprocess('sudo service %s start' % redis_name) super(BuildApiDocsCommand, self)._do_command() finally: if start_redis: # pylint: disable=broad-except try: check_subprocess('sudo service %s stop' % redis_name) except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping temporary ' + redis_name)
def _do_call_do_repository(repository, command): """Run the command's _do_repository on the given repository. This will track the invocation and its outcome. """ # pylint: disable=protected-access logging.info('%s processing %s', command.name, repository.name) try: metric_labels = command.determine_metric_labels() metric_labels['repository'] = repository.name result = command.metrics.track_and_time_call( 'RunRepositoryCommand', metric_labels, command.metrics.default_determine_outcome_labels, command._do_repository_wrapper, repository) logging.info('%s finished %s', command.name, repository.name) return result except Exception as ex: maybe_log_exception( '{command} on repo={repo}'.format( command=command.name, repo=repository.name), ex) raise