def _do_call_do_repository(repository, command):
"""Run the command's _do_repository on the given repository.
This will track the invocation and its outcome.
"""
# pylint: disable=protected-access
logging.info("%s processing %s", command.name, repository.name)
try:
metric_labels = command.determine_metric_labels()
metric_labels["repository"] = repository.name
result = command.metrics.track_and_time_call(
"RunRepositoryCommand",
metric_labels,
command.metrics.default_determine_outcome_labels,
command._do_repository_wrapper,
repository,
)
logging.info("%s finished %s", command.name, repository.name)
return result
except Exception as ex:
maybe_log_exception(
"{command} on repo={repo}".format(command=command.name,
repo=repository.name),
ex,
)
raise
def _do_call_do_repository(repository, command):
"""Run the command's _do_repository on the given repository.
This will track the invocation and its outcome.
"""
# pylint: disable=protected-access
def determine_outcome_labels():
"""Add a repository label in addition to whatever the command wanted."""
result = command.determine_outcome_metric_labels()
result['repository'] = repository.name
return result
logging.info('%s processing %s', command.name, repository.name)
try:
tracking_labels = command.determine_tracking_metric_labels()
tracking_labels['repository'] = repository.name
result = command.metrics.instrument_track_and_outcome(
'RunRepositoryCommand',
'Command invocations scoped to one repository', tracking_labels,
determine_outcome_labels, command._do_repository_wrapper,
repository)
logging.info('%s finished %s', command.name, repository.name)
return result
except Exception as ex:
maybe_log_exception(
'{command} on repo={repo}'.format(command=command.name,
repo=repository.name), ex)
raise
def load_bom_from_url(self, url):
"""Returns the bom specification dict from a gcs url."""
logging.debug('Loading %s', url)
try:
text = check_subprocess('gsutil cat ' + url)
return yaml.load(text)
except Exception as ex:
self.__bad_files[self.url_to_bom_name(url)] = exception_to_message(ex)
maybe_log_exception('load_from_from_url', ex,
action_msg='Skipping %s' % url)
return None
def __call__(self):
logging.debug('Running command=%s...', self.name)
try:
metric_labels = self.determine_metric_labels()
result = self.metrics.track_and_time_call(
'RunCommand', metric_labels,
self.metrics.default_determine_outcome_labels,
self._do_command)
logging.debug('Finished command=%s', self.name)
return result
except Exception as ex:
maybe_log_exception(self.name, ex)
raise
def wrapped_main():
"""Run main and dump outstanding threads when done."""
# pylint: disable=broad-except
try:
retcode = main()
except Exception as ex:
sys.stdout.flush()
maybe_log_exception('main()', ex, action_msg='Terminating')
logging.error("FAILED")
retcode = -1
dump_threads()
return retcode
def wrapped_main():
"""Run main and dump outstanding threads when done."""
# pylint: disable=broad-except
try:
retcode = main()
except Exception as ex:
sys.stdout.flush()
maybe_log_exception('main()', ex, action_msg='Terminating')
logging.error("FAILED")
retcode = -1
dump_threads()
return retcode
def __call__(self):
logging.debug('Running command=%s...', self.name)
try:
tracking_labels = self.determine_tracking_metric_labels()
result = self.metrics.instrument_track_and_outcome(
'RunCommand', 'Command Invocations', tracking_labels,
self.determine_outcome_metric_labels,
self._do_command)
logging.debug('Finished command=%s', self.name)
return result
except Exception as ex:
maybe_log_exception(self.name, ex)
raise
def __build_from_live_server(self, repository):
"""Implements CommandProcessor interface."""
docs_url_path = SWAGGER_URL_PATHS[repository.name]
env = dict(os.environ)
port = unused_port()
env['SERVER_PORT'] = str(port)
base_url = 'http://localhost:' + str(port)
gate_logfile = self.get_logfile_path(repository.name +
'-apidocs-server')
logging.info(
'Starting up prototype %s so we can extract docs from it.'
' We will log this instance to %s', repository.name, gate_logfile)
boot_run_cmd = './gradlew' # default will run
ensure_dir_exists(os.path.dirname(gate_logfile))
gate_logstream = open(gate_logfile, 'w')
process = start_subprocess(boot_run_cmd,
stream=gate_logstream,
stdout=gate_logstream,
cwd=repository.git_dir,
env=env)
max_wait_secs = self.options.max_wait_secs_startup
# pylint: disable=broad-except
try:
logging.info('Waiting up to %s secs for %s to be ready on port %d',
max_wait_secs, repository.name, port)
self.wait_for_url(base_url + '/health', max_wait_secs)
json_path = os.path.join(self.get_output_dir(), 'docs.json')
self.__generate_json_from_url(repository,
base_url + '/' + docs_url_path,
json_path)
self.build_swagger_docs(repository, json_path)
finally:
try:
gate_logstream.flush()
gate_logstream.write(
'\n' + log_timestring() +
' ***** buildtool is killing subprocess *****\n')
logging.info(
'Killing %s subprocess %s now that we are done with it',
repository.name, process.pid)
process.kill()
wait_subprocess(process)
gate_logstream.close()
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping {name} subprocess {pid}.'
.format(name=repository.name, pid=process.pid))
def ingest_bom(self, line):
"""Function to ingest a single bom into the result map."""
bom = self.load_bom_from_url(line)
if not bom:
return
try:
if bom['version'] + '.yml' != line[line.rfind('/') + 1:]:
message = 'BOM version "%s" != filename "%s"' % (bom['version'], line)
self.__bad_files[self.url_to_bom_name(line.strip())] = message
logging.warning(message)
raise_and_log_error(UnexpectedError(message))
self.analyze_bom(bom)
except Exception as ex:
self.__bad_files[self.url_to_bom_name(line.strip())] = ex.message
maybe_log_exception('analyze_bom', ex,
action_msg='Skipping %s' % line)
def __build_from_live_server(self, repository):
"""Implements CommandProcessor interface."""
docs_url_path = SWAGGER_URL_PATHS[repository.name]
env = dict(os.environ)
port = unused_port()
env['SERVER_PORT'] = str(port)
base_url = 'http://localhost:' + str(port)
gate_logfile = self.get_logfile_path(repository.name + '-apidocs-server')
logging.info('Starting up prototype %s so we can extract docs from it.'
' We will log this instance to %s',
repository.name, gate_logfile)
boot_run_cmd = './gradlew' # default will run
ensure_dir_exists(os.path.dirname(gate_logfile))
gate_logstream = open(gate_logfile, 'w')
process = start_subprocess(
boot_run_cmd, stream=gate_logstream, stdout=gate_logstream,
cwd=repository.git_dir, env=env)
max_wait_secs = self.options.max_wait_secs_startup
# pylint: disable=broad-except
try:
logging.info('Waiting up to %s secs for %s to be ready on port %d',
max_wait_secs, repository.name, port)
self.wait_for_url(base_url + '/health', max_wait_secs)
json_path = os.path.join(self.get_output_dir(), 'docs.json')
self.__generate_json_from_url(
repository, base_url + '/' + docs_url_path, json_path)
self.build_swagger_docs(repository, json_path)
finally:
try:
gate_logstream.flush()
gate_logstream.write(
'\n' + log_timestring()
+ ' ***** buildtool is killing subprocess *****\n')
logging.info('Killing %s subprocess %s now that we are done with it',
repository.name, process.pid)
process.kill()
wait_subprocess(process)
gate_logstream.close()
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping {name} subprocess {pid}.'
.format(name=repository.name, pid=process.pid))
def _do_command(self):
"""Wraps base method so we can start/stop redis if needed."""
self._ensure_templates_directory()
redis_name = 'redis-server'
start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0
logging.debug('redis-server %s running.',
'IS NOT' if start_redis else 'IS')
try:
if start_redis:
check_subprocess('sudo service %s start' % redis_name)
super(BuildApiDocsCommand, self)._do_command()
finally:
if start_redis:
# pylint: disable=broad-except
try:
check_subprocess('sudo service %s stop' % redis_name)
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping temporary ' + redis_name)
def _do_command(self):
"""Wraps base method so we can start/stop redis if needed."""
self._ensure_templates_directory()
redis_name = 'redis-server'
start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0
logging.debug('redis-server %s running.',
'IS NOT' if start_redis else 'IS')
try:
if start_redis:
check_subprocess('sudo service %s start' % redis_name)
super(BuildApiDocsCommand, self)._do_command()
finally:
if start_redis:
# pylint: disable=broad-except
try:
check_subprocess('sudo service %s stop' % redis_name)
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping temporary ' + redis_name)
def _do_call_do_repository(repository, command):
"""Run the command's _do_repository on the given repository.
This will track the invocation and its outcome.
"""
# pylint: disable=protected-access
logging.info('%s processing %s', command.name, repository.name)
try:
metric_labels = command.determine_metric_labels()
metric_labels['repository'] = repository.name
result = command.metrics.track_and_time_call(
'RunRepositoryCommand',
metric_labels, command.metrics.default_determine_outcome_labels,
command._do_repository_wrapper, repository)
logging.info('%s finished %s', command.name, repository.name)
return result
except Exception as ex:
maybe_log_exception(
'{command} on repo={repo}'.format(
command=command.name, repo=repository.name),
ex)
raise
