def start_subprocess(cmd, stream=None, stdout=None, echo=False, **kwargs):
"""Starts a subprocess and returns handle to it."""
split_cmd = shlex.split(cmd)
actual_command = cmd if kwargs.get('shell') else split_cmd
log_level = logging.INFO if echo else logging.DEBUG
extra_log_info = ''
if 'cwd' in kwargs:
extra_log_info += ' in cwd="%s"' % kwargs['cwd']
logging.log(log_level, 'Running %s%s...', repr(cmd), extra_log_info)
start_date = datetime.datetime.now()
if stream:
stream.write('{time} Spawning {cmd!r}{extra}\n----\n\n'.format(
time=log_timestring(now=start_date), cmd=cmd,
extra=extra_log_info))
stream.flush()
process = subprocess.Popen(actual_command,
close_fds=True,
stdout=stdout or subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
logging.log(log_level, 'Running %s as pid %s', split_cmd[0], process.pid)
process.start_date = start_date
time.sleep(0) # yield this thread
return process
def wait_subprocess(process, stream=None, echo=False, postprocess_hook=None):
"""Waits for subprocess to finish and returns (final status, stdout).
This will also consume the remaining output to return it.
Returns:
Process exit code, stdout remaining in process prior to this invocation.
Any previously read output from the process will not be included.
"""
text_lines = []
if process.stdout is not None:
# stdout isnt going to another stream; collect it from the pipe.
for raw_line in iter(process.stdout.readline, ''):
if not raw_line:
break
decoded_line = raw_line.decode(encoding='utf-8')
text_lines.append(decoded_line)
if stream:
stream.write(raw_line)
stream.flush()
process.wait()
if stream is None:
# Close stdout pipe if we didnt give a stream.
# Otherwise caller owns the stream.
process.stdout.close()
if hasattr(process, 'start_date'):
end_date = datetime.datetime.now()
delta_time_str = timedelta_string(end_date - process.start_date)
else:
delta_time_str = 'UNKNOWN'
returncode = process.returncode
stdout = ''.join(text_lines)
if stream:
stream.write('\n\n----\n{time} Spawned process completed'
' with returncode {returncode} in {delta_time}.\n'.format(
time=log_timestring(now=end_date),
returncode=returncode,
delta_time=delta_time_str))
stream.flush()
if echo:
logging.info('%s returned %d with output:\n%s', process.pid,
returncode, stdout)
logging.debug('Finished %s with returncode=%d in %s', process.pid,
returncode, delta_time_str)
if postprocess_hook:
postprocess_hook(returncode, stdout)
return returncode, stdout.strip()
def __build_from_live_server(self, repository):
"""Implements CommandProcessor interface."""
docs_url_path = SWAGGER_URL_PATHS[repository.name]
env = dict(os.environ)
port = unused_port()
env['SERVER_PORT'] = str(port)
base_url = 'http://localhost:' + str(port)
gate_logfile = self.get_logfile_path(repository.name +
'-apidocs-server')
logging.info(
'Starting up prototype %s so we can extract docs from it.'
' We will log this instance to %s', repository.name, gate_logfile)
boot_run_cmd = './gradlew' # default will run
ensure_dir_exists(os.path.dirname(gate_logfile))
gate_logstream = open(gate_logfile, 'w')
process = start_subprocess(boot_run_cmd,
stream=gate_logstream,
stdout=gate_logstream,
cwd=repository.git_dir,
env=env)
max_wait_secs = self.options.max_wait_secs_startup
# pylint: disable=broad-except
try:
logging.info('Waiting up to %s secs for %s to be ready on port %d',
max_wait_secs, repository.name, port)
self.wait_for_url(base_url + '/health', max_wait_secs)
json_path = os.path.join(self.get_output_dir(), 'docs.json')
self.__generate_json_from_url(repository,
base_url + '/' + docs_url_path,
json_path)
self.build_swagger_docs(repository, json_path)
finally:
try:
gate_logstream.flush()
gate_logstream.write(
'\n' + log_timestring() +
' ***** buildtool is killing subprocess *****\n')
logging.info(
'Killing %s subprocess %s now that we are done with it',
repository.name, process.pid)
process.kill()
wait_subprocess(process)
gate_logstream.close()
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping {name} subprocess {pid}.'
.format(name=repository.name, pid=process.pid))
def __build_from_live_server(self, repository):
"""Implements CommandProcessor interface."""
docs_url_path = SWAGGER_URL_PATHS[repository.name]
env = dict(os.environ)
port = unused_port()
env['SERVER_PORT'] = str(port)
base_url = 'http://localhost:' + str(port)
gate_logfile = self.get_logfile_path(repository.name + '-apidocs-server')
logging.info('Starting up prototype %s so we can extract docs from it.'
' We will log this instance to %s',
repository.name, gate_logfile)
boot_run_cmd = './gradlew' # default will run
ensure_dir_exists(os.path.dirname(gate_logfile))
gate_logstream = open(gate_logfile, 'w')
process = start_subprocess(
boot_run_cmd, stream=gate_logstream, stdout=gate_logstream,
cwd=repository.git_dir, env=env)
max_wait_secs = self.options.max_wait_secs_startup
# pylint: disable=broad-except
try:
logging.info('Waiting up to %s secs for %s to be ready on port %d',
max_wait_secs, repository.name, port)
self.wait_for_url(base_url + '/health', max_wait_secs)
json_path = os.path.join(self.get_output_dir(), 'docs.json')
self.__generate_json_from_url(
repository, base_url + '/' + docs_url_path, json_path)
self.build_swagger_docs(repository, json_path)
finally:
try:
gate_logstream.flush()
gate_logstream.write(
'\n' + log_timestring()
+ ' ***** buildtool is killing subprocess *****\n')
logging.info('Killing %s subprocess %s now that we are done with it',
repository.name, process.pid)
process.kill()
wait_subprocess(process)
gate_logstream.close()
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping {name} subprocess {pid}.'
.format(name=repository.name, pid=process.pid))
def wait_subprocess(process, stream=None, echo=False, postprocess_hook=None):
"""Waits for subprocess to finish and returns (final status, stdout).
This will also consume the remaining output to return it.
Returns:
Process exit code, stdout remaining in process prior to this invocation.
Any previously read output from the process will not be included.
"""
text_lines = []
if process.stdout is not None:
# stdout isnt going to another stream; collect it from the pipe.
for raw_line in iter(process.stdout.readline, ""):
if not raw_line:
break
decoded_line = raw_line.decode(encoding="utf-8")
text_lines.append(decoded_line)
if stream:
stream.write(decoded_line)
stream.flush()
if process.stderr is not None:
log_level = logging.INFO if echo else logging.DEBUG
# stderr isn't going to another file handle; log it
for raw_line in iter(process.stderr.readline, ""):
decoded_line = raw_line.decode(encoding="utf-8").rstrip("\n")
logging.log(
log_level, "PID %s wrote to stderr: %s", process.pid, decoded_line
)
process.wait()
if stream is None and process.stdout is not None:
# Close stdout pipe if we didnt give a stream.
# Otherwise caller owns the stream.
process.stdout.close()
if hasattr(process, "start_date"):
end_date = datetime.datetime.now()
delta_time_str = timedelta_string(end_date - process.start_date)
else:
delta_time_str = "UNKNOWN"
returncode = process.returncode
stdout = "".join(text_lines)
if stream:
stream.write(
"\n\n----\n{time} Spawned process completed"
" with returncode {returncode} in {delta_time}.\n".format(
time=log_timestring(now=end_date),
returncode=returncode,
delta_time=delta_time_str,
)
)
stream.flush()
if echo:
logging.info("%s returned %d with output:\n%s", process.pid, returncode, stdout)
logging.debug(
"Finished %s with returncode=%d in %s", process.pid, returncode, delta_time_str
)
if postprocess_hook:
postprocess_hook(returncode, stdout)
return returncode, stdout.strip()
