def run_wsgi(conf_path, app_section, *args, **kwargs):
"""
Runs the server according to some strategy. The default strategy runs a
specified number of workers in pre-fork model. The object-server (only)
may use a servers-per-port strategy if its config has a servers_per_port
setting with a value greater than zero.
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
:returns: 0 if successful, nonzero otherwise
"""
# Load configuration, Set logger and Load request processor
try:
(conf, logger, log_name) = \
_initrp(conf_path, app_section, *args, **kwargs)
except ConfigFileError as e:
print(e)
return 1
# optional nice/ionice priority scheduling
utils.modify_priority(conf, logger)
servers_per_port = int(conf.get('servers_per_port', '0') or 0)
# NOTE: for now servers_per_port is object-server-only; future work could
# be done to test and allow it to be used for account and container
# servers, but that has not been done yet.
if servers_per_port and app_section == 'object-server':
strategy = ServersPerPortStrategy(conf,
logger,
servers_per_port=servers_per_port)
else:
strategy = WorkersStrategy(conf, logger)
error_msg = strategy.bind_ports()
if error_msg:
logger.error(error_msg)
print(error_msg)
return 1
# Ensure the configuration and application can be loaded before proceeding.
global_conf = {'log_name': log_name}
if 'global_conf_callback' in kwargs:
kwargs['global_conf_callback'](conf, global_conf)
loadapp(conf_path, global_conf=global_conf)
# set utils.FALLOCATE_RESERVE if desired
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value(conf.get('fallocate_reserve', '1%'))
# redirect errors to logger and close stdio
capture_stdio(logger)
no_fork_sock = strategy.no_fork_sock()
if no_fork_sock:
run_server(conf, logger, no_fork_sock, global_conf=global_conf)
return 0
def kill_children(*args):
"""Kills the entire process group."""
logger.error('SIGTERM received')
signal.signal(signal.SIGTERM, signal.SIG_IGN)
running[0] = False
os.killpg(0, signal.SIGTERM)
def hup(*args):
"""Shuts down the server, but allows running requests to complete"""
logger.error('SIGHUP received')
signal.signal(signal.SIGHUP, signal.SIG_IGN)
running[0] = False
running = [True]
signal.signal(signal.SIGTERM, kill_children)
signal.signal(signal.SIGHUP, hup)
while running[0]:
for sock, sock_info in strategy.new_worker_socks():
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
strategy.post_fork_hook()
run_server(conf, logger, sock)
strategy.log_sock_exit(sock, sock_info)
return 0
else:
strategy.register_worker_start(sock, sock_info, pid)
# The strategy may need to pay attention to something in addition to
# child process exits (like new ports showing up in a ring).
#
# NOTE: a timeout value of None will just instantiate the Timeout
# object and not actually schedule it, which is equivalent to no
# timeout for the green_os.wait().
loop_timeout = strategy.loop_timeout()
with Timeout(loop_timeout, exception=False):
try:
try:
pid, status = green_os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
strategy.register_worker_exit(pid)
except OSError as err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
if err.errno == errno.ECHILD:
# If there are no children at all (ECHILD), then
# there's nothing to actually wait on. We sleep
# for a little bit to avoid a tight CPU spin
# and still are able to catch any KeyboardInterrupt
# events that happen. The value of 0.01 matches the
# value in eventlet's waitpid().
sleep(0.01)
except KeyboardInterrupt:
logger.notice('User quit')
running[0] = False
break
strategy.shutdown_sockets()
logger.notice('Exited')
return 0
def updatedJobWorker(self):
"""
We use the parasol results to update the status of jobs, adding them
to the list of updated jobs.
Results have the following structure.. (thanks Mark D!)
int status; /* Job status - wait() return format. 0 is good. */
char *host; /* Machine job ran on. */
char *jobId; /* Job queuing system job ID */
char *exe; /* Job executable file (no path) */
int usrTicks; /* 'User' CPU time in ticks. */
int sysTicks; /* 'System' CPU time in ticks. */
unsigned submitTime; /* Job submission time in seconds since 1/1/1970 */
unsigned startTime; /* Job start time in seconds since 1/1/1970 */
unsigned endTime; /* Job end time in seconds since 1/1/1970 */
char *user; /* User who ran job */
char *errFile; /* Location of stderr file on host */
Plus you finally have the command name.
"""
resultsFiles = set()
resultsFileHandles = []
try:
while self.running:
# Look for any new results files that have been created, and open them
newResultsFiles = set(os.listdir(
self.parasolResultsDir)).difference(resultsFiles)
for newFile in newResultsFiles:
newFilePath = os.path.join(self.parasolResultsDir, newFile)
resultsFileHandles.append(open(newFilePath, 'r'))
resultsFiles.add(newFile)
for fileHandle in resultsFileHandles:
while self.running:
line = fileHandle.readline()
if not line:
break
assert line[-1] == '\n'
(status, host, jobId, exe, usrTicks, sysTicks,
submitTime, startTime, endTime, user, errFile,
command) = line[:-1].split(None, 11)
status = int(status)
jobId = int(jobId)
if os.WIFEXITED(status):
status = os.WEXITSTATUS(status)
else:
status = -status
self.cpuUsageQueue.put(jobId)
startTime = int(startTime)
endTime = int(endTime)
if endTime == startTime:
# Both, start and end time is an integer so to get sub-second
# accuracy we use the ticks reported by Parasol as an approximation.
# This isn't documented but what Parasol calls "ticks" is actually a
# hundredth of a second. Parasol does the unit conversion early on
# after a job finished. Search paraNode.c for ticksToHundreths. We
# also cheat a little by always reporting at least one hundredth of a
# second.
usrTicks = int(usrTicks)
sysTicks = int(sysTicks)
wallTime = float(max(1,
usrTicks + sysTicks)) * 0.01
else:
wallTime = float(endTime - startTime)
self.updatedJobsQueue.put((jobId, status, wallTime))
time.sleep(1)
except:
logger.warn("Error occurred while parsing parasol results files.")
raise
finally:
for fileHandle in resultsFileHandles:
fileHandle.close()
def do_ldap_backup(tar_output: tarfile.TarFile) -> int:
# Produce an ldif of all users and groups. All other ldap objects, such as
# the organizational units and the Manager entity, are managed by puppet in
# the future.
result = 0
handle, tmpfilename1 = tempfile.mkstemp()
os.close(handle)
ret = os.system(
'ldapsearch -LLL -z 0 -D cn=Manager,o=sr -y /etc/ldap.secret -x -h localhost "(objectClass=posixAccount)" -b ou=users,o=sr > {0}'
.format(tmpfilename1))
if not os.WIFEXITED(ret) or os.WEXITSTATUS(ret) != 0:
print("Couldn't backup ldap users", file=sys.stderr)
result = 1
ret = os.system(
'ldapsearch -LLL -z 0 -D cn=Manager,o=sr -y /etc/ldap.secret -x -h localhost "(objectClass=posixGroup)" -b ou=groups,o=sr >> {0}'
.format(tmpfilename1))
if not os.WIFEXITED(ret) or os.WEXITSTATUS(ret) != 0:
print("Couldn't backup ldap groups", file=sys.stderr)
result = 1
# Code below procured from ldif parser documentation. Is fed an ldap,
# reformats a couple of entries to be modifications rather than additions.
# This is so that some special, puppet-created and configured groups, can be
# backed up and restored. Without this, adding groups like shell-users
# during backup restore would be an error.
make_modify = [
"cn=shell-users,ou=groups,o=sr", "cn=mentors,ou=groups,o=sr",
"cn=srusers,ou=groups,o=sr", "cn=withdrawn,ou=groups,o=sr",
"cn=media-consent,ou=groups,o=sr"
]
remove = ["uid=ide,ou=users,o=sr", "uid=anon,ou=users,o=sr"]
# This class hooks into processing an ldif
class MyLDIF(LDIFParser):
def __init__(self, input, output):
LDIFParser.__init__(self, input)
self.writer = LDIFWriter(output)
# Encode special dn-specific backup logic here.
def handle(self, dn, entry):
if dn in make_modify:
if not 'memberUid' in entry:
# No members in this group, discard
return
members = entry['memberUid']
self.writer.unparse(dn,
[(ldap.MOD_REPLACE, 'memberUid', members)])
return
elif dn in remove:
return
elif dn == None:
return
else:
self.writer.unparse(dn, entry)
# Open the ldif generated before, dump it into another tmpe file with
# relevant modification.
handle, tmpfilename2 = tempfile.mkstemp()
os.close(handle)
infile = open(tmpfilename1, 'r')
outfile = open(tmpfilename2, 'w')
parser = MyLDIF(infile, outfile)
parser.parse()
infile.close()
outfile.close()
tar_output.add(tmpfilename2, arcname="ldap/ldap_backup")
os.unlink(tmpfilename1)
os.unlink(tmpfilename2)
return result
def _run_in_child(
self,
*,
chroot_dir: Path,
network_config: Optional[pyspawner.NetworkConfig],
compiled_module: CompiledModule,
timeout: float,
result: Any,
function: str,
args: List[Any],
) -> None:
"""Fork a child process to run `function` with `args`.
`args` must be Thrift data types. `result` must also be a Thrift type --
its `.read()` function will be called, which may produce an error if
the child process has a bug. (EOFError is very likely.)
Raise ModuleExitedError if the child process did not behave as expected.
Raise ModuleTimeoutError if it did not exit after a delay -- or if it
closed its file descriptors long before it exited.
"""
limit_time = time.time() + timeout
module_process = self._pyspawner.spawn_child(
args=[compiled_module, function, args],
process_name=compiled_module.module_slug,
sandbox_config=pyspawner.SandboxConfig(chroot_dir=chroot_dir,
network=network_config),
)
# stdout is Thrift package; stderr is logs
output_reader = ChildReader(module_process.stdout.fileno(),
OUTPUT_BUFFER_MAX_BYTES)
log_reader = ChildReader(module_process.stderr.fileno(),
LOG_BUFFER_MAX_BYTES)
# Read until the child closes its stdout and stderr
with selectors.DefaultSelector() as selector:
selector.register(output_reader.fileno, selectors.EVENT_READ)
selector.register(log_reader.fileno, selectors.EVENT_READ)
timed_out = False
while selector.get_map():
remaining = limit_time - time.time()
if remaining <= 0:
if not timed_out:
timed_out = True
module_process.kill(
) # untrusted code could ignore SIGTERM
remaining = None # wait as long as it takes for everything to die
# Fall through. After SIGKILL the child will close each fd,
# sending EOF to us. That means the selector _must_ return.
events = selector.select(timeout=remaining)
ready = frozenset(key.fd for key, _ in events)
for reader in (output_reader, log_reader):
if reader.fileno in ready:
reader.ingest()
if reader.eof:
selector.unregister(reader.fileno)
# The child closed its fds, so it should die soon. If it doesn't, that's
# a bug -- so kill -9 it!
#
# os.wait() has no timeout option, and asyncio messes with signals so
# we won't use those. Spin until the process dies, and force-kill if we
# spin too long.
for _ in range(DEAD_PROCESS_N_WAITS):
pid, exit_status = module_process.wait(os.WNOHANG)
if pid != 0: # pid==0 means process is still running
break
time.sleep(DEAD_PROCESS_WAIT_POLL_INTERVAL)
else:
# we waited and waited. No luck. Dead module. Kill it.
timed_out = True
module_process.kill()
_, exit_status = module_process.wait(0)
if os.WIFEXITED(exit_status):
exit_code = os.WEXITSTATUS(exit_status)
elif os.WIFSIGNALED(exit_status):
exit_code = -os.WTERMSIG(exit_status)
else:
raise RuntimeError("Unhandled wait() status: %r" % exit_status)
if timed_out:
raise ModuleTimeoutError(compiled_module.module_slug, timeout)
if exit_code != 0:
raise ModuleExitedError(compiled_module.module_slug, exit_code,
log_reader.to_str())
transport = thrift.transport.TTransport.TMemoryBuffer(
output_reader.buffer)
protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(transport)
try:
result.read(protocol)
except EOFError: # TODO handle other errors Thrift may throw
raise ModuleExitedError(compiled_module.module_slug, exit_code,
log_reader.to_str()) from None
# We should be at the end of the output now. If we aren't, that means
# the child wrote too much.
if transport.read(1) != b"":
raise ModuleExitedError(compiled_module.module_slug, exit_code,
log_reader.to_str())
if log_reader.buffer:
logger.info("Output from module process: %s", log_reader.to_str())
return result
def RunProgram(self, program, arguments, context, result):
"""Run the 'program'.
'program' -- The path to the program to run.
'arguments' -- A list of the arguments to the program. This
list must contain a first argument corresponding to 'argv[0]'.
'context' -- A 'Context' giving run-time parameters to the
test.
'result' -- A 'Result' object. The outcome will be
'Result.PASS' when this method is called. The 'result' may be
modified by this method to indicate outcomes other than
'Result.PASS' or to add annotations."""
# Construct the environment.
environment = self.MakeEnvironment(context)
# Create the executable.
if self.timeout >= 0:
timeout = self.timeout
else:
# If no timeout was specified, we sill run this process in a
# separate process group and kill the entire process group
# when the child is done executing. That means that
# orphaned child processes created by the test will be
# cleaned up.
timeout = -2
e = qm.executable.Filter(self.stdin, timeout)
# Run it.
# print arguments
# print " path "+ program
exit_status = e.Run(arguments, environment, path=program)
# Get the output generated by the program regardless of how program finished
stdout = e.stdout
stderr = e.stderr
# Record the results.
#Stdout is too big we need to just discard it I guess or save it as stdout.out in output
result["ExecTest.stdout"] = result.Quote(stdout)
result["ExecTest.stderr"] = result.Quote(stderr)
if not re.search(r'End run of OMNeT', e.stdout) and not re.search(
r'Calling finish', e.stdout):
return result.Fail("Simulation did not end properly")
# If the process terminated normally, check the outputs.
if sys.platform == "win32" or os.WIFEXITED(exit_status):
# There are no causes of failure yet.
causes = []
# The target program terminated normally. Extract the
# exit code, if this test checks it.
if self.exit_code is None:
exit_code = None
elif sys.platform == "win32":
exit_code = exit_status
else:
exit_code = os.WEXITSTATUS(exit_status)
## result["ExecTest.exit_code"] = str(exit_code)
## # Check to see if the exit code matches.
## if exit_code != self.exit_code:
## causes.append("exit_code")
## result["ExecTest.expected_exit_code"] \
## = str(self.exit_code)
## # Check to see if the standard output matches.
## if not self.__CompareText(stdout, self.stdout):
## causes.append("standard output")
## result["ExecTest.expected_stdout"] \
## = result.Quote(self.stdout)
## # Check to see that the standard error matches.
## if not self.__CompareText(stderr, self.stderr):
## causes.append("standard error")
## result["ExecTest.expected_stderr"] \
## = result.Quote(self.stderr)
## # If anything went wrong, the test failed.
## if causes:
## result.Fail("Unexpected %s." % string.join(causes, ", "))
elif os.WIFSIGNALED(exit_status):
# The target program terminated with a signal. Construe
# that as a test failure.
signal_number = str(os.WTERMSIG(exit_status))
## result.Fail("Program terminated by signal.")
result["ExecTest.signal_number"] = signal_number
# Get the output generated by the program.
elif os.WIFSTOPPED(exit_status):
# The target program was stopped. Construe that as a
# test failure.
signal_number = str(os.WSTOPSIG(exit_status))
result.Fail("Program stopped by signal.")
result["ExecTest.signal_number"] = signal_number
else:
# The target program terminated abnormally in some other
# manner. (This shouldn't normally happen...)
result.Fail("Program did not terminate normally.")
elapsedWaits = 0
maxWaits = 5
waitTime = 0.1
retpid = 0
try:
while elapsedWaits < maxWaits:
(retpid, status) = os.waitpid(pid, os.WNOHANG)
if retpid == pid and os.WIFEXITED(status):
break
time.sleep(waitTime)
elapsedWaits += 1
except OSError, e:
log.debug("_terminate_process( %i ): %s", pid, e)
if retpid == pid:
if os.WIFEXITED(status):
rc = os.WEXITSTATUS(status)
elif os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
if pid in self.processes:
self.processes.remove(pid)
else:
self.KillProcess(pid)
def KillAllProcesses(self):
"""
Kill all processes this manager has created.
@warning: this is not a clean shutdown, but a forced shutdown
that may result in system cruft.
"""
def run_wsgi(conf_path, app_section, *args, **kwargs):
"""
Runs the server using the specified number of workers.
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
"""
# Load configuration, Set logger and Load request processor
print "run wsgi..."
try:
(conf, logger, log_name) = \
_initrp(conf_path, app_section, *args, **kwargs)
except ConfigFileError as e:
print e
return
# bind to address and port
sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
# remaining tasks should not require elevated privileges
drop_privileges(conf.get('user', 'swift'))
# Ensure the configuration and application can be loaded before proceeding.
global_conf = {'log_name': log_name}
if 'global_conf_callback' in kwargs:
kwargs['global_conf_callback'](conf, global_conf)
loadapp(conf_path, global_conf=global_conf)
# set utils.FALLOCATE_RESERVE if desired
reserve = int(conf.get('fallocate_reserve', 0))
if reserve > 0:
utils.FALLOCATE_RESERVE = reserve
# redirect errors to logger and close stdio
capture_stdio(logger)
worker_count = config_auto_int_value(conf.get('workers'), CPU_COUNT)
# Useful for profiling [no forks].
if worker_count == 0:
run_server(conf, logger, sock, global_conf=global_conf)
return
def kill_children(*args):
"""Kills the entire process group."""
logger.error('SIGTERM received')
signal.signal(signal.SIGTERM, signal.SIG_IGN)
running[0] = False
os.killpg(0, signal.SIGTERM)
def hup(*args):
"""Shuts down the server, but allows running requests to complete"""
logger.error('SIGHUP received')
signal.signal(signal.SIGHUP, signal.SIG_IGN)
running[0] = False
running = [True]
signal.signal(signal.SIGTERM, kill_children)
signal.signal(signal.SIGHUP, hup)
children = []
while running[0]:
while len(children) < worker_count:
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
run_server(conf, logger, sock)
logger.notice('Child %d exiting normally' % os.getpid())
return
else:
logger.notice('Started child %s' % pid)
children.append(pid)
try:
pid, status = os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
logger.error('Removing dead child %s' % pid)
children.remove(pid)
except OSError as err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
logger.notice('User quit')
break
greenio.shutdown_safe(sock)
sock.close()
logger.notice('Exited')
def execWithCallback(command,
argv,
stdin=None,
stdout=None,
stderr=None,
echo=True,
callback=None,
callback_data=None,
root='/'):
def chroot():
os.chroot(root)
def closefds():
stdinclose()
stdoutclose()
stderrclose()
if command.startswith('/'):
log.warning("'%s' specified as full path" % (command, ))
stdinclose = stdoutclose = stderrclose = lambda: None
argv = list(argv)
if isinstance(stdin, str):
if os.access(stdin, os.R_OK):
stdin = os.open(stdin, os.O_RDONLY)
stdinclose = lambda: os.close(stdin)
else:
stdin = sys.stdin.fileno()
elif isinstance(stdin, int):
pass
elif stdin is None or not isinstance(stdin, file):
stdin = sys.stdin.fileno()
if isinstance(stdout, str):
stdout = os.open(stdout, os.O_RDWR | os.O_CREAT)
stdoutclose = lambda: os.close(stdout)
elif isinstance(stdout, int):
pass
elif stdout is None or not isinstance(stdout, file):
stdout = sys.stdout.fileno()
if isinstance(stderr, str):
stderr = os.open(stderr, os.O_RDWR | os.O_CREAT)
stderrclose = lambda: os.close(stderr)
elif isinstance(stderr, int):
pass
elif stderr is None or not isinstance(stderr, file):
stderr = sys.stderr.fileno()
program_log.info("Running... %s" % ([command] + argv, ))
p = os.pipe()
p_stderr = os.pipe()
childpid = os.fork()
if not childpid:
os.close(p[0])
os.close(p_stderr[0])
os.dup2(p[1], 1)
os.dup2(p_stderr[1], 2)
os.dup2(stdin, 0)
os.close(stdin)
os.close(p[1])
os.close(p_stderr[1])
os.execvp(command, [command] + argv)
os._exit(1)
os.close(p[1])
os.close(p_stderr[1])
logline = ''
while 1:
try:
s = os.read(p[0], 1)
except OSError as e:
if e.errno != 4:
raise IOError, e.args
if echo:
os.write(stdout, s)
if s == '\n':
program_log.info(logline)
logline = ''
else:
logline += s
if callback:
callback(s, callback_data=callback_data)
# break out early if the sub-process changes status.
# no need to flush the stream if the process has exited
try:
(pid, status) = os.waitpid(childpid, os.WNOHANG)
if pid != 0:
break
except OSError as e:
log.critical("exception from waitpid: %s %s" %
(e.errno, e.strerror))
if len(s) < 1:
break
if len(logline) > 0:
program_log.info(logline)
log_errors = ''
while 1:
try:
err = os.read(p_stderr[0], 128)
except OSError as e:
if e.errno != 4:
raise IOError, e.args
break
log_errors += err
if len(err) < 1:
break
map(program_log.error, log_errors.splitlines())
os.close(p[0])
os.close(p_stderr[0])
try:
#if we didn't already get our child's exit status above, do so now.
if not pid:
(pid, status) = os.waitpid(childpid, 0)
except OSError as e:
log.critical("exception from waitpid: %s %s" % (e.errno, e.strerror))
closefds()
# *shrug* no clue why this would happen, but hope that things are fine
if status is None:
return 0
if os.WIFEXITED(status):
return os.WEXITSTATUS(status)
return 1
def __sig_child_handler(self, signum, frame):
# Our child exits with sig 9 when all is good... so map that to 0
ret = 0
pid = None
try:
status = None
sig = None
core = False
self.logger.debug("Running children: %s" % self.executor_pids)
self.logger.debug("Got signal %s" % signum)
pid, ret = os.wait()
self.logger.debug("After wait")
msg = "Child %s: wait returned code %s which means:" % (pid, ret)
if os.WIFSIGNALED(ret):
sig = os.WTERMSIG(ret)
msg += " signalled %s" % sig
if os.WIFEXITED(ret):
status = os.WEXITSTATUS(ret)
msg += " exited %s" % status
if os.WIFSTOPPED(ret):
msg += " stopped %s" % os.WSTOPSIG(ret)
if os.WCOREDUMP(ret):
core = True
msg += " core dumped"
if os.WIFCONTINUED(ret):
msg += " contunied"
self.logger.debug(msg)
if pid in self.executor_pids:
self.executor_pids.remove(pid)
self.executor_rets.append((status, sig, core))
else:
self.logger.error("Pid %s is not a child" % pid)
# sometimes signal handler is not called, clean here zombies
for pid in self.executor_pids:
p, r = os.waitpid(pid, os.WNOHANG)
if p != 0:
self.logger.debug(
"Zombie with pid %d found, exit code=%d" % (p, r))
self.executor_pids.remove(pid)
#self.executor_rets.append((status, sig, core))
ret = 0
if len(self.executor_pids) == 0:
self.logger.trace("Statuses of all executors: %s" %
self.executor_rets)
for st, sg, co in self.executor_rets:
if st is not None and st != 0:
ret = st
if co:
ret = 1
self.logger.info("Exit with code %s" % ret)
sys.exit(ret)
except Exception, ex:
self.logger.error("Error waiting for child process: %s" % ex)
if len(self.executor_pids) <= 1:
self.logger.warn(
"No more child processes, exit with success: pids=%s, last pid=%s, ret=%s"
% (self.executor_pids, pid, ret))
sys.exit(0)
else:
self.logger.info("Children left: %s" % self.executor_pids)
def run_wsgi(conf_path, app_section, *args, **kwargs):
"""
Runs the server according to some strategy. The default strategy runs a
specified number of workers in pre-fork model. The object-server (only)
may use a servers-per-port strategy if its config has a servers_per_port
setting with a value greater than zero.
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
:returns: 0 if successful, nonzero otherwise
"""
# Load configuration, Set logger and Load request processor
# conf: 包含配置信息的字典
# logger: log对象
# log_name: log名，eg:proxy_logging
try:
(conf, logger, log_name) = \
_initrp(conf_path, app_section, *args, **kwargs)
except ConfigFileError as e:
print(e)
return 1
# optional nice/ionice priority scheduling
utils.modify_priority(conf, logger)
servers_per_port = int(conf.get('servers_per_port', '0') or 0)
# NOTE: for now servers_per_port is object-server-only; future work could
# be done to test and allow it to be used for account and container
# servers, but that has not been done yet.
if servers_per_port and app_section == 'object-server':
strategy = ServersPerPortStrategy(
conf, logger, servers_per_port=servers_per_port)
else:
# 多个 server 进程监听一个端口，默认为逻辑cpu的个数
strategy = WorkersStrategy(conf, logger)
# patch event before loadapp
# 运行时修改已有的代码，动态替换已有的标准库
utils.eventlet_monkey_patch()
# Ensure the configuration and application can be loaded before proceeding.
global_conf = {'log_name': log_name}
if 'global_conf_callback' in kwargs:
kwargs['global_conf_callback'](conf, global_conf)
loadapp(conf_path, global_conf=global_conf)
# set utils.FALLOCATE_RESERVE if desired
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value(conf.get('fallocate_reserve', '1%'))
# Start listening on bind_addr/port
error_msg = strategy.do_bind_ports()
if error_msg:
logger.error(error_msg)
print(error_msg)
return 1
# Redirect errors to logger and close stdio. Do this *after* binding ports;
# we use this to signal that the service is ready to accept connections.
capture_stdio(logger)
no_fork_sock = strategy.no_fork_sock()
if no_fork_sock:
run_server(conf, logger, no_fork_sock, global_conf=global_conf)
return 0
def stop_with_signal(signum, *args):
"""Set running flag to False and capture the signum"""
running_context[0] = False
running_context[1] = signum
# context to hold boolean running state and stop signum
running_context = [True, None]
signal.signal(signal.SIGTERM, stop_with_signal)
signal.signal(signal.SIGHUP, stop_with_signal)
while running_context[0]:
for sock, sock_info in strategy.new_worker_socks():
pid = os.fork()
if pid == 0: # in child process
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
strategy.post_fork_hook()
run_server(conf, logger, sock)
strategy.log_sock_exit(sock, sock_info)
return 0
else: # in parent process
strategy.register_worker_start(sock, sock_info, pid)
# The strategy may need to pay attention to something in addition to
# child process exits (like new ports showing up in a ring).
#
# NOTE: a timeout value of None will just instantiate the Timeout
# object and not actually schedule it, which is equivalent to no
# timeout for the green_os.wait().
loop_timeout = strategy.loop_timeout()
with Timeout(loop_timeout, exception=False):
try:
try:
pid, status = green_os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
strategy.register_worker_exit(pid)
except OSError as err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
if err.errno == errno.ECHILD:
# If there are no children at all (ECHILD), then
# there's nothing to actually wait on. We sleep
# for a little bit to avoid a tight CPU spin
# and still are able to catch any KeyboardInterrupt
# events that happen. The value of 0.01 matches the
# value in eventlet's waitpid().
sleep(0.01)
except KeyboardInterrupt:
logger.notice('User quit')
running_context[0] = False
break
if running_context[1] is not None:
try:
signame = SIGNUM_TO_NAME[running_context[1]]
except KeyError:
logger.error('Stopping with unexpected signal %r' %
running_context[1])
else:
logger.error('%s received', signame)
if running_context[1] == signal.SIGTERM:
os.killpg(0, signal.SIGTERM)
strategy.shutdown_sockets()
signal.signal(signal.SIGTERM, signal.SIG_IGN)
logger.notice('Exited (%s)', os.getpid())
return 0
nextIsIncludeFile = True
result = map(lambda x: "-I" + x, includes)
result += map(lambda x: "-D" + x, defines)
result += map(lambda x: "-include " + x, include_file)
return result
def mergeLists(base, new):
result = list(base)
for newLine in new:
try:
result.index(newLine)
except ValueError:
result += [newLine]
return result
configuration = readConfiguration()
args = parseArguments(sys.argv)
result = mergeLists(configuration, args)
writeConfiguration(map(lambda x: x + "\n", result))
status = os.system(" ".join(sys.argv[1:]))
if not os.WIFEXITED(status):
sys.exit(1)
sys.exit(os.WEXITSTATUS(status))
# vim: set ts=2 sts=2 sw=2 expandtab :
printFixed("PASSED")
break
except CheckFailed, e:
if j == RETRIES-1:
# Save checkpoint images for later diagnosis.
if os.path.isdir(dmtcp_tmpdir()) and os.path.isdir(ckptDir):
if subprocess.call( ("cp -pr " + ckptDir + ' '
+ dmtcp_tmpdir()).split() ) == 0:
print "\n***** Copied checkpoint images to " + dmtcp_tmpdir() \
+ "/" + ckptDir
raise e
else:
printFixed("FAILED ")
(oldpid, oldstatus) = os.waitpid(procs[-1].pid, os.WNOHANG)
if oldpid == procs[-1].pid:
if os.WIFEXITED(oldstatus):
printFixed("(first process exited: oldstatus "
+ str(os.WEXITSTATUS(oldstatus)) + ")")
if os.WIFSIGNALED(oldstatus):
printFixed("(first process rec'd signal "
+ str(os.WTERMSIG(oldstatus)) + ")")
if os.WCOREDUMP(oldstatus):
coredump = "core." + str(oldpid)
if os.path.isdir(dmtcp_tmpdir()) and os.path.isfile(coredump):
if subprocess.call( ("cp -pr " + coredump + ' '
+ dmtcp_tmpdir()).split() ) == 0:
printFixed(" (" + coredump + " copied to DMTCP_TMPDIR:" +
dmtcp_tmpdir() + "/)")
else:
printFixed("(first process didn't die)")
printFixed(" retry:")
def dojob(job, userimage, log=None, solve_command=None, solve_locally=None):
jobdir = job.make_dir()
#print('Created job dir', jobdir)
#log = create_job_logger(job)
#jobdir = job.get_dir()
if log is None:
log = create_job_logger(job)
log.msg('Starting Job processing for', job)
job.set_start_time()
job.save()
#os.chdir(dirnm) - not thread safe (working directory is global)!
log.msg('Creating directory', jobdir)
axyfn = 'job.axy'
axypath = os.path.join(jobdir, axyfn)
sub = userimage.submission
log.msg('submission id', sub.id)
df = userimage.image.disk_file
img = userimage.image
# Build command-line arguments for the augment-xylist program, which
# detects sources in the image and adds processing arguments to the header
# to produce a "job.axy" file.
slo,shi = sub.get_scale_bounds()
# Note, this must match Job.get_wcs_file().
wcsfile = 'wcs.fits'
corrfile = 'corr.fits'
axyflags = []
axyargs = {
'--out': axypath,
'--scale-low': slo,
'--scale-high': shi,
'--scale-units': sub.scale_units,
'--wcs': wcsfile,
'--corr': corrfile,
'--rdls': 'rdls.fits',
'--pixel-error': sub.positional_error,
'--ra': sub.center_ra,
'--dec': sub.center_dec,
'--radius': sub.radius,
'--downsample': sub.downsample_factor,
# tuning-up maybe fixed; if not, turn it off with:
#'--odds-to-tune': 1e9,
# Other things we might want include...
# --invert
# -g / --guess-scale: try to guess the image scale from the FITS headers
# --crpix-x <pix>: set the WCS reference point to the given position
# --crpix-y <pix>: set the WCS reference point to the given position
# -w / --width <pixels>: specify the field width
# -e / --height <pixels>: specify the field height
# -X / --x-column <column-name>: the FITS column name
# -Y / --y-column <column-name>
}
if hasattr(img,'sourcelist'):
# image is a source list; use --xylist
axyargs['--xylist'] = img.sourcelist.get_fits_path()
w,h = img.width, img.height
if sub.image_width:
w = sub.image_width
if sub.image_height:
h = sub.image_height
axyargs['--width' ] = w
axyargs['--height'] = h
else:
axyargs['--image'] = df.get_path()
# UGLY
if sub.parity == 0:
axyargs['--parity'] = 'pos'
elif sub.parity == 1:
axyargs['--parity'] = 'neg'
if sub.tweak_order == 0:
axyflags.append('--no-tweak')
else:
axyargs['--tweak-order'] = '%i' % sub.tweak_order
if sub.use_sextractor:
axyflags.append('--use-sextractor')
if sub.crpix_center:
axyflags.append('--crpix-center')
if sub.invert:
axyflags.append('--invert')
cmd = 'augment-xylist '
for (k,v) in axyargs.items():
if v:
cmd += k + ' ' + str(v) + ' '
for k in axyflags:
cmd += k + ' '
log.msg('running: ' + cmd)
(rtn, out, err) = run_command(cmd)
if rtn:
log.msg('out: ' + out)
log.msg('err: ' + err)
logmsg('augment-xylist failed: rtn val', rtn, 'err', err)
raise Exception
log.msg('created axy file', axypath)
# shell into compute server...
logfn = job.get_log_file()
# the "tar" commands both use "-C" to chdir, and the ssh command
# and redirect uses absolute paths.
if solve_locally is not None:
cmd = (('cd %(jobdir)s && %(solvecmd)s %(jobid)s %(axyfile)s >> ' +
'%(logfile)s') %
dict(jobid='job-%s-%i' % (settings.sitename, job.id),
solvecmd=solve_locally,
axyfile=axyfn, jobdir=jobdir,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val',
rtn)
raise Exception
log.msg('Solver completed successfully.')
else:
if solve_command is None:
solve_command = 'ssh -x -T %(sshconfig)s'
cmd = (('(echo %(jobid)s; '
'tar cf - --ignore-failed-read -C %(jobdir)s %(axyfile)s) | '
+ solve_command + ' 2>>%(logfile)s | '
'tar xf - --atime-preserve -m --exclude=%(axyfile)s -C %(jobdir)s '
'>>%(logfile)s 2>&1') %
dict(jobid='job-%s-%i' % (settings.sitename, job.id),
axyfile=axyfn, jobdir=jobdir,
sshconfig=settings.ssh_solver_config,
logfile=logfn))
log.msg('command:', cmd)
w = os.system(cmd)
if not os.WIFEXITED(w):
log.msg('Solver failed (sent signal?)')
logmsg('Call to solver failed for job', job.id)
raise Exception
rtn = os.WEXITSTATUS(w)
if rtn:
log.msg('Solver failed with return value %i' % rtn)
logmsg('Call to solver failed for job', job.id, 'with return val',
rtn)
raise Exception
log.msg('Solver completed successfully.')
# Solved?
wcsfn = os.path.join(jobdir, wcsfile)
log.msg('Checking for WCS file', wcsfn)
if os.path.exists(wcsfn):
log.msg('WCS file exists')
# Parse the wcs.fits file
wcs = Tan(wcsfn, 0)
# Convert to database model...
tan = TanWCS(crval1=wcs.crval[0], crval2=wcs.crval[1],
crpix1=wcs.crpix[0], crpix2=wcs.crpix[1],
cd11=wcs.cd[0], cd12=wcs.cd[1],
cd21=wcs.cd[2], cd22=wcs.cd[3],
imagew=img.width, imageh=img.height)
tan.save()
log.msg('Created TanWCS:', tan)
# Find field's healpix nside and index
ra, dec, radius = tan.get_center_radecradius()
nside = anutil.healpix_nside_for_side_length_arcmin(radius*60)
nside = int(2**round(math.log(nside, 2)))
healpix = anutil.radecdegtohealpix(ra, dec, nside)
sky_location, created = SkyLocation.objects.get_or_create(nside=nside, healpix=healpix)
log.msg('SkyLocation:', sky_location)
# Find bounds for the Calibration object.
r0,r1,d0,d1 = wcs.radec_bounds()
# Find cartesian coordinates
ra *= math.pi/180
dec *= math.pi/180
tempr = math.cos(dec)
x = tempr*math.cos(ra)
y = tempr*math.sin(ra)
z = math.sin(dec)
r = radius/180*math.pi
calib = Calibration(raw_tan=tan, ramin=r0, ramax=r1, decmin=d0, decmax=d1,
x=x,y=y,z=z,r=r,
sky_location=sky_location)
calib.save()
log.msg('Created Calibration', calib)
job.calibration = calib
job.save() # save calib before adding machine tags
job.status = 'S'
job.user_image.add_machine_tags(job)
job.user_image.add_sky_objects(job)
else:
job.status = 'F'
job.set_end_time()
job.save()
log.msg('Finished job', job.id)
logmsg('Finished job',job.id)
return job.id
def createUser (self, user_name, *args, **kwargs):
"""Create a new user on the system with the given name. Optional kwargs:
algo -- The password algorithm to use in case isCrypted=True.
If none is given, the cryptPassword default is used.
gecos -- The GECOS information (full name, office, phone, etc.).
Defaults to "".
groups -- A list of existing group names the user should be
added to. Defaults to [].
homedir -- The home directory for the new user. Defaults to
/home/<name>.
isCrypted -- Is the password kwargs already encrypted? Defaults
to False.
lock -- Is the new account locked by default? Defaults to
False.
password -- The password. See isCrypted for how this is interpreted.
If the password is "" then the account is created
with a blank password. If None or False the account will
be left in its initial state (locked)
root -- The directory of the system to create the new user
in. homedir will be interpreted relative to this.
Defaults to /mnt/sysimage.
shell -- The shell for the new user. If none is given, the
libuser default is used.
uid -- The UID for the new user. If none is given, the next
available one is used.
gid -- The GID for the new user. If none is given, the next
available one is used.
"""
childpid = os.fork()
root = kwargs.get("root", "/mnt/sysimage")
if not childpid:
if not root in ["","/"]:
os.chroot(root)
os.chdir("/")
del(os.environ["LIBUSER_CONF"])
self.admin = libuser.admin()
if self.admin.lookupUserByName(user_name):
log.error("User %s already exists, not creating.", user_name)
os._exit(1)
userEnt = self.admin.initUser(user_name)
groupEnt = self.admin.initGroup(user_name)
if kwargs.get("gid", -1) >= 0:
groupEnt.set(libuser.GIDNUMBER, kwargs["gid"])
grpLst = filter(lambda grp: grp,
map(self.admin.lookupGroupByName, kwargs.get("groups", [])))
userEnt.set(libuser.GIDNUMBER, [groupEnt.get(libuser.GIDNUMBER)[0]] +
map(lambda grp: grp.get(libuser.GIDNUMBER)[0], grpLst))
if kwargs.get("homedir", False):
userEnt.set(libuser.HOMEDIRECTORY, kwargs["homedir"])
else:
iutil.mkdirChain(root+'/home')
userEnt.set(libuser.HOMEDIRECTORY, "/home/" + user_name)
if kwargs.get("shell", False):
userEnt.set(libuser.LOGINSHELL, kwargs["shell"])
if kwargs.get("uid", -1) >= 0:
userEnt.set(libuser.UIDNUMBER, kwargs["uid"])
if kwargs.get("gecos", False):
userEnt.set(libuser.GECOS, kwargs["gecos"])
# need to create home directory for the user or does it already exist?
# userEnt.get returns lists (usually with a single item)
mk_homedir = not os.path.exists(userEnt.get(libuser.HOMEDIRECTORY)[0])
try:
self.admin.addUser(userEnt, mkmailspool=kwargs.get("mkmailspool", True),
mkhomedir=mk_homedir)
except RuntimeError as e:
log.critical("Error when creating new user: %s", e)
os._exit(1)
try:
self.admin.addGroup(groupEnt)
except RuntimeError as e:
log.critical("Error when creating new group: %s", e)
os._exit(1)
if not mk_homedir:
try:
stats = os.stat(userEnt.get(libuser.HOMEDIRECTORY)[0])
orig_uid = stats.st_uid
orig_gid = stats.st_gid
log.info("Home directory for the user %s already existed, "
"fixing the owner.", user_name)
# home directory already existed, change owner of it properly
iutil.chown_dir_tree(userEnt.get(libuser.HOMEDIRECTORY)[0],
userEnt.get(libuser.UIDNUMBER)[0],
groupEnt.get(libuser.GIDNUMBER)[0],
orig_uid, orig_gid)
except OSError as e:
log.critical("Unable to change owner of existing home directory: %s",
os.strerror)
os._exit(1)
pw = kwargs.get("password", False)
try:
if pw:
if kwargs.get("isCrypted", False):
password = kwargs["password"]
else:
password = cryptPassword(kwargs["password"], algo=kwargs.get("algo", None))
self.admin.setpassUser(userEnt, password, True)
userEnt.set(libuser.SHADOWLASTCHANGE, "")
self.admin.modifyUser(userEnt)
elif pw == "":
# Setup the account with *NO* password
self.admin.unlockUser(userEnt)
log.info("user account %s setup with no password", user_name)
if kwargs.get("lock", False):
self.admin.lockUser(userEnt)
log.info("user account %s locked", user_name)
# setpassUser raises SystemError on failure, while unlockUser and lockUser
# raise RuntimeError
except (RuntimeError, SystemError) as e:
log.critical("Unable to set password for new user: %s", e)
os._exit(1)
# Add the user to all the groups they should be part of.
grpLst.append(self.admin.lookupGroupByName(user_name))
try:
for grp in grpLst:
grp.add(libuser.MEMBERNAME, user_name)
self.admin.modifyGroup(grp)
except RuntimeError as e:
log.critical("Unable to add user to groups: %s", e)
os._exit(1)
os._exit(0)
try:
status = os.waitpid(childpid, 0)[1]
except OSError as e:
log.critical("exception from waitpid while creating a user: %s %s", e.errno, e.strerror)
return False
if os.WIFEXITED(status) and (os.WEXITSTATUS(status) == 0):
return True
else:
return False
def isalive(self):
'''This tests if the child process is running or not. This is
non-blocking. If the child was terminated then this will read the
exitstatus or signalstatus of the child. This returns True if the child
process appears to be running or False if not. It can take literally
SECONDS for Solaris to return the right status. '''
if self.terminated:
return False
if self.flag_eof:
# This is for Linux, which requires the blocking form
# of waitpid to get the status of a defunct process.
# This is super-lame. The flag_eof would have been set
# in read_nonblocking(), so this should be safe.
waitpid_options = 0
else:
waitpid_options = os.WNOHANG
try:
pid, status = os.waitpid(self.pid, waitpid_options)
except OSError as e:
# No child processes
if e.errno == errno.ECHILD:
raise PtyProcessError('isalive() encountered condition ' +
'where "terminated" is 0, but there was no child ' +
'process. Did someone else call waitpid() ' +
'on our process?')
else:
raise
# I have to do this twice for Solaris.
# I can't even believe that I figured this out...
# If waitpid() returns 0 it means that no child process
# wishes to report, and the value of status is undefined.
if pid == 0:
try:
### os.WNOHANG) # Solaris!
pid, status = os.waitpid(self.pid, waitpid_options)
except OSError as e: # pragma: no cover
# This should never happen...
if e.errno == errno.ECHILD:
raise PtyProcessError('isalive() encountered condition ' +
'that should never happen. There was no child ' +
'process. Did someone else call waitpid() ' +
'on our process?')
else:
raise
# If pid is still 0 after two calls to waitpid() then the process
# really is alive. This seems to work on all platforms, except for
# Irix which seems to require a blocking call on waitpid or select,
# so I let read_nonblocking take care of this situation
# (unfortunately, this requires waiting through the timeout).
if pid == 0:
return True
if pid == 0:
return True
if os.WIFEXITED(status):
self.status = status
self.exitstatus = os.WEXITSTATUS(status)
self.signalstatus = None
self.terminated = True
elif os.WIFSIGNALED(status):
self.status = status
self.exitstatus = None
self.signalstatus = os.WTERMSIG(status)
self.terminated = True
elif os.WIFSTOPPED(status):
raise PtyProcessError('isalive() encountered condition ' +
'where child process is stopped. This is not ' +
'supported. Is some other process attempting ' +
'job control with our child pid?')
return False
running = [True]
signal.signal(signal.SIGTERM, kill_children)
signal.signal(signal.SIGHUP, hup)
children = []
while running[0]:
while len(children) < worker_count:
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
run_server()
logger.notice('Child %d exiting normally' % os.getpid())
return
else:
logger.notice('Started child %s' % pid)
children.append(pid)
try:
pid, status = os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
logger.error('Removing dead child %s' % pid)
children.remove(pid)
except OSError, err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
logger.notice('User quit')
break
greenio.shutdown_safe(sock)
sock.close()
logger.notice('Exited')
def failed(self, exit_code):
"""
Notify the component it crashed. This will be called from b10_init
object.
If you try to call failed on a component that is not running,
a ValueError is raised.
If it is a core component or needed component and it was started only
recently, the component will become dead and will ask b10_init to shut
down with error exit status. A dead component can't be started again.
Otherwise the component will try to restart.
The exit code is used for logging. It might be None.
It calls _failed_internal internally.
Returns True if the process was immediately restarted, returns
False is the process was not restarted, either because
it is considered a core or needed component, or because
the component is to be restarted later.
"""
if exit_code is not None:
if os.WIFEXITED(exit_code):
exit_str = "process exited normally with exit status %d" % (
exit_code)
elif os.WIFSIGNALED(exit_code):
sig = os.WTERMSIG(exit_code)
signame = get_signame(sig)
if os.WCOREDUMP(exit_code):
exit_str = "process dumped core with exit status %d (killed by signal %d: %s)" % (
exit_code, sig, signame)
else:
exit_str = "process terminated with exit status %d (killed by signal %d: %s)" % (
exit_code, sig, signame)
else:
exit_str = "unknown condition with exit status %d" % (
exit_code)
else:
exit_str = "unknown condition"
logger.error(BIND10_COMPONENT_FAILED, self.name(), self.pid(),
exit_str)
if not self.is_running():
raise ValueError("Can't fail component that isn't running")
self.__state = STATE_RESTARTING # tentatively set, maybe changed to DEAD
self._failed_internal()
# If it is a core component or the needed component failed to start
# (including it stopped really soon)
if self._kind == 'core' or \
(self._kind == 'needed' and time.time() - STARTED_OK_TIME <
self._original_start_time):
self.__state = STATE_DEAD
logger.fatal(BIND10_COMPONENT_UNSATISFIED, self.name())
self._b10_init.component_shutdown(1)
return False
# This means we want to restart
else:
# if the component was only running for a short time, don't
# restart right away, but set a time it wants to restarted,
# and return that it wants to be restarted later
self.set_restart_time()
return self.restart()
skt.close()
exit()
#client code ended
#error creating client thread
elif pid_Client < 0:
print("Error creating Client Thread")
#main code
else:
#wait for server thread to terminate
#try:
while True:
wpid, status = os.waitpid(pid_Server, os.WUNTRACED)
if (os.WIFEXITED(status) or os.WIFSIGNALED(status)):
break
#except:
#connect_other_client.close()
#skt.close()
#try:
while True:
wpid, status = os.waitpid(pid_Client, os.WUNTRACED)
if (os.WIFEXITED(status) or os.WIFSIGNALED(status)):
break
#except:
#connect_other_client.close()
#skt.close()
#connect_other_client.close()
#skt.close()
return 0
def exit_status(s):
if os.WIFEXITED(s):
return os.WEXITSTATUS(s)
return 1
def loop(self):
"""Main loop.
Wait until all the runner subprocesses have exited, restarting them if
necessary and configured to do so.
"""
log = logging.getLogger('mailman.runner')
log.info('Master started')
self._pause()
while True:
try:
pid, status = os.wait()
except OSError as error:
# No children? We're done.
if error.errno == errno.ECHILD:
break
# If the system call got interrupted, just restart it.
elif error.errno == errno.EINTR:
continue
else:
raise
# Find out why the subprocess exited by getting the signal
# received or exit status.
if os.WIFSIGNALED(status):
why = os.WTERMSIG(status)
elif os.WIFEXITED(status):
why = os.WEXITSTATUS(status)
else:
why = None
# We'll restart the subprocess if it exited with a SIGUSR1 or
# because of a failure (i.e. no exit signal), and the no-restart
# command line switch was not given. This lets us better handle
# runaway restarts (e.g. if the subprocess had a syntax error!)
rname, slice_number, count, restarts = self._kids.pop(pid)
config_name = 'runner.' + rname
restart = False
if why == signal.SIGUSR1 and self._restartable:
restart = True
# Have we hit the maximum number of restarts?
restarts += 1
max_restarts = int(getattr(config, config_name).max_restarts)
if restarts > max_restarts:
restart = False
# Are we permanently non-restartable?
log.debug("""\
Master detected subprocess exit
(pid: {0:d}, why: {1}, class: {2}, slice: {3:d}/{4:d}) {5}""".format(
pid, why, rname, slice_number + 1, count,
('[restarting]' if restart else '')))
# See if we've reached the maximum number of allowable restarts.
if restarts > max_restarts:
log.info(
"""\
Runner {0} reached maximum restart limit of {1:d}, not restarting.""", rname,
max_restarts)
# Now perhaps restart the process unless it exited with a
# SIGTERM or we aren't restarting.
if restart:
spec = '{0}:{1:d}:{2:d}'.format(rname, slice_number, count)
new_pid = self._start_runner(spec)
new_info = (rname, slice_number, count, restarts)
self._kids.add(new_pid, new_info)
log.info('Master stopped')
def ikos_analyzer(db_path, pp_path, opt):
if settings.BUILD_MODE == 'Debug':
log.warning('ikos was built in debug mode, the analysis might be slow')
# Fix huge slow down when ikos-analyzer uses DROP TABLE on an existing db
if os.path.isfile(db_path):
os.remove(db_path)
cmd = [settings.ikos_analyzer()]
# analysis options
cmd += ['-a=%s' % ','.join(opt.analyses),
'-d=%s' % opt.domain,
'-entry-points=%s' % ','.join(opt.entry_points),
'-globals-init=%s' % opt.globals_init,
'-prec=%s' % opt.precision_level,
'-proc=%s' % opt.procedural]
if opt.no_init_globals:
cmd.append('-no-init-globals=%s' % ','.join(opt.no_init_globals))
if opt.no_liveness:
cmd.append('-no-liveness')
if opt.no_pointer:
cmd.append('-no-pointer')
if opt.no_fixpoint_profiles:
cmd.append('-no-fixpoint-profiles')
if opt.no_fixpoint_cache:
cmd.append('-no-fixpoint-cache')
if opt.hardware_addresses:
cmd.append('-hardware-addresses=%s' % ','.join(opt.hardware_addresses))
if opt.hardware_addresses_file:
cmd.append('-hardware-addresses-file=%s' % opt.hardware_addresses_file)
if opt.argc is not None:
cmd.append('-argc=%d' % opt.argc)
# import options
cmd.append('-allow-dbg-mismatch')
if opt.no_bc_verify:
cmd.append('-no-verify')
if opt.no_libc:
cmd.append('-no-libc')
if opt.no_libcpp:
cmd.append('-no-libcpp')
if opt.no_libikos:
cmd.append('-no-libikos')
# AR passes options
if opt.no_type_check:
cmd.append('-no-type-check')
if opt.no_simplify_cfg:
cmd.append('-no-simplify-cfg')
if opt.no_simplify_upcast_comparison:
cmd.append('-no-simplify-upcast-comparison')
if 'gauge' in opt.domain:
cmd.append('-add-loop-counters')
# debug options
cmd += ['-display-checks=%s' % opt.display_checks,
'-display-inv=%s' % opt.display_inv]
if opt.display_ar:
cmd.append('-display-ar')
if opt.display_liveness:
cmd.append('-display-liveness')
if opt.display_function_pointer:
cmd.append('-display-function-pointer')
if opt.display_pointer:
cmd.append('-display-pointer')
if opt.display_fixpoint_profiles:
cmd.append('-display-fixpoint-profiles')
if opt.generate_dot:
cmd += ['-generate-dot', '-generate-dot-dir', opt.generate_dot_dir]
# add -name-values if necessary
if (opt.display_checks in ('all', 'fail') or
opt.display_inv in ('all', 'fail') or
opt.display_liveness or
opt.display_fixpoint_profiles or
opt.display_function_pointer or
opt.display_pointer or
opt.display_raw_checks):
cmd.append('-name-values')
# misc. options
if opt.color == 'yes':
cmd.append('-color=1')
elif opt.color == 'no':
cmd.append('-color=0')
cmd.append('-log=%s' % opt.log_level)
cmd.append('-progress=%s' % opt.progress)
# input/output
cmd += [pp_path, '-o', db_path]
# set resource limit, if requested
if opt.mem > 0:
import resource # fails on Windows
def set_limits():
mem_bytes = opt.mem * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, [mem_bytes, mem_bytes])
else:
set_limits = None
# called after timeout
def kill(p):
try:
log.error('Timeout')
p.send_signal(signal.SIGALRM)
except OSError:
pass
log.info('Running ikos analyzer')
log.debug('Running %s' % command_string(cmd))
p = subprocess.Popen(cmd, preexec_fn=set_limits)
timer = threading.Timer(opt.cpu, kill, [p])
if opt.cpu > 0:
timer.start()
try:
if sys.platform.startswith('win'):
return_status = p.wait()
else:
_, return_status = os.waitpid(p.pid, 0)
finally:
# kill the timer if the process has terminated already
if timer.isAlive():
timer.cancel()
# special case for Windows, since it does not define WIFEXITED & co.
if sys.platform.startswith('win'):
if return_status != 0:
raise AnalyzerError('a run-time error occurred', cmd, return_status)
else:
return
# if it did not terminate properly, propagate this error code
if os.WIFEXITED(return_status) and os.WEXITSTATUS(return_status) != 0:
exit_status = os.WEXITSTATUS(return_status)
raise AnalyzerError('a run-time error occurred', cmd, exit_status)
if os.WIFSIGNALED(return_status):
signum = os.WTERMSIG(return_status)
raise AnalyzerError('exited with signal %s' % signal_name(signum),
cmd,
signum)
if os.WIFSTOPPED(return_status):
signum = os.WSTOPSIG(return_status)
raise AnalyzerError('exited with signal %d' % signal_name(signum),
cmd,
signum)
def test_terminate_sigterm(self):
self._terminate_with_signal(signal.SIGTERM)
status = self._reap_test()
self.assertTrue(os.WIFEXITED(status))
self.assertEqual(os.WEXITSTATUS(status), 0)
def SetUp(self, context, result):
# if not context.has_key("IPv6Suite.srcDir"):
# By default we assume there is a compiler.
srcDir = context["IPv6Suite.srcDir"]
srcDir = os.path.expanduser(srcDir)
context["IPv6Suite.srcDir"] = srcDir
if not os.path.exists(srcDir):
result.SetOutcome(
result.ERROR, "srcDir does not exist. Where will we get source"
"files from?")
return
buildDir = context["IPv6Suite.buildDir"]
buildDir = os.path.abspath(buildDir)
context["IPv6Suite.buildDir"] = buildDir
buildDir = os.path.join(buildDir, IdToDir(self))
self.wipe_build_dir = qm.parse_boolean(
context["IPv6Suite.wipe_build_dir"])
#Can't use bool since the boolean_value is a string of 'true|false' and since bool
# thinks of any string as true except empty or none.
#self.wipe_build_dir = bool(self.wipe_build_dir)
#We want clean builds especially when previously failed ones may have broken generated code
if self.wipe_build_dir and os.path.exists(buildDir):
shutil.rmtree(buildDir)
mkdirs(buildDir)
if self.wipe_build_dir:
print "Wiping all files"
context["IPv6Suite.myBuildDir"] = buildDir
self.myBuildDir = buildDir
#srcdir/test is where we store the related input files like ini/xml etc.
#Well database of qmtest would be where we store these input files but what structure underneath?
make_executable = RedirectedExecutable()
#We actually want a gui to relace this value if possible?
cmake_defines = string.split(self.ipv6suite_build_options)
cmake_options = "-DOPP_USE_TK:BOOL=OFF"
## "-DLIBRARY_OUTPUT_PATH:PATH=lib -DEXECUTABLE_OUTPUT_PATH:PATH=exe"
for o in cmake_defines:
cmake_options = "-D" + o + ":BOOL=ON" + " " + cmake_options
if not os.path.exists(os.path.join(buildDir, "CMakeCache.txt")):
cmake_command = ["cmake"] + string.split(cmake_options) + [srcDir]
#print cmake_command
status = make_executable.Run(cmake_command, dir=buildDir)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
q_stdout = result.Quote(make_executable.stdout)
q_stderr = result.Quote(make_executable.stderr)
result.SetOutcome(
result.ERROR, "Error in cmake Makefile generation", {
"status": str(status),
"stdout": q_stdout,
"stderr": q_stderr,
"command": " ".join(cmake_command),
})
return
if os.environ.has_key("RPM_BUILD_NCPUS"):
cpus = os.environ["RPM_BUILD_NCPUS"]
else:
cpus = "1"
make_command = ["make", "-j" + cpus] + string.split(self.make_options)
status = make_executable.Run(make_command, dir=buildDir)
if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
q_stdout = result.Quote(make_executable.stdout)
q_stderr = result.Quote(make_executable.stderr)
result.SetOutcome(
result.ERROR, "Error building IPv6Suite", {
"status": str(status),
"stdout": q_stdout,
"stderr": q_stderr,
"command": " ".join(make_command),
})
return
def _get_global(k):
if k in _initialized_globals:
return globals()[k]
if k == 'secpass':
unprivileged = False
if hasattr(portage, 'settings'):
unprivileged = "unprivileged" in portage.settings.features
else:
# The config class has equivalent code, but we also need to
# do it here if _disable_legacy_globals() has been called.
eroot_or_parent = first_existing(
os.path.join(_target_root(),
_target_eprefix().lstrip(os.sep)))
try:
eroot_st = os.stat(eroot_or_parent)
except OSError:
pass
else:
unprivileged = _unprivileged_mode(eroot_or_parent, eroot_st)
v = 0
if uid == 0:
v = 2
elif unprivileged:
v = 2
elif portage_gid in os.getgroups():
v = 1
elif k in ('portage_gid', 'portage_uid'):
#Discover the uid and gid of the portage user/group
keyerror = False
try:
portage_uid = pwd.getpwnam(_get_global('_portage_username')).pw_uid
except KeyError:
keyerror = True
portage_uid = 0
try:
portage_gid = grp.getgrnam(_get_global('_portage_grpname')).gr_gid
except KeyError:
keyerror = True
portage_gid = 0
# Suppress this error message if both PORTAGE_GRPNAME and
# PORTAGE_USERNAME are set to "root", for things like
# Android (see bug #454060).
if keyerror and not (_get_global('_portage_username') == "root"
and _get_global('_portage_grpname') == "root"):
writemsg(colorize(
"BAD", _("portage: 'portage' user or group missing.")) + "\n",
noiselevel=-1)
writemsg(_(" For the defaults, line 1 goes into passwd, "
"and 2 into group.\n"),
noiselevel=-1)
writemsg(colorize("GOOD",
" portage:x:250:250:portage:/var/tmp/portage:/bin/false") \
+ "\n", noiselevel=-1)
writemsg(colorize("GOOD", " portage::250:portage") + "\n",
noiselevel=-1)
portage_group_warning()
globals()['portage_gid'] = portage_gid
_initialized_globals.add('portage_gid')
globals()['portage_uid'] = portage_uid
_initialized_globals.add('portage_uid')
if k == 'portage_gid':
return portage_gid
elif k == 'portage_uid':
return portage_uid
else:
raise AssertionError('unknown name: %s' % k)
elif k == 'userpriv_groups':
v = [_get_global('portage_gid')]
if secpass >= 2:
# Get a list of group IDs for the portage user. Do not use
# grp.getgrall() since it is known to trigger spurious
# SIGPIPE problems with nss_ldap.
cmd = ["id", "-G", _portage_username]
if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000:
# Python 3.1 _execvp throws TypeError for non-absolute executable
# path passed as bytes (see http://bugs.python.org/issue8513).
fullname = portage.process.find_binary(cmd[0])
if fullname is None:
globals()[k] = v
_initialized_globals.add(k)
return v
cmd[0] = fullname
encoding = portage._encodings['content']
cmd = [
portage._unicode_encode(x, encoding=encoding, errors='strict')
for x in cmd
]
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
myoutput = proc.communicate()[0]
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
for x in portage._unicode_decode(myoutput,
encoding=encoding,
errors='strict').split():
try:
v.append(int(x))
except ValueError:
pass
v = sorted(set(v))
# Avoid instantiating portage.settings when the desired
# variable is set in os.environ.
elif k in ('_portage_grpname', '_portage_username'):
v = None
if k == '_portage_grpname':
env_key = 'PORTAGE_GRPNAME'
else:
env_key = 'PORTAGE_USERNAME'
if env_key in os.environ:
v = os.environ[env_key]
elif hasattr(portage, 'settings'):
v = portage.settings.get(env_key)
else:
# The config class has equivalent code, but we also need to
# do it here if _disable_legacy_globals() has been called.
eroot_or_parent = first_existing(
os.path.join(_target_root(),
_target_eprefix().lstrip(os.sep)))
try:
eroot_st = os.stat(eroot_or_parent)
except OSError:
pass
else:
if _unprivileged_mode(eroot_or_parent, eroot_st):
if k == '_portage_grpname':
try:
grp_struct = grp.getgrgid(eroot_st.st_gid)
except KeyError:
pass
else:
v = grp_struct.gr_name
else:
try:
pwd_struct = pwd.getpwuid(eroot_st.st_uid)
except KeyError:
pass
else:
v = pwd_struct.pw_name
if v is None:
v = 'portage'
else:
raise AssertionError('unknown name: %s' % k)
globals()[k] = v
_initialized_globals.add(k)
return v
# Running in parent process.
os.close(wfd)
ovs.fatal_signal.fork()
while True:
try:
s = os.read(rfd, 1)
error = 0
except OSError, e:
s = ""
error = e.errno
if error != errno.EINTR:
break
if len(s) != 1:
retval, status = _waitpid(pid, 0)
if retval == pid:
if os.WIFEXITED(status) and os.WEXITSTATUS(status):
# Child exited with an error. Convey the same error to
# our parent process as a courtesy.
sys.exit(os.WEXITSTATUS(status))
else:
sys.stderr.write("fork child failed to signal "
"startup (%s)\n" %
ovs.process.status_msg(status))
else:
assert retval < 0
sys.stderr.write("waitpid failed (%s)\n" %
os.strerror(-retval))
sys.exit(1)
os.close(rfd)
else:
def wait_pid(pid, timeout=None, proc_name=None,
_waitpid=os.waitpid,
_timer=getattr(time, 'monotonic', time.time),
_min=min,
_sleep=time.sleep,
_pid_exists=pid_exists):
"""Wait for a process PID to terminate.
If the process terminated normally by calling exit(3) or _exit(2),
or by returning from main(), the return value is the positive integer
passed to *exit().
If it was terminated by a signal it returns the negated value of the
signal which caused the termination (e.g. -SIGTERM).
If PID is not a children of os.getpid() (current process) just
wait until the process disappears and return None.
If PID does not exist at all return None immediately.
If *timeout* != None and process is still alive raise TimeoutExpired.
timeout=0 is also possible (either return immediately or raise).
"""
if pid <= 0:
raise ValueError("can't wait for PID 0") # see "man waitpid"
interval = 0.0001
flags = 0
if timeout is not None:
flags |= os.WNOHANG
stop_at = _timer() + timeout
def sleep(interval):
# Sleep for some time and return a new increased interval.
if timeout is not None:
if _timer() >= stop_at:
raise TimeoutExpired(timeout, pid=pid, name=proc_name)
_sleep(interval)
return _min(interval * 2, 0.04)
# See: https://linux.die.net/man/2/waitpid
while True:
try:
retpid, status = os.waitpid(pid, flags)
except InterruptedError:
interval = sleep(interval)
except ChildProcessError:
# This has two meanings:
# - PID is not a child of os.getpid() in which case
# we keep polling until it's gone
# - PID never existed in the first place
# In both cases we'll eventually return None as we
# can't determine its exit status code.
while _pid_exists(pid):
interval = sleep(interval)
return
else:
if retpid == 0:
# WNOHANG flag was used and PID is still running.
interval = sleep(interval)
continue
elif os.WIFEXITED(status):
# Process terminated normally by calling exit(3) or _exit(2),
# or by returning from main(). The return value is the
# positive integer passed to *exit().
return os.WEXITSTATUS(status)
elif os.WIFSIGNALED(status):
# Process exited due to a signal. Return the negative value
# of that signal.
return negsig_to_enum(-os.WTERMSIG(status))
# elif os.WIFSTOPPED(status):
# # Process was stopped via SIGSTOP or is being traced, and
# # waitpid() was called with WUNTRACED flag. PID is still
# # alive. From now on waitpid() will keep returning (0, 0)
# # until the process state doesn't change.
# # It may make sense to catch/enable this since stopped PIDs
# # ignore SIGTERM.
# interval = sleep(interval)
# continue
# elif os.WIFCONTINUED(status):
# # Process was resumed via SIGCONT and waitpid() was called
# # with WCONTINUED flag.
# interval = sleep(interval)
# continue
else:
# Should never happen.
raise ValueError("unknown process exit status %r" % status)
def exitstatus(self):
if os.WIFEXITED(self.status):
return os.WEXITSTATUS(self.status)
# '/user/jrivadeneira/daloflow/dataset32x32/:/mnt/local-storage/daloflow/dataset-cache/dataset32x32/'
cache_parts = cache_path.split(':')
can_continue_with_cache = (len(cache_parts) == 2)
if can_continue_with_cache:
# param to choose if we want local copy or not
hdfs_dir = cache_parts[0]
cache_dir = cache_parts[1]
hdfs_list = cache_dir + "/list.txt"
# add the container name at the end cache_dir
container_name = os.uname()[1]
cache_dir = cache_dir + "/" + container_name
if can_continue_with_cache:
status = os.system("mkdir -p " + cache_dir)
can_continue_with_cache = os.WIFEXITED(status) and (
os.WEXITSTATUS(status) == 0)
# list of files to copy in local
if can_continue_with_cache:
with open(hdfs_list, "w") as f:
f.write('labels.p\n')
for item in partition['train']:
fname = '/'.join(item.split('/')[1:]) + '.tar.gz\n'
f.write(fname[1:])
f.close()
# copy from hdfs to local
if can_continue_with_cache:
status = os.system("hdfs/hdfs-cp.sh" + " " + "hdfs2local" + " " +
hdfs_dir + " " + hdfs_list + " " + cache_dir)
def test_execve():
if os.name != 'posix':
py.test.skip('posix specific function')
ll_execve = getllimpl(os.execve)
def run_execve(program, args=None, env=None, do_path_lookup=False):
if args is None:
args = [program]
else:
args = [program] + args
if env is None:
env = {}
# we cannot directly call ll_execve() because it replaces the
# current process.
fd_read, fd_write = os.pipe()
childpid = os.fork()
if childpid == 0:
# in the child
os.close(fd_read)
os.dup2(fd_write, 1) # stdout
os.close(fd_write)
if do_path_lookup:
os.execvp(program, args)
else:
ll_execve(program, args, env)
assert 0, "should not arrive here"
else:
# in the parent
os.close(fd_write)
child_stdout = []
while True:
data = os.read(fd_read, 4096)
if not data: break # closed
child_stdout.append(data)
pid, status = os.waitpid(childpid, 0)
os.close(fd_read)
return status, ''.join(child_stdout)
# Test exit status and code
result, child_stdout = run_execve("/usr/bin/which", ["true"],
do_path_lookup=True)
result, child_stdout = run_execve(
child_stdout.strip()) # /bin/true or /usr/bin/true
assert os.WIFEXITED(result)
assert os.WEXITSTATUS(result) == 0
result, child_stdout = run_execve("/usr/bin/which", ["false"],
do_path_lookup=True)
result, child_stdout = run_execve(
child_stdout.strip()) # /bin/false or /usr/bin/false
assert os.WIFEXITED(result)
assert os.WEXITSTATUS(result) == 1
# Test environment
result, child_stdout = run_execve("/usr/bin/env", env=EXECVE_ENV)
assert os.WIFEXITED(result)
assert os.WEXITSTATUS(result) == 0
assert dict([line.split('=')
for line in child_stdout.splitlines()]) == EXECVE_ENV
# The following won't actually execute anything, so they don't need
# a child process helper.
# If the target does not exist, an OSError should result
info = py.test.raises(OSError, ll_execve, "this/file/is/non/existent", [],
{})
assert info.value.errno == errno.ENOENT
# If the target is not executable, an OSError should result
info = py.test.raises(OSError, ll_execve, "/etc/passwd", [], {})
assert info.value.errno == errno.EACCES
def launch_refarch_env(self):
with open(self.inventory_file, 'r') as f:
print yaml.safe_dump(json.load(f), default_flow_style=False)
if not self.args.no_confirm:
if not click.confirm('Continue adding nodes with these values?'):
sys.exit(0)
if (self.container_storage in ('cns', 'crs')
and 'storage' in self.node_type):
if 'None' in self.tag:
# do the full install and config minus the cleanup
self.tag = 'vms,node-setup'
playbooks = ['playbooks/%s-storage.yaml' % self.container_storage]
else:
if 'None' in self.tag:
# do the full install and config minus the cleanup
self.tag = 'all'
playbooks = ['playbooks/add-node.yaml']
playbook_vars_dict = {
'add_node':
'yes',
'vcenter_host':
self.vcenter_host,
'vcenter_username':
self.vcenter_username,
'vcenter_password':
self.vcenter_password,
'vcenter_template_name':
self.vcenter_template_name,
'vcenter_folder':
self.vcenter_folder,
'vcenter_datastore':
self.vcenter_datastore,
'vcenter_cluster':
self.vcenter_cluster,
'vcenter_datacenter':
self.vcenter_datacenter,
'vcenter_resource_pool':
self.vcenter_resource_pool,
'dns_zone':
self.dns_zone,
'wildcard_zone':
self.wildcard_zone,
'app_dns_prefix':
self.app_dns_prefix,
'vm_network':
self.vm_network,
'cns_automation_config_file_path':
(self.cns_automation_config_file_path),
'console_port':
self.console_port,
'cluster_id':
self.cluster_id,
'container_storage':
self.container_storage,
'container_storage_disks':
self.container_storage_disks,
'container_storage_disk_type':
self.container_storage_disk_type,
'additional_disks_to_storage_nodes':
(self.additional_disks_to_storage_nodes),
'dp_tool_heketi_admin_key':
self.heketi_admin_key,
'dp_tool_heketi_user_key':
self.heketi_user_key,
'ose_puddle_repo':
self.ose_puddle_repo,
'gluster_puddle_repo':
self.gluster_puddle_repo,
'deployment_type':
self.deployment_type,
'openshift_deployment_type':
self.deployment_type,
'openshift_vers':
self.openshift_vers,
'admin_key':
self.admin_key,
'user_key':
self.user_key,
'rhel_subscription_user':
self.rhel_subscription_user,
'rhel_subscription_pass':
self.rhel_subscription_pass,
'rhsm_satellite':
self.rhel_subscription_server,
'rhsm_pool':
self.rhel_subscription_pool,
'openshift_sdn':
self.openshift_sdn,
'openshift_use_openshift_sdn':
True,
'lb_host':
self.lb_host,
'node_type':
self.node_type,
'ocp_hostname_prefix':
self.ocp_hostname_prefix,
'disable_yum_update_and_reboot':
self.disable_yum_update_and_reboot
}
if self.openshift_disable_check_data:
playbook_vars_dict["openshift_disable_check"] = (','.join(
self.openshift_disable_check_data))
if self.container_storage_block_hosting_volume_size:
playbook_vars_dict[
'openshift_storage_glusterfs_block_host_vol_size'] = (
self.container_storage_block_hosting_volume_size)
if self.container_storage_glusterfs_timeout:
playbook_vars_dict['openshift_storage_glusterfs_timeout'] = (
self.container_storage_glusterfs_timeout)
if self.docker_registry_url:
playbook_vars_dict['oreg_url'] = self.docker_registry_url
if self.docker_additional_registries:
playbook_vars_dict['openshift_docker_additional_registries'] = (
self.docker_additional_registries)
playbook_vars_dict['openshift_docker_ent_reg'] = ''
if self.docker_insecure_registries:
playbook_vars_dict['openshift_docker_insecure_registries'] = (
self.docker_insecure_registries)
if self.docker_image_tag:
playbook_vars_dict['openshift_image_tag'] = self.docker_image_tag
if self.openshift_vers in ("v3_6", "v3_7", "v3_9"):
for key in ('image', 'version', 'block_image', 'block_version',
'heketi_image', 'heketi_version'):
value = getattr(self, 'cns_glusterfs_%s' % key)
if not value:
continue
playbook_vars_dict['openshift_storage_glusterfs_%s' %
key] = (value)
if self.openshift_vers in ('v3_6', 'v3_7'):
playbook_vars_dict['docker_version'] = '1.12.6'
elif self.openshift_vers != "v3_9":
if self.cns_glusterfs_version:
playbook_vars_dict['openshift_storage_glusterfs_image'] = (
"%s:%s" %
(self.cns_glusterfs_image
or 'rhgs3/rhgs-server-rhel7', self.cns_glusterfs_version))
elif self.cns_glusterfs_image:
playbook_vars_dict['openshift_storage_glusterfs_image'] = (
"%s:latest" % self.cns_glusterfs_image)
if self.cns_glusterfs_block_version:
playbook_vars_dict[
'openshift_storage_glusterfs_block_image'] = (
"%s:%s" % (self.cns_glusterfs_block_image
or 'rhgs3/rhgs-gluster-block-prov-rhel7',
self.cns_glusterfs_block_version))
elif self.cns_glusterfs_block_image:
playbook_vars_dict[
"openshift_storage_glusterfs_block_image"] = (
"%s:latest" % self.cns_glusterfs_block_image)
if self.cns_glusterfs_heketi_version:
playbook_vars_dict[
'openshift_storage_glusterfs_heketi_image'] = (
"%s:%s" % (self.cns_glusterfs_heketi_image
or 'rhgs3/rhgs-volmanager-rhel7',
self.cns_glusterfs_heketi_version))
elif self.cns_glusterfs_heketi_image:
playbook_vars_dict[
"openshift_storage_glusterfs_heketi_image"] = (
"%s:latest" % self.cns_glusterfs_heketi_image)
playbook_vars_str = ' '.join('%s=%s' % (k, v)
for (k, v) in playbook_vars_dict.items())
for playbook in playbooks:
devnull = '' if self.verbose > 0 else '> /dev/null'
# refresh the inventory cache to prevent stale hosts from
# interferring with re-running
command = 'inventory/vsphere/vms/vmware_inventory.py %s' % (
devnull)
os.system(command)
# remove any cached facts to prevent stale data during a re-run
command = 'rm -rf .ansible/cached_facts'
os.system(command)
command = (
"ansible-playbook"
" --extra-vars '@./%s'"
" --tags %s"
" -e '%s' %s" %
(self.inventory_file, self.tag, playbook_vars_str, playbook))
if self.verbose > 0:
command += " -vvvvv"
click.echo('We are running: %s' % command)
status = os.system(command)
if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0:
sys.exit(os.WEXITSTATUS(status))
command = ("ansible-playbook "
"-i %smaster-0, playbooks/get_ocp_info.yaml") % (
"%s-" % self.ocp_hostname_prefix
if self.ocp_hostname_prefix else "")
os.system(command)
print "Successful run!"
if click.confirm('Update INI?'):
self.update_ini_file()
if click.confirm('Delete inventory file?'):
print "Removing the existing %s file" % self.inventory_file
os.remove(self.inventory_file)
sys.exit(0)