def process_command(sock, args, instance):
"""Create a process.
You need to specify a name, followed by name=value pairs for the process
options, followed by the command and args to execute. The name must not contain
spaces.
Process options are:
uid - the username or user ID to use when starting the process
gid - the group name or group ID to use when starting the process
working_dir - must be an absolute path if specified
output - size of each output buffer (default is 1m)
You can also specify environment variables by prefixing the name with 'env.' and
rlimits by prefixing the name with 'rlimit.'
Examples:
make process sf uid=1001 gid=2000 working_dir=/sf/bin/ output=1m /sf/bin/uwsgi --ini uwsgi-live.ini --socket fd://27 --stats 127.0.0.1:8090
make process nginx /usr/local/nginx/sbin/nginx
"""
if not args:
raise Error('Process requires a name')
name = args.pop(0)
env = {}
rlimits = {}
kwargs = {}
for key, value in extract_name_value_pairs(args).items():
if key.startswith('env.'):
env[key[4:]] = value
elif key.startswith('rlimit.'):
key = key[7:]
try:
rlimits[getattr(resource,
'RLIMIT_%s' % key.upper())] = int(value)
except AttributeError:
raise utils.Error('Unknown rlimit "%s"' % key)
except ValueError:
raise utils.Error(
'The rlimit value for "%s" must be an integer, not "%s"' %
(key, value))
else:
kwargs[key] = value
watch = int(kwargs.pop('watch', 0))
p = Process(name, args, env, rlimits, instance, **kwargs)
with instance['globals']['lock']:
result = p.spawn()
if watch:
send_with_retry(sock, cast_bytes('{0}\n'.format(result)))
return _do_watch(sock, {name: {
'p': result,
't': 0,
'closed': False
}}, instance)
return str(result)
def __init__(self,
name,
instance,
family=None,
type='stream',
backlog=5,
path=None,
umask=None,
host=None,
port=0,
interface=None,
reuseport=False):
if path and unix_socket is None:
raise NotImplemented(
'Unix sockets are not supported on this system')
instance_globals = instance['globals']
self._sockets_by_name = instance_globals['sockets']['by_name']
self._sockets_by_path = instance_globals['sockets']['by_path']
self.name = name
if family:
self.family = utils.valid_families[family]
else:
self.family = unix_socket if path else socket.AF_INET
self.socket_type = utils.valid_types[type]
self.backlog = int(backlog)
self.path = self.umask = None
self.host = self.port = self.interface = self.reuseport = None
self.socket = None
if self.family == unix_socket:
if not path or not os.path.isabs(path):
raise utils.Error('Absolute path required for Unix sockets')
self.path = path
self.umask = None if umask is None else int(umask)
else:
self.port = int(port)
self.interface = interface
if host:
self.host = self._host = host
target_length = 4 if self.family == socket.AF_INET6 else 2
for info in socket.getaddrinfo(host, self.port):
if len(info[-1]) == target_length:
self._host = info[-1][0]
break
else:
if self.family == socket.AF_INET6:
self.host = '::' if interface else '::1'
else:
self.host = '0.0.0.0' if interface else '127.0.0.1'
self._host = self.host
self.reuseport = reuseport if reuseport and hasattr(
socket, 'SO_REUSEPORT') else False
def watch_command(sock, args, instance):
"""Watch a process"""
instance_globals = instance['globals']
all_processes = instance_globals['processes']
with instance_globals['lock']:
procs = dict((name, {
'p': proc,
't': 0,
'closed': False
}) for name, proc in wildcard_iter(all_processes, args, True))
if not procs:
raise utils.Error('Nothing to watch')
send_with_retry(sock, cast_bytes('Watching {0}\n'.format(len(procs))))
return _do_watch(sock, procs, instance)
def clone_for_reuseport(self):
s = socket.socket(self.family, self.socket_type)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if self.interface:
import IN
if hasattr(IN, 'SO_BINDTODEVICE'):
s.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE,
self.interface + '\0')
try:
s.bind((self._host, self.port))
except socket.error as e:
raise utils.Error('Bind failed on {0}:{1}: {2}'.format(
self.host, self.port, e))
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
s.listen(self.backlog)
try:
s.set_inheritable(True)
except Exception:
pass
return s
def start(self):
existing = self._sockets_by_name.get(self.name)
if existing:
if self == existing:
self.socket = existing.socket
self.port = existing.port
else:
raise utils.Error(
'Socket for {0} has already been created - {1}'.format(
self.name, str(existing)))
else:
if self.family == unix_socket:
if self.path in self._sockets_by_path:
raise utils.Error(
'Socket for {0} has already been created'.format(
self.path))
try:
os.unlink(self.path)
except OSError:
if os.path.exists(self.path):
raise
s = socket.socket(self.family, self.socket_type)
try:
if self.umask is None:
s.bind(self.path)
else:
old_mask = os.umask(self.umask)
s.bind(self.path)
os.umask(old_mask)
except socket.error as e:
raise utils.Error('Bind failed: {0}'.format(e))
self._sockets_by_path[self.path] = self
else:
s = socket.socket(self.family, self.socket_type)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if self.interface:
import IN
if hasattr(IN, 'SO_BINDTODEVICE'):
s.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE,
self.interface + '\0')
try:
s.bind((self._host, self.port))
except socket.error as e:
raise utils.Error('Bind failed on {0}:{1}: {2}'.format(
self.host, self.port, e))
if not self.port:
self.port = s.getsockname()[1]
if self.reuseport:
try:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
s.close()
s = None
except socket.error:
self.reuseport = False
# noinspection PyUnresolvedReferences
if s:
s.listen(self.backlog)
try:
s.set_inheritable(True)
except Exception:
pass
self.socket = s
self._sockets_by_name[self.name] = self
log.info('Created socket %s', self)
return self
def spawn(self):
existing = self._processes.get(self.name)
if existing:
if self == existing:
self.pid = existing.pid
self.running = existing.running
self.started = existing.started
else:
raise utils.Error(
'Process for {0} has already been created - {1}'.format(
self.name, str(existing)))
else:
managed_sockets = []
fixed_args = []
self.started = time()
for arg in self.args:
if '$(socket.' in arg:
start = arg.find('$(socket.') + 9
end = arg.find(')', start)
if end == -1:
raise utils.Error(
'Process for {0} argument starts with "$(socket." but has no closing parenthesis'
.format(self.name))
socket_and_part = arg[start:end]
socket_name, part = socket_and_part.rpartition('.')[::2]
if not part or part not in ('port', 'fileno'):
raise utils.Error(
'You forgot to specify either ".port" or ".fileno" after the name'
)
try:
s = find_socket(socket_name, self.instance)
except Exception:
raise utils.Error(
'Socket {0} not found'.format(socket_name))
if part == 'port':
replacement = s.port
elif s.reuseport:
sock = s.clone_for_reuseport()
managed_sockets.append(sock)
replacement = sock.fileno()
else:
replacement = s.socket.fileno()
arg = '{0}{1}{2}'.format(arg[:start - 9], replacement,
arg[end + 1:])
fixed_args.append(arg)
if not fixed_args:
raise utils.Error('No command')
def preexec():
streams = [sys.stdin]
if not self.out:
streams.append(sys.stdout)
if not self.err:
streams.append(sys.stderr)
for stream in streams:
if hasattr(stream, 'fileno'):
try:
stream.flush()
devnull = os.open(os.devnull, os.O_RDWR)
# noinspection PyTypeChecker
os.dup2(devnull, stream.fileno())
# noinspection PyTypeChecker
os.close(devnull)
except IOError:
# some streams, like stdin - might be already closed.
pass
# noinspection PyArgumentList
os.setsid()
if resource:
for limit, value in self.rlimits.items():
resource.setrlimit(limit, (value, value))
if self.gid:
try:
# noinspection PyTypeChecker
os.setgid(self.gid)
except OverflowError:
if not ctypes:
raise
# versions of python < 2.6.2 don't manage unsigned int for
# groups like on osx or fedora
os.setgid(-ctypes.c_int(-self.gid).value)
if self.username is not None:
try:
# noinspection PyTypeChecker
os.initgroups(self.username, self.gid)
except (OSError, AttributeError):
# not support on Mac or 2.6
pass
if self.uid:
# noinspection PyTypeChecker
os.setuid(self.uid)
extra = {}
if self.out:
extra['stdout'] = PIPE
if self.err:
if self.err == 'stdout':
extra['stderr'] = STDOUT
else:
extra['stderr'] = PIPE
try:
self._worker = Popen(fixed_args,
preexec_fn=preexec,
close_fds=False,
shell=self.shell,
cwd=self.working_dir,
env=self.env,
bufsize=-1,
**extra)
except FileNotFoundError as file_not_found_exception:
if not os.path.exists(fixed_args[0]):
raise utils.Error(
'Bad command - {0}'.format(file_not_found_exception))
if self.working_dir and not os.path.isdir(self.working_dir):
raise utils.Error('Bad working_dir - {0}'.format(
file_not_found_exception))
raise
# let go of sockets created only for self.worker to inherit
for sock in managed_sockets:
sock.close()
self._processes[self.name] = self
self.pid = self._worker.pid
self._output = OutputQueue(self.bufsize)
log.info('Created process %s', self)
self.running = True
self._thread = Thread(target=self._watch)
self._thread.daemon = True
self._thread.start()
return self
def __init__(self,
name,
args,
env,
rlimits,
instance,
working_dir=None,
shell=False,
uid=None,
gid=None,
stdout=1,
stderr=1,
bufsize='1m'):
self.instance = instance
instance_globals = instance['globals']
self._processes = instance_globals['processes']
self.name = name
self.args = args
self.env = env
self.rlimits = rlimits
self.working_dir = working_dir
self.shell = shell
self.bufsize = convert_size_string_to_bytes(bufsize)
self.pid = 0
self.running = False
self.started = 0
if self.bufsize:
self.out = int(stdout)
self.err = stderr if stderr == 'stdout' else int(stderr)
else:
self.out = self.err = 0
if uid:
if pwd:
try:
self.uid = int(uid)
self.username = pwd.getpwuid(self.uid).pw_name
except KeyError:
raise utils.Error('%r is not a valid user id' % uid)
except ValueError:
try:
self.username = uid
self.uid = pwd.getpwnam(uid).pw_uid
except KeyError:
raise utils.Error('%r is not a valid user name' % uid)
else:
raise utils.Error('uid is not supported on this platform')
else:
self.username = None
self.uid = None
if gid:
if grp:
try:
self.gid = int(gid)
grp.getgrgid(self.gid)
except (KeyError, OverflowError):
raise utils.Error('No such group: %r' % gid)
except ValueError:
try:
self.gid = grp.getgrnam(gid).gr_gid
except KeyError:
raise utils.Error('No such group: %r' % gid)
else:
raise utils.Error('gid is not supported on this platform')
elif self.uid:
self.gid = pwd.getpwuid(self.uid).pw_gid
else:
self.gid = None
# sockets created before fork, should be let go after.
self._worker = None
self._thread = None
self._output = None
self._auto_close = False