def process_command(sock, args, instance): """Create a process. You need to specify a name, followed by name=value pairs for the process options, followed by the command and args to execute. The name must not contain spaces. Process options are: uid - the username or user ID to use when starting the process gid - the group name or group ID to use when starting the process working_dir - must be an absolute path if specified output - size of each output buffer (default is 1m) You can also specify environment variables by prefixing the name with 'env.' and rlimits by prefixing the name with 'rlimit.' Examples: make process sf uid=1001 gid=2000 working_dir=/sf/bin/ output=1m /sf/bin/uwsgi --ini uwsgi-live.ini --socket fd://27 --stats 127.0.0.1:8090 make process nginx /usr/local/nginx/sbin/nginx """ if not args: raise Error('Process requires a name') name = args.pop(0) env = {} rlimits = {} kwargs = {} for key, value in extract_name_value_pairs(args).items(): if key.startswith('env.'): env[key[4:]] = value elif key.startswith('rlimit.'): key = key[7:] try: rlimits[getattr(resource, 'RLIMIT_%s' % key.upper())] = int(value) except AttributeError: raise utils.Error('Unknown rlimit "%s"' % key) except ValueError: raise utils.Error( 'The rlimit value for "%s" must be an integer, not "%s"' % (key, value)) else: kwargs[key] = value watch = int(kwargs.pop('watch', 0)) p = Process(name, args, env, rlimits, instance, **kwargs) with instance['globals']['lock']: result = p.spawn() if watch: send_with_retry(sock, cast_bytes('{0}\n'.format(result))) return _do_watch(sock, {name: { 'p': result, 't': 0, 'closed': False }}, instance) return str(result)
def __init__(self, name, instance, family=None, type='stream', backlog=5, path=None, umask=None, host=None, port=0, interface=None, reuseport=False): if path and unix_socket is None: raise NotImplemented( 'Unix sockets are not supported on this system') instance_globals = instance['globals'] self._sockets_by_name = instance_globals['sockets']['by_name'] self._sockets_by_path = instance_globals['sockets']['by_path'] self.name = name if family: self.family = utils.valid_families[family] else: self.family = unix_socket if path else socket.AF_INET self.socket_type = utils.valid_types[type] self.backlog = int(backlog) self.path = self.umask = None self.host = self.port = self.interface = self.reuseport = None self.socket = None if self.family == unix_socket: if not path or not os.path.isabs(path): raise utils.Error('Absolute path required for Unix sockets') self.path = path self.umask = None if umask is None else int(umask) else: self.port = int(port) self.interface = interface if host: self.host = self._host = host target_length = 4 if self.family == socket.AF_INET6 else 2 for info in socket.getaddrinfo(host, self.port): if len(info[-1]) == target_length: self._host = info[-1][0] break else: if self.family == socket.AF_INET6: self.host = '::' if interface else '::1' else: self.host = '0.0.0.0' if interface else '127.0.0.1' self._host = self.host self.reuseport = reuseport if reuseport and hasattr( socket, 'SO_REUSEPORT') else False
def watch_command(sock, args, instance): """Watch a process""" instance_globals = instance['globals'] all_processes = instance_globals['processes'] with instance_globals['lock']: procs = dict((name, { 'p': proc, 't': 0, 'closed': False }) for name, proc in wildcard_iter(all_processes, args, True)) if not procs: raise utils.Error('Nothing to watch') send_with_retry(sock, cast_bytes('Watching {0}\n'.format(len(procs)))) return _do_watch(sock, procs, instance)
def clone_for_reuseport(self): s = socket.socket(self.family, self.socket_type) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if self.interface: import IN if hasattr(IN, 'SO_BINDTODEVICE'): s.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, self.interface + '\0') try: s.bind((self._host, self.port)) except socket.error as e: raise utils.Error('Bind failed on {0}:{1}: {2}'.format( self.host, self.port, e)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) s.listen(self.backlog) try: s.set_inheritable(True) except Exception: pass return s
def start(self): existing = self._sockets_by_name.get(self.name) if existing: if self == existing: self.socket = existing.socket self.port = existing.port else: raise utils.Error( 'Socket for {0} has already been created - {1}'.format( self.name, str(existing))) else: if self.family == unix_socket: if self.path in self._sockets_by_path: raise utils.Error( 'Socket for {0} has already been created'.format( self.path)) try: os.unlink(self.path) except OSError: if os.path.exists(self.path): raise s = socket.socket(self.family, self.socket_type) try: if self.umask is None: s.bind(self.path) else: old_mask = os.umask(self.umask) s.bind(self.path) os.umask(old_mask) except socket.error as e: raise utils.Error('Bind failed: {0}'.format(e)) self._sockets_by_path[self.path] = self else: s = socket.socket(self.family, self.socket_type) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if self.interface: import IN if hasattr(IN, 'SO_BINDTODEVICE'): s.setsockopt(socket.SOL_SOCKET, IN.SO_BINDTODEVICE, self.interface + '\0') try: s.bind((self._host, self.port)) except socket.error as e: raise utils.Error('Bind failed on {0}:{1}: {2}'.format( self.host, self.port, e)) if not self.port: self.port = s.getsockname()[1] if self.reuseport: try: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) s.close() s = None except socket.error: self.reuseport = False # noinspection PyUnresolvedReferences if s: s.listen(self.backlog) try: s.set_inheritable(True) except Exception: pass self.socket = s self._sockets_by_name[self.name] = self log.info('Created socket %s', self) return self
def spawn(self): existing = self._processes.get(self.name) if existing: if self == existing: self.pid = existing.pid self.running = existing.running self.started = existing.started else: raise utils.Error( 'Process for {0} has already been created - {1}'.format( self.name, str(existing))) else: managed_sockets = [] fixed_args = [] self.started = time() for arg in self.args: if '$(socket.' in arg: start = arg.find('$(socket.') + 9 end = arg.find(')', start) if end == -1: raise utils.Error( 'Process for {0} argument starts with "$(socket." but has no closing parenthesis' .format(self.name)) socket_and_part = arg[start:end] socket_name, part = socket_and_part.rpartition('.')[::2] if not part or part not in ('port', 'fileno'): raise utils.Error( 'You forgot to specify either ".port" or ".fileno" after the name' ) try: s = find_socket(socket_name, self.instance) except Exception: raise utils.Error( 'Socket {0} not found'.format(socket_name)) if part == 'port': replacement = s.port elif s.reuseport: sock = s.clone_for_reuseport() managed_sockets.append(sock) replacement = sock.fileno() else: replacement = s.socket.fileno() arg = '{0}{1}{2}'.format(arg[:start - 9], replacement, arg[end + 1:]) fixed_args.append(arg) if not fixed_args: raise utils.Error('No command') def preexec(): streams = [sys.stdin] if not self.out: streams.append(sys.stdout) if not self.err: streams.append(sys.stderr) for stream in streams: if hasattr(stream, 'fileno'): try: stream.flush() devnull = os.open(os.devnull, os.O_RDWR) # noinspection PyTypeChecker os.dup2(devnull, stream.fileno()) # noinspection PyTypeChecker os.close(devnull) except IOError: # some streams, like stdin - might be already closed. pass # noinspection PyArgumentList os.setsid() if resource: for limit, value in self.rlimits.items(): resource.setrlimit(limit, (value, value)) if self.gid: try: # noinspection PyTypeChecker os.setgid(self.gid) except OverflowError: if not ctypes: raise # versions of python < 2.6.2 don't manage unsigned int for # groups like on osx or fedora os.setgid(-ctypes.c_int(-self.gid).value) if self.username is not None: try: # noinspection PyTypeChecker os.initgroups(self.username, self.gid) except (OSError, AttributeError): # not support on Mac or 2.6 pass if self.uid: # noinspection PyTypeChecker os.setuid(self.uid) extra = {} if self.out: extra['stdout'] = PIPE if self.err: if self.err == 'stdout': extra['stderr'] = STDOUT else: extra['stderr'] = PIPE try: self._worker = Popen(fixed_args, preexec_fn=preexec, close_fds=False, shell=self.shell, cwd=self.working_dir, env=self.env, bufsize=-1, **extra) except FileNotFoundError as file_not_found_exception: if not os.path.exists(fixed_args[0]): raise utils.Error( 'Bad command - {0}'.format(file_not_found_exception)) if self.working_dir and not os.path.isdir(self.working_dir): raise utils.Error('Bad working_dir - {0}'.format( file_not_found_exception)) raise # let go of sockets created only for self.worker to inherit for sock in managed_sockets: sock.close() self._processes[self.name] = self self.pid = self._worker.pid self._output = OutputQueue(self.bufsize) log.info('Created process %s', self) self.running = True self._thread = Thread(target=self._watch) self._thread.daemon = True self._thread.start() return self
def __init__(self, name, args, env, rlimits, instance, working_dir=None, shell=False, uid=None, gid=None, stdout=1, stderr=1, bufsize='1m'): self.instance = instance instance_globals = instance['globals'] self._processes = instance_globals['processes'] self.name = name self.args = args self.env = env self.rlimits = rlimits self.working_dir = working_dir self.shell = shell self.bufsize = convert_size_string_to_bytes(bufsize) self.pid = 0 self.running = False self.started = 0 if self.bufsize: self.out = int(stdout) self.err = stderr if stderr == 'stdout' else int(stderr) else: self.out = self.err = 0 if uid: if pwd: try: self.uid = int(uid) self.username = pwd.getpwuid(self.uid).pw_name except KeyError: raise utils.Error('%r is not a valid user id' % uid) except ValueError: try: self.username = uid self.uid = pwd.getpwnam(uid).pw_uid except KeyError: raise utils.Error('%r is not a valid user name' % uid) else: raise utils.Error('uid is not supported on this platform') else: self.username = None self.uid = None if gid: if grp: try: self.gid = int(gid) grp.getgrgid(self.gid) except (KeyError, OverflowError): raise utils.Error('No such group: %r' % gid) except ValueError: try: self.gid = grp.getgrnam(gid).gr_gid except KeyError: raise utils.Error('No such group: %r' % gid) else: raise utils.Error('gid is not supported on this platform') elif self.uid: self.gid = pwd.getpwuid(self.uid).pw_gid else: self.gid = None # sockets created before fork, should be let go after. self._worker = None self._thread = None self._output = None self._auto_close = False