class Context(object): def __init__(self): self.server = None self.client = None self.jobs = {} self.provides = set() self.lock = RLock() self.kq = select.kqueue() self.devnull = os.open('/dev/null', os.O_RDWR) self.logger = logging.getLogger('Context') self.rpc = RpcContext() self.rpc.register_service_instance('serviced.management', ManagementService(self)) self.rpc.register_service_instance('serviced.job', JobService(self)) def init_dispatcher(self): if self.client and self.client.connected: return def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_server(self, address): self.server = Server(self) self.server.rpc = self.rpc self.server.streaming = True self.server.start(address, transport_options={'permissions': 0o777}) thread = Thread(target=self.server.serve_forever) thread.name = 'ServerThread' thread.daemon = True thread.start() def provide(self, targets): def doit(): self.logger.debug('Adding dependency targets: {0}'.format( ', '.join(targets))) with self.lock: self.provides |= targets for job in list(self.jobs.values()): if job.state == JobState.STOPPED and job.requires <= self.provides: job.start() if targets: Timer(2, doit).start() def job_by_pid(self, pid): job = first_or_default(lambda j: j.pid == pid, self.jobs.values()) return job def event_loop(self): while True: with contextlib.suppress(InterruptedError): for ev in self.kq.control(None, MAX_EVENTS): self.logger.log(TRACE, 'New event: {0}'.format(ev)) if ev.filter == select.KQ_FILTER_PROC: job = self.job_by_pid(ev.ident) if job: job.pid_event(ev) continue if ev.fflags & select.KQ_NOTE_CHILD: if ev.fflags & select.KQ_NOTE_EXIT: continue pjob = self.job_by_pid(ev.data) if not pjob: self.untrack_pid(ev.ident) continue # Stop tracking at session ID boundary try: if pjob.pgid != os.getpgid(ev.ident): self.untrack_pid(ev.ident) continue except ProcessLookupError: continue with self.lock: job = Job(self) job.load_anonymous(pjob, ev.ident) self.jobs[job.id] = job self.logger.info('Added job {0}'.format( job.label)) def track_pid(self, pid): ev = select.kevent( pid, select.KQ_FILTER_PROC, select.KQ_EV_ADD | select.KQ_EV_ENABLE, select.KQ_NOTE_EXIT | select.KQ_NOTE_EXEC | select.KQ_NOTE_FORK | select.KQ_NOTE_TRACK, 0, 0) self.kq.control([ev], 0) def untrack_pid(self, pid): ev = select.kevent(pid, select.KQ_FILTER_PROC, select.KQ_EV_DELETE, 0, 0, 0) with contextlib.suppress(FileNotFoundError): self.kq.control([ev], 0) def emit_event(self, name, args): self.server.broadcast_event(name, args) if self.client and self.client.connected: self.client.emit_event(name, args) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('serviced') self.client.enable_server(self.rpc) self.client.resume_service('serviced.job') self.client.resume_service('serviced.management') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def bootstrap(self): def doit(): with self.lock: job = Job(self) job.load({ 'Label': 'org.freenas.serviced.bootstrap', 'ProgramArguments': BOOTSTRAP_JOB, 'OneShot': True, 'RunAtLoad': True, }) self.jobs[job.id] = job Thread(target=doit).start() def shutdown(self): self.client.disconnect() self.server.close() sys.exit(0) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on') args = parser.parse_args() configure_logging('/var/log/serviced.log', 'DEBUG', file=True) bsd.setproctitle('serviced') self.logger.info('Started') self.init_server(args.s) self.bootstrap() self.event_loop()
class Context(object): def __init__(self): self.store = collections.deque() self.lock = threading.Lock() self.seqno = 0 self.rpc_server = Server(self) self.boot_id = str(uuid.uuid4()) self.exiting = False self.server = None self.servers = [] self.klog_reader = None self.flush = False self.flush_thread = None self.datastore = None self.started_at = datetime.utcnow() self.rpc = RpcContext() self.rpc.register_service_instance('logd.logging', LoggingService(self)) self.rpc.register_service_instance('logd.debug', DebugService()) self.cv = threading.Condition() def init_datastore(self): try: self.datastore = datastore.get_datastore(log=True) self.datastore.insert( 'boots', { 'id': self.boot_id, 'booted_at': self.started_at, 'hostname': socket.gethostname() }) except datastore.DatastoreException as err: logging.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) def init_rpc_server(self): self.server = Server(self) self.server.rpc = self.rpc self.rpc.streaming_enabled = True self.rpc.streaming_burst = 16 self.server.streaming = True self.server.start(DEFAULT_SOCKET_ADDRESS, transport_options={'permissions': 0o666}) thread = threading.Thread(target=self.server.serve_forever, name='RPC server thread', daemon=True) thread.start() def init_syslog_server(self): for path, perm in SYSLOG_SOCKETS.items(): server = SyslogServer(path, perm, self) server.start() self.servers.append(server) def init_klog(self): self.klog_reader = KernelLogReader(self) thread = threading.Thread(target=self.klog_reader.process, name='klog reader', daemon=True) thread.start() def init_flush(self): self.flush_thread = threading.Thread(target=self.do_flush, name='Flush thread') self.flush_thread.start() def push(self, item): if not item: return if 'message' not in item or 'priority' not in item: return if 'timestamp' not in item: item['timestamp'] = datetime.now() if 'pid' in item: try: job = get_job_by_pid(item['pid'], True) item['service'] = job['Label'] except ServicedException: pass with self.lock: priority, facility = parse_priority(item['priority']) item.update({ 'id': str(uuid.uuid4()), 'seqno': self.seqno, 'boot_id': self.boot_id, 'priority': priority.name, 'facility': facility.name if facility else None }) self.store.append(item) self.seqno += 1 self.server.broadcast_event('logd.logging.message', item) def do_flush(self): logging.debug('Flush thread initialized') while True: # Flush immediately after getting wakeup or when timeout expires with self.cv: self.cv.wait(FLUSH_INTERVAL) if not self.flush: continue if not self.datastore: try: self.init_datastore() logging.info('Datastore initialized') except BaseException as err: logging.warning( 'Cannot initialize datastore: {0}'.format(err)) logging.warning('Flush skipped') continue logging.debug('Attempting to flush logs') with self.lock: for i in self.store: self.datastore.insert('syslog', i) self.store.clear() if self.exiting: return def sigusr1(self, signo, frame): with self.cv: logging.info('Flushing logs on signal') self.flush = True self.cv.notify_all() def main(self): setproctitle('logd') self.init_syslog_server() self.init_klog() self.init_rpc_server() self.init_flush() checkin() signal.signal(signal.SIGUSR1, signal.SIG_DFL) while True: sig = signal.sigwait([signal.SIGTERM, signal.SIGUSR1]) if sig == signal.SIGUSR1: with self.cv: logging.info('Flushing logs on signal') self.flush = True self.cv.notify_all() continue if sig == signal.SIGTERM: logging.info('Got SIGTERM, exiting') with self.cv: self.exiting = True self.cv.notify_all() self.flush_thread.join() break
class Context(object): def __init__(self): self.server = None self.client = None self.jobs = {} self.provides = set() self.lock = RLock() self.kq = select.kqueue() self.devnull = os.open('/dev/null', os.O_RDWR) self.logger = logging.getLogger('Context') self.rpc = RpcContext() self.rpc.register_service_instance('serviced.management', ManagementService(self)) self.rpc.register_service_instance('serviced.job', JobService(self)) def init_dispatcher(self): if self.client and self.client.connected: return def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_server(self, address): self.server = Server(self) self.server.rpc = self.rpc self.server.streaming = True self.server.start(address, transport_options={'permissions': 0o777}) thread = Thread(target=self.server.serve_forever) thread.name = 'ServerThread' thread.daemon = True thread.start() def provide(self, targets): def doit(): self.logger.debug('Adding dependency targets: {0}'.format(', '.join(targets))) with self.lock: self.provides |= targets for job in list(self.jobs.values()): if job.state == JobState.STOPPED and job.requires <= self.provides: job.start() if targets: Timer(2, doit).start() def job_by_pid(self, pid): job = first_or_default(lambda j: j.pid == pid, self.jobs.values()) return job def event_loop(self): while True: with contextlib.suppress(InterruptedError): for ev in self.kq.control(None, MAX_EVENTS): self.logger.log(TRACE, 'New event: {0}'.format(ev)) if ev.filter == select.KQ_FILTER_PROC: job = self.job_by_pid(ev.ident) if job: job.pid_event(ev) continue if ev.fflags & select.KQ_NOTE_CHILD: if ev.fflags & select.KQ_NOTE_EXIT: continue pjob = self.job_by_pid(ev.data) if not pjob: self.untrack_pid(ev.ident) continue # Stop tracking at session ID boundary try: if pjob.pgid != os.getpgid(ev.ident): self.untrack_pid(ev.ident) continue except ProcessLookupError: continue with self.lock: job = Job(self) job.load_anonymous(pjob, ev.ident) self.jobs[job.id] = job self.logger.info('Added job {0}'.format(job.label)) def track_pid(self, pid): ev = select.kevent( pid, select.KQ_FILTER_PROC, select.KQ_EV_ADD | select.KQ_EV_ENABLE, select.KQ_NOTE_EXIT | select.KQ_NOTE_EXEC | select.KQ_NOTE_FORK | select.KQ_NOTE_TRACK, 0, 0 ) self.kq.control([ev], 0) def untrack_pid(self, pid): ev = select.kevent( pid, select.KQ_FILTER_PROC, select.KQ_EV_DELETE, 0, 0, 0 ) with contextlib.suppress(FileNotFoundError): self.kq.control([ev], 0) def emit_event(self, name, args): self.server.broadcast_event(name, args) if self.client and self.client.connected: self.client.emit_event(name, args) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('serviced') self.client.enable_server(self.rpc) self.client.resume_service('serviced.job') self.client.resume_service('serviced.management') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def bootstrap(self): def doit(): with self.lock: job = Job(self) job.load({ 'Label': 'org.freenas.serviced.bootstrap', 'ProgramArguments': BOOTSTRAP_JOB, 'OneShot': True, 'RunAtLoad': True, }) self.jobs[job.id] = job Thread(target=doit).start() def shutdown(self): self.client.disconnect() self.server.close() sys.exit(0) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on') args = parser.parse_args() configure_logging('/var/log/serviced.log', 'DEBUG', file=True) bsd.setproctitle('serviced') self.logger.info('Started') self.init_server(args.s) self.bootstrap() self.event_loop()
class Context(object): def __init__(self): self.store = collections.deque() self.lock = threading.Lock() self.seqno = 0 self.rpc_server = Server(self) self.boot_id = str(uuid.uuid4()) self.exiting = False self.server = None self.servers = [] self.forwarders = [] self.klog_reader = None self.flush = False self.flush_thread = None self.datastore = None self.configstore = None self.started_at = datetime.utcnow() self.rpc = RpcContext() self.rpc.register_service_instance('logd.logging', LoggingService(self)) self.rpc.register_service_instance('logd.debug', DebugService()) self.cv = threading.Condition() def init_configstore(self): ds = datastore.get_datastore() self.configstore = datastore.config.ConfigStore(ds) def init_datastore(self): try: self.datastore = datastore.get_datastore(log=True) self.datastore.insert('boots', { 'id': self.boot_id, 'booted_at': self.started_at, 'hostname': socket.gethostname() }) except datastore.DatastoreException as err: logging.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) def init_rpc_server(self): self.server = Server(self) self.server.rpc = self.rpc self.rpc.streaming_enabled = True self.rpc.streaming_burst = 16 self.server.streaming = True self.server.start(DEFAULT_SOCKET_ADDRESS, transport_options={'permissions': 0o666}) thread = threading.Thread(target=self.server.serve_forever, name='RPC server thread', daemon=True) thread.start() def init_syslog_server(self): for path, perm in SYSLOG_SOCKETS.items(): server = SyslogServer(path, perm, self) server.start() self.servers.append(server) def init_klog(self): self.klog_reader = KernelLogReader(self) thread = threading.Thread(target=self.klog_reader.process, name='klog reader', daemon=True) thread.start() def init_flush(self): self.flush_thread = threading.Thread(target=self.do_flush, name='Flush thread') self.flush_thread.start() def load_configuration(self): syslog_server = self.configstore.get('system.syslog_server') if not syslog_server: if self.forwarders: for i in self.forwarders: i.close() self.forwarders.clear() return if ':' in syslog_server: host, port = syslog_server.split(':') else: host = syslog_server port = 514 self.forwarders.append(SyslogForwarder(host, port, self)) def push(self, item): if not item: return if 'message' not in item or 'priority' not in item: return if 'timestamp' not in item: item['timestamp'] = datetime.now() if 'pid' in item: try: job = get_job_by_pid(item['pid'], True) item['service'] = job['Label'] except ServicedException: pass with self.lock: priority, facility = parse_priority(item['priority']) item.update({ 'id': str(uuid.uuid4()), 'seqno': self.seqno, 'boot_id': self.boot_id, 'priority': priority.name, 'facility': facility.name if facility else None }) self.store.append(item) self.seqno += 1 self.server.broadcast_event('logd.logging.message', item) self.forward(item) def forward(self, item): hostname = socket.gethostname() prio = SyslogPriority.INFO try: prio = int(getattr(SyslogPriority, item['priority'])) except: pass msg = f'<{prio}>{item["timestamp"]:%b %d %H:%M:%S} {hostname} {item["identifier"]}: {item["message"]}' for i in self.forwarders: i.forward(msg.encode('utf-8', 'ignore')) def do_flush(self): logging.debug('Flush thread initialized') while True: # Flush immediately after getting wakeup or when timeout expires with self.cv: self.cv.wait(FLUSH_INTERVAL) if not self.flush: continue if not self.datastore: try: self.init_datastore() logging.info('Datastore initialized') except BaseException as err: logging.warning('Cannot initialize datastore: {0}'.format(err)) logging.warning('Flush skipped') continue logging.debug('Attempting to flush logs') with self.lock: for i in self.store: self.datastore.insert('syslog', i) self.store.clear() if self.exiting: return def sigusr1(self, signo, frame): with self.cv: logging.info('Flushing logs on signal') self.flush = True self.cv.notify_all() def main(self): setproctitle('logd') self.init_configstore() self.init_syslog_server() self.init_klog() self.init_rpc_server() self.init_flush() self.load_configuration() checkin() signal.signal(signal.SIGUSR1, signal.SIG_DFL) signal.signal(signal.SIGHUP, signal.SIG_DFL) while True: sig = signal.sigwait([signal.SIGTERM, signal.SIGUSR1, signal.SIGHUP]) if sig == signal.SIGUSR1: with self.cv: logging.info('Flushing logs on signal') self.flush = True self.cv.notify_all() continue if sig == signal.SIGHUP: logging.info('Reloading configuration on SIGHUP') self.load_configuration() continue if sig == signal.SIGTERM: logging.info('Got SIGTERM, exiting') with self.cv: self.exiting = True self.cv.notify_all() self.flush_thread.join() break