def test_back_to_back(self): a, b = socket.socketpair() self.assertGreaterEqual(a.fileno(), 0) self.assertGreaterEqual(b.fileno(), 0) c1 = Client() c1.standalone_server = True c1.enable_server() c1.register_service('test', TestService()) c1.connect('fd://{0}'.format(a.fileno())) self.assertTrue(c1.connected) c2 = Client() c2.connect('fd://{0}'.format(b.fileno())) self.assertTrue(c2.connected) self.assertEqual(c2.call_sync('test.hello', 'freenas'), 'Hello World, freenas') c2.disconnect() a.close() c1.disconnect() b.close()
def test_back_to_back(self): a, b = socket.socketpair() self.assertGreaterEqual(a.fileno(), 0) self.assertGreaterEqual(b.fileno(), 0) c1 = Client() c1.standalone_server = True c1.enable_server() c1.register_service('test', TestService()) c1.connect('fd://{0}'.format(a.fileno())) self.assertTrue(c1.connected) c2 = Client() c2.connect('fd://{0}'.format(b.fileno())) self.assertTrue(c2.connected) self.assertEqual(c2.call_sync('test.hello', 'freenas'), 'Hello World, freenas') c2.disconnect() a.close() c1.disconnect() b.close()
def setup_back_to_back(self, streaming=False): a, b = socket.socketpair() self.assertGreaterEqual(a.fileno(), 0) self.assertGreaterEqual(b.fileno(), 0) c1 = Client() c1._s = a c1.enable_server() c1.standalone_server = True if streaming: c1.streaming = True c1.rpc.streaming_enabled = True c1.register_service('test', TestService()) c1.connect('fd://{0}'.format(a.fileno())) self.assertTrue(c1.connected) c2 = Client() c2._s = b c2.streaming = True c2.connect('fd://{0}'.format(b.fileno())) self.assertTrue(c2.connected) return c1, c2
class Main(object): def __init__(self): self.logger = logging.getLogger('dscached') self.config = None self.datastore = None self.configstore = None self.rpc = RpcContext() self.rpc.streaming_enabled = True self.rpc.streaming_burst = 16 self.client = None self.server = None self.plugin_dirs = [] self.plugins = {} self.directories = [] self.users_cache = TTLCacheStore() self.groups_cache = TTLCacheStore() self.hosts_cache = TTLCacheStore() self.cache_ttl = 7200 self.search_order = [] self.cache_enumerations = True self.cache_lookups = True self.rpc.register_service_instance('dscached.account', AccountService(self)) self.rpc.register_service_instance('dscached.group', GroupService(self)) self.rpc.register_service_instance('dscached.host', HostService(self)) self.rpc.register_service_instance('dscached.management', ManagementService(self)) self.rpc.register_service_instance('dscached.debug', DebugService()) def get_enabled_directories(self): return list(filter(None, (self.get_directory_by_name(n) for n in self.get_search_order()))) def get_search_order(self): return ['local', 'system'] + self.search_order def get_directory_by_domain(self, domain_name): return first_or_default(lambda d: d.domain_name == domain_name, self.directories) def get_directory_by_name(self, name): return first_or_default(lambda d: d.name == name, self.directories) def get_directory_for_id(self, uid=None, gid=None): if uid is not None: if uid == 0: # Special case for root user return first_or_default(lambda d: d.plugin_type == 'local', self.directories) return first_or_default( lambda d: d.max_uid and d.max_uid >= uid >= d.min_uid, self.directories ) if gid is not None: if gid == 0: # Special case for wheel group return first_or_default(lambda d: d.plugin_type == 'local', self.directories) return first_or_default( lambda d: d.max_gid and d.max_gid >= gid >= d.min_gid, self.directories ) def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_server(self, address): self.server = Server(self) self.server.rpc = self.rpc self.server.start(address) thread = Thread(target=self.server.serve_forever) thread.name = 'ServerThread' thread.daemon = True thread.start() def parse_config(self, filename): try: with open(filename, 'r') as f: self.config = json.load(f) except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error('Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['dscached']['plugin-dirs'] def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('dscached') self.client.enable_server(self.rpc) self.client.resume_service('dscached.account') self.client.resume_service('dscached.group') self.client.resume_service('dscached.host') self.client.resume_service('dscached.management') self.client.resume_service('dscached.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = imp.load_source(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def register_plugin(self, name, cls): self.plugins[name] = cls self.logger.info('Registered plugin {0} (class {1})'.format(name, cls)) def register_schema(self, name, schema): self.client.register_schema(name, schema) def init_directories(self): for i in self.datastore.query('directories'): try: directory = Directory(self, i) directory.configure() self.directories.append(directory) except BaseException as err: continue def load_config(self): self.search_order = self.configstore.get('directory.search_order') self.cache_ttl = self.configstore.get('directory.cache_ttl') self.cache_enumerations = self.configstore.get('directory.cache_enumerations') self.cache_lookups = self.configstore.get('directory.cache_lookups') def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on') args = parser.parse_args() configure_logging('/var/log/dscached.log', 'DEBUG') setproctitle.setproctitle('dscached') self.config = args.c self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.load_config() self.init_server(args.s) self.scan_plugins() self.init_directories() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('etcd') self.root = None self.configfile = None self.config = None self.datastore = None self.configstore = None self.client = None self.plugin_dirs = [] self.renderers = {} self.managed_files = {} def init_datastore(self): try: self.datastore = datastore.get_datastore(self.configfile) except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('etcd') self.client.enable_server() self.client.register_service('etcd.generation', FileGenerationService(self)) self.client.register_service('etcd.management', ManagementService(self)) self.client.register_service('etcd.debug', DebugService()) self.client.resume_service('etcd.generation') self.client.resume_service('etcd.management') self.client.resume_service('etcd.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def init_renderers(self): for name, impl in TEMPLATE_RENDERERS.items(): self.renderers[name] = impl(self) def parse_config(self, filename): try: f = open(filename, 'r') self.config = json.load(f) f.close() except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error('Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['etcd']['plugin-dirs'] def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for root, dirs, files in os.walk(dir): for name in files: abspath = os.path.join(root, name) path = os.path.relpath(abspath, dir) name, ext = os.path.splitext(path) if name in self.managed_files.keys(): continue if ext in TEMPLATE_RENDERERS.keys(): self.managed_files[name] = abspath self.logger.info('Adding managed file %s [%s]', name, ext) def generate_file(self, file_path): if file_path not in self.managed_files.keys(): raise RpcException(errno.ENOENT, 'No such file') template_path = self.managed_files[file_path] name, ext = os.path.splitext(template_path) if ext not in self.renderers.keys(): raise RuntimeError("Can't find renderer for {0}".format(file_path)) renderer = self.renderers[ext] try: return renderer.render_template(template_path) except Exception as e: self.logger.warn('Cannot generate file {0}: {1}'.format(file_path, str(e))) return "# FILE GENERATION FAILED: {0}\n".format(str(e)) def emit_event(self, name, params): self.client.emit_event(name, params) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-f', action='store_true', default=False, help='Run in foreground') parser.add_argument('mountpoint', metavar='MOUNTPOINT', default='/etc', help='/etc mount point') args = parser.parse_args() configure_logging('/var/log/etcd.log', 'DEBUG') setproctitle.setproctitle('etcd') self.root = args.mountpoint self.configfile = args.c self.parse_config(args.c) self.scan_plugins() self.init_renderers() self.init_datastore() self.init_dispatcher() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('alertd') self.config = None self.datastore = None self.configstore = None self.client = None self.plugin_dirs = [] self.emitters = {} def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_reminder(self): t = threading.Thread(target=self.reminder_thread) t.daemon = True t.start() def parse_config(self, filename): try: f = open(filename, 'r') self.config = json.load(f) f.close() except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error('Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['alertd']['plugin-dirs'] def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('alertd') self.client.enable_server() self.client.register_service('alertd.management', ManagementService(self)) self.client.register_service('alertd.alert', AlertService(self)) self.client.register_service('alertd.debug', DebugService()) self.client.resume_service('alertd.management') self.client.resume_service('alertd.alert') self.client.resume_service('alertd.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = imp.load_source(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def emit_alert(self, alert): self.logger.debug('Emitting alert <id:{0}> (class {1})'.format(alert['id'], alert['class'])) for i in self.datastore.query('alert.filters'): for predicate in i.get('predicates', []): if predicate['operator'] not in operators_table: continue if not operators_table[predicate['operator']](alert[predicate['property']], predicate['value']): break else: try: emitter = self.emitters.get(i['emitter']) if not emitter: self.logger.warning('Invalid emitter {0} for alert filter {1}'.format(i['emitter'], i['id'])) continue self.logger.debug('Alert <id:{0}> matched filter {1}'.format(alert['id'], i['id'])) if alert['send_count'] > 0: emitter.emit_again(alert, i['parameters']) else: emitter.emit_first(alert, i['parameters']) except BaseException as err: # Failed to emit alert using alert emitter # XXX: generate another alert about that self.logger.error('Cannot emit alert <id:{0}> using {1}: {2}'.format( alert['id'], i['emitter'], str(err)) ) alert['send_count'] += 1 alert['last_emitted_at'] = datetime.utcnow() if alert['one_shot']: alert['active'] = False self.datastore.update('alerts', alert['id'], alert) def cancel_alert(self, alert): self.logger.debug('Cancelling alert <id:{0}> (class {1})'.format(alert['id'], alert['class'])) alert.update({ 'active': False, 'cancelled': datetime.utcnow() }) self.datastore.update('alerts', alert['id'], alert) def register_emitter(self, name, cls): self.emitters[name] = cls(self) self.logger.info('Registered emitter {0} (class {1})'.format(name, cls)) def reminder_thread(self): while True: time.sleep(REMINDER_SECONDS) for i in self.datastore.query('alerts'): if not i['active'] or i['dismissed']: continue last_emission = i.get('last_emitted_at') or i['created_at'] interval = REMINDER_SCHEDULE[i['severity']] if not interval: continue if last_emission + timedelta(seconds=interval) <= datetime.utcnow(): self.emit_alert(i) def checkin(self): checkin() def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() configure_logging('/var/log/alertd.log', 'DEBUG') setproctitle('alertd') self.config = args.c self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.scan_plugins() self.init_reminder() self.checkin() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('clid') self.config = None self.datastore = None self.configstore = None self.client = None self.config = None self.logger = logging.getLogger() self.plugin_dirs = [] self.ml = None self.context = None def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_cli(self): self.logger.info('Initializing CLI instance') self.context = Context() self.context.connection = self.client self.context.plugin_dirs = PLUGIN_DIRS self.context.discover_plugins() self.context.start_entity_subscribers() self.context.login_plugins() self.ml = MainLoop(self.context) self.logger.info('CLI instance ready') def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('clid') self.client.enable_server() self.client.call_sync('management.enable_features', ['streaming_responses']) self.client.register_service('clid.management', ManagementService(self)) self.client.register_service('clid.eval', EvalService(self)) self.client.register_service('clid.debug', DebugService()) self.client.resume_service('clid.management') self.client.resume_service('clid.eval') self.client.resume_service('clid.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() self.config = args.c configure_logging('/var/log/clid.log', 'DEBUG') setproctitle('clid') self.init_dispatcher() self.init_cli() self.client.wait_forever()
class Context(object): def __init__(self): self.logger = logging.getLogger('schedulerd') self.config = None self.datastore = None self.configstore = None self.client = None self.scheduler = None self.active_tasks = {} def init_datastore(self): try: self.datastore = get_datastore(self.config) except DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_scheduler(self): store = MongoDBJobStore(database='freenas', collection='calendar_tasks', client=self.datastore.client) self.scheduler = BackgroundScheduler(jobstores={'default': store}, timezone=pytz.utc) self.scheduler.start() def register_schemas(self): self.client.register_schema('calendar-task', { 'type': 'object', 'additionalProperties': False, 'properties': { 'id': {'type': 'string'}, 'name': {'type': 'string'}, 'args': {'type': 'array'}, 'description': {'type': 'string'}, 'enabled': {'type': 'boolean'}, 'hidden': {'type': 'boolean'}, 'protected': {'type': 'boolean'}, 'status': {'$ref': 'calendar-task-status'}, 'schedule': { 'type': 'object', 'additionalProperties': False, 'properties': { 'coalesce': {'type': ['boolean', 'integer', 'null']}, 'year': {'type': ['string', 'integer', 'null']}, 'month': {'type': ['string', 'integer', 'null']}, 'day': {'type': ['string', 'integer', 'null']}, 'week': {'type': ['string', 'integer', 'null']}, 'day_of_week': {'type': ['string', 'integer', 'null']}, 'hour': {'type': ['string', 'integer', 'null']}, 'minute': {'type': ['string', 'integer', 'null']}, 'second': {'type': ['string', 'integer', 'null']}, 'timezone': {'type': ['string', 'null']} } } } }) self.client.register_schema('calendar-task-status', { 'type': 'object', 'properties': { 'next_run_time': {'type': 'string'}, 'last_run_status': {'type': 'string'}, 'current_run_status': {'type': ['string', 'null']}, 'current_run_progress': {'type': ['object', 'null']} } }) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('schedulerd') self.client.enable_server() self.client.register_service('scheduler.management', ManagementService(self)) self.client.register_service('scheduler.debug', DebugService()) self.client.resume_service('scheduler.management') self.client.resume_service('scheduler.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def run_job(self, *args, **kwargs): tid = self.client.submit_task(*args) self.active_tasks[kwargs['id']] = tid self.client.call_sync('task.wait', tid, timeout=None) result = self.client.call_sync('task.status', tid) if result['state'] != 'FINISHED': try: self.client.call_sync('alerts.emit', { 'name': 'scheduler.task.failed', 'severity': 'CRITICAL', 'description': 'Task {0} has failed: {1}'.format(kwargs['name'], result['error']['message']), }) except RpcException as e: self.logger.error('Failed to emit alert', exc_info=True) del self.active_tasks[kwargs['id']] self.datastore.insert('schedulerd.runs', { 'job_id': kwargs['id'], 'task_id': result['id'] }) def emit_event(self, name, params): self.client.emit_event(name, params) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-f', action='store_true', default=False, help='Run in foreground') args = parser.parse_args() configure_logging('/var/log/schedulerd.log', 'DEBUG') setproctitle.setproctitle('schedulerd') self.config = args.c self.init_datastore() self.init_scheduler() self.init_dispatcher() self.register_schemas() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('neighbord') self.config = None self.datastore = None self.configstore = None self.client = None self.config = None self.logger = logging.getLogger() self.plugin_dirs = [] self.plugins = {} def parse_config(self, filename): try: with open(filename, 'r') as f: self.config = json.load(f) except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error( 'Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['neighbord']['plugin-dirs'] def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = load_module_from_file(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def register_plugin(self, name, cls): self.plugins[name] = cls(self) self.logger.info('Registered plugin {0} (class {1})'.format(name, cls)) def register_service(self, name, regtype, port, properties=None): for plugin in self.plugins.values(): plugin.register(regtype, name, port, properties) def register(self): try: hostname = socket.gethostname() general = self.client.call_sync('system.general.get_config') properties = { 'version': self.client.call_sync('system.info.version'), 'description': general['description'], 'tags': ','.join(general['tags']) } self.register_service(hostname, 'freenas', 80, properties) self.register_service(hostname, 'http', 80) self.register_service(hostname, 'ssh', 22) self.register_service(hostname, 'sftp-ssh', 22) except BaseException as err: self.logger.error('Failed to register services: {0}'.format( str(err))) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('neighbord') self.client.enable_server() self.client.register_service('neighbord.management', ManagementService(self)) self.client.register_service('neighbord.discovery', DiscoveryService(self)) self.client.register_service('neighbord.debug', DebugService()) self.client.resume_service('neighbord.management') self.client.resume_service('neighbord.discovery') self.client.resume_service('neighbord.debug') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() self.config = args.c configure_logging('/var/log/neighbord.log', 'DEBUG') setproctitle.setproctitle('neighbord') self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.scan_plugins() self.register() self.client.wait_forever()
class Context(object): def __init__(self): self.service = TaskProxyService(self) self.task = queue.Queue(1) self.datastore = None self.configstore = None self.conn = None self.instance = None self.running = Event() def put_status(self, state, result=None, exception=None): obj = { 'status': state, 'result': None } if result is not None: obj['result'] = result if exception is not None: obj['error'] = serialize_error(exception) self.conn.call_sync('task.put_status', obj) def main(self): if len(sys.argv) != 2: print("Invalid number of arguments", file=sys.stderr) sys.exit(errno.EINVAL) key = sys.argv[1] logging.basicConfig(level=logging.DEBUG) self.datastore = get_datastore() self.configstore = ConfigStore(self.datastore) self.conn = Client() self.conn.connect('unix:') self.conn.login_service('task.{0}'.format(os.getpid())) self.conn.enable_server() self.conn.rpc.register_service_instance('taskproxy', self.service) self.conn.call_sync('task.checkin', key) setproctitle.setproctitle('task executor (idle)') while True: try: task = self.task.get() setproctitle.setproctitle('task executor (tid {0})'.format(task['id'])) if task['debugger']: sys.path.append('/usr/local/lib/dispatcher/pydev') import pydevd host, port = task['debugger'] pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True) name, _ = os.path.splitext(os.path.basename(task['filename'])) module = load_module_from_file(name, task['filename']) setproctitle.setproctitle('task executor (tid {0})'.format(task['id'])) try: self.instance = getattr(module, task['class'])(DispatcherWrapper(self.conn), self.datastore) self.instance.configstore = self.configstore self.instance.environment = task['environment'] self.running.set() result = self.instance.run(*task['args']) except BaseException as err: print("Task exception: {0}".format(str(err)), file=sys.stderr) traceback.print_exc(file=sys.stderr) if hasattr(self.instance, 'rollback'): self.put_status('ROLLBACK') try: self.instance.rollback(*task['args']) except BaseException as rerr: print("Task exception during rollback: {0}".format(str(rerr)), file=sys.stderr) traceback.print_exc(file=sys.stderr) self.put_status('FAILED', exception=err) else: self.put_status('FINISHED', result=result) finally: self.running.clear() except RpcException as err: print("RPC failed: {0}".format(str(err)), file=sys.stderr) sys.exit(errno.EBADMSG) except socket.error as err: print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr) sys.exit(errno.ETIMEDOUT) if task['debugger']: import pydevd pydevd.stoptrace() setproctitle.setproctitle('task executor (idle)')
class Main(object): def __init__(self): self.logger = logging.getLogger('dscached') self.config = None self.datastore = None self.configstore = None self.client = None self.plugin_dirs = [] self.plugins = {} def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def parse_config(self, filename): try: with open(filename, 'r') as f: self.config = json.load(f) except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error('Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['dscached']['plugin-dirs'] def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('dscached') self.client.enable_server() self.client.register_service('dscached.account', AccountService(self)) self.client.register_service('dscached.group', GroupService(self)) self.client.register_service('dscached.debug', DebugService()) self.client.resume_service('dscached.account') self.client.resume_service('dscached.group') self.client.resume_service('dscached.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = imp.load_source(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def register_plugin(self, name, cls): self.plugins[name] = cls(self) self.logger.info('Registered plugin {0} (class {1})'.format(name, cls)) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() configure_logging('/var/log/dscached.log', 'DEBUG') setproctitle.setproctitle('dscached') self.config = args.c self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.scan_plugins() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('neighbord') self.config = None self.datastore = None self.configstore = None self.client = None self.config = None self.logger = logging.getLogger() self.plugin_dirs = [] self.plugins = {} def parse_config(self, filename): try: with open(filename, 'r') as f: self.config = json.load(f) except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error('Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['neighbord']['plugin-dirs'] def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = load_module_from_file(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def register_plugin(self, name, cls): self.plugins[name] = cls(self) self.logger.info('Registered plugin {0} (class {1})'.format(name, cls)) def register_service(self, name, regtype, port, properties=None): for plugin in self.plugins.values(): plugin.register(regtype, name, port, properties) def register(self): try: hostname = socket.gethostname() general = self.client.call_sync('system.general.get_config') properties = { 'version': self.client.call_sync('system.info.version'), 'description': general['description'], 'tags': ','.join(general['tags']) } self.register_service(hostname, 'freenas', 80, properties) self.register_service(hostname, 'http', 80) self.register_service(hostname, 'ssh', 22) self.register_service(hostname, 'sftp-ssh', 22) except BaseException as err: self.logger.error('Failed to register services: {0}'.format(str(err))) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('neighbord') self.client.enable_server() self.client.register_service('neighbord.management', ManagementService(self)) self.client.register_service('neighbord.discovery', DiscoveryService(self)) self.client.register_service('neighbord.debug', DebugService()) self.client.resume_service('neighbord.management') self.client.resume_service('neighbord.discovery') self.client.resume_service('neighbord.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def checkin(self): checkin() def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() self.config = args.c configure_logging('/var/log/neighbord.log', 'DEBUG') setproctitle('neighbord') self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.scan_plugins() self.register() self.checkin() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('alertd') self.config = None self.datastore = None self.configstore = None self.client = None self.plugin_dirs = [] self.emitters = {} def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_reminder(self): t = threading.Thread(target=self.reminder_thread) t.daemon = True t.start() def parse_config(self, filename): try: f = open(filename, 'r') self.config = json.load(f) f.close() except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error( 'Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['alertd']['plugin-dirs'] def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('alertd') self.client.enable_server() self.client.register_service('alertd.management', ManagementService(self)) self.client.register_service('alertd.alert', AlertService(self)) self.client.register_service('alertd.debug', DebugService()) self.client.resume_service('alertd.management') self.client.resume_service('alertd.alert') self.client.resume_service('alertd.debug') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = imp.load_source(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def emit_alert(self, alert): self.logger.debug('Emitting alert <id:{0}> (class {1})'.format( alert['id'], alert['clazz'])) for i in self.datastore.query( 'alert.filters', ('or', [('clazz', '=', None), ('clazz', '=', alert['clazz'])])): for pr in i.get('predicates', []): if pr['operator'] not in operators_table: continue try: if not operators_table[pr['operator']]( alert.properties.get(pr['property']), pr['value']): break except: continue else: try: emitter = self.emitters.get(i['emitter']) if not emitter: self.logger.warning( 'Invalid emitter {0} for alert filter {1}'.format( i['emitter'], i['id'])) continue self.logger.debug( 'Alert <id:{0}> matched filter {1}'.format( alert['id'], i['id'])) if alert['send_count'] > 0: if not alert['one_shot']: emitter.emit_again(alert, i['parameters']) else: emitter.emit_first(alert, i['parameters']) except BaseException as err: # Failed to emit alert using alert emitter # XXX: generate another alert about that self.logger.error( 'Cannot emit alert <id:{0}> using {1}: {2}'.format( alert['id'], i['emitter'], str(err))) alert['send_count'] += 1 alert['last_emitted_at'] = datetime.utcnow() self.datastore.update('alerts', alert['id'], alert) def cancel_alert(self, alert): self.logger.debug('Cancelling alert <id:{0}> (class {1})'.format( alert['id'], alert['clazz'])) alert.update({'active': False, 'cancelled': datetime.utcnow()}) self.datastore.update('alerts', alert['id'], alert) def register_emitter(self, name, cls): self.emitters[name] = cls(self) self.logger.info('Registered emitter {0} (class {1})'.format( name, cls)) def reminder_thread(self): while True: time.sleep(REMINDER_SECONDS) for i in self.datastore.query('alerts'): if not i['active'] or i['dismissed']: continue last_emission = i.get('last_emitted_at') or i['created_at'] interval = REMINDER_SCHEDULE[i['severity']] if not interval: continue if last_emission + timedelta( seconds=interval) <= datetime.utcnow(): self.emit_alert(i) def checkin(self): checkin() def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() configure_logging('alertd', 'DEBUG') setproctitle('alertd') self.config = args.c self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.scan_plugins() self.init_reminder() self.checkin() self.client.wait_forever()
class Context(object): def __init__(self): self.logger = logging.getLogger('schedulerd') self.config = None self.datastore = None self.configstore = None self.client = None self.scheduler = None self.active_tasks = {} def init_datastore(self): try: self.datastore = get_datastore(self.config) except DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_scheduler(self): store = FreeNASJobStore() self.scheduler = BackgroundScheduler(jobstores={'default': store, 'temp': MemoryJobStore()}, timezone=pytz.utc) self.scheduler.start() def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('schedulerd') self.client.enable_server() self.client.register_service('scheduler.management', ManagementService(self)) self.client.register_service('scheduler.debug', DebugService()) self.client.resume_service('scheduler.management') self.client.resume_service('scheduler.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def run_job(self, *args, **kwargs): tid = self.client.call_sync('task.submit_with_env', args[0], args[1:], { 'RUN_AS_USER': '******', 'CALENDAR_TASK_NAME': kwargs.get('name') }) self.active_tasks[kwargs['id']] = tid self.client.call_sync('task.wait', tid, timeout=None) result = self.client.call_sync('task.status', tid) if result['state'] != 'FINISHED': try: self.client.call_sync('alert.emit', { 'name': 'scheduler.task.failed', 'severity': 'CRITICAL', 'description': 'Task {0} has failed: {1}'.format( kwargs.get('name', tid), result['error']['message'] ), }) except RpcException as e: self.logger.error('Failed to emit alert', exc_info=True) del self.active_tasks[kwargs['id']] self.datastore.insert('schedulerd.runs', { 'job_id': kwargs['id'], 'task_id': result['id'] }) def emit_event(self, name, params): self.client.emit_event(name, params) def checkin(self): checkin() def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-f', action='store_true', default=False, help='Run in foreground') args = parser.parse_args() configure_logging('/var/log/schedulerd.log', 'DEBUG') setproctitle('schedulerd') self.config = args.c self.init_datastore() self.init_scheduler() self.init_dispatcher() self.checkin() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('clid') self.config = None self.datastore = None self.configstore = None self.client = None self.config = None self.logger = logging.getLogger() self.plugin_dirs = [] self.ml = None self.context = None def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_cli(self): self.logger.info('Initializing CLI instance') self.context = Context() self.context.connection = self.client self.context.plugin_dirs = PLUGIN_DIRS self.context.discover_plugins() self.context.start_entity_subscribers() self.context.login_plugins() self.ml = MainLoop(self.context) self.logger.info('CLI instance ready') def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('clid') self.client.enable_server() self.client.call_sync('management.enable_features', ['streaming_responses']) self.client.register_service('clid.management', ManagementService(self)) self.client.register_service('clid.eval', EvalService(self)) self.client.register_service('clid.debug', DebugService()) self.client.resume_service('clid.management') self.client.resume_service('clid.eval') self.client.resume_service('clid.debug') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') args = parser.parse_args() self.config = args.c configure_logging('clid', 'DEBUG') setproctitle('clid') self.init_dispatcher() self.init_cli() self.client.wait_forever()
class Context(object): def __init__(self): self.service = TaskProxyService(self) self.task = queue.Queue(1) self.datastore = None self.configstore = None self.conn = None self.instance = None self.running = Event() def put_status(self, state, result=None, exception=None): obj = { 'status': state, 'result': None } if result is not None: obj['result'] = result if exception is not None: obj['error'] = serialize_error(exception) self.conn.call_sync('task.put_status', obj) def task_progress_handler(self, args): if self.instance: self.instance.task_progress_handler(args) def collect_fds(self, obj): if isinstance(obj, dict): for v in obj.values(): if isinstance(v, FileDescriptor): yield v else: yield from self.collect_fds(v) if isinstance(obj, (list, tuple)): for o in obj: if isinstance(o, FileDescriptor): yield o else: yield from self.collect_fds(o) def close_fds(self, fds): for i in fds: try: os.close(i.fd) except OSError: pass def main(self): if len(sys.argv) != 2: print("Invalid number of arguments", file=sys.stderr) sys.exit(errno.EINVAL) key = sys.argv[1] configure_logging(None, logging.DEBUG) self.datastore = get_datastore() self.configstore = ConfigStore(self.datastore) self.conn = Client() self.conn.connect('unix:') self.conn.login_service('task.{0}'.format(os.getpid())) self.conn.enable_server() self.conn.rpc.register_service_instance('taskproxy', self.service) self.conn.register_event_handler('task.progress', self.task_progress_handler) self.conn.call_sync('task.checkin', key) setproctitle.setproctitle('task executor (idle)') while True: try: task = self.task.get() logging.root.setLevel(self.conn.call_sync('management.get_logging_level')) setproctitle.setproctitle('task executor (tid {0})'.format(task['id'])) if task['debugger']: sys.path.append('/usr/local/lib/dispatcher/pydev') import pydevd host, port = task['debugger'] pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True) name, _ = os.path.splitext(os.path.basename(task['filename'])) module = load_module_from_file(name, task['filename']) setproctitle.setproctitle('task executor (tid {0})'.format(task['id'])) fds = list(self.collect_fds(task['args'])) try: self.instance = getattr(module, task['class'])(DispatcherWrapper(self.conn), self.datastore) self.instance.configstore = self.configstore self.instance.user = task['user'] self.instance.environment = task['environment'] self.running.set() result = self.instance.run(*task['args']) except BaseException as err: print("Task exception: {0}".format(str(err)), file=sys.stderr) traceback.print_exc(file=sys.stderr) if hasattr(self.instance, 'rollback'): self.put_status('ROLLBACK') try: self.instance.rollback(*task['args']) except BaseException as rerr: print("Task exception during rollback: {0}".format(str(rerr)), file=sys.stderr) traceback.print_exc(file=sys.stderr) self.put_status('FAILED', exception=err) else: self.put_status('FINISHED', result=result) finally: self.close_fds(fds) self.running.clear() except RpcException as err: print("RPC failed: {0}".format(str(err)), file=sys.stderr) print(traceback.format_exc(), flush=True) sys.exit(errno.EBADMSG) except socket.error as err: print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr) sys.exit(errno.ETIMEDOUT) if task['debugger']: import pydevd pydevd.stoptrace() setproctitle.setproctitle('task executor (idle)')
class Context(object): def __init__(self): self.server = None self.client = None self.jobs = {} self.provides = set() self.lock = RLock() self.kq = select.kqueue() self.devnull = os.open('/dev/null', os.O_RDWR) self.logger = logging.getLogger('Context') self.rpc = RpcContext() self.rpc.register_service_instance('serviced.management', ManagementService(self)) self.rpc.register_service_instance('serviced.job', JobService(self)) def init_dispatcher(self): if self.client and self.client.connected: return def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_server(self, address): self.server = Server(self) self.server.rpc = self.rpc self.server.streaming = True self.server.start(address, transport_options={'permissions': 0o777}) thread = Thread(target=self.server.serve_forever) thread.name = 'ServerThread' thread.daemon = True thread.start() def provide(self, targets): def doit(): self.logger.debug('Adding dependency targets: {0}'.format( ', '.join(targets))) with self.lock: self.provides |= targets for job in list(self.jobs.values()): if job.state == JobState.STOPPED and job.requires <= self.provides: job.start() if targets: Timer(2, doit).start() def job_by_pid(self, pid): job = first_or_default(lambda j: j.pid == pid, self.jobs.values()) return job def event_loop(self): while True: with contextlib.suppress(InterruptedError): for ev in self.kq.control(None, MAX_EVENTS): self.logger.log(TRACE, 'New event: {0}'.format(ev)) if ev.filter == select.KQ_FILTER_PROC: job = self.job_by_pid(ev.ident) if job: job.pid_event(ev) continue if ev.fflags & select.KQ_NOTE_CHILD: if ev.fflags & select.KQ_NOTE_EXIT: continue pjob = self.job_by_pid(ev.data) if not pjob: self.untrack_pid(ev.ident) continue # Stop tracking at session ID boundary try: if pjob.pgid != os.getpgid(ev.ident): self.untrack_pid(ev.ident) continue except ProcessLookupError: continue with self.lock: job = Job(self) job.load_anonymous(pjob, ev.ident) self.jobs[job.id] = job self.logger.info('Added job {0}'.format( job.label)) def track_pid(self, pid): ev = select.kevent( pid, select.KQ_FILTER_PROC, select.KQ_EV_ADD | select.KQ_EV_ENABLE, select.KQ_NOTE_EXIT | select.KQ_NOTE_EXEC | select.KQ_NOTE_FORK | select.KQ_NOTE_TRACK, 0, 0) self.kq.control([ev], 0) def untrack_pid(self, pid): ev = select.kevent(pid, select.KQ_FILTER_PROC, select.KQ_EV_DELETE, 0, 0, 0) with contextlib.suppress(FileNotFoundError): self.kq.control([ev], 0) def emit_event(self, name, args): self.server.broadcast_event(name, args) if self.client and self.client.connected: self.client.emit_event(name, args) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('serviced') self.client.enable_server(self.rpc) self.client.resume_service('serviced.job') self.client.resume_service('serviced.management') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def bootstrap(self): def doit(): with self.lock: job = Job(self) job.load({ 'Label': 'org.freenas.serviced.bootstrap', 'ProgramArguments': BOOTSTRAP_JOB, 'OneShot': True, 'RunAtLoad': True, }) self.jobs[job.id] = job Thread(target=doit).start() def shutdown(self): self.client.disconnect() self.server.close() sys.exit(0) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on') args = parser.parse_args() configure_logging('/var/log/serviced.log', 'DEBUG', file=True) bsd.setproctitle('serviced') self.logger.info('Started') self.init_server(args.s) self.bootstrap() self.event_loop()
class Context(object): def __init__(self): self.logger = logging.getLogger('schedulerd') self.config = None self.datastore = None self.configstore = None self.client = None self.scheduler = None self.active_tasks = {} def init_datastore(self): try: self.datastore = get_datastore(self.config) except DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_scheduler(self): store = MongoDBJobStore(database='freenas', collection='calendar_tasks', client=self.datastore.client) self.scheduler = BackgroundScheduler(jobstores={'default': store}, timezone=pytz.utc) self.scheduler.start() def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('schedulerd') self.client.enable_server() self.client.register_service('scheduler.management', ManagementService(self)) self.client.register_service('scheduler.debug', DebugService()) self.client.resume_service('scheduler.management') self.client.resume_service('scheduler.debug') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def run_job(self, *args, **kwargs): tid = self.client.call_sync( 'task.submit_with_env', args[0], args[1:], { 'RUN_AS_USER': '******', 'CALENDAR_TASK_NAME': kwargs.get('name') }) self.active_tasks[kwargs['id']] = tid self.client.call_sync('task.wait', tid, timeout=None) result = self.client.call_sync('task.status', tid) if result['state'] != 'FINISHED': try: self.client.call_sync( 'alert.emit', { 'name': 'scheduler.task.failed', 'severity': 'CRITICAL', 'description': 'Task {0} has failed: {1}'.format( kwargs.get('name', tid), result['error']['message']), }) except RpcException as e: self.logger.error('Failed to emit alert', exc_info=True) del self.active_tasks[kwargs['id']] self.datastore.insert('schedulerd.runs', { 'job_id': kwargs['id'], 'task_id': result['id'] }) def emit_event(self, name, params): self.client.emit_event(name, params) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-f', action='store_true', default=False, help='Run in foreground') args = parser.parse_args() configure_logging('/var/log/schedulerd.log', 'DEBUG') setproctitle.setproctitle('schedulerd') self.config = args.c self.init_datastore() self.init_scheduler() self.init_dispatcher() self.client.wait_forever()
class Context(object): def __init__(self): self.logger = logging.getLogger(self.__class__.__name__) self.msock = msock.client.Client() self.msock.on_closed = self.on_msock_close self.rpc_fd = -1 self.connection_id = None self.jobs = [] self.state = ConnectionState.OFFLINE self.config = None self.keepalive = None self.connected_at = None self.cv = Condition() self.rpc = RpcContext() self.client = Client() self.server = Server() self.middleware_endpoint = None def start(self, configpath, sockpath): signal.signal(signal.SIGUSR2, lambda signo, frame: self.connect()) self.read_config(configpath) self.server.rpc = RpcContext() self.server.rpc.register_service_instance('control', ControlService(self)) self.server.start(sockpath) threading.Thread(target=self.server.serve_forever, name='server thread', daemon=True).start() def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect_dispatcher() self.middleware_endpoint = Client() self.middleware_endpoint.on_error(on_error) self.connect_dispatcher() def connect_dispatcher(self): while True: try: self.middleware_endpoint.connect('unix:') self.middleware_endpoint.login_service('debugd') self.middleware_endpoint.enable_server() self.middleware_endpoint.register_service( 'debugd.management', ControlService(self)) self.middleware_endpoint.resume_service('debugd.management') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def read_config(self, path): try: with open(path) as f: self.config = json.load(f) except (IOError, OSError, ValueError) as err: self.logger.fatal('Cannot open config file: {0}'.format(str(err))) self.logger.fatal('Exiting.') sys.exit(1) def connect(self, discard=False): if discard: self.connection_id = None self.keepalive = threading.Thread(target=self.connect_keepalive, daemon=True) self.keepalive.start() def connect_keepalive(self): while True: try: if not self.connection_id: self.connection_id = uuid.uuid4() self.msock.connect(SUPPORT_PROXY_ADDRESS) self.logger.info( 'Connecting to {0}'.format(SUPPORT_PROXY_ADDRESS)) self.rpc_fd = self.msock.create_channel(0) time.sleep(1) # FIXME self.client = Client() self.client.connect('fd://', fobj=self.rpc_fd) self.client.channel_serializer = MSockChannelSerializer( self.msock) self.client.standalone_server = True self.client.enable_server() self.client.register_service('debug', DebugService(self)) self.client.call_sync('server.login', str(self.connection_id), socket.gethostname(), get_version(), 'none') self.set_state(ConnectionState.CONNECTED) except BaseException as err: self.logger.warning( 'Failed to initiate support connection: {0}'.format(err), exc_info=True) self.msock.disconnect() else: self.connected_at = datetime.now() with self.cv: self.cv.wait_for(lambda: self.state in ( ConnectionState.LOST, ConnectionState.OFFLINE)) if self.state == ConnectionState.OFFLINE: return self.logger.warning( 'Support connection lost, retrying in 10 seconds') time.sleep(10) def disconnect(self): self.connected_at = None self.set_state(ConnectionState.OFFLINE) self.client.disconnect() self.msock.destroy_channel(0) self.msock.disconnect() self.jobs.clear() def on_msock_close(self): self.connected_at = None self.set_state(ConnectionState.LOST) def run_job(self, job): self.jobs.append(job) job.context = self job.start() def set_state(self, state): with self.cv: self.state = state self.cv.notify_all()
class Context(object): def __init__(self): self.server = None self.client = None self.jobs = {} self.provides = set() self.lock = RLock() self.kq = select.kqueue() self.devnull = os.open('/dev/null', os.O_RDWR) self.logger = logging.getLogger('Context') self.rpc = RpcContext() self.rpc.register_service_instance('serviced.management', ManagementService(self)) self.rpc.register_service_instance('serviced.job', JobService(self)) def init_dispatcher(self): if self.client and self.client.connected: return def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_server(self, address): self.server = Server(self) self.server.rpc = self.rpc self.server.streaming = True self.server.start(address, transport_options={'permissions': 0o777}) thread = Thread(target=self.server.serve_forever) thread.name = 'ServerThread' thread.daemon = True thread.start() def provide(self, targets): def doit(): self.logger.debug('Adding dependency targets: {0}'.format(', '.join(targets))) with self.lock: self.provides |= targets for job in list(self.jobs.values()): if job.state == JobState.STOPPED and job.requires <= self.provides: job.start() if targets: Timer(2, doit).start() def job_by_pid(self, pid): job = first_or_default(lambda j: j.pid == pid, self.jobs.values()) return job def event_loop(self): while True: with contextlib.suppress(InterruptedError): for ev in self.kq.control(None, MAX_EVENTS): self.logger.log(TRACE, 'New event: {0}'.format(ev)) if ev.filter == select.KQ_FILTER_PROC: job = self.job_by_pid(ev.ident) if job: job.pid_event(ev) continue if ev.fflags & select.KQ_NOTE_CHILD: if ev.fflags & select.KQ_NOTE_EXIT: continue pjob = self.job_by_pid(ev.data) if not pjob: self.untrack_pid(ev.ident) continue # Stop tracking at session ID boundary try: if pjob.pgid != os.getpgid(ev.ident): self.untrack_pid(ev.ident) continue except ProcessLookupError: continue with self.lock: job = Job(self) job.load_anonymous(pjob, ev.ident) self.jobs[job.id] = job self.logger.info('Added job {0}'.format(job.label)) def track_pid(self, pid): ev = select.kevent( pid, select.KQ_FILTER_PROC, select.KQ_EV_ADD | select.KQ_EV_ENABLE, select.KQ_NOTE_EXIT | select.KQ_NOTE_EXEC | select.KQ_NOTE_FORK | select.KQ_NOTE_TRACK, 0, 0 ) self.kq.control([ev], 0) def untrack_pid(self, pid): ev = select.kevent( pid, select.KQ_FILTER_PROC, select.KQ_EV_DELETE, 0, 0, 0 ) with contextlib.suppress(FileNotFoundError): self.kq.control([ev], 0) def emit_event(self, name, args): self.server.broadcast_event(name, args) if self.client and self.client.connected: self.client.emit_event(name, args) def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('serviced') self.client.enable_server(self.rpc) self.client.resume_service('serviced.job') self.client.resume_service('serviced.management') return except (OSError, RpcException) as err: self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err))) time.sleep(1) def bootstrap(self): def doit(): with self.lock: job = Job(self) job.load({ 'Label': 'org.freenas.serviced.bootstrap', 'ProgramArguments': BOOTSTRAP_JOB, 'OneShot': True, 'RunAtLoad': True, }) self.jobs[job.id] = job Thread(target=doit).start() def shutdown(self): self.client.disconnect() self.server.close() sys.exit(0) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on') args = parser.parse_args() configure_logging('/var/log/serviced.log', 'DEBUG', file=True) bsd.setproctitle('serviced') self.logger.info('Started') self.init_server(args.s) self.bootstrap() self.event_loop()
class Main(object): def __init__(self): self.logger = logging.getLogger('dscached') self.config = None self.datastore = None self.configstore = None self.rpc = RpcContext() self.rpc.streaming_enabled = True self.rpc.streaming_burst = 16 self.client = None self.server = None self.plugin_dirs = [] self.plugins = {} self.directories = [] self.users_cache = TTLCacheStore() self.groups_cache = TTLCacheStore() self.hosts_cache = TTLCacheStore() self.cache_ttl = 7200 self.search_order = [] self.cache_enumerations = True self.cache_lookups = True self.home_directory_root = None self.account_service = AccountService(self) self.group_service = GroupService(self) self.rpc.register_service_instance('dscached.account', self.account_service) self.rpc.register_service_instance('dscached.group', self.group_service) self.rpc.register_service_instance('dscached.host', HostService(self)) self.rpc.register_service_instance('dscached.idmap', IdmapService(self)) self.rpc.register_service_instance('dscached.management', ManagementService(self)) self.rpc.register_service_instance('dscached.debug', DebugService()) def get_active_directories(self): return list( filter(lambda d: d and d.state == DirectoryState.BOUND, self.directories)) def get_searched_directories(self): return list( filter(lambda d: d and d.state == DirectoryState.BOUND, (self.get_directory_by_name(n) for n in self.get_search_order()))) def get_search_order(self): return self.search_order def get_directory_by_domain(self, domain_name): return first_or_default(lambda d: d.domain_name == domain_name, self.directories) def get_directory_by_name(self, name): return first_or_default(lambda d: d.name == name, self.directories) def get_directory_for_id(self, uid=None, gid=None): if uid is not None: if uid == 0: # Special case for root user return first_or_default(lambda d: d.plugin_type == 'local', self.directories) return first_or_default( lambda d: d.max_uid and d.max_uid >= uid >= d.min_uid, self.directories) if gid is not None: if gid == 0: # Special case for wheel group return first_or_default(lambda d: d.plugin_type == 'local', self.directories) return first_or_default( lambda d: d.max_gid and d.max_gid >= gid >= d.min_gid, self.directories) def get_home_directory(self, directory, username): if not self.home_directory_root: return '/nonexistent' return os.path.join(self.home_directory_root, f'{username}@{directory.domain_name}') def wait_for_etcd(self): self.client.test_or_wait_for_event( 'plugin.service_resume', lambda args: args['name'] == 'etcd.generation', lambda: 'etcd.generation' in self.client.call_sync( 'discovery.get_services')) def init_datastore(self): try: self.datastore = datastore.get_datastore() except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def init_server(self, address): self.server = Server(self) self.server.rpc = self.rpc self.server.streaming = True self.server.start(address, transport_options={'permissions': 0o777}) thread = Thread(target=self.server.serve_forever) thread.name = 'ServerThread' thread.daemon = True thread.start() def parse_config(self, filename): try: with open(filename, 'r') as f: self.config = json.load(f) except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error( 'Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['dscached']['plugin-dirs'] def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('dscached') self.client.enable_server(self.rpc) self.client.resume_service('dscached.account') self.client.resume_service('dscached.group') self.client.resume_service('dscached.host') self.client.resume_service('dscached.idmap') self.client.resume_service('dscached.management') self.client.resume_service('dscached.debug') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for f in os.listdir(dir): name, ext = os.path.splitext(os.path.basename(f)) if ext != '.py': continue try: plugin = load_module_from_file(name, os.path.join(dir, f)) plugin._init(self) except: self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True) def register_plugin(self, name, cls): self.plugins[name] = cls self.logger.info('Registered plugin {0} (class {1})'.format(name, cls)) def register_schema(self, name, schema): self.client.register_schema(name, schema) def register_schemas(self): from freenas.dispatcher.model import context for name, schema in (s.__named_json_schema__() for s in context.local_json_schema_objects): self.logger.debug(f'Registering schema: {name}') self.client.register_schema(name, schema) def init_directories(self): for i in self.datastore.query('directories'): try: directory = Directory(self, i) self.directories.append(directory) directory.configure() except: continue def load_config(self): self.search_order = self.configstore.get('directory.search_order') self.cache_ttl = self.configstore.get('directory.cache_ttl') self.cache_enumerations = self.configstore.get( 'directory.cache_enumerations') self.cache_lookups = self.configstore.get('directory.cache_lookups') self.home_directory_root = self.configstore.get( 'system.home_directory_root') def checkin(self): checkin() def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on') args = parser.parse_args() configure_logging('dscached', 'DEBUG') setproctitle('dscached') self.config = args.c self.parse_config(self.config) self.init_datastore() self.init_dispatcher() self.load_config() self.init_server(args.s) self.scan_plugins() self.register_schemas() self.wait_for_etcd() self.init_directories() self.checkin() self.client.wait_forever()
class Main(object): def __init__(self): self.logger = logging.getLogger('etcd') self.root = None self.configfile = None self.config = None self.datastore = None self.configstore = None self.client = None self.plugin_dirs = [] self.renderers = {} self.managed_files = {} def init_datastore(self): try: self.datastore = datastore.get_datastore(self.configfile) except datastore.DatastoreException as err: self.logger.error('Cannot initialize datastore: %s', str(err)) sys.exit(1) self.configstore = ConfigStore(self.datastore) def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning('Connection to dispatcher lost') self.connect() self.client = Client() self.client.on_error(on_error) self.connect() def connect(self): while True: try: self.client.connect('unix:') self.client.login_service('etcd') self.client.enable_server() self.client.register_service('etcd.generation', FileGenerationService(self)) self.client.register_service('etcd.management', ManagementService(self)) self.client.register_service('etcd.debug', DebugService()) self.client.resume_service('etcd.generation') self.client.resume_service('etcd.management') self.client.resume_service('etcd.debug') return except (OSError, RpcException) as err: self.logger.warning( 'Cannot connect to dispatcher: {0}, retrying in 1 second'. format(str(err))) time.sleep(1) def init_renderers(self): for name, impl in TEMPLATE_RENDERERS.items(): self.renderers[name] = impl(self) def parse_config(self, filename): try: f = open(filename, 'r') self.config = json.load(f) f.close() except IOError as err: self.logger.error('Cannot read config file: %s', err.message) sys.exit(1) except ValueError: self.logger.error( 'Config file has unreadable format (not valid JSON)') sys.exit(1) self.plugin_dirs = self.config['etcd']['plugin-dirs'] def scan_plugins(self): for i in self.plugin_dirs: self.scan_plugin_dir(i) def scan_plugin_dir(self, dir): self.logger.debug('Scanning plugin directory %s', dir) for root, dirs, files in os.walk(dir): for name in files: abspath = os.path.join(root, name) path = os.path.relpath(abspath, dir) name, ext = os.path.splitext(path) if name in self.managed_files.keys(): continue if ext in TEMPLATE_RENDERERS.keys(): self.managed_files[name] = abspath self.logger.info('Adding managed file %s [%s]', name, ext) def generate_file(self, file_path): if file_path not in self.managed_files.keys(): raise RpcException(errno.ENOENT, 'No such file') template_path = self.managed_files[file_path] name, ext = os.path.splitext(template_path) if ext not in self.renderers.keys(): raise RuntimeError("Can't find renderer for {0}".format(file_path)) renderer = self.renderers[ext] try: return renderer.render_template(template_path) except Exception as e: self.logger.warn('Cannot generate file {0}: {1}'.format( file_path, str(e))) return "# FILE GENERATION FAILED: {0}\n".format(str(e)) def emit_event(self, name, params): self.client.emit_event(name, params) def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-f', action='store_true', default=False, help='Run in foreground') parser.add_argument('mountpoint', metavar='MOUNTPOINT', default='/etc', help='/etc mount point') args = parser.parse_args() configure_logging('/var/log/etcd.log', 'DEBUG') setproctitle.setproctitle('etcd') self.root = args.mountpoint self.configfile = args.c self.parse_config(args.c) self.scan_plugins() self.init_renderers() self.init_datastore() self.init_dispatcher() self.client.wait_forever()
class Context(object): def __init__(self): self.service = TaskProxyService(self) self.task = queue.Queue(1) self.datastore = None self.configstore = None self.conn = None self.instance = None self.module_cache = {} self.running = Event() def put_status(self, state, result=None, exception=None): obj = {'status': state, 'result': None} if result is not None: obj['result'] = result if exception is not None: obj['error'] = serialize_error(exception) self.conn.call_sync('task.put_status', obj) def task_progress_handler(self, args): if self.instance: self.instance.task_progress_handler(args) def collect_fds(self, obj): if isinstance(obj, dict): for v in obj.values(): if isinstance(v, FileDescriptor): yield v else: yield from self.collect_fds(v) if isinstance(obj, (list, tuple)): for o in obj: if isinstance(o, FileDescriptor): yield o else: yield from self.collect_fds(o) def close_fds(self, fds): for i in fds: try: os.close(i.fd) except OSError: pass def run_task_hooks(self, instance, task, type, **extra_env): for hook, props in task['hooks'].get(type, {}).items(): try: if props['condition'] and not props['condition']( *task['args']): continue except BaseException as err: print(err) continue instance.join_subtasks( instance.run_subtask(hook, *task['args'], **extra_env)) def main(self): if len(sys.argv) != 2: print("Invalid number of arguments", file=sys.stderr) sys.exit(errno.EINVAL) key = sys.argv[1] configure_logging(None, logging.DEBUG) self.datastore = get_datastore() self.configstore = ConfigStore(self.datastore) self.conn = Client() self.conn.connect('unix:') self.conn.login_service('task.{0}'.format(os.getpid())) self.conn.enable_server() self.conn.call_sync('management.enable_features', ['streaming_responses']) self.conn.rpc.register_service_instance('taskproxy', self.service) self.conn.register_event_handler('task.progress', self.task_progress_handler) self.conn.call_sync('task.checkin', key) setproctitle.setproctitle('task executor (idle)') while True: try: task = self.task.get() logging.root.setLevel( self.conn.call_sync('management.get_logging_level')) setproctitle.setproctitle('task executor (tid {0})'.format( task['id'])) if task['debugger']: sys.path.append('/usr/local/lib/dispatcher/pydev') import pydevd host, port = task['debugger'] pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True) name, _ = os.path.splitext(os.path.basename(task['filename'])) module = self.module_cache.get(task['filename']) if not module: module = load_module_from_file(name, task['filename']) self.module_cache[task['filename']] = module setproctitle.setproctitle('task executor (tid {0})'.format( task['id'])) fds = list(self.collect_fds(task['args'])) try: dispatcher = DispatcherWrapper(self.conn) self.instance = getattr(module, task['class'])(dispatcher, self.datastore) self.instance.configstore = self.configstore self.instance.user = task['user'] self.instance.environment = task['environment'] self.running.set() self.run_task_hooks(self.instance, task, 'before') result = self.instance.run(*task['args']) self.run_task_hooks(self.instance, task, 'after', result=result) except BaseException as err: print("Task exception: {0}".format(str(err)), file=sys.stderr) traceback.print_exc(file=sys.stderr) if hasattr(self.instance, 'rollback'): self.put_status('ROLLBACK') try: self.instance.rollback(*task['args']) except BaseException as rerr: print("Task exception during rollback: {0}".format( str(rerr)), file=sys.stderr) traceback.print_exc(file=sys.stderr) # Main task is already failed at this point, so ignore hook errors with contextlib.suppress(RpcException): self.run_task_hooks(self.instance, task, 'error', error=serialize_error(err)) self.put_status('FAILED', exception=err) else: self.put_status('FINISHED', result=result) finally: self.close_fds(fds) self.running.clear() except RpcException as err: print("RPC failed: {0}".format(str(err)), file=sys.stderr) print(traceback.format_exc(), flush=True) sys.exit(errno.EBADMSG) except socket.error as err: print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr) sys.exit(errno.ETIMEDOUT) if task['debugger']: import pydevd pydevd.stoptrace() setproctitle.setproctitle('task executor (idle)')
class Context(object): def __init__(self): self.logger = logging.getLogger(self.__class__.__name__) self.msock = msock.client.Client() self.msock.on_closed = self.on_msock_close self.rpc_fd = -1 self.connection_id = None self.jobs = [] self.state = ConnectionState.OFFLINE self.config = None self.keepalive = None self.connected_at = None self.cv = Condition() self.rpc = RpcContext() self.client = Client() self.server = Server() self.middleware_endpoint = None def start(self, configpath, sockpath): signal.signal(signal.SIGUSR2, lambda signo, frame: self.connect()) self.read_config(configpath) self.server.rpc = RpcContext() self.server.rpc.register_service_instance("control", ControlService(self)) self.server.start(sockpath) threading.Thread(target=self.server.serve_forever, name="server thread", daemon=True).start() def init_dispatcher(self): def on_error(reason, **kwargs): if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT): self.logger.warning("Connection to dispatcher lost") self.connect_dispatcher() self.middleware_endpoint = Client() self.middleware_endpoint.on_error(on_error) self.connect_dispatcher() def connect_dispatcher(self): while True: try: self.middleware_endpoint.connect("unix:") self.middleware_endpoint.login_service("debugd") self.middleware_endpoint.enable_server() self.middleware_endpoint.register_service("debugd.management", ControlService(self)) self.middleware_endpoint.resume_service("debugd.management") return except (OSError, RpcException) as err: self.logger.warning("Cannot connect to dispatcher: {0}, retrying in 1 second".format(str(err))) time.sleep(1) def read_config(self, path): try: with open(path) as f: self.config = json.load(f) except (IOError, OSError, ValueError) as err: self.logger.fatal("Cannot open config file: {0}".format(str(err))) self.logger.fatal("Exiting.") sys.exit(1) def connect(self, discard=False): if discard: self.connection_id = None self.keepalive = threading.Thread(target=self.connect_keepalive, daemon=True) self.keepalive.start() def connect_keepalive(self): while True: try: if not self.connection_id: self.connection_id = uuid.uuid4() self.msock.connect(SUPPORT_PROXY_ADDRESS) self.logger.info("Connecting to {0}".format(SUPPORT_PROXY_ADDRESS)) self.rpc_fd = self.msock.create_channel(0) time.sleep(1) # FIXME self.client = Client() self.client.connect("fd://", fobj=self.rpc_fd) self.client.channel_serializer = MSockChannelSerializer(self.msock) self.client.standalone_server = True self.client.enable_server() self.client.register_service("debug", DebugService(self)) self.client.call_sync( "server.login", str(self.connection_id), socket.gethostname(), get_version(), "none" ) self.set_state(ConnectionState.CONNECTED) except BaseException as err: self.logger.warning("Failed to initiate support connection: {0}".format(err), exc_info=True) self.msock.disconnect() else: self.connected_at = datetime.now() with self.cv: self.cv.wait_for(lambda: self.state in (ConnectionState.LOST, ConnectionState.OFFLINE)) if self.state == ConnectionState.OFFLINE: return self.logger.warning("Support connection lost, retrying in 10 seconds") time.sleep(10) def disconnect(self): self.connected_at = None self.set_state(ConnectionState.OFFLINE) self.client.disconnect() self.msock.destroy_channel(0) self.msock.disconnect() self.jobs.clear() def on_msock_close(self): self.connected_at = None self.set_state(ConnectionState.LOST) def run_job(self, job): self.jobs.append(job) job.context = self job.start() def set_state(self, state): with self.cv: self.state = state self.cv.notify_all()