def server(register, port, auth, modules, title, cors_origin): """Runs nodeinfo server.""" if port == 0: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('0.0.0.0', 0)) port = sock.getsockname()[1] sock.close() hostname = sysinfo.hostname() hostport = '%s:%s' % (hostname, port) if register: zkclient = context.GLOBAL.zk.conn zkclient.add_listener(zkutils.exit_on_lost) appname = 'root.%s#%010d' % (hostname, os.getpid()) path = z.path.endpoint(appname, 'tcp', 'nodeinfo') _LOGGER.info('register endpoint: %s %s', path, hostport) zkutils.create(zkclient, path, hostport, acl=[_SERVERS_ACL], ephemeral=True) _LOGGER.info('Starting nodeinfo server on port: %s', port) utils.drop_privileges() api_paths = [] if modules: api_paths = api.init(modules, title.replace('_', ' '), cors_origin) rest_server = rest.TcpRestServer(port, auth_type=auth, protect=api_paths) rest_server.run()
def _safe_create(self, rsrc_id, path, data): """Create ephemeral node in Zookeeper. If the node is present, check if the owner session id is ours, if not, fail. """ try: zkutils.create(self.zkclient, path, data, ephemeral=True) _LOGGER.info('Created node: %s', path) except kazoo.client.NodeExistsError: content, metadata = zkutils.get_with_metadata(self.zkclient, path) session_id, _pwd = self.zkclient.client_id if metadata.owner_session_id != session_id: _LOGGER.info('Node exists, owned by other: %s - %s - %s', path, content, metadata.owner_session_id) self._watch(rsrc_id, path) return False if content != data: _LOGGER.info('Content different: %s - old: %s, new: %s', path, content, data) zkutils.update(self.zkclient, path, data) _LOGGER.info('Node is up to date: %s - %s', path, session_id) return True
def server(approot, register, port, auth, modules, title, cors_origin): """Runs nodeinfo server.""" if port == 0: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind(('0.0.0.0', 0)) port = sock.getsockname()[1] sock.close() hostname = sysinfo.hostname() hostport = '%s:%s' % (hostname, port) if register: zkclient = context.GLOBAL.zk.conn zkclient.add_listener(zkutils.exit_on_lost) appname = 'root.%s#%010d' % (hostname, os.getpid()) app_pattern = 'root.%s#*' % (hostname) path = z.path.endpoint(appname, 'tcp', 'nodeinfo') _LOGGER.info('register endpoint: %s %s', path, hostport) zkutils.create(zkclient, path, hostport, acl=[zkclient.make_servers_acl()], ephemeral=True) # TODO: remove "legacy" endpoint registration once conversion is # complete. tm_env = appenv.AppEnvironment(approot) # TODO: need to figure out how to handle windows. assert os.name != 'nt' endpoints_mgr = endpoints.EndpointsMgr(tm_env.endpoints_dir) endpoints_mgr.unlink_all( app_pattern, endpoint='nodeinfo', proto='tcp' ) endpoints_mgr.create_spec( appname=appname, endpoint='nodeinfo', proto='tcp', real_port=port, pid=os.getpid(), port=port, owner='/proc/{}'.format(os.getpid()), ) _LOGGER.info('Starting nodeinfo server on port: %s', port) utils.drop_privileges() api_paths = [] if modules: api_modules = {module: None for module in modules} api_paths = api.init( api_modules, title.replace('_', ' '), cors_origin ) rest_server = rest.TcpRestServer(port, auth_type=auth, protect=api_paths) rest_server.run()
def _upload_batch(zkclient, db_node_path, table, batch): """Generate snapshot DB and upload to zk. """ with tempfile.NamedTemporaryFile(delete=False) as f: pass conn = sqlite3.connect(f.name) with conn: conn.execute( """ CREATE TABLE {table} ( path text, timestamp real, data text, directory text, name text ) """.format(table=table) ) conn.executemany( """ INSERT INTO {table} ( path, timestamp, data, directory, name ) VALUES(?, ?, ?, ?, ?) """.format(table=table), batch ) conn.executescript( """ CREATE INDEX name_idx ON {table} (name); CREATE INDEX path_idx ON {table} (path); """.format(table=table) ) conn.close() with io.open(f.name, 'rb') as f: db_node = zkutils.create( zkclient, db_node_path, zlib.compress(f.read()), sequence=True ) _LOGGER.info( 'Uploaded compressed snapshot DB: %s to: %s', f.name, db_node ) os.unlink(f.name) # Delete uploaded nodes from zk. for path, _timestamp, _data, _directory, _name in batch: zkutils.with_retry(zkutils.ensure_deleted, zkclient, path)
def _create_ephemeral_with_retry(zkclient, path, data): """Create ephemeral node with retry.""" prev_data = None for _ in range(0, _EPHEMERAL_RETRY_COUNT): try: return zkutils.create(zkclient, path, data, acl=[_SERVERS_ACL], ephemeral=True) except kazoo.client.NodeExistsError: prev_data = zkutils.get_default(zkclient, path) _LOGGER.warning('Node exists, will retry: %s, data: %r', path, prev_data) time.sleep(_EPHEMERAL_RETRY_INTERVAL) raise exc.ContainerSetupError('%s:%s' % (path, prev_data), app_abort.AbortedReason.PRESENCE)
def _create_ephemeral_with_retry(zkclient, path, data): """Create ephemeral node with retry.""" prev_data = None for _ in range(0, 5): try: return zkutils.create(zkclient, path, data, acl=[_SERVERS_ACL], ephemeral=True) except kazoo.client.NodeExistsError: prev_data, metadata = zkutils.get_default(zkclient, path, need_metadata=True) _LOGGER.warn('Node exists, will retry: %s, data: %r, metadata: %r', path, prev_data, metadata) time.sleep(_EPHEMERAL_RETRY_INTERVAL) raise exc.ContainerSetupError('presence.%s:%s' % (path, prev_data))
def _upload_batch(zkclient, db_node_path, dbname, batch): """Generate snapshot DB and upload to zk.""" with tempfile.NamedTemporaryFile(delete=False) as f: pass with sqlite3.connect(f.name) as conn: conn.execute( 'create table %s (path text, timestamp integer, data text)' % ( dbname ) ) conn.executemany( 'insert into %s (path, timestamp, data) values(?, ?, ?)' % ( dbname ), batch ) conn.executescript( """ CREATE INDEX path_timestamp_idx on %s (path, timestamp); PRAGMA query_only = TRUE; """ % dbname ) conn.close() with open(f.name, 'rb') as f: db_node = zkutils.create( zkclient, db_node_path, zlib.compress(f.read()), sequence=True ) _LOGGER.info( 'Uploaded compressed trace_db snapshot: %s to: %s', f.name, db_node ) os.unlink(f.name) # Delete uploaded nodes from zk. for path, _ts, _data in batch: zkutils.with_retry(zkutils.ensure_deleted, zkclient, path)
def _close(self): """Close database, compress and upload to ZK.""" _LOGGER.info('Closing task_db snapshot: %s, total rows: %s', self.name, self.rows_count) self.conn.commit() self.conn.close() if self.rows_count > 0: db_node_path = z.path.tasks_history('tasks.db.gzip-') with open(self.name, 'rb') as f: db_node = zkutils.create( self.zkclient, db_node_path, zlib.compress( f.read() ), sequence=True ) _LOGGER.info('Uploaded compressed task_db snapshot: %s to: %s', self.name, db_node) os.unlink(self.name) self.name = None self.conn = None self.cur = None self.rows_count = 0
def server(approot, register, port, auth, modules, config, title, cors_origin, rate_limit_global, rate_limit_module, rate_limit_by): """Runs nodeinfo server.""" rate_limit = _get_rate_limit( rate_limit_global, rate_limit_module, rate_limit_by ) rest_server = rest.TcpRestServer(port, auth_type=auth, rate_limit=rate_limit) port = rest_server.port hostname = sysinfo.hostname() hostport = '%s:%s' % (hostname, port) if register: zkclient = context.GLOBAL.zk.conn zkclient.add_listener(zkutils.exit_on_lost) appname = 'root.%s#%010d' % (hostname, os.getpid()) app_pattern = 'root.%s#*' % (hostname) path = z.path.endpoint(appname, 'tcp', 'nodeinfo') _LOGGER.info('register endpoint: %s %s', path, hostport) zkutils.create(zkclient, path, hostport, acl=[zkclient.make_servers_acl()], ephemeral=True) # TODO: remove "legacy" endpoint registration once conversion is # complete. tm_env = appenv.AppEnvironment(approot) endpoints_mgr = endpoints.EndpointsMgr(tm_env.endpoints_dir) endpoints_mgr.unlink_all( app_pattern, endpoint='nodeinfo', proto='tcp' ) # On Linux endpoint for nodeinfo is a symlink pointing to # /proc/{pid}, on Windows it's just a regular file owner = '/proc/{}'.format(os.getpid()) if os.name == 'posix' \ else None endpoints_mgr.create_spec( appname=appname, endpoint='nodeinfo', proto='tcp', real_port=port, pid=os.getpid(), port=port, owner=owner, ) _LOGGER.info('Starting nodeinfo server on port: %s', port) utils.drop_privileges() if modules: api_modules = {module: None for module in modules} for module, cfg in config: if module not in api_modules: raise click.UsageError( 'Orphan config: %s, not in: %r' % (module, api_modules) ) api_modules[module] = yaml.load(stream=cfg) cfg.close() rest_server.protect = api.init( api_modules, title.replace('_', ' '), cors_origin ) rest_server.run()