def request_keytabs(zkclient, spool_dir): """Request keytabs from the locker. """ lockers = zkutils.with_retry(zkclient.get_children, z.KEYTAB_LOCKER) random.shuffle(lockers) for locker in lockers: _LOGGER.info('Connecting to keytab locker: %s', locker) host, port = locker.split(':') fs.mkdir_safe(spool_dir) if _get_keytabs_from(host, port, spool_dir): return
def _upload_batch(zkclient, db_node_path, table, batch): """Generate snapshot DB and upload to zk. """ with tempfile.NamedTemporaryFile(delete=False) as f: pass conn = sqlite3.connect(f.name) with conn: conn.execute(""" CREATE TABLE {table} ( path text, timestamp real, data text, directory text, name text ) """.format(table=table)) conn.executemany( """ INSERT INTO {table} ( path, timestamp, data, directory, name ) VALUES(?, ?, ?, ?, ?) """.format(table=table), batch) conn.executescript(""" CREATE INDEX name_idx ON {table} (name); CREATE INDEX path_idx ON {table} (path); """.format(table=table)) conn.close() with io.open(f.name, 'rb') as f: db_node = zkutils.create(zkclient, db_node_path, zlib.compress(f.read()), sequence=True) _LOGGER.info('Uploaded compressed snapshot DB: %s to: %s', f.name, db_node) os.unlink(f.name) # Delete uploaded nodes from zk. for path, _timestamp, _data, _directory, _name in batch: zkutils.with_retry(zkutils.ensure_deleted, zkclient, path)
def request_tickets(zkclient, appname, tkt_spool_dir, principals): """Request tickets from the locker for the given app. """ # Too many nested blocks. # # pylint: disable=R0101 lockers = zkutils.with_retry(zkclient.get_children, z.TICKET_LOCKER) random.shuffle(lockers) expected = set(principals) for locker in lockers: if not expected: _LOGGER.info('Done: all tickets retrieved.') return host, port = locker.split(':') service = 'host@%s' % host _LOGGER.info('connecting: %s:%s, %s', host, port, service) client = gssapiprotocol.GSSAPILineClient(host, int(port), service) try: if client.connect(): _LOGGER.debug('connected to: %s:%s, %s', host, port, service) client.write(appname.encode()) _LOGGER.debug('sent: %r', appname) while True: line = client.read() if not line: _LOGGER.debug('Got empty response.') break princ, encoded = line.split(b':', 1) princ = princ.decode() ticket_data = base64.standard_b64decode(encoded) if ticket_data: _LOGGER.info('got ticket %s:%s', princ, hashlib.sha1(encoded).hexdigest()) store_ticket(Ticket(princ, ticket_data), tkt_spool_dir) expected.discard(princ) else: _LOGGER.info('got ticket %s:None', princ) else: _LOGGER.warning('Cannot connect to %s:%s, %s', host, port, service) finally: client.disconnect()
def _send_container_archive(zkclient, app, archive_file): """This sends the archives of the container to warm storage. It sends the archive (tarball) up to WARM storage if the archive is configured for the cell. If it is not configured or it fails for any reason, it continues without exception. This ensures that failures do not cause disk to fill up.""" try: # Connect to zk to get the WARM name and auth key config = zkutils.with_retry(zkutils.get, zkclient, z.ARCHIVE_CONFIG) plugin = importlib.import_module('treadmill.plugins.archive') # yes, we want to call with ** uploader = plugin.Uploader(**config) uploader(archive_file, app) except kazoo.client.NoNodeError: _LOGGER.error('Archive not configured in zookeeper.')
def process_request(self, princ, appname): """Process ticket request. - Assuming princ host/<hostname>@realm, extract the host name. - Read application name, and verify that the app indeed is placed on the host. - Read list of principals from the application manifest. - Send back ticket files for each princ, base64 encoded. """ _LOGGER.info('Processing request from %s: %s', princ, appname) if not princ or not princ.startswith('host/'): _LOGGER.error('Host principal expected, got: %s.', princ) return hostname = princ[len('host/'):princ.rfind('@')] if not self.zkclient.exists(z.path.placement(hostname, appname)): _LOGGER.error('App %s not scheduled on node %s', appname, hostname) return tkt_dict = dict() try: appnode = z.path.scheduled(appname) app = zkutils.with_retry(zkutils.get, self.zkclient, appnode) tickets = set(app.get('tickets', [])) _LOGGER.info('App tickets: %s: %r', appname, tickets) for ticket in tickets: tkt_file = os.path.join(self.tkt_spool_dir, ticket) try: with io.open(tkt_file, 'rb') as f: encoded = base64.standard_b64encode(f.read()) tkt_dict[ticket] = encoded except (IOError, OSError) as err: if err.errno == errno.ENOENT: _LOGGER.warning('Ticket file does not exist: %s', tkt_file) else: raise except kazoo.client.NoNodeError: _LOGGER.info('App does not exist: %s', appname) return tkt_dict
def _on_created(self, path): """This is the handler function when new files are seen""" if not os.path.exists(path): return localpath = os.path.basename(path) if localpath.startswith('.'): return _LOGGER.info('New event file - %r', path) eventtime, appname, event, data = localpath.split(',', 4) with open(path, mode='rb') as f: eventnode = '%s,%s,%s,%s' % (eventtime, _HOSTNAME, event, data) _LOGGER.debug('Creating %s', z.path.task(appname, eventnode)) try: zkutils.with_retry( self.zkclient.create, z.path.task(appname, eventnode), f.read(), acl=[_SERVERS_ACL], makepath=True ) except kazoo.client.NodeExistsError: pass if event in ['aborted', 'killed', 'finished']: scheduled_node = z.path.scheduled(appname) _LOGGER.info('Unscheduling, event=%s: %s', event, scheduled_node) zkutils.with_retry( zkutils.ensure_deleted, self.zkclient, scheduled_node ) # For terminal state, update the task node with exit summary. try: zkutils.with_retry( zkutils.update, self.zkclient, z.path.task(appname), {'state': event, 'when': eventtime, 'host': _HOSTNAME, 'data': data} ) except kazoo.client.NoNodeError: _LOGGER.warn('Task node not found: %s', z.path.task(appname)) os.unlink(path)
def _get_app_keytabs(self, princ, appname): """Get keytabs required for the given app.""" # TODO: the logic is duplicated with ticket locker # we need to refactor to implement generic locker in the future if not princ or not princ.startswith('host/'): raise keytabs2.KeytabLockerError( 'princ "{}" not accepted'.format(princ)) hostname = princ[len('host/'):princ.rfind('@')] if not self.zkclient.exists(z.path.placement(hostname, appname)): _LOGGER.error('App %s not scheduled on node %s', appname, hostname) return [] try: appnode = z.path.scheduled(appname) app = zkutils.with_retry(zkutils.get, self.zkclient, appnode) return app.get('keytabs', []) except kazoo.client.NoNodeError: _LOGGER.info('App does not exist: %s', appname) return []
def _get_app_tickets(self, princ, appname): """Get tickets required for the given app.""" hostname = princ[len('host/'):princ.rfind('@')] if (hostname, appname) in self.trusted: tkts = self.trusted[(hostname, appname)] _LOGGER.info('Trusted app: %s/%s, tickets: %r', hostname, appname, tkts) return tkts if not self.zkclient.exists(z.path.placement(hostname, appname)): _LOGGER.error('App %s not scheduled on node %s', appname, hostname) return set() try: appnode = z.path.scheduled(appname) app = zkutils.with_retry(zkutils.get, self.zkclient, appnode) return set(app.get('tickets', [])) except kazoo.client.NoNodeError: _LOGGER.info('App does not exist: %s', appname) return set()
def lockers(zkclient): """Get registered ticket lockers.""" endpoints = zkutils.with_retry(zkclient.get_children, z.path.ticket_locker()) random.shuffle(endpoints) return endpoints