def request_keytabs(zkclient, spool_dir):
"""Request keytabs from the locker.
"""
lockers = zkutils.with_retry(zkclient.get_children, z.KEYTAB_LOCKER)
random.shuffle(lockers)
for locker in lockers:
_LOGGER.info('Connecting to keytab locker: %s', locker)
host, port = locker.split(':')
fs.mkdir_safe(spool_dir)
if _get_keytabs_from(host, port, spool_dir):
return
def _upload_batch(zkclient, db_node_path, table, batch):
"""Generate snapshot DB and upload to zk.
"""
with tempfile.NamedTemporaryFile(delete=False) as f:
pass
conn = sqlite3.connect(f.name)
with conn:
conn.execute("""
CREATE TABLE {table} (
path text, timestamp real, data text,
directory text, name text
)
""".format(table=table))
conn.executemany(
"""
INSERT INTO {table} (
path, timestamp, data, directory, name
) VALUES(?, ?, ?, ?, ?)
""".format(table=table), batch)
conn.executescript("""
CREATE INDEX name_idx ON {table} (name);
CREATE INDEX path_idx ON {table} (path);
""".format(table=table))
conn.close()
with io.open(f.name, 'rb') as f:
db_node = zkutils.create(zkclient,
db_node_path,
zlib.compress(f.read()),
sequence=True)
_LOGGER.info('Uploaded compressed snapshot DB: %s to: %s', f.name,
db_node)
os.unlink(f.name)
# Delete uploaded nodes from zk.
for path, _timestamp, _data, _directory, _name in batch:
zkutils.with_retry(zkutils.ensure_deleted, zkclient, path)
def request_tickets(zkclient, appname, tkt_spool_dir, principals):
"""Request tickets from the locker for the given app.
"""
# Too many nested blocks.
#
# pylint: disable=R0101
lockers = zkutils.with_retry(zkclient.get_children, z.TICKET_LOCKER)
random.shuffle(lockers)
expected = set(principals)
for locker in lockers:
if not expected:
_LOGGER.info('Done: all tickets retrieved.')
return
host, port = locker.split(':')
service = 'host@%s' % host
_LOGGER.info('connecting: %s:%s, %s', host, port, service)
client = gssapiprotocol.GSSAPILineClient(host, int(port), service)
try:
if client.connect():
_LOGGER.debug('connected to: %s:%s, %s', host, port, service)
client.write(appname.encode())
_LOGGER.debug('sent: %r', appname)
while True:
line = client.read()
if not line:
_LOGGER.debug('Got empty response.')
break
princ, encoded = line.split(b':', 1)
princ = princ.decode()
ticket_data = base64.standard_b64decode(encoded)
if ticket_data:
_LOGGER.info('got ticket %s:%s', princ,
hashlib.sha1(encoded).hexdigest())
store_ticket(Ticket(princ, ticket_data), tkt_spool_dir)
expected.discard(princ)
else:
_LOGGER.info('got ticket %s:None', princ)
else:
_LOGGER.warning('Cannot connect to %s:%s, %s', host, port,
service)
finally:
client.disconnect()
def _send_container_archive(zkclient, app, archive_file):
"""This sends the archives of the container to warm storage.
It sends the archive (tarball) up to WARM storage if the archive is
configured for the cell. If it is not configured or it fails for any
reason, it continues without exception. This ensures that failures do not
cause disk to fill up."""
try:
# Connect to zk to get the WARM name and auth key
config = zkutils.with_retry(zkutils.get, zkclient, z.ARCHIVE_CONFIG)
plugin = importlib.import_module('treadmill.plugins.archive')
# yes, we want to call with **
uploader = plugin.Uploader(**config)
uploader(archive_file, app)
except kazoo.client.NoNodeError:
_LOGGER.error('Archive not configured in zookeeper.')
def process_request(self, princ, appname):
"""Process ticket request.
- Assuming princ host/<hostname>@realm, extract the host name.
- Read application name, and verify that the app indeed is placed on
the host.
- Read list of principals from the application manifest.
- Send back ticket files for each princ, base64 encoded.
"""
_LOGGER.info('Processing request from %s: %s', princ, appname)
if not princ or not princ.startswith('host/'):
_LOGGER.error('Host principal expected, got: %s.', princ)
return
hostname = princ[len('host/'):princ.rfind('@')]
if not self.zkclient.exists(z.path.placement(hostname, appname)):
_LOGGER.error('App %s not scheduled on node %s', appname, hostname)
return
tkt_dict = dict()
try:
appnode = z.path.scheduled(appname)
app = zkutils.with_retry(zkutils.get, self.zkclient, appnode)
tickets = set(app.get('tickets', []))
_LOGGER.info('App tickets: %s: %r', appname, tickets)
for ticket in tickets:
tkt_file = os.path.join(self.tkt_spool_dir, ticket)
try:
with io.open(tkt_file, 'rb') as f:
encoded = base64.standard_b64encode(f.read())
tkt_dict[ticket] = encoded
except (IOError, OSError) as err:
if err.errno == errno.ENOENT:
_LOGGER.warning('Ticket file does not exist: %s',
tkt_file)
else:
raise
except kazoo.client.NoNodeError:
_LOGGER.info('App does not exist: %s', appname)
return tkt_dict
def _on_created(self, path):
"""This is the handler function when new files are seen"""
if not os.path.exists(path):
return
localpath = os.path.basename(path)
if localpath.startswith('.'):
return
_LOGGER.info('New event file - %r', path)
eventtime, appname, event, data = localpath.split(',', 4)
with open(path, mode='rb') as f:
eventnode = '%s,%s,%s,%s' % (eventtime, _HOSTNAME, event, data)
_LOGGER.debug('Creating %s', z.path.task(appname, eventnode))
try:
zkutils.with_retry(
self.zkclient.create,
z.path.task(appname, eventnode),
f.read(),
acl=[_SERVERS_ACL],
makepath=True
)
except kazoo.client.NodeExistsError:
pass
if event in ['aborted', 'killed', 'finished']:
scheduled_node = z.path.scheduled(appname)
_LOGGER.info('Unscheduling, event=%s: %s', event, scheduled_node)
zkutils.with_retry(
zkutils.ensure_deleted, self.zkclient,
scheduled_node
)
# For terminal state, update the task node with exit summary.
try:
zkutils.with_retry(
zkutils.update,
self.zkclient,
z.path.task(appname),
{'state': event,
'when': eventtime,
'host': _HOSTNAME,
'data': data}
)
except kazoo.client.NoNodeError:
_LOGGER.warn('Task node not found: %s', z.path.task(appname))
os.unlink(path)
def _get_app_keytabs(self, princ, appname):
"""Get keytabs required for the given app."""
# TODO: the logic is duplicated with ticket locker
# we need to refactor to implement generic locker in the future
if not princ or not princ.startswith('host/'):
raise keytabs2.KeytabLockerError(
'princ "{}" not accepted'.format(princ))
hostname = princ[len('host/'):princ.rfind('@')]
if not self.zkclient.exists(z.path.placement(hostname, appname)):
_LOGGER.error('App %s not scheduled on node %s', appname, hostname)
return []
try:
appnode = z.path.scheduled(appname)
app = zkutils.with_retry(zkutils.get, self.zkclient, appnode)
return app.get('keytabs', [])
except kazoo.client.NoNodeError:
_LOGGER.info('App does not exist: %s', appname)
return []
def _get_app_tickets(self, princ, appname):
"""Get tickets required for the given app."""
hostname = princ[len('host/'):princ.rfind('@')]
if (hostname, appname) in self.trusted:
tkts = self.trusted[(hostname, appname)]
_LOGGER.info('Trusted app: %s/%s, tickets: %r', hostname, appname,
tkts)
return tkts
if not self.zkclient.exists(z.path.placement(hostname, appname)):
_LOGGER.error('App %s not scheduled on node %s', appname, hostname)
return set()
try:
appnode = z.path.scheduled(appname)
app = zkutils.with_retry(zkutils.get, self.zkclient, appnode)
return set(app.get('tickets', []))
except kazoo.client.NoNodeError:
_LOGGER.info('App does not exist: %s', appname)
return set()
def lockers(zkclient):
"""Get registered ticket lockers."""
endpoints = zkutils.with_retry(zkclient.get_children,
z.path.ticket_locker())
random.shuffle(endpoints)
return endpoints
