def get(name):
"""Get an App Group entry"""
try:
admin_app_group = context.GLOBAL.admin.app_group()
cli.out(formatter(admin_app_group.get(name)))
except admin_exceptions.NoSuchObjectResult:
cli.bad_exit('App group does not exist: %s', name)
def run_putty(host, port, sshcmd, command):
"""Runs plink/putty (windows)."""
if not host or not port:
return -2
# Trick putty into storing ssh key automatically.
plink = os.path.join(os.path.dirname(sshcmd), 'plink.exe')
if not shutil.which(plink):
cli.bad_exit('{} cannot be found in the PATH'.format(plink))
store_key_cmd = [
plink, '-P', port,
'%s@%s' % (os.environ['USERNAME'], host), 'exit'
]
_LOGGER.debug('Importing host key: %s', store_key_cmd)
store_key_proc = subprocess.Popen(store_key_cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = store_key_proc.communicate(input='y\n\n\n\n\n\n\n\n\n'.encode())
_LOGGER.debug('plink STDOUT: %s', out)
_LOGGER.debug('plink STDERR: %s', err)
if command:
sshcmd = plink
ssh = [sshcmd, '-P', port, '%s@%s' % (os.environ['USERNAME'], host)]
if command:
ssh.extend(command)
devnull = {}
def _get_devnull():
"""Gets handle to the null device."""
if not devnull:
devnull['fd'] = os.open(os.devnull, os.O_RDWR)
return devnull['fd']
if not shutil.which(sshcmd):
cli.bad_exit('{} cannot be found in the PATH'.format(sshcmd))
_LOGGER.debug('Starting ssh: %s', ssh)
try:
if os.path.basename(sshcmd).lower() == 'putty.exe':
utils.sane_execvp(ssh[0], ssh)
else:
# Call plink. Redirect to devnull if std streams are empty/invalid.
subprocess.call(
ssh,
stdin=None if _check_handle(sys.stdin) else _get_devnull(),
stdout=None if _check_handle(sys.stdout) else _get_devnull(),
stderr=None if _check_handle(sys.stderr) else _get_devnull())
except KeyboardInterrupt:
sys.exit(0)
finally:
if devnull:
os.close(devnull['fd'])
def get(name):
"""Get a Treadmill App Group entry"""
try:
admin_app_group = admin.AppGroup(context.GLOBAL.ldap.conn)
cli.out(formatter(admin_app_group.get(name)))
except ldap3.LDAPNoSuchObjectResult:
cli.bad_exit('App group does not exist: %s', name)
def configure(name, group_type, cell, pattern, endpoints, data):
"""Create, get or modify an App Group"""
admin_app_group = context.GLOBAL.admin.app_group()
data_struct = {}
if group_type:
data_struct['group-type'] = group_type
if cell:
data_struct['cells'] = cell
if pattern is not None:
data_struct['pattern'] = pattern
if data is not None:
data_struct['data'] = data
if endpoints is not None:
data_struct['endpoints'] = endpoints
if data_struct:
try:
admin_app_group.create(name, data_struct)
_LOGGER.debug('Created app group %s', name)
except admin_exceptions.AlreadyExistsResult:
_LOGGER.debug('Updating app group %s', name)
admin_app_group.update(name, data_struct)
try:
cli.out(formatter(admin_app_group.get(name, dirty=True)))
except admin_exceptions.NoSuchObjectResult:
cli.bad_exit('App group does not exist: %s', name)
def ssh(wsapi, api, ssh, app, command, wait):
"""SSH into Treadmill container."""
if ssh is None:
ssh = _DEFAULT_SSH
else:
ssh = ssh.name
if wait:
return _wait_for_app(wsapi, ssh, app, command)
apis = context.GLOBAL.state_api(api)
url = '/endpoint/{}/tcp/ssh'.format(urllib.quote(app))
response = restclient.get(apis, url)
endpoints = response.json()
_LOGGER.debug('endpoints: %r', endpoints)
if not endpoints:
cli.bad_exit('No ssh endpoint(s) found for %s', app)
# Take the first one, if there are more than one, then this is
# consistent with when 1 is returned.
endpoint = endpoints[0]
run_ssh(endpoint['host'], str(endpoint['port']), ssh, list(command))
def configure(cell, traits, server, partition, data):
"""Create, get or modify server configuration"""
admin_srv = context.GLOBAL.admin.server()
attrs = {}
if cell:
attrs['cell'] = cell
if traits:
attrs['traits'] = cli.combine(traits)
if partition:
if partition == '-':
partition = None
attrs['partition'] = partition
if data:
with io.open(data, 'r') as fd:
attrs['data'] = json.loads(fd.read())
if attrs:
try:
admin_srv.create(server, attrs)
except admin_exceptions.AlreadyExistsResult:
admin_srv.update(server, attrs)
try:
cli.out(formatter(admin_srv.get(server, dirty=bool(attrs))))
except admin_exceptions.NoSuchObjectResult:
cli.bad_exit('Server does not exist: %s', server)
def create_cmd(cell, hostname, instance_profile, instance_type, subnet,
image, disk):
"""Create cell ZooKeeper server(s)."""
ec2_conn = awscontext.GLOBAL.ec2
ipa_client = awscontext.GLOBAL.ipaclient
admin_cell = admin.Cell(context.GLOBAL.ldap.conn)
masters = admin_cell.get(cell, dirty=True)['masters']
if hostname:
masters = [
master for master in masters if master['hostname'] == hostname
]
if not masters:
cli.bad_exit('%s not found in the cell config', hostname)
for master in masters:
try:
ec2_instance = ec2client.get_instance(
ec2_conn, hostnames=[master['hostname']]
)
cli.out('%s EC2 instance already exists', master['hostname'])
_LOGGER.debug(ec2_instance)
except exc.NotFoundError:
hostmanager.create_zk(
ec2_conn=ec2_conn,
ipa_client=ipa_client,
master=master,
subnet_id=subnet,
instance_type=instance_type,
instance_profile=instance_profile,
image_id=image,
disk=disk
)
cli.out('Created: %s', master['hostname'])
def _wait_for_app(wsapi, ssh, app, command):
"""Use websockets to wait for the app to start"""
def on_message(result):
"""Callback to process trace message."""
host = result['host']
port = result['port']
run_ssh(host, port, ssh, list(command))
return False
def on_error(result):
"""Callback to process errors."""
click.echo('Error: %s' % result['_error'], err=True)
try:
return ws_client.ws_loop(
wsapi,
{'topic': '/endpoints',
'filter': app,
'proto': 'tcp',
'endpoint': 'ssh'},
False,
on_message,
on_error
)
except ws_client.ConnectionError:
cli.bad_exit('Could not connect to any Websocket APIs')
def get(partition, server):
"""Get partition"""
cell = context.GLOBAL.cell
admin_part = context.GLOBAL.admin.partition()
admin_srv = context.GLOBAL.admin.server()
if partition:
# Get partition by name.
cli.out(formatter(admin_part.get([partition, cell])))
elif server:
# Get partition by server name.
srv = admin_srv.get(server)
if srv['cell'] != cell:
cli.bad_exit('Server does not belong to %s: %s',
cell, srv['cell'])
# The server checks out (otherwise there will be exception already)
#
# If partition is not explicitely defined, return empty dict.
try:
partition_obj = admin_part.get([srv['partition'], cell])
except admin_exceptions.NoSuchObjectResult:
partition_obj = {}
cli.out(formatter(partition_obj))
else:
cli.bad_exit('Partition or server name is required')
def _wait_for_app(ssh, app, command, queue=None):
"""Use websockets to wait for the app to start"""
# JoinableQueue is filled with a dummy item otherwise queue.join() unblocks
# immediately wo/ actually letting the ws_loop and _wait_for_ssh to run.
queue = queue or g_queue.JoinableQueue(items=[('dummy.host', 1234)])
def on_message(result, queue=queue):
"""Callback to process trace message."""
_LOGGER.debug('Endpoint trase msg: %r', result)
queue.put((result['host'], result['port']))
return False
def on_error(result):
"""Callback to process errors."""
click.echo('Error: %s' % result['_error'], err=True)
try:
gevent.spawn(_wait_for_ssh, queue, ssh, command)
gevent.spawn(
ws_client.ws_loop, context.GLOBAL.ws_api(), {
'topic': '/endpoints',
'filter': app,
'proto': 'tcp',
'endpoint': 'ssh'
}, False, on_message, on_error)
queue.join()
except ws_client.WSConnectionError:
cli.bad_exit('Could not connect to any Websocket APIs')
def configure(name, group_type, cell, pattern, endpoints, data):
"""Create or modify Treadmill App Group entry"""
admin_app_group = admin.AppGroup(context.GLOBAL.ldap.conn)
data_struct = {}
if group_type:
data_struct['group-type'] = group_type
if cell:
data_struct['cells'] = cell
if pattern is not None:
data_struct['pattern'] = pattern
if data is not None:
data_struct['data'] = data
if endpoints is not None:
data_struct['endpoints'] = endpoints
if data_struct:
try:
admin_app_group.create(name, data_struct)
_LOGGER.debug('Created app group %s', name)
except ldap3.LDAPEntryAlreadyExistsResult:
_LOGGER.debug('Updating app group %s', name)
admin_app_group.update(name, data_struct)
try:
cli.out(formatter(admin_app_group.get(name)))
except ldap3.LDAPNoSuchObjectResult:
cli.bad_exit('App group does not exist: %s', name)
def _get_endpoint_for_host(endpoints, host):
"""Return the nodeinfo endpoint running on the host in parameter."""
try:
rv = endpoints[host]
except KeyError:
cli.bad_exit('Nodeinfo endpoint not found on %s', host)
return rv
def configure(api, match, manifest, delete, appname):
"""Configure a Treadmill app"""
restapi = context.GLOBAL.admin_api(api)
if appname:
if delete:
return _delete(restapi, appname)
return _configure(restapi, manifest, appname)
else:
if not match:
cli.bad_exit('You must supply a --match option')
return _list(restapi, match)
def running(app_pattern):
"""Get the metrics of running instances."""
instances = _find_running_instance(app_pattern, ctx['ws_api'])
if not instances:
cli.bad_exit('No running instance matched the pattern.')
_LOGGER.debug('Found instance(s): %s', instances)
for inst, host in instances.items():
endpoint = _get_endpoint_for_host(ctx['nodeinf_eps'], host)
_get_app_metrics(endpoint, inst, outdir=ctx['outdir'])
def placement(instance, mode):
"""Explain application placement"""
cell_master = make_readonly_master()
if instance not in cell_master.cell.apps:
cli.bad_exit('Instance not found.')
app = cell_master.cell.apps[instance]
if app.server:
cli.bad_exit('Instace already placed on %s' % app.server)
frame = reports.explain_placement(cell_master.cell, app, mode)
_print(frame, explain=True)
def _download_rrd(nodeinfo_url, metrics_url, rrdfile):
"""Get rrd file and store in output directory."""
_LOGGER.info('Download metrics from %s/%s', nodeinfo_url, metrics_url)
try:
resp = restclient.get(nodeinfo_url, metrics_url, stream=True)
with open(rrdfile, 'w+b') as f:
for chunk in resp.iter_content(chunk_size=128):
f.write(chunk)
rrdutils.gen_graph(rrdfile, rrdutils.RRDTOOL, show_mem_limit=False)
except restclient.NotFoundError as err:
_LOGGER.error('%s', err)
cli.bad_exit('Metrics not found: %s', err)
except rrdutils.RRDToolNotFoundError:
cli.bad_exit('The rrdtool utility cannot be found in the PATH')
def _find_endpoints(pattern, proto, endpoint):
"""Return all the matching endpoints in the cell.
The return value is a dict with host-endpoint assigments as key-value
pairs.
"""
apis = context.GLOBAL.state_api()
url = '/endpoint/{}/{}/{}'.format(pattern, proto, endpoint)
endpoints = restclient.get(apis, url).json()
if not endpoints:
cli.bad_exit('Nodeinfo API couldn\'t be found')
return endpoints
def run_unix(host, port, ssh, command):
"""Runs standard ssh (non-windows)."""
if not host or not port:
return -2
if not shutil.which(ssh):
cli.bad_exit('{} cannot be found in the PATH'.format(ssh))
ssh = [
ssh, '-o', 'UserKnownHostsFile=/dev/null', '-o',
'StrictHostKeyChecking=no', '-p', port, host
] + command
_LOGGER.debug('Starting ssh: %s', ssh)
return utils.sane_execvp(ssh[0], ssh)
def get(rec_dn, cls, attrs):
"""List all defined DNs."""
if not attrs:
attrs = []
try:
# TODO: it is porbably possible to derive class from DN.
klass = getattr(admin, cls)
attrs.extend([elem[0] for elem in klass.schema()])
except AttributeError:
cli.bad_exit('Invalid admin type: %s', cls)
return
entry = context.GLOBAL.ldap.conn.get(rec_dn, '(objectClass=*)',
list(set(attrs)))
formatter = cli.make_formatter(None)
cli.out(formatter(entry))
def app(app):
"""Get the metrics of the application in params."""
instance, uniq = app.split('/')
if uniq == 'running':
instances = _find_running_instance(instance, ctx['ws_api'])
else:
instances = _find_uniq_instance(instance, uniq, ctx['ws_api'])
if not instances:
cli.bad_exit('No instance found with the application name.')
_LOGGER.debug('Found instance(s): %s', instances)
for inst, host in instances.items():
endpoint = _get_endpoint_for_host(ctx['nodeinf_eps'], host)
_get_app_metrics(endpoint, inst, uniq, outdir=ctx['outdir'])
def running(app_pattern, long):
"""Get the metrics of running instances."""
instances = _find_running_instance(app_pattern, ctx['ws_api'])
if not instances:
cli.bad_exit('No running instance matched the pattern.')
_LOGGER.debug('Found instance(s): %s', instances)
timeframe = 'long' if long else 'short'
for inst, host in instances.items():
endpoint = _get_endpoint_for_host(ctx['nodeinf_eps'], host)
_LOGGER.debug("getting metrics from endpoint %r", endpoint)
_get_app_metrics(endpoint,
inst,
timeframe,
outdir=ctx['outdir'],
cell_api=ctx['cell_api'])
def _get_endpoints(api=None):
"""Return all the nodeinfo endpoints for the given cell."""
apis = context.GLOBAL.state_api(api)
url = '/endpoint/{}/tcp/nodeinfo'.format(urllib.parse.quote('root.*'))
response = restclient.get(apis, url)
endpoints = [{
'name': end['name'],
'proto': end['proto'],
'endpoint': end['endpoint'],
'hostport': '{0}:{1}'.format(end['host'], end['port'])
} for end in response.json()]
if not endpoints:
cli.bad_exit("Nodeinfo API couldn't be found")
return endpoints
def logs(api, host, service):
"""View application logs.
"""
try:
appname, uniq, logtype, service = service.split('/', 3)
except ValueError:
cli.bad_exit('Invalid service format: '
'expect <appname>/<uniq>/service/<servicename>')
if bool(host) ^ bool(uniq != 'running'):
cli.bad_exit('Usage: Either request logs from a running service'
' with <appname>/running/service/<servicename> or'
' provide a hostname with --host and'
' <appname>/<uniq>/service/<servicename>')
apis = context.GLOBAL.state_api(api)
if uniq == 'running':
state_url = '/state?%s' % urllib.parse.urlencode(
[('match', appname)]
)
where = restclient.get(apis, state_url).json()
if not where:
cli.bad_exit('%s not running.', appname)
host = where[0]['host']
if not host:
cli.bad_exit('%s is pending.', appname)
nodeinfo_url = '/endpoint/root.%s/tcp/nodeinfo' % host
nodeinfo = restclient.get(apis, nodeinfo_url).json()
if not nodeinfo:
cli.bad_exit('Nodeinfo api not found: %s', host)
nodeinfo_api = ['http://%s:%s' % (nodeinfo[0]['host'],
nodeinfo[0]['port'])]
logurl = '/app/%s/%s/%s/%s' % (
urllib.parse.quote(appname),
urllib.parse.quote(uniq),
logtype,
urllib.parse.quote(service))
log = restclient.get(nodeinfo_api, logurl)
print(log.text)
def logs(app_or_svc, host, uniq, service):
"""View application's service logs."""
try:
app, uniq, logtype, logname = app_or_svc.split('/', 3)
except ValueError:
app, uniq, logtype, logname = app_or_svc, uniq, 'service', service
if any(param is None for param in [app, uniq, logtype, logname]):
cli.bad_exit('Incomplete parameter list')
_host, port = _nodeinfo_endpoint(host)
api = 'http://{0}:{1}'.format(host, port)
logurl = '/local-app/%s/%s/%s/%s' % (urllib_parse.quote(app),
urllib_parse.quote(uniq), logtype,
urllib_parse.quote(logname))
log = restclient.get(api, logurl)
click.echo(log.text)
def logs(api, app_or_svc, host, service, uniq, ws_api):
"""View application's service logs.
Arguments are expected to be specified a) either as one string or b)
parts defined one-by-one ie.:
a) <appname>/<uniq or running>/service/<servicename>
b) <appname> --uniq <uniq> --service <servicename>
Eg.:
a) proid.foo#1234/xz9474as8/service/my-echo
b) proid.foo#1234 --uniq xz9474as8 --service my-echo
For the latest log simply omit 'uniq':
proid.foo#1234 --service my-echo
"""
try:
app, uniq, logtype, logname = app_or_svc.split('/', 3)
except ValueError:
app, uniq, logtype, logname = app_or_svc, uniq, 'service', service
if logname is None:
cli.bad_exit("Please specify the 'service' parameter.")
if host is None:
instance = None
if uniq == 'running':
instance = _find_running_instance(app, ws_api)
if not instance:
instance = _find_uniq_instance(app, uniq, ws_api)
if not instance:
cli.bad_exit('No {}instance could be found.'.format(
'running ' if uniq == 'running' else ''))
_LOGGER.debug('Found instance: %s', instance)
host = instance['host']
uniq = instance['uniq']
try:
endpoint, = (ep for ep in _find_endpoints(
urllib.parse.quote('root.*'), 'tcp', 'nodeinfo', api)
if ep['host'] == host)
except ValueError as err:
_LOGGER.exception(err)
cli.bad_exit('No endpoint found on %s', host)
api = 'http://{0}:{1}'.format(endpoint['host'], endpoint['port'])
logurl = '/app/%s/%s/%s/%s' % (urllib.parse.quote(app),
urllib.parse.quote(uniq), logtype,
urllib.parse.quote(logname))
log = restclient.get(api, logurl)
click.echo(log.text)
def rotate_cmd(cell, hostname, instance_profile, instance_type, subnet,
image, disk):
"""Rotate cell ZooKeeper server."""
ec2_conn = awscontext.GLOBAL.ec2
ipa_client = awscontext.GLOBAL.ipaclient
admin_cell = admin.Cell(context.GLOBAL.ldap.conn)
masters = admin_cell.get(cell, dirty=True)['masters']
try:
master = next(
master for master in masters if master['hostname'] == hostname
)
except StopIteration:
cli.bad_exit('%s not found in the cell config', hostname)
try:
ec2_instance = ec2client.get_instance(
ec2_conn, hostnames=[hostname]
)
_LOGGER.debug(ec2_instance)
except exc.NotFoundError:
cli.bad_exit('%s EC2 instance does not exist', hostname)
hostmanager.delete_hosts(ec2_conn, ipa_client, [hostname])
cli.out('Deleted: %s', hostname)
# Copy subnet, type and image from the old instance unless we override.
hostmanager.create_zk(
ec2_conn=ec2_conn,
ipa_client=ipa_client,
master=master,
subnet_id=subnet or ec2_instance['SubnetId'],
instance_type=instance_type or ec2_instance['InstanceType'],
instance_profile=instance_profile,
image_id=image or ec2_instance['ImageId'],
disk=disk
)
cli.out('Created: %s', hostname)
def _wait_for_ssh(queue, ssh, command, timeout=1, attempts=40):
"""Wait until a successful connection to the ssh endpoint can be made."""
try:
host, port = queue.get(timeout=timeout * attempts)
except g_queue.Empty:
cli.bad_exit('No SSH endpoint found.')
for _ in six.moves.range(attempts):
_LOGGER.debug('Checking SSH endpoint %s:%s', host, port)
if _connect(host, port):
run_ssh(host, port, ssh, list(command))
break # if run_ssh doesn't end with os.execvp()...
try:
host, port = queue.get(timeout=timeout)
queue.task_done()
except g_queue.Empty:
pass
# Either all the connection attempts failed or we're after run_ssh
# (not resulting in os.execvp) so let's "clear the queue" so the thread
# can join
queue.task_done()
def krb5keytab(keytab, sock_path):
"""The client utility to get krb5keytab from the local proxy."""
if not sock_path:
sock_path = _DEFAULT_SOCK_PATH
client = peercredprotocol.PeerCredLineClient(sock_path)
try:
client.connect()
# If we write keytab ourselvs, do not ask server to write the
# file.
request = {
'keytab': (not keytab)
}
client.write(json.dumps(request).encode('utf8'))
reply = client.read()
if not reply:
cli.bad_exit('Connection closed.')
response = json.loads(reply.decode())
if response.get('status') != 'success':
cli.bad_exit(response.get('why', 'Unknown error'))
if keytab:
keytab_entries = base64.standard_b64decode(
response['result']['keytab_entries']
)
_LOGGER.info('Writing keytab: %s', keytab)
fs.write_safe(
keytab,
lambda f: f.write(keytab_entries),
)
except FileNotFoundError:
cli.bad_exit(
'Failed connecting to %s, krb5keytab proxy is not running.',
sock_path
)
finally:
client.disconnect()
