def test_get_424(self, resp_mock): """Test treadmill.restclient.get FAILED_DEPENDENCY (424)""" resp_mock.return_value.status_code = http_client.FAILED_DEPENDENCY resp_mock.return_value.json.return_value = {} with self.assertRaises(restclient.ValidationError): restclient.get('http://foo.com', '/')
def test_retry_on_request_timeout(self, _): """Test retry on request timeout""" with self.assertRaises(restclient.MaxRequestRetriesError) as cm: restclient.get('http://foo.com', '/') err = cm.exception self.assertEqual(len(err.attempts), 5)
def test_retry_on_connection_error(self, _): """Test retry on connection error""" with self.assertRaises(restclient.MaxRequestRetriesError) as cm: restclient.get('http://foo.com', '/') err = cm.exception self.assertEquals(len(err.attempts), 5)
def test_get_401(self, resp_mock): """Test treadmill.restclient.get UNAUTHORIZED (401)""" resp_mock.return_value.status_code = http_client.UNAUTHORIZED resp_mock.return_value.json.return_value = {} with self.assertRaises(restclient.NotAuthorizedError): restclient.get('http://foo.com', '/')
def configure(count, name, policy): """Configure application monitor""" restapi = context.GLOBAL.cell_api() url = _REST_PATH + name options = {} if count is not None: options['count'] = count if policy is not None: options['policy'] = policy # reconfigure if any of the parameters is specified if options: existing = None try: existing = restclient.get(restapi, url).json() except restclient.NotFoundError: _LOGGER.debug('App monitor not found: %s', name) if existing is None: _check_configure_usage(count) _LOGGER.debug('Creating app monitor: %s', name) restclient.post(restapi, url, payload=options) else: existing.update(options) _LOGGER.debug('Updating app monitor: %s', name) restclient.put(restapi, url, payload=existing) _LOGGER.debug('Retrieving app monitor: %s', name) monitor_entry = restclient.get(restapi, url) cli.out(formatter(monitor_entry.json()))
def test_get_403(self, resp_mock): """Test treadmill.restclient.get FORBIDDEN (403)""" resp_mock.return_value.status_code = http_client.FORBIDDEN resp_mock.return_value.json.return_value = {} with self.assertRaises(restclient.NotAuthorizedError): restclient.get('http://foo.com', '/')
def test_retry(self): """Tests retry logic.""" with self.assertRaises(restclient.MaxRequestRetriesError) as cm: restclient.get(['http://foo.com', 'http://bar.com'], '/baz', retries=3) err = cm.exception self.assertEqual(len(err.attempts), 6) # Requests are done in order, by because other methods are being # callled, to make test simpler, any_order is set to True so that # test will pass. requests.get.assert_has_calls([ mock.call('http://foo.com/baz', json=None, proxies=None, headers=None, auth=mock.ANY, timeout=(.5, 10), stream=None), mock.call('http://bar.com/baz', json=None, proxies=None, headers=None, auth=mock.ANY, timeout=(.5, 10), stream=None), mock.call('http://foo.com/baz', json=None, proxies=None, headers=None, auth=mock.ANY, timeout=(1.5, 10), stream=None), mock.call('http://bar.com/baz', json=None, proxies=None, headers=None, auth=mock.ANY, timeout=(1.5, 10), stream=None), mock.call('http://foo.com/baz', json=None, proxies=None, headers=None, auth=mock.ANY, timeout=(2.5, 10), stream=None), mock.call('http://bar.com/baz', json=None, proxies=None, headers=None, auth=mock.ANY, timeout=(2.5, 10), stream=None), ], any_order=True) self.assertEqual(requests.get.call_count, 6)
def test_default_timeout_get(self, resp_mock): """Tests that default timeout for get request is set correctly.""" resp_mock.return_value.status_code = http.client.OK resp_mock.return_value.text = 'foo' restclient.get('http://foo.com', '/') resp_mock.assert_called_with( 'http://foo.com/', stream=None, auth=mock.ANY, headers=None, json=None, timeout=(0.5, 10), proxies=None )
def test_get_401(self, resp_mock): """Test treadmill.restclient.get UNAUTHORIZED (401)""" # XXX: Correct code in FORBIDDEN. Support invalid UNAUTHORIZED during # migration. resp_mock.return_value.status_code = http_client.UNAUTHORIZED resp_mock.return_value.json.return_value = {} with self.assertRaises(restclient.NotAuthorizedError): restclient.get('http://foo.com', '/')
def _get_app_rsrc(instance, admin_api=None, cell_api=None): """Return the application's reserved resources from the manifest.""" try: mf = restclient.get(context.GLOBAL.cell_api(cell_api), '/instance/%s' % urllib.quote(instance)).json() except restclient.NotFoundError: mf = restclient.get(context.GLOBAL.admin_api(admin_api), '/app/%s' % instance).json() return {rsrc: mf[rsrc] for rsrc in ('cpu', 'disk', 'memory') if rsrc in mf}
def _check_tenant_exists(restapi, allocation): """Check if tenant exist.""" tenant_url = '/tenant/{}'.format(allocation) # Check if tenant exists. try: restclient.get(restapi, tenant_url).json() except restclient.NotFoundError: raise click.UsageError( 'Allocation not found, ' 'run allocation configure {} --systems ...'.format(allocation))
def _display_tenant(restapi, tenant): """Display allocations for the given tenant.""" tenant_url = '/tenant/{}'.format(tenant) alloc_url = '/allocation/{}'.format(tenant) tenant_obj = restclient.get(restapi, tenant_url).json() allocations_obj = restclient.get(restapi, alloc_url).json() tenant_obj['allocations'] = allocations_obj tenant_formatter = cli.make_formatter('tenant') cli.out(tenant_formatter(tenant_obj))
def assign(allocation, env, cell, pattern, priority, delete): """Assign application pattern:priority to the allocation. Application pattern must start with <PROID>. and is a glob expression. Environments of the proid and one specified in command line using --env option must match. Once scheduled, Treadmill scheduler will match application against all available patterns and assign application to a reserved capacity. All application assigned to a capacity are ordered by priority from high to low. """ restapi = context.GLOBAL.admin_api(ctx.get('api')) _check_tenant_exists(restapi, allocation) _make_allocation(restapi, allocation, env) reservation_url = '/allocation/{}/{}/reservation/{}'.format( allocation, env, cell) try: restclient.get(restapi, reservation_url) except restclient.NotFoundError: # TODO: default partition should be resolved in API, not in CLI. restclient.post(restapi, reservation_url, payload={ 'memory': '0M', 'disk': '0M', 'cpu': '0%', 'partition': admin.DEFAULT_PARTITION }) url = '/allocation/{}/{}/assignment/{}/{}'.format( allocation, env, cell, pattern) if delete: restclient.delete(restapi, url) else: default_prio = None existing = restclient.get(restapi, url).json() for assignment in existing: if assignment['pattern'] == pattern: default_prio = assignment['priority'] if default_prio is None: default_prio = _DEFAULT_PRIORITY data = {'priority': priority if priority else default_prio} restclient.put(restapi, url, payload=data) _display_tenant(restapi, allocation)
def _display_tenant(restapi, tenant): """Display allocations for the given tenant.""" tenant_url = '/tenant/%s' % tenant alloc_url = '/allocation/%s' % tenant tenant_obj = restclient.get(restapi, tenant_url).json() allocations_obj = restclient.get(restapi, alloc_url).json() tenant_obj['allocations'] = allocations_obj tenant_formatter = cli.make_formatter(cli.TenantPrettyFormatter) cli.out(tenant_formatter(tenant_obj))
def test_verify_get(self, resp_mock): """Tests that 'verify' for get request is set correctly.""" resp_mock.return_value.status_code = http_client.OK resp_mock.return_value.text = 'foo' restclient.get('http://foo.com', '/', verify='/path/to/ca/certs') resp_mock.assert_called_with('http://foo.com/', stream=None, auth=mock.ANY, headers=None, json=None, timeout=(0.5, 10), proxies=None, verify='/path/to/ca/certs')
def assign(allocation, env, cell, pattern, priority, delete): """Assign application pattern:priority to the allocation. Application pattern must start with <PROID>. and is a glob expression. Environments of the proid and one specified in command line using --env option must match. Once scheduled, Treadmill scheduler will match application against all available patterns and assign application to a reserved capacity. All application assigned to a capacity are ordered by priority from high to low. """ restapi = context.GLOBAL.admin_api() _check_tenant_exists(restapi, allocation) _make_allocation(restapi, allocation, env) reservation_url = '/allocation/{}/{}/reservation/{}'.format( allocation, env, cell) try: restclient.get(restapi, reservation_url) except restclient.NotFoundError: restclient.post(restapi, reservation_url, payload={ 'memory': '0M', 'disk': '0M', 'cpu': '0%' }) url = '/allocation/{}/{}/assignment/{}/{}'.format( allocation, env, cell, pattern) if delete: restclient.delete(restapi, url) else: data = {} if priority: data['priority'] = priority existing = restclient.get(restapi, url).json() rest_func = restclient.post for assignment in existing: if assignment['pattern'] == pattern: rest_func = restclient.put break rest_func(restapi, url, payload=data) _display_tenant(restapi, allocation)
def _health_check(pattern, proto, endpoint, command): """Invoke instance health check.""" stateapi = context.GLOBAL.state_api() stateurl = '/endpoint/%s/%s/%s' % (urllib.parse.quote(pattern), proto, endpoint) response = restclient.get(stateapi, stateurl) lines = [ '%s %s' % (end['name'], '%s:%s' % (end['host'], end['port'])) for end in response.json() ] cmd_input = '\n'.join(lines) bad = [] try: proc = subprocess.Popen( command, close_fds=_CLOSE_FDS, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) (out, _err) = proc.communicate(cmd_input) retcode = proc.returncode if proc.returncode == 0: for instance in out.splitlines(): _LOGGER.info('not ok: %s', instance) bad.append(instance) else: _LOGGER.warn('Health check ignored. %r, rc: %s.', command, retcode) except Exception: # pylint: disable=W0703 _LOGGER.exception('Error invoking: %r', command) return bad
def configure(tenant, systems, name, env): """Configure allocation tenant.""" restapi = context.GLOBAL.admin_api(ctx.get('api')) tenant_url = '/tenant/%s' % tenant if systems: try: existing = restclient.get(restapi, tenant_url).json() all_systems = set(existing['systems']) all_systems.update(map(int, systems)) restclient.put(restapi, tenant_url, payload={'systems': list(all_systems)}) except restclient.NotFoundError: restclient.post(restapi, tenant_url, payload={'systems': map(int, systems)}) if env: if name is None: name = env alloc_url = '/allocation/%s/%s' % (tenant, name) try: restclient.post(restapi, alloc_url, payload={'environment': env}) except restclient.AlreadyExistsError: pass _display_tenant(restapi, tenant)
def configure(job_id, event, resource, expression, count): """Create or modify an existing app start schedule""" restapi = context.GLOBAL.cell_api(ctx['api']) url = _REST_PATH + job_id data = {} if event: data['event'] = event if resource: data['resource'] = resource if expression: data['expression'] = expression if count is not None: data['count'] = count if data: try: _LOGGER.debug('Creating cron job: %s', job_id) restclient.post(restapi, url, payload=data) except restclient.AlreadyExistsError: _LOGGER.debug('Updating cron job: %s', job_id) restclient.put(restapi, url, payload=data) _LOGGER.debug('Retrieving cron job: %s', job_id) job = restclient.get(restapi, url).json() _LOGGER.debug('job: %r', job) cli.out(_FORMATTER(job))
def ssh(wsapi, api, ssh, app, command, wait): """SSH into Treadmill container.""" if ssh is None: ssh = _DEFAULT_SSH else: ssh = ssh.name if wait: return _wait_for_app(wsapi, ssh, app, command) apis = context.GLOBAL.state_api(api) url = '/endpoint/{}/tcp/ssh'.format(urllib.quote(app)) response = restclient.get(apis, url) endpoints = response.json() _LOGGER.debug('endpoints: %r', endpoints) if not endpoints: cli.bad_exit('No ssh endpoint(s) found for %s', app) # Take the first one, if there are more than one, then this is # consistent with when 1 is returned. endpoint = endpoints[0] run_ssh(endpoint['host'], str(endpoint['port']), ssh, list(command))
def logs(api, app_or_svc, host, service, uniq, ws_api): """View application's service logs. Arguments are expected to be specified a) either as one string or b) parts defined one-by-one ie.: a) <appname>/<uniq or running>/service/<servicename> b) <appname> --uniq <uniq> --service <servicename> Eg.: a) proid.foo#1234/xz9474as8/service/my-echo b) proid.foo#1234 --uniq xz9474as8 --service my-echo For the latest log simply omit 'uniq': proid.foo#1234 --service my-echo """ try: app, uniq, logtype, logname = app_or_svc.split('/', 3) except ValueError: app, uniq, logtype, logname = app_or_svc, uniq, 'service', service if logname is None: cli.bad_exit("Please specify the 'service' parameter.") if host is None: instance = None if uniq == 'running': instance = _find_running_instance(app, ws_api) if not instance: instance = _find_uniq_instance(app, uniq, ws_api) if not instance: cli.bad_exit('No {}instance could be found.'.format( 'running ' if uniq == 'running' else '')) _LOGGER.debug('Found instance: %s', instance) host = instance['host'] uniq = instance['uniq'] try: endpoint, = (ep for ep in _find_endpoints( urllib.parse.quote('root.*'), 'tcp', 'nodeinfo', api) if ep['host'] == host) except ValueError as err: _LOGGER.exception(err) cli.bad_exit('No endpoint found on %s', host) api = 'http://{0}:{1}'.format(endpoint['host'], endpoint['port']) logurl = '/app/%s/%s/%s/%s' % (urllib.parse.quote(app), urllib.parse.quote(uniq), logtype, urllib.parse.quote(logname)) log = restclient.get(api, logurl) click.echo(log.text)
def _count(cell, appname): """Get number of instances scheduled/running on the cell.""" try: ctx = context.Context() ctx.cell = cell ctx.dns_domain = context.GLOBAL.dns_domain stateapi = ctx.state_api() url = '/state/?' + urllib.urlencode([('match', appname)]) response = restclient.get(stateapi, url) state = response.json() for instance in state: _LOGGER.info('cell: %s - %s %s %s', cell, instance['name'], instance['state'], instance['host']) return len([instance for instance in state if instance['state'] == 'running']) except Exception: # pylint: disable=W0703 _LOGGER.exception('Unable to get instance count for cell %s, app: %s', cell, appname) return 0
def configure(name, cell, pattern, endpoints, alias, scope): """Create, modify or get Treadmill App DNS entry""" restapi = context.GLOBAL.admin_api() url = _REST_PATH + name data = {} if cell: data['cells'] = cell if pattern is not None: data['pattern'] = pattern if endpoints is not None: data['endpoints'] = endpoints if alias is not None: data['alias'] = alias if scope is not None: data['scope'] = scope if data: try: _LOGGER.debug('Trying to create app-dns entry %s', name) restclient.post(restapi, url, data) except restclient.AlreadyExistsError: _LOGGER.debug('Updating app-dns entry %s', name) restclient.put(restapi, url, data) _LOGGER.debug('Retrieving App DNS entry %s', name) app_dns_entry = restclient.get(restapi, url).json() cli.out(formatter(app_dns_entry))
def _list(cell, partition): """List all servers.""" query = {'cell': cell} if partition is not None: query['partition'] = partition url = '/server/?{}'.format(urllib_parse.urlencode(query)) restapi = context.GLOBAL.admin_api(ctx.get('api')) cli.out(server_formatter(restclient.get(restapi, url).json()))
def _list(): """List out all cron events""" restapi = context.GLOBAL.cell_api(ctx['api']) response = restclient.get(restapi, _REST_PATH) jobs = response.json() _LOGGER.debug('jobs: %r', jobs) cli.out(_FORMATTER(jobs))
def configure(name): """Display details of the server.""" restapi = context.GLOBAL.admin_api() cli.out( server_formatter( restclient.get(restapi, '/server/{}'.format(name)).json() ) )
def _show_endpoints(apis, pattern, endpoint): """Show cell endpoints.""" url = '/v3/endpoint/%s' % urllib.quote(pattern) if endpoint: url += '/' + endpoint response = restclient.get(apis, url) cli.out(_ENDPOINT_FORMATTER(response.json()))
def _show_state(apis, match): """Show cell state.""" url = '/v3/state/' if match: url += '?' + urllib.urlencode([('match', match)]) response = restclient.get(apis, url) cli.out(_STATE_FORMATTER(response.json()))
def reserve(allocation, env, cell, partition, rank, rank_adjustment, max_utilization, empty, memory, cpu, disk, traits): """Reserve capacity on the cell for given environment.""" _check_reserve_usage(empty, memory, cpu, disk) restapi = context.GLOBAL.admin_api(ctx.get('api')) _check_tenant_exists(restapi, allocation) _make_allocation(restapi, allocation, env) data = {} if empty: data['memory'] = '0M' data['disk'] = '0M' data['cpu'] = '0%' if memory: data['memory'] = memory if cpu: data['cpu'] = cpu if disk: data['disk'] = disk if rank is not None: data['rank'] = rank if rank_adjustment is not None: data['rank_adjustment'] = rank_adjustment if max_utilization is not None: data['max_utilization'] = max_utilization if partition: data['partition'] = partition if traits: data['traits'] = cli.combine(traits) if data: reservation_url = '/allocation/{}/{}/reservation/{}'.format( allocation, env, cell) try: existing = restclient.get(restapi, reservation_url).json() # TODO: need cleaner way of deleting attributes that are not # valid for update. It is a hack. for attr in list(existing): if (attr not in ['memory', 'cpu', 'disk', 'partition']): del existing[attr] existing.update(data) restclient.put(restapi, reservation_url, payload=existing) except restclient.NotFoundError: # some attributes need default values when creating if not partition: data['partition'] = admin.DEFAULT_PARTITION restclient.post(restapi, reservation_url, payload=data) _display_tenant(restapi, allocation)
def test_get_ok(self, resp_mock): """Test treadmill.restclient.get OK (200)""" resp_mock.return_value.status_code = http_client.OK resp_mock.return_value.text = 'foo' resp = restclient.get('http://foo.com', '/') self.assertIsNotNone(resp) self.assertEqual(resp.text, 'foo')