def test_adjust_apps_request_path_no_aaditional_paths(self):
with application.test_request_context("/v2/apps/my-app",
method="GET") as ctx:
request_wrapper = Request(ctx.request)
original_app = MarathonApp(id="/dev/my-app")
request_wrapper._adjust_request_path_if_needed(
request_wrapper.request, original_app)
self.assertEqual("/v2/apps/dev/my-app",
request_wrapper.request.path)
def test_adjust_apps_versions_request_path(self):
with application.test_request_context('/v2/apps/my-app/versions',
method='GET') as ctx:
request_wrapper = Request(ctx.request)
original_app = MarathonApp(id="/dev/my-app")
request_wrapper._adjust_request_path_if_needed(
request_wrapper.request, original_app)
self.assertEqual("/v2/apps/dev/my-app/versions",
request_wrapper.request.path)
def test_adjust_apps_tasks_DELETE_request_path(self):
with application.test_request_context("/v2/apps/my-app/tasks",
method="DELETE") as ctx:
request_wrapper = Request(ctx.request)
original_app = MarathonApp(id="/dev/my-app")
request_wrapper._adjust_request_path_if_needed(
request_wrapper.request, original_app)
self.assertEqual("/v2/apps/dev/my-app/tasks",
request_wrapper.request.path)
def test_adjust_apps_request_path_repeating_non_final_path(self):
with application.test_request_context('/v2/apps/app0/app0/app1',
method='GET') as ctx:
request_wrapper = Request(ctx.request)
original_app = MarathonApp(id="/dev/app0/app0/app1")
request_wrapper._adjust_request_path_if_needed(
request_wrapper.request, original_app)
self.assertEqual("/v2/apps/dev/app0/app0/app1",
request_wrapper.request.path)
def run_paasta_metastatus_high_disk(context, app_id):
context.marathon_client.create_app(
app_id,
MarathonApp(
cmd='/bin/sleep 100000',
disk=95,
instances=3,
container=CONTAINER,
),
)
def run_paasta_metastatus_high_cpu(context, app_id):
context.marathon_clients.current[0].create_app(
app_id,
MarathonApp(
cmd='/bin/sleep 100000',
cpus=9.1,
instances=3,
container=CONTAINER,
),
)
def setUp(self, single_full_app_fixture):
self.request_apps = [
(MarathonApp(id='/xablau'), MarathonApp(id='/xena')),
(MarathonApp(id='/foo'), MarathonApp(id='/bar')),
]
rebuild_schema()
self.session = HollowmanSession()
self.user = User(tx_email="*****@*****.**",
tx_name="John Doe",
tx_authkey="69ed620926be4067a36402c3f7e9ddf0")
self.account_dev = Account(id=4,
name="Dev Team",
namespace="dev",
owner="company")
self.user.accounts = [self.account_dev]
self.session.add(self.user)
self.session.add(self.account_dev)
self.session.commit()
def test_split_groups_read_on_root_group(self,
group_dev_namespace_fixture):
with application.test_request_context("/v2/groups",
method="GET") as ctx:
ctx.request.user = self.user
request_parser = Request(ctx.request)
with RequestsMock() as rsps:
rsps.add(
method="GET",
url=conf.MARATHON_ADDRESSES[0] + "/v2/groups//dev/",
body=json.dumps(group_dev_namespace_fixture),
status=200,
)
apps = list(request_parser.split())
self.assertEqual(3, len(apps))
original_app_one = MarathonApp.from_json({"id": "/dev/a/app0"})
original_app_two = MarathonApp.from_json(
{"id": "/dev/group-b/appb0"})
original_app_three = MarathonApp.from_json(
{"id": "/dev/group-b/group-b0/app0"})
expected_apps = [
(
request_parser.merge_marathon_apps(
MarathonApp(), original_app_one),
original_app_one,
),
(
request_parser.merge_marathon_apps(
MarathonApp(), original_app_two),
original_app_two,
),
(
request_parser.merge_marathon_apps(
MarathonApp(), original_app_three),
original_app_three,
),
]
self.assertEqual(apps, expected_apps)
def test_adjust_apps_request_path_keep_additional_paths_multiple_paths(
self):
with application.test_request_context(
'/v2/apps/my-app/versions/2017-10-31T13:01:07.768Z',
method='GET') as ctx:
request_wrapper = Request(ctx.request)
original_app = MarathonApp(id="/dev/my-app")
request_wrapper._adjust_request_path_if_needed(
request_wrapper.request, original_app)
self.assertEqual(
"/v2/apps/dev/my-app/versions/2017-10-31T13:01:07.768Z",
request_wrapper.request.path)
def __init__(self,
scheduler,
executable='dask-worker',
docker_image='mrocklin/dask-distributed',
marathon_address='http://localhost:8080',
username=None,
password=None,
auth_token=None,
app_name=None,
**kwargs):
self.scheduler = scheduler
self.executor = ThreadPoolExecutor(1)
# Create Marathon App to run dask-worker
args = [
executable,
scheduler.address,
'--name',
'$MESOS_TASK_ID', # use Mesos task ID as worker name
'--worker-port',
'$PORT_WORKER',
'--nanny-port',
'$PORT_NANNY',
'--http-port',
'$PORT_HTTP'
]
ports = [{
'port': 0,
'protocol': 'tcp',
'name': port_name
} for port_name in ['worker', 'nanny', 'http']]
if 'mem' in kwargs:
args.extend(
['--memory-limit',
str(int(kwargs['mem'] * 0.6 * 1e6))])
kwargs['cmd'] = ' '.join(args)
container = MarathonContainer({'image': docker_image})
app = MarathonApp(instances=0,
container=container,
port_definitions=ports,
**kwargs)
# Connect and register app
self.client = MarathonClient(servers=marathon_address,
username=username,
password=password,
auth_token=auth_token)
self.app = self.client.create_app(app_name or 'dask-%s' % uuid.uuid4(),
app)
def register_services(self, service_registry="conf/marathon"):
for app_def in glob.glob(os.path.join(service_registry, "*json")):
with open(app_def, "r") as stream:
args = json.loads(stream.read())
app_id = args['id']
args = Names.snake_case(args)
logger.debug("Creating service: %s", json.dumps(args,
indent=2))
args['tasks'] = []
app = MarathonApp(**args)
try:
logging.info("Creating app [id=>{0}]".format(app_id))
self.marathon.create_app(app_id, app)
except:
traceback.print_exc()
def test_a_read_single_app_request_returns_a_single_marathonapp_if_app_exists(
self, fixture):
with application.test_request_context('/v2/apps//foo',
method='GET',
data=b'') as ctx:
ctx.request.user = self.user
request_parser = Request(ctx.request)
with patch.object(request_parser, 'marathon_client') as client:
original_app = MarathonApp.from_json(fixture)
client.get_app.return_value = original_app
apps = list(request_parser.split())
self.assertEqual(apps, [(request_parser.merge_marathon_apps(
MarathonApp(), original_app), client.get_app.return_value)])
def get_marathon_app(self):
env = self.get_env()
for key, value in env.items():
if isinstance(value, unicode):
env[key] = value.encode('utf-8')
return MarathonApp(args=self.get_args(),
container=self.get_docker_container(),
constraints=self.get_constraints(),
dependencies=self.get_dependencies(),
env=env,
uris=self.get_uris(),
labels=self.get_labels(),
cpus=self.get_cpus(),
mem=self.get_mem(),
instances=self.get_instances(),
upgrade_strategy=self.get_upgrade_strategy(),
health_checks=self.get_health_checks())
def test_a_request_for_restart_operation_with_appid_in_url_path_returns_a_tuple_of_marathonapp(
self, fixture):
with application.test_request_context('/v2/apps/xablau/restart',
method='PUT',
data=b'{"force": true}') as ctx:
ctx.request.user = self.user
request_parser = Request(ctx.request)
with RequestsMock() as rsps:
rsps.add(method='GET',
url=conf.MARATHON_ADDRESSES[0] +
'/v2/apps//dev/xablau',
body=json.dumps({'app': fixture}),
status=200)
apps = list(request_parser.split())
original_app = MarathonApp.from_json(fixture)
expected_app = (request_parser.merge_marathon_apps(
MarathonApp(), original_app), original_app)
self.assertEqual(apps, [expected_app])
def start(self, nworkers=0):
# address = self.scheduler.address.replace('tcp://', '')
args = ['dask-worker', self.scheduler.address,
'--name', '$MESOS_TASK_ID', # use Mesos task ID as worker name
'--worker-port', '$PORT_WORKER',
'--bokeh-port', '$PORT_BOKEH',
'--nanny-port', '$PORT_NANNY',
'--nprocs', str(self.nprocs),
'--nthreads', str(self.nthreads)]
ports = [{'port': 0,
'protocol': 'tcp',
'name': name}
for name in ['worker', 'nanny', 'http', 'bokeh']]
# healths = [{'portIndex': i,
# 'protocol': 'TCP',
# 'gracePeriodSeconds': 300,
# 'intervalSeconds': 60,
# 'timeoutSeconds': 20,
# 'maxConsecutiveFailures': 3}
# for i, name in enumerate(['worker', 'nanny', 'http', 'bokeh'])]
healths = []
if 'mem' in self.options:
args.extend(['--memory-limit',
str(int(self.options['mem'] * 0.8 * 1e6))])
docker_parameters = [{"key": "volume", "value": v}
for v in self.volumes]
container = MarathonContainer({'image': self.docker,
'forcePullImage': True,
'parameters': docker_parameters})
command = ' '.join(args)
app = MarathonApp(instances=nworkers, container=container,
port_definitions=ports, cmd=command,
health_checks=healths,
**self.options)
self.client.update_app(self.name, app)
logger.info('Started marathon workers {}'.format(self.name))
def test_can_read_app_if_already_migrated(self, single_full_app_fixture):
"""
Conferimos que é possível fazer um GET em
/v2/apps/<app-id> para uma app que já está migrada.
O <app-id> usado é sempre *sem* namespace
"""
request_data = deepcopy(single_full_app_fixture)
single_full_app_fixture['id'] = '/dev/foo'
with application.test_request_context('/v2/apps/foo',
method='GET') as ctx:
ctx.request.user = self.user
request_parser = Request(ctx.request)
with RequestsMock() as rsps:
rsps.add(method='GET',
url=conf.MARATHON_ADDRESSES[0] + '/v2/apps//dev/foo',
body=json.dumps({'app': single_full_app_fixture}),
status=200)
apps = list(request_parser.split())
original_app = MarathonApp.from_json(single_full_app_fixture)
expected_app = (request_parser.merge_marathon_apps(
MarathonApp(), original_app), original_app)
self.assertEqual(apps, [expected_app])
def deploy(marathon, name, docker, volume, scheduler_cpus, scheduler_mem,
adaptive, port, bokeh_port, constraint, maximum_over_capacity,
minimum_health_capacity, label, uri, jupyter, **kwargs):
name = name or 'daskathon-{}'.format(str(uuid.uuid4())[-4:])
kwargs['name'] = '{}-workers'.format(name)
kwargs['docker'] = docker
kwargs['port'] = port
kwargs['bokeh_port'] = bokeh_port
args = [('--{}'.format(k.replace('_', '-')), str(v))
for k, v in kwargs.items() if v not in (None, '')]
for c in constraint:
args.append(('--constraint', c))
for u in uri:
args.append(('--uri', u))
for v in volume:
args.append(('--volume', v))
if maximum_over_capacity:
args.append(('--maximum-over-capacity', str(maximum_over_capacity)))
if minimum_health_capacity:
args.append(
('--minimum-health-capacity', str(minimum_health_capacity)))
args = list(concat(args))
if adaptive:
args.append('--adaptive')
client = MarathonClient(marathon)
docker_parameters = [{"key": "volume", "value": v} for v in volume]
container = MarathonContainer({
'image': docker,
'forcePullImage': True,
'parameters': docker_parameters
})
args = ['daskathon', 'run'] + args + [marathon]
cmd = ' '.join(args)
healths = [{
'portIndex': i,
'protocol': 'TCP'
} for i, _ in enumerate(['scheduler', 'bokeh'])]
services = [('scheduler', port), ('bokeh', bokeh_port)]
ports = [{
'port': p,
'protocol': 'tcp',
'name': service
} for (service, p) in services]
constraints = [c.split(':')[:3] for c in constraint]
labels = dict([l.split(':') for l in label])
upgrade_strategy = {
'maximum_over_capacity': maximum_over_capacity,
'minimum_health_capacity': minimum_health_capacity
}
scheduler = MarathonApp(instances=1,
container=container,
cpus=scheduler_cpus,
mem=scheduler_mem,
task_kill_grace_period_seconds=20,
port_definitions=ports,
health_checks=healths,
constraints=constraints,
upgrade_strategy=upgrade_strategy,
labels=labels,
uris=uri,
require_ports=True,
cmd=cmd)
client.update_app('{}-scheduler'.format(name), scheduler)
if jupyter:
cmd = ('jupyter notebook --allow-root --no-browser '
'--NotebookApp.token=\'\' --ip 0.0.0.0 --port $PORT_NOTEBOOK')
ports = [{'port': 0, 'protocol': 'tcp', 'name': 'notebook'}]
jupyter = deepcopy(scheduler)
jupyter.cmd = cmd
jupyter.port_definitions = ports
client.update_app('{}-jupyter'.format(name), jupyter)
def run_paasta_metastatus_high_mem(context, app_id):
context.marathon_clients.current[0].create_app(
app_id,
MarathonApp(cmd="/bin/sleep 100000", mem=490, instances=3, container=CONTAINER),
)
def run_paasta_metastatus_high_cpu(context, app_id):
context.marathon_client.create_app(
app_id, MarathonApp(cmd='/bin/sleep infinity', cpus=9, instances=1))
def get_marathon_app(app_json):
app_dict = json.loads(app_json)
app_marathon = MarathonApp()
for key, value in app_dict.items():
setattr(app_marathon, key, value)
return app_marathon