def start_deployd(context):
try:
os.makedirs('/nail/etc/services')
except OSError as e:
if e.errno == errno.EEXIST:
pass
with ZookeeperPool() as zk:
try:
zk.create('/autoscaling')
except NodeExistsError:
pass
context.zk_hosts = '%s/mesos-testcluster' % get_service_connection_string(
'zookeeper')
context.soa_dir = '/nail/etc/services'
if not hasattr(context, 'daemon'):
context.daemon = Popen('paasta-deployd', stderr=PIPE)
output = context.daemon.stderr.readline().decode('utf-8')
start = time.time()
timeout = start + 60
while "Startup finished!" not in output:
output = context.daemon.stderr.readline().decode('utf-8')
if not output:
raise Exception("deployd exited prematurely")
print(output.rstrip('\n'))
if time.time() > timeout:
raise Exception("deployd never ran")
time.sleep(5)
def check_mesos_utilization(context, percent):
config = {"master": "%s" % get_service_connection_string("mesosmaster"), "scheme": "http"}
with contextlib.nested(
mock.patch("paasta_tools.check_mesos_resource_utilization.send_event"),
mock.patch.object(mesos.cli.master, "CFG", config),
) as (mock_events, mock_cfg):
context.mesos_util_check = check_mesos_resource_utilization.check_thresholds(int(percent))
def zookeeper_rmr_keys(context):
context.zk_hosts = "%s/mesos-testcluster" % get_service_connection_string(
"zookeeper"
)
with mock.patch.object(
SystemPaastaConfig, "get_zk_hosts", autospec=True, return_value=context.zk_hosts
):
with ZookeeperPool() as zookeeper_client:
zookeeper_client.delete("/autoscaling/test-service", recursive=True)
def create_app_with_instances(context, job_id, number):
set_number_instances(context, number)
context.job_id = job_id
(service, instance, _, __) = decompose_job_id(job_id)
context.service = service
context.instance = instance
context.zk_hosts = '%s/mesos-testcluster' % get_service_connection_string('zookeeper')
update_context_marathon_config(context)
context.app_id = context.marathon_complete_config['id']
run_setup_marathon_job(context)
def _clean_up_zookeeper_autoscaling(context):
"""If max_instances was set for autoscaling, clean up zookeeper"""
client = KazooClient(hosts='%s/mesos-testcluster' % get_service_connection_string('zookeeper'), read_only=True)
client.start()
try:
client.delete('/autoscaling', recursive=True)
except NoNodeError:
pass
client.stop()
client.close()
def when_setup_service_initiated(context):
config = {
'master': '%s' % get_service_connection_string('mesosmaster'),
'scheme': 'http',
'response_timeout': 5,
}
with contextlib.nested(
mock.patch(
'paasta_tools.bounce_lib.get_happy_tasks',
autospec=True,
# Wrap function call so we can select a subset of tasks or test
# intermediate steps, like when an app is not completely up
side_effect=lambda app, _, __, ___, **kwargs: get_happy_tasks(
app, context.service, "fake_nerve_ns", context.system_paasta_config)[:context.max_tasks],
),
mock.patch('paasta_tools.bounce_lib.bounce_lock_zookeeper', autospec=True),
mock.patch('paasta_tools.bounce_lib.create_app_lock', autospec=True),
mock.patch('paasta_tools.bounce_lib.time.sleep', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.marathon_tools.get_config_hash', autospec=True, return_value='confighash'),
mock.patch('paasta_tools.marathon_tools.get_code_sha_from_dockerurl', autospec=True, return_value='newapp'),
mock.patch('paasta_tools.marathon_tools.get_docker_url', autospec=True, return_value='busybox'),
mock.patch('paasta_tools.paasta_maintenance.load_credentials', autospec=True),
mock.patch.object(mesos.cli.master, 'CFG', config),
) as (
_,
_,
_,
_,
mock_load_system_paasta_config,
_,
_,
_,
_,
mock_load_credentials,
_,
):
mock_load_credentials.side_effect = paasta_maintenance.load_credentials(mesos_secrets='/etc/mesos-slave-secret')
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value=context.cluster)
# 120 * 0.5 = 60 seconds
for _ in xrange(120):
try:
(code, message) = setup_marathon_job.setup_service(
service=context.service,
instance=context.instance,
client=context.marathon_client,
service_marathon_config=context.new_marathon_service_config,
soa_dir='/nail/etc/services',
)
assert code == 0, message
return
except MarathonHttpError:
time.sleep(0.5)
raise Exception("Unable to acquire app lock for setup_marathon_job.setup_service")
def _clean_up_zookeeper_autoscaling(context):
"""If max_instances was set for autoscaling, clean up zookeeper"""
if 'max_instances' in context:
client = KazooClient(hosts='%s/mesos-testcluster' % get_service_connection_string('zookeeper'), read_only=True)
client.start()
try:
client.delete('/autoscaling', recursive=True)
except NoNodeError:
pass
client.stop()
client.close()
def create_app_with_instances_constraints(context, job_id, number, constraints):
set_number_instances(context, number)
context.job_id = job_id
(service, instance, _, __) = decompose_job_id(job_id)
context.service = service
context.instance = instance
context.zk_hosts = "%s/mesos-testcluster" % get_service_connection_string("zookeeper")
context.constraints = constraints
update_context_marathon_config(context)
context.app_id = context.marathon_complete_config["id"]
run_setup_marathon_job(context)
def delete_apps(context, job_id, cluster_name):
context.job_id = job_id
(service, instance, _, __) = decompose_job_id(job_id)
context.service = service
context.instance = instance
context.zk_hosts = "%s/mesos-testcluster" % get_service_connection_string(
"zookeeper")
update_context_marathon_config(context)
context.app_id = context.marathon_complete_config["id"]
os.remove(f"{context.soa_dir}/{service}/marathon-{cluster_name}.yaml")
os.remove(f"{context.soa_dir}/{service}/deployments.json")
os.rmdir(f"{context.soa_dir}/{service}")
def delete_apps(context, job_id, cluster_name):
context.job_id = job_id
(service, instance, _, __) = decompose_job_id(job_id)
context.service = service
context.instance = instance
context.zk_hosts = '%s/mesos-testcluster' % get_service_connection_string('zookeeper')
update_context_marathon_config(context)
context.app_id = context.marathon_complete_config['id']
os.remove("{0}/{1}/marathon-{2}.yaml".format(context.soa_dir, service,
cluster_name))
os.remove("{0}/{1}/deployments.json".format(context.soa_dir, service,
cluster_name))
os.rmdir("{0}/{1}".format(context.soa_dir, service))
def create_app_with_instances_constraints(context, job_id, number, constraints, no_apps_running=False):
set_number_instances(context, number)
context.job_id = job_id
(service, instance, _, __) = decompose_job_id(job_id)
context.service = service
context.instance = instance
context.zk_hosts = '%s/mesos-testcluster' % get_service_connection_string('zookeeper')
context.constraints = constraints
update_context_marathon_config(context)
context.app_id = context.marathon_complete_config['id']
if no_apps_running:
run_setup_marathon_job_no_apps_found(context)
else:
run_setup_marathon_job(context)
def check_mesos_utilization(context, percent):
config = {
"master": "%s" % get_service_connection_string('mesosmaster'),
"scheme": "http"
}
with contextlib.nested(
mock.patch('paasta_tools.check_mesos_resource_utilization.send_event'),
mock.patch.object(mesos.cli.master, 'CFG', config),
) as (
mock_events,
mock_cfg,
):
context.mesos_util_check = check_mesos_resource_utilization.check_thresholds(int(percent))
def check_mesos_utilization(context, percent):
config = {
"master": "%s" % get_service_connection_string('mesosmaster'),
"scheme": "http",
"response_timeout": 5,
}
with contextlib.nested(
mock.patch('paasta_tools.check_mesos_resource_utilization.send_event'),
mock.patch.object(mesos.cli.master, 'CFG', config),
) as (
mock_events,
mock_cfg,
):
context.mesos_util_check = check_mesos_resource_utilization.check_thresholds(int(percent))
def mark_host_at_risk(context, host):
start = paasta_maintenance.datetime_to_nanoseconds(paasta_maintenance.now())
duration = paasta_maintenance.parse_timedelta('1h')
config = {
'master': '%s' % get_service_connection_string('mesosmaster'),
'scheme': 'http',
'response_timeout': 5,
}
with contextlib.nested(
mock.patch('paasta_tools.paasta_maintenance.load_credentials', autospec=True),
mock.patch.object(mesos.cli.master, 'CFG', config),
) as (
mock_load_credentials,
_,
):
mock_load_credentials.side_effect = paasta_maintenance.load_credentials(mesos_secrets='/etc/mesos-slave-secret')
paasta_maintenance.drain([host], start, duration)
context.at_risk_host = host
def start_deployd(context):
try:
os.makedirs("/nail/etc/services")
except OSError as e:
if e.errno == errno.EEXIST:
pass
with ZookeeperPool() as zk:
try:
zk.create("/autoscaling")
except NodeExistsError:
pass
context.zk_hosts = "%s/mesos-testcluster" % get_service_connection_string(
"zookeeper")
context.soa_dir = "/nail/etc/services"
if not hasattr(context, "daemon"):
context.daemon = Popen("paasta-deployd", stderr=PIPE)
output = context.daemon.stderr.readline().decode("utf-8")
start = time.time()
timeout = start + 60
while "Startup finished!" not in output:
output = context.daemon.stderr.readline().decode("utf-8")
if not output:
raise Exception("deployd exited prematurely")
print(output.rstrip("\n"))
if time.time() > timeout:
raise Exception("deployd never ran")
context.num_workers_crashed = 0
def dont_let_stderr_buffer():
while True:
line = context.daemon.stderr.readline()
if not line:
return
if DEAD_DEPLOYD_WORKER_MESSAGE.encode("utf-8") in line:
context.num_workers_crashed += 1
paasta_print(f"deployd stderr: {line}")
threading.Thread(target=dont_let_stderr_buffer).start()
time.sleep(5)
def create_app_with_instances_constraints(context,
job_id,
number,
constraints,
no_apps_running=False):
set_number_instances(context, number)
context.job_id = job_id
(service, instance, _, __) = decompose_job_id(job_id)
context.service = service
context.instance = instance
context.zk_hosts = "%s/mesos-testcluster" % get_service_connection_string(
"zookeeper")
context.constraints = constraints
update_context_marathon_config(context)
context.app_id = context.marathon_complete_config["id"]
context.new_id = (
context.app_id
) # for compatibility with bounces_steps.there_are_num_which_tasks
if no_apps_running:
run_setup_marathon_job_no_apps_found(context)
else:
run_setup_marathon_job(context)
def run_until_number_tasks(context, number):
config = {
'master': '%s' % get_service_connection_string('mesosmaster'),
'scheme': 'http',
'response_timeout': 5,
}
for _ in xrange(20):
with contextlib.nested(
mock.patch('paasta_tools.paasta_maintenance.load_credentials', autospec=True),
mock.patch.object(mesos.cli.master, 'CFG', config),
) as (
mock_load_credentials,
_,
):
mock_load_credentials.side_effect = paasta_maintenance.load_credentials(
mesos_secrets='/etc/mesos-slave-secret',
)
run_setup_marathon_job(context)
sleep(0.5)
if context.marathon_client.get_app(context.app_id).instances == number:
return
assert context.marathon_client.get_app(context.app_id).instances == number
def _get_chronos_connection_string():
return 'http://%s' % get_service_connection_string('chronos')
def _get_marathon_connection_string(service='marathon'):
return 'http://%s' % get_service_connection_string(service)
def get_paasta_api_url():
return "http://%s/%s" % (get_service_connection_string('api'),
'swagger.json')
def setup_chronos_client():
connection_string = get_service_connection_string('chronos')
return chronos.connect(connection_string, scheduler_api_version=None)
def setup_paasta_api_client():
return SwaggerClient.from_url(
"http://%s/%s" %
(get_service_connection_string('api'), 'swagger.json'))
def a_working_hacheck_container(context):
connection_string = get_service_connection_string('hacheck')
context.hacheck_host, context.hacheck_port = connection_string.split(':')
context.hacheck_port = int(context.hacheck_port)
def get_paasta_api_url():
return "http://{}/{}".format(get_service_connection_string('api'),
'swagger.json')
def _get_zookeeper_connection_string(chroot):
return 'zk://%s/%s' % (get_service_connection_string('zookeeper'), chroot)
def _get_chronos_connection_string():
return 'http://%s' % get_service_connection_string('chronos')
def _get_marathon_connection_string():
return 'http://%s' % get_service_connection_string('marathon')
def a_working_httpdrain_container(context):
connection_string = get_service_connection_string("httpdrain")
context.hacheck_host, context.hacheck_port = connection_string.split(":")
context.hacheck_port = int(context.hacheck_port)
def get_paasta_api_url():
return "http://{}/{}".format(get_service_connection_string("api"), "swagger.json")
def _get_chronos_connection_string():
return "http://%s" % get_service_connection_string("chronos")
def _get_zookeeper_connection_string(chroot):
return 'zk://%s/%s' % (get_service_connection_string('zookeeper'), chroot)
def _get_zookeeper_connection_string(chroot):
return 'zk://{}/{}'.format(get_service_connection_string('zookeeper'),
chroot)
def setup_chronos_client():
connection_string = get_service_connection_string('chronos')
return chronos.connect(connection_string)
def _get_marathon_connection_string(service="marathon"):
return "http://%s" % get_service_connection_string(service)
def setup_chronos_client():
connection_string = get_service_connection_string('chronos')
return chronos.connect(connection_string)
def a_working_hacheck_container(context):
connection_string = get_service_connection_string('hacheck')
context.hacheck_host, context.hacheck_port = connection_string.split(':')
context.hacheck_port = int(context.hacheck_port)
def _get_zookeeper_connection_string(chroot):
return "zk://{}/{}".format(get_service_connection_string("zookeeper"), chroot)