def _connect(self):
# Start the local load balancer in front of Marathon
service = Service(
'marathon.local', 'marathon:%s' % self._urls, self._socketpath, 'unix',
'http', healthcheck=True, healthcheckurl='/ping')
for url in self._urls:
parsed = urlparse(url)
# Resolve hostnames since HAproxy wants IP addresses
ipaddr = socket.gethostbyname(parsed.hostname or '127.0.0.1')
server = Server(ipaddr, parsed.port or 80)
service._add(server)
self._backend.update(self._marathonService, {self._socketpath: service})
def testTimeout(self):
"""
Verifies that client and server timeouts can be overridden
"""
services = {
'1234/tcp':
Service('example.demo',
self,
1234,
'tcp',
timeoutclient='300',
timeoutserver='500').addServer(
Server('1.2.3.4', 31001, 'worker1'))
}
expected = """# example.demo (testTimeout (proxymatic.test.haproxy_test.HAproxyTest))
listen demo.example-1234
bind 0.0.0.0:1234
balance leastconn
mode tcp
timeout client 300s
timeout server 500s
default-server inter 15s
server backend-worker1-31001 1.2.3.4:31001 weight 128
"""
self._check(services, expected)
def _parse(self, content):
services = {}
state = json.loads(content)
for node in util.rget(state, 'node', 'nodes') or []:
for backend in util.rget(node, 'nodes') or []:
try:
parts = backend['key'].split(':')
port = int(parts[2])
protocol = parts[3] if len(parts) > 3 else 'tcp'
key = '%s/%s' % (port, protocol.lower())
# Resolve hostnames since HAproxy wants IP addresses
endpoint = backend['value'].split(':')
ipaddr = socket.gethostbyname(endpoint[0])
server = Server(ipaddr, endpoint[1], endpoint[0])
# Append backend to service
if key not in services:
name = node['key'].split('/')[-1]
services[key] = Service(
name, 'registrator:%s' % self._url.geturl(), port,
protocol)
services[key] = services[key].addServer(server)
except Exception as e:
logging.warn(
"Failed to parse service %s backend %s/%s: %s",
node['key'], backend['key'], backend['value'], str(e))
logging.debug(traceback.format_exc())
return services
def _connect(self):
# Start the local load balancer in front of Marathon
service = Service('marathon',
'marathon:%s' % self._urls,
self._socketpath,
'unix',
'http',
healthcheck=True,
healthcheckurl='/ping')
for url in self._urls:
parsed = urlparse(url)
# Resolve hostnames since HAproxy wants IP addresses
ipaddr = socket.gethostbyname(parsed.hostname or '127.0.0.1')
server = Server(ipaddr, parsed.port or 80, parsed.hostname)
service._add(server)
self._backend.update(self._marathonService,
{self._socketpath: service})
def testHAProxyReload(self):
"""
Verifies that the HAproxy process is reloaded gracefully
"""
services = {
'1234/tcp':
Service('example.demo', self, 1234,
'tcp').addServer(Server('1.2.3.4', 31001, 'worker1'))
}
expected = """# example.demo (testHAProxyReload (proxymatic.test.haproxy_test.HAproxyTest))
listen demo.example-1234
bind 0.0.0.0:1234
balance leastconn
mode tcp
default-server inter 15s
server backend-worker1-31001 1.2.3.4:31001 weight 128
"""
self._check(services, expected)
services = {
'1234/tcp':
Service('example.demo', self, 1234,
'tcp').addServer(Server('1.2.3.4', 31001,
'worker1')).addServer(
Server('2.2.3.4', 31002,
'worker2'))
}
expected = """# example.demo (testHAProxyReload (proxymatic.test.haproxy_test.HAproxyTest))
listen demo.example-1234
bind 0.0.0.0:1234
balance leastconn
mode tcp
default-server inter 15s
server backend-worker2-31002 2.2.3.4:31002 weight 128
server backend-worker1-31001 1.2.3.4:31001 weight 128
"""
self._check(services, expected, pid=567)
def _refresh(self):
services = {}
# Start the local load balancer in front of Marathon
service = Service('marathon.local', 'marathon:%s' % self._urls, self._socketpath, 'unix')
services[self._socketpath] = service
for url in self._urls:
parsed = urlparse(url)
# Resolve hostnames since HAproxy wants IP addresses
ipaddr = socket.gethostbyname(parsed.hostname or '127.0.0.1')
server = Server(ipaddr, parsed.port or 80)
service._add(server)
# Poll Marathon for running tasks
try:
logging.debug("GET Marathon services from %s", self._socketpath)
response = unixrequest('GET', self._socketpath, '/v2/tasks', None, {'Accept': 'application/json'})
services.update(self._parse(response))
finally:
self._backend.update(self, services)
logging.debug("Refreshed services from Marathon at %s", self._urls)
def testLoadBalancerMode(self):
"""
Verifies that HTTP mode can be enabled
"""
services = {
'1234/tcp':
Service('example.demo', self, 1234,
'tcp').setApplication('http').addServer(
Server('1.2.3.4', 31001, 'worker1'))
}
expected = """# example.demo (testLoadBalancerMode (proxymatic.test.haproxy_test.HAproxyTest))
listen demo.example-1234
bind 0.0.0.0:1234
balance leastconn
mode http
default-server inter 15s
server backend-worker1-31001 1.2.3.4:31001 weight 128
"""
self._check(services, expected)
def testWeight(self):
"""
Verifies that backend weights are processed correctly
"""
services = {
'1234/tcp':
Service('example.demo', self, 1234, 'tcp').addServer(
Server('1.2.3.4', 31001, 'worker1').setWeight(250)).addServer(
Server('2.2.3.4', 31002, 'worker2'))
}
expected = """# example.demo (testWeight (proxymatic.test.haproxy_test.HAproxyTest))
listen demo.example-1234
bind 0.0.0.0:1234
balance leastconn
mode tcp
default-server inter 15s
server backend-worker1-31001 1.2.3.4:31001 weight 64
server backend-worker2-31002 2.2.3.4:31002 weight 128
"""
self._check(services, expected)
def _parse(self, content):
services = {}
try:
# logging.debug(content)
document = json.loads(content)
except ValueError as e:
raise RuntimeError(
"Failed to parse HTTP JSON response from Marathon (%s): %s" %
(str(e), str(content)[0:150]))
def failed(check):
alive = check.get('alive', False)
if not alive:
cause = check.get('lastFailureCause', '')
if cause:
logging.info(
"Task %s is failing health check with result '%s'",
check.get('taskId', ''), cause)
else:
logging.debug("Skipping task %s which is not alive (yet)",
check.get('taskId', ''))
return not alive
for task in document.get('tasks', []):
# Fetch exact config for this app version
taskConfig = getAppVersion(self._socketpath, task.get('appId'),
task.get('version'))
exposedPorts = task.get('ports', [])
servicePorts = task.get('servicePorts', [])
seenServicePorts = set()
# Apply servicePort overrides
servicePorts = self._applyServicePortOverrides(
taskConfig, servicePorts)
# Skip tasks that are being killed
if task.get('state') == 'TASK_KILLING':
logging.debug(
"Skipping task %s as it's currently being killed",
task.get('id'))
continue
for servicePort, portIndex in zip(servicePorts,
range(len(servicePorts))):
protocol = 'tcp'
key = '%s/%s' % (servicePort, protocol.lower())
# Marathon has been observed to sometimes return servicePort=0 failure cases
if str(servicePort) == '0':
logging.warn("Skipping task with servicePort=0")
continue
# Marathon returns multiple entries for services that expose both TCP and UDP using the same
# port number. There's no way to separate TCP and UDP service ports at the moment.
if servicePort in seenServicePorts:
continue
seenServicePorts.add(servicePort)
# Verify that all health checks pass
healthChecks = taskConfig.get('healthChecks', [])
healthResults = task.get('healthCheckResults', [])
# Skip any task that isn't alive according to its health checks
if any(failed(check) for check in healthResults):
continue
# Skip tasks that hasn't yet responded to at least one of their health checks. Note that Marathon
# considers tasks ready as soon as they respond OK to one of their defined health checks.
if len(healthChecks) > 0 and len(healthResults) == 0:
logging.debug(
"Skipping task %s which hasn't responded to health checks yet",
task.get('id', ''))
continue
try:
exposedPort = exposedPorts[portIndex]
# Resolve hostnames since HAproxy wants IP addresses
ipaddr = socket.gethostbyname(task['host'])
server = Server(ipaddr, exposedPort, task['host'])
# Set backend load balancer options
self._applyAttributeInt('weight', taskConfig, portIndex,
server)
self._applyAttributeInt('maxconn', taskConfig, portIndex,
server, self._groupsize)
# Append backend to service
if key not in services:
name = '.'.join(
reversed(filter(bool, task['appId'].split('/'))))
services[key] = Service(name,
'marathon:%s' % self._urls,
servicePort, protocol)
services[key]._add(server)
# Set load balancer protocol mode
self._applyLoadBalancerMode(taskConfig, portIndex,
services[key])
# Apply timeout parameters
self._applyAttributeInt('timeout.client', taskConfig,
portIndex, services[key])
self._applyAttributeInt('timeout.server', taskConfig,
portIndex, services[key])
except Exception as e:
logging.warn("Failed parse service %s backend %s: %s",
task.get('appId', ''), task.get('id', ''),
str(e))
logging.debug(traceback.format_exc())
return services