def test_proxy(self):
config = {
"endpoints": {"https://app.datadoghq.com": ["foo"]},
"proxy_settings": {
"host": "localhost",
"port": PROXY_PORT,
"user": None,
"password": None
}
}
app = Application()
app.skip_ssl_validation = True
app._agentConfig = config
trManager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
CustomAgentTransaction.set_tr_manager(trManager)
app.use_simple_http_client = False # We need proxy capabilities
app.agent_dns_caching = False
# _test is the instance of this class. It is needed to call the method stop() and deal with the asynchronous
# calls as described here : http://www.tornadoweb.org/en/stable/testing.html
CustomAgentTransaction._test = self
CustomAgentTransaction.set_application(app)
CustomAgentTransaction.set_endpoints(config['endpoints'])
CustomAgentTransaction('body', {}, "") # Create and flush the transaction
self.wait()
del CustomAgentTransaction._test
access_log = self.docker_client.exec_start(
self.docker_client.exec_create(CONTAINER_NAME, 'cat /var/log/squid/access.log')['Id'])
self.assertTrue("CONNECT" in access_log) # There should be an entry in the proxy access log
self.assertEquals(len(trManager._endpoints_errors), 1) # There should be an error since we gave a bogus api_key
def testCustomEndpoint(self):
MetricTransaction._endpoints = []
config = {
"dd_url": "https://foo.bar.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {}, "msgtype")
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = ['https://foo.bar.com/intake/msgtype?api_key=foo']
self.assertEqual(endpoints, expected, (endpoints, expected))
def testCustomEndpoint(self):
MetricTransaction._endpoints = []
config = {
"endpoints": {
"https://foo.bar.com": ["foo"]
},
"dd_url": "https://foo.bar.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app.agent_dns_caching = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "msgtype")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://foo.bar.com/intake/msgtype?api_key=foo']
self.assertEqual(endpoints, expected, (endpoints, expected))
def test_multiple_endpoints(self):
config = {
"endpoints": {
"https://app.datadoghq.com": ['api_key'],
"https://app.example.com": ['api_key']
},
"dd_url": "https://app.datadoghq.com",
"api_key": 'api_key',
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
MetricTransaction({}, {})
# 2 endpoints = 2 transactions
self.assertEqual(len(trManager._transactions), 2)
self.assertEqual(trManager._transactions[0]._endpoint,
'https://app.datadoghq.com')
self.assertEqual(trManager._transactions[1]._endpoint,
'https://app.example.com')
def testCustomEndpoint(self):
MetricTransaction._endpoints = []
config = {
"sd_url": "https://foo.bar.com",
"agent_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {}, "msgtype")
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
# Direct metric submission is not being enabled.
#expected = ['https://foo.bar.com/intake/msgtype?agent_key=foo']
expected = []
self.assertEqual(endpoints, expected, (endpoints, expected))
def testCustomEndpoint(self):
MetricTransaction._endpoints = []
config = {
"endpoints": {"https://foo.bar.com": ["foo"]},
"dd_url": "https://foo.bar.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY, max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "msgtype")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://foo.bar.com/intake/msgtype?api_key=foo']
self.assertEqual(endpoints, expected, (endpoints, expected))
def test_proxy(self):
config = {
"endpoints": {"https://app.datadoghq.com": ["foo"]},
"proxy_settings": {
"host": "localhost",
"port": PROXY_PORT,
"user": None,
"password": None
}
}
app = Application()
app.skip_ssl_validation = True
app._agentConfig = config
trManager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
CustomAgentTransaction.set_tr_manager(trManager)
app.use_simple_http_client = False # We need proxy capabilities
app.agent_dns_caching = False
# _test is the instance of this class. It is needed to call the method stop() and deal with the asynchronous
# calls as described here : http://www.tornadoweb.org/en/stable/testing.html
CustomAgentTransaction._test = self
CustomAgentTransaction.set_application(app)
CustomAgentTransaction.set_endpoints(config['endpoints'])
CustomAgentTransaction('body', {}, "") # Create and flush the transaction
self.wait()
del CustomAgentTransaction._test
access_log = self.docker_client.exec_start(
self.docker_client.exec_create(CONTAINER_NAME, 'cat /var/log/squid/access.log')['Id'])
self.assertTrue("CONNECT" in access_log) # There should be an entry in the proxy access log
self.assertEquals(len(trManager._endpoints_errors), 1) # There should be an error since we gave a bogus api_key
def test_multiple_endpoints(self):
raise SkipTest("This test doesn't apply to Server Density.")
config = {
"endpoints": {
"https://app.datadoghq.com": ['api_key'],
"https://app.example.com": ['api_key']
},
"dd_url": "https://app.datadoghq.com",
"api_key": 'api_key',
"use_sd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY, max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
MetricTransaction({}, {})
# 2 endpoints = 2 transactions
self.assertEqual(len(trManager._transactions), 2)
self.assertEqual(trManager._transactions[0]._endpoint, 'https://app.datadoghq.com')
self.assertEqual(trManager._transactions[1]._endpoint, 'https://app.example.com')
def testEndpoints(self):
"""Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
MetricTransaction._endpoints = []
config = {
"dd_url": "https://app.datadoghq.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datadoghq.com/intake?api_key=foo'.format(
get_version().replace(".", "-"))
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
transaction = APIMetricTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key=foo'.
format(get_version().replace(".", "-"))
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def testThrottling(self):
"""Test throttling while flushing"""
# No throttling, no delay for replay
trManager = TransactionManager(timedelta(seconds = 0), MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
# Add 3 transactions, make sure no memory limit is in the way
oneTrSize = MAX_QUEUE_SIZE / 10
for i in xrange(3):
tr = memTransaction(oneTrSize, trManager)
trManager.append(tr)
# Try to flush them, time it
before = datetime.now()
trManager.flush()
after = datetime.now()
self.assertTrue( (after-before) > 3 * THROTTLING_DELAY)
def testEndpoints(self):
"""
Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
raise SkipTest("This test doesn't apply to Server Density.")
MetricTransaction._endpoints = []
api_key = "a" * 32
config = {
"sd_url": "https://agent.serverdensity.io",
"api_key": api_key,
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {}, "")
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = ['https://{0}-app.agent.datadoghq.com/intake/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Metric Transaction
transaction = APIMetricTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = ['https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Service Check Transaction
APIServiceCheckTransaction._trManager = trManager
APIServiceCheckTransaction.set_application(app)
APIServiceCheckTransaction.set_endpoints()
transaction = APIServiceCheckTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = ['https://{0}-app.agent.datadoghq.com/api/v1/check_run/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({'check': 'test', 'status': 0}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def testEndpoints(self):
"""
Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
raise SkipTest("This test doesn't apply to Server Density.")
MetricTransaction._endpoints = []
api_key = "a" * 32
config = {
"sd_url": "https://agent.serverdensity.io",
"api_key": api_key,
"use_sd": True
}
app = Application()
app.skip_ssl_validation = False
app.agent_dns_caching = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = [
'https://{0}-app.agent.datadoghq.com/intake/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Metric Transaction
transaction = APIMetricTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = [
'https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key={1}'.
format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
try:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
except requests.HTTPError:
if r.status_code != 400 or 'No series present in the payload' not in r.content:
raise
# API Service Check Transaction
APIServiceCheckTransaction._trManager = trManager
APIServiceCheckTransaction.set_application(app)
APIServiceCheckTransaction.set_endpoints(config['endpoints'])
transaction = APIServiceCheckTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = [
'https://{0}-app.agent.datadoghq.com/api/v1/check_run/?api_key={1}'
.format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({
'check': 'test',
'status': 0
}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def testEndpoints(self):
"""
Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
MetricTransaction._endpoints = []
api_key = "a" * 32
config = {
"endpoints": {"https://app.datadoghq.com": [api_key]},
"dd_url": "https://app.datadoghq.com",
"api_key": api_key,
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app.agent_dns_caching = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY, max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://{0}-app.agent.datadoghq.com/intake/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Metric Transaction
transaction = APIMetricTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
try:
r = requests.post(url, data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
except requests.HTTPError:
if r.status_code != 400 or 'No series present in the payload' not in r.content:
raise
# API Service Check Transaction
APIServiceCheckTransaction._trManager = trManager
APIServiceCheckTransaction.set_application(app)
APIServiceCheckTransaction.set_endpoints(config['endpoints'])
transaction = APIServiceCheckTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://{0}-app.agent.datadoghq.com/api/v1/check_run/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({'check': 'test', 'status': 0}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
