def test_explicit_mode(self):
from .. import TransactionManager
from ..interfaces import AlreadyInTransaction, NoTransaction
tm = TransactionManager()
self.assertFalse(tm.explicit)
tm = TransactionManager(explicit=True)
self.assertTrue(tm.explicit)
for name in 'get', 'commit', 'abort', 'doom', 'isDoomed', 'savepoint':
with self.assertRaises(NoTransaction):
getattr(tm, name)()
t = tm.begin()
with self.assertRaises(AlreadyInTransaction):
tm.begin()
self.assertTrue(t is tm.get())
self.assertFalse(tm.isDoomed())
tm.doom()
self.assertTrue(tm.isDoomed())
tm.abort()
for name in 'get', 'commit', 'abort', 'doom', 'isDoomed', 'savepoint':
with self.assertRaises(NoTransaction):
getattr(tm, name)()
t = tm.begin()
self.assertFalse(tm.isDoomed())
with self.assertRaises(AlreadyInTransaction):
tm.begin()
tm.savepoint()
tm.commit()
def checkResolve(self, resolvable=True):
db = DB(self._storage)
t1 = TransactionManager()
c1 = db.open(t1)
o1 = c1.root()['p'] = (PCounter if resolvable else PCounter2)()
o1.inc()
t1.commit()
t2 = TransactionManager()
c2 = db.open(t2)
o2 = c2.root()['p']
o2.inc(2)
t2.commit()
o1.inc(3)
try:
t1.commit()
except ConflictError as err:
self.assertIn(".PCounter2,", str(err))
self.assertEqual(o1._value, 3)
else:
self.assertTrue(resolvable, "Expected ConflictError")
self.assertEqual(o1._value, 6)
t2.begin()
self.assertEqual(o2._value, o1._value)
db.close()
def test_endpoint_error(self):
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
timedelta(seconds=0), max_endpoint_errors=2)
step = 10
oneTrSize = (MAX_QUEUE_SIZE / step) - 1
for i in xrange(step):
trManager.append(memTransaction(oneTrSize, trManager))
trManager.flush()
# There should be exactly step transaction in the list,
# and only 2 of them with a flush count of 1
self.assertEqual(len(trManager._transactions), step)
flush_count = 0
for tr in trManager._transactions:
flush_count += tr._flush_count
self.assertEqual(flush_count, 2)
# If we retry to flush, two OTHER transactions should be tried
trManager.flush()
self.assertEqual(len(trManager._transactions), step)
flush_count = 0
for tr in trManager._transactions:
flush_count += tr._flush_count
self.assertIn(tr._flush_count, [0, 1])
self.assertEqual(flush_count, 4)
# Finally when it's possible to flush, everything should go smoothly
for tr in trManager._transactions:
tr.is_flushable = True
trManager.flush()
self.assertEqual(len(trManager._transactions), 0)
def testCustomEndpoint(self):
MetricTransaction._endpoints = []
config = {
"dd_url": "https://foo.bar.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {}, "msgtype")
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = ['https://foo.bar.com/intake/msgtype?api_key=foo']
self.assertEqual(endpoints, expected, (endpoints, expected))
def test_proxy(self):
config = {
"endpoints": {"https://app.datadoghq.com": ["foo"]},
"proxy_settings": {
"host": "localhost",
"port": PROXY_PORT,
"user": None,
"password": None
}
}
app = Application()
app.skip_ssl_validation = True
app._agentConfig = config
trManager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
CustomAgentTransaction.set_tr_manager(trManager)
app.use_simple_http_client = False # We need proxy capabilities
app.agent_dns_caching = False
# _test is the instance of this class. It is needed to call the method stop() and deal with the asynchronous
# calls as described here : http://www.tornadoweb.org/en/stable/testing.html
CustomAgentTransaction._test = self
CustomAgentTransaction.set_application(app)
CustomAgentTransaction.set_endpoints(config['endpoints'])
CustomAgentTransaction('body', {}, "") # Create and flush the transaction
self.wait()
del CustomAgentTransaction._test
access_log = self.docker_client.exec_start(
self.docker_client.exec_create(CONTAINER_NAME, 'cat /var/log/squid/access.log')['Id'])
self.assertTrue("CONNECT" in access_log) # There should be an entry in the proxy access log
self.assertEquals(len(trManager._endpoints_errors), 1) # There should be an error since we gave a bogus api_key
def __init__(self,
port,
agentConfig,
watchmonitor=True,
skip_ssl_validation=False,
use_simple_http_client=False):
self.ip = get_ip(agentConfig, log)
self._port = int(port)
self._agentConfig = agentConfig
self._metrics = {}
AgentTransaction.set_application(self)
AgentTransaction.set_endpoints()
self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY,
MAX_QUEUE_SIZE, THROTTLING_DELAY)
AgentTransaction.set_tr_manager(self._tr_manager)
self._watchmonitor = None
self.skip_ssl_validation = skip_ssl_validation or agentConfig.get(
'skip_ssl_validation', False)
self.use_simple_http_client = use_simple_http_client
self._send_controler = 0 # control self.postAgentInfoToServer run once a minute
if self.skip_ssl_validation:
log.info(
"Skipping SSL hostname validation, useful when using a transparent proxy"
)
if watchmonitor:
watchmonitor_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHmonitor_INTERVAL_MULTIPLIER / 1000
self._watchmonitor = Watchmonitor(watchmonitor_timeout,
max_mem_mb=agentConfig.get(
'limit_memory_consumption',
None))
def __init__(self,
port,
agentConfig,
watchdog=True,
skip_ssl_validation=False,
use_simple_http_client=False):
self._port = int(port)
self._agentConfig = agentConfig
self._metrics = {}
MetricTransaction.set_application(self)
MetricTransaction.set_endpoints()
self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY,
MAX_QUEUE_SIZE, THROTTLING_DELAY)
MetricTransaction.set_tr_manager(self._tr_manager)
self._watchdog = None
self.skip_ssl_validation = skip_ssl_validation or agentConfig.get(
'skip_ssl_validation', False)
self.use_simple_http_client = use_simple_http_client
if self.skip_ssl_validation:
log.info(
"Skipping SSL hostname validation, useful when using a transparent proxy"
)
if watchdog:
watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER
self._watchdog = Watchdog(watchdog_timeout,
max_mem_mb=agentConfig.get(
'limit_memory_consumption', None))
def testCustomEndpoint(self):
MetricTransaction._endpoints = []
config = {
"endpoints": {
"https://foo.bar.com": ["foo"]
},
"dd_url": "https://foo.bar.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app.agent_dns_caching = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "msgtype")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://foo.bar.com/intake/msgtype?api_key=foo']
self.assertEqual(endpoints, expected, (endpoints, expected))
def test_multiple_endpoints(self):
config = {
"endpoints": {
"https://app.datadoghq.com": ['api_key'],
"https://app.example.com": ['api_key']
},
"dd_url": "https://app.datadoghq.com",
"api_key": 'api_key',
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
MetricTransaction({}, {})
# 2 endpoints = 2 transactions
self.assertEqual(len(trManager._transactions), 2)
self.assertEqual(trManager._transactions[0]._endpoint,
'https://app.datadoghq.com')
self.assertEqual(trManager._transactions[1]._endpoint,
'https://app.example.com')
def make_routable_request(dbsession: Optional[Session],
registry: Registry,
path="/") -> IRequest:
"""Creates a dummy request that has route_url and other routing methods.
As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL.
See also :func:`make_dummy_request`.
:param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``.
"""
base_url = registry.get("websauna.site_url", None)
# TODO: Honour request_factory here
request = Request.blank(path, base_url=base_url)
# apply_request_extensions()?
request.registry = registry
request.user = None
if dbsession:
request.dbsession = dbsession
else:
tm = TransactionManager()
dbsession = create_dbsession(request.registry, tm)
request.dbsession = dbsession
request.tm = request.transaction_manager = tm
def terminate_session(request):
# Close db session at the end of the request and return the db connection back to the pool
dbsession.close()
request.add_finished_callback(terminate_session)
return request
def exec_eager(self, *args, **kwargs):
"""Run transaction aware task in eager mode."""
# We are run in a post-commit hook, so there is no transaction manager available
tm = TransactionManager()
# Do not attempt any transaction retries in eager mode
tm.retry_attempt_count = 1
self.request.update(request=self.make_faux_request(tm=tm))
return self.run(*args, **kwargs)
def exec_eager(self, *args, **kwargs):
"""Run transaction aware task in eager mode."""
# We are run in a post-commit hook, so there is no transaction manager available
tm = TransactionManager()
self.request.update(request=self.make_faux_request(
transaction_manager=tm))
return self.run(*args, **kwargs)
def update_schema_manager(event):
# Use a local transaction manager to sidestep any issues
# with an active transaction or explicit mode.
txm = TransactionManager(explicit=True)
txm.begin()
with contextlib.closing(event.database.open(txm)) as conn:
state = conn.root().get(PersistentWebhookSchemaManager.key, State())
txm.abort()
schema = get_schema_manager()
schema.compareSubscriptionsAndComputeGeneration(state)
def exec_eager(self, *args, **kwargs):
"""Run transaction aware task in eager mode."""
# We are run in a post-commit hook, so there is no transaction manager available
tm = TransactionManager()
self.request.update(request=self.make_faux_request(
transaction_manager=tm))
with tm:
# This doesn't do transaction retry attempts, but should be good enough for eager
return self.run(*args, **kwargs)
class Replica(ServiceBase):
transaction_manager = TransactionManager()
@rpc(String)
def set_server(ctx, server):
logging.info(' set_server()')
Replica.transaction_manager.set_server(server)
@rpc(String)
def add_replica(ctx, replica):
logging.info(' add_replica()')
Replica.transaction_manager.add_replica(replica)
@srpc(String, _returns=Boolean)
def tpc_vote_replica(msg):
logging.info(' tpc_vote_replica()')
return Replica.transaction_manager.tpc_vote_replica(msg)
@srpc(String, _returns=Boolean)
def tpc_commit_replica(msg,):
logging.info(' tpc_commit_replica()')
return Replica.transaction_manager.tpc_commit_replica(msg)
@srpc(String, _returns=Boolean)
def tpc_abort_replica(msg):
logging.info(' tpc_abort_replica()')
return Replica.transaction_manager.tpc_abort_replica(msg)
@rpc(String)
def tpc_ack(ctx, msg):
logging.info(' tpc_ack()')
Replica.transaction_manager.tpc_ack(msg)
@rpc(String, String)
def put(ctx, key, value):
logging.info(' put()')
return Replica.transaction_manager.put(key, value)
@srpc(String, _returns=Boolean)
def delete(key):
logging.info(' delete()')
return Replica.transaction_manager.delete(key)
@srpc(String, _returns=String)
def get(key):
logging.info(' get()')
return Replica.transaction_manager.get(key)
@srpc(String, _returns=String)
def get_value_replica(key):
logging.info(' get_value_replica()')
return Replica.transaction_manager.get_value_replica(key)
def testEndpoints(self):
"""Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
MetricTransaction._endpoints = []
config = {
"dd_url": "https://app.datadoghq.com",
"api_key": "foo",
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datadoghq.com/intake?api_key=foo'.format(
get_version().replace(".", "-"))
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
transaction = APIMetricTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key=foo'.
format(get_version().replace(".", "-"))
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def __init__(self, port, agentConfig, watchdog=True):
self._port = int(port)
self._agentConfig = agentConfig
self._metrics = {}
MetricTransaction.set_application(self)
MetricTransaction.set_endpoints()
self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY,
MAX_QUEUE_SIZE, THROTTLING_DELAY)
MetricTransaction.set_tr_manager(self._tr_manager)
self._watchdog = None
if watchdog:
watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER
self._watchdog = Watchdog(watchdog_timeout)
def __init__(self,
port,
agentConfig,
watchdog=True,
skip_ssl_validation=False,
use_simple_http_client=False):
self._port = int(port)
self._agentConfig = agentConfig
self._metrics = {}
self._dns_cache = None
AgentTransaction.set_application(self)
AgentTransaction.set_endpoints(agentConfig['endpoints'])
if agentConfig['endpoints'] == {}:
log.warning(
u"No valid endpoint found. Forwarder will drop all incoming payloads."
)
AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout'])
max_parallelism = self.NO_PARALLELISM
# Multiple endpoints => enable parallelism
if len(agentConfig['endpoints']) > 1:
max_parallelism = self.DEFAULT_PARALLELISM
self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY,
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_parallelism=max_parallelism)
AgentTransaction.set_tr_manager(self._tr_manager)
self._watchdog = None
self.skip_ssl_validation = skip_ssl_validation or _is_affirmative(
agentConfig.get('skip_ssl_validation'))
self.agent_dns_caching = _is_affirmative(
agentConfig.get('dns_caching'))
self.agent_dns_ttl = int(agentConfig.get('dns_ttl', DEFAULT_DNS_TTL))
if self.agent_dns_caching:
self._dns_cache = DNSCache(ttl=self.agent_dns_ttl)
self.use_simple_http_client = use_simple_http_client
if self.skip_ssl_validation:
log.info(
"Skipping SSL hostname validation, useful when using a transparent proxy"
)
# Monitor activity
if watchdog:
watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000
self._watchdog = Watchdog.create(
watchdog_timeout, max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD)
def test_no_parallelism(self):
step = 2
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
timedelta(seconds=0),
max_parallelism=1,
max_endpoint_errors=100)
for i in xrange(step):
trManager.append(SleepingTransaction(trManager, delay=1))
trManager.flush()
# Flushes should be sequential
for i in xrange(step):
self.assertEqual(trManager._running_flushes, 1)
self.assertEqual(trManager._finished_flushes, i)
self.assertEqual(len(trManager._trs_to_flush), step - (i + 1))
time.sleep(1)
def test_drop_repeated_error(self):
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
timedelta(seconds=0),
max_endpoint_errors=1)
# Fail it once
oneTrSize = 10
trManager.append(memTransaction(oneTrSize, trManager))
# It should still be there after flush
trManager.flush()
self.assertEqual(len(trManager._transactions), 1)
#Try again, now it should be gone
trManager.flush()
self.assertEqual(len(trManager._transactions), 0)
def make_routable_request(dbsession: t.Optional[Session] = None,
registry: t.Optional[Registry] = None,
path='/') -> IRequest:
"""Creates a dummy request that has route_url and other routing methods.
As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL.
See also :func:`make_dummy_request`.
TODO: Split this to two different functions: one for existing dbsession and one for where dbsession is connected.
:param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``.
:param registry: Configuration registry
:param path: Path being requested.
:return: Current request.
"""
base_url = registry.settings.get("websauna.site_url", None)
# TODO: Honour request_factory here
request = Request.blank(path, base_url=base_url)
request.registry = registry
request.user = None
request.view_name = ''
# This will create request.tm, others
apply_request_extensions(request)
if dbsession:
# Use the provided dbsession for this request
request.dbsession = dbsession
if hasattr(dbsession, "transaction_manager"):
request.tm = request.transaction_manager = dbsession.transaction_manager
else:
# Create a new dbsession and transaction manager for this request
tm = TransactionManager()
dbsession = create_dbsession(request.registry, tm)
request.dbsession = dbsession
request.tm = request.transaction_manager = tm
def terminate_session(request):
# Close db session at the end of the request and return the db connection back to the pool
dbsession.close()
request.add_finished_callback(terminate_session)
return request
def testThrottling(self):
"""Test throttling while flushing"""
# No throttling, no delay for replay
trManager = TransactionManager(timedelta(seconds = 0), MAX_QUEUE_SIZE, THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
# Add 3 transactions, make sure no memory limit is in the way
oneTrSize = MAX_QUEUE_SIZE / 10
for i in xrange(3):
tr = memTransaction(oneTrSize, trManager)
trManager.append(tr)
# Try to flush them, time it
before = datetime.now()
trManager.flush()
after = datetime.now()
self.assertTrue( (after-before) > 3 * THROTTLING_DELAY)
def testMemoryLimit(self):
"""Test memory limit as well as simple flush"""
# No throttling, no delay for replay
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
timedelta(seconds=0))
step = 10
oneTrSize = (MAX_QUEUE_SIZE / step) - 1
for i in xrange(step):
tr = memTransaction(oneTrSize, trManager)
trManager.append(tr)
trManager.flush()
# There should be exactly step transaction in the list, with
# a flush count of 1
self.assertEqual(len(trManager._transactions), step)
for tr in trManager._transactions:
self.assertEqual(tr._flush_count, 1)
# Try to add one more
tr = memTransaction(oneTrSize + 10, trManager)
trManager.append(tr)
# At this point, transaction one (the oldest) should have been removed from the list
self.assertEqual(len(trManager._transactions), step)
for tr in trManager._transactions:
self.assertNotEqual(tr._id, 1)
trManager.flush()
self.assertEqual(len(trManager._transactions), step)
# Check and allow transactions to be flushed
for tr in trManager._transactions:
tr.is_flushable = True
# Last transaction has been flushed only once
if tr._id == step + 1:
self.assertEqual(tr._flush_count, 1)
else:
self.assertEqual(tr._flush_count, 2)
trManager.flush()
self.assertEqual(len(trManager._transactions), 0)
def test_parallelism(self):
step = 4
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
timedelta(seconds=0), max_parallelism=step,
max_endpoint_errors=100)
for i in xrange(step):
trManager.append(SleepingTransaction(trManager))
trManager.flush()
self.assertEqual(trManager._running_flushes, step)
self.assertEqual(trManager._finished_flushes, 0)
# If _trs_to_flush != None, it means that it's still running as it should be
self.assertEqual(trManager._trs_to_flush, [])
time.sleep(1)
# It should be finished
self.assertEqual(trManager._running_flushes, 0)
self.assertEqual(trManager._finished_flushes, step)
self.assertIs(trManager._trs_to_flush, None)
def __init__(self, cache_backend=None, keyhandler=None, keygen=None):
self.__dict__ = self.__shared_state
self.prefix = settings.MIDDLEWARE_KEY_PREFIX
if keyhandler:
self.kh_class = keyhandler
if keygen:
self.kg_class = keygen
if not cache_backend and not hasattr(self, 'cache_backend'):
cache_backend = settings._get_backend()
if not keygen and not hasattr(self, 'kg_class'):
self.kg_class = KeyGen
if keyhandler is None and not hasattr(self, 'kh_class'):
self.kh_class = KeyHandler
if cache_backend:
self.cache_backend = TransactionManager(cache_backend,
self.kg_class)
self.keyhandler = self.kh_class(self.cache_backend, self.kg_class,
self.prefix)
self._patched = getattr(self, '_patched', False)
def test_notify_transaction_late_comers(self):
# If a datamanager registers for synchonization after a
# transaction has started, we should call newTransaction so it
# can do necessry setup.
import mock
from .. import TransactionManager
manager = TransactionManager()
sync1 = mock.MagicMock()
manager.registerSynch(sync1)
sync1.newTransaction.assert_not_called()
t = manager.begin()
sync1.newTransaction.assert_called_with(t)
sync2 = mock.MagicMock()
manager.registerSynch(sync2)
sync2.newTransaction.assert_called_with(t)
# for, um, completeness
t.commit()
for s in sync1, sync2:
s.beforeCompletion.assert_called_with(t)
s.afterCompletion.assert_called_with(t)
def __init__(self,
port,
agentConfig,
watchdog=True,
skip_ssl_validation=False,
use_simple_http_client=False):
self._port = int(port)
self._agentConfig = agentConfig
self._metrics = {}
AgentTransaction.set_application(self)
AgentTransaction.set_endpoints(agentConfig['endpoints'])
AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout'])
max_parallelism = self.NO_PARALLELISM
# Multiple endpoints => enable parallelism
if len(agentConfig['endpoints']) > 1:
max_parallelism = self.DEFAULT_PARALLELISM
self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY,
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_parallelism=max_parallelism)
AgentTransaction.set_tr_manager(self._tr_manager)
self._watchdog = None
self.skip_ssl_validation = skip_ssl_validation or agentConfig.get(
'skip_ssl_validation', False)
self.use_simple_http_client = use_simple_http_client
if self.skip_ssl_validation:
log.info(
"Skipping SSL hostname validation, useful when using a transparent proxy"
)
# Monitor activity
if watchdog:
watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000
self._watchdog = Watchdog(
watchdog_timeout,
max_mem_mb=agentConfig.get('limit_memory_consumption', None),
max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD)
def get_tm():
if 'tm' not in g:
if debug:
print('creating transactionmanager')
g.tm = TransactionManager()
return g.tm
def testEndpoints(self):
"""
Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
raise SkipTest("This test doesn't apply to Server Density.")
MetricTransaction._endpoints = []
api_key = "a" * 32
config = {
"sd_url": "https://agent.serverdensity.io",
"api_key": api_key,
"use_sd": True
}
app = Application()
app.skip_ssl_validation = False
app.agent_dns_caching = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0),
MAX_QUEUE_SIZE,
THROTTLING_DELAY,
max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = [
'https://{0}-app.agent.datadoghq.com/intake/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Metric Transaction
transaction = APIMetricTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = [
'https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key={1}'.
format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
try:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
except requests.HTTPError:
if r.status_code != 400 or 'No series present in the payload' not in r.content:
raise
# API Service Check Transaction
APIServiceCheckTransaction._trManager = trManager
APIServiceCheckTransaction.set_application(app)
APIServiceCheckTransaction.set_endpoints(config['endpoints'])
transaction = APIServiceCheckTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = [
'https://{0}-app.agent.datadoghq.com/api/v1/check_run/?api_key={1}'
.format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({
'check': 'test',
'status': 0
}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def __init__(self, *args, **kwargs):
self.made_seekable = 0
self.tm = TransactionManager()
super(DummyRequest, self).__init__(self, *args, **kwargs)
