def test_multiple_endpoints(self): raise SkipTest("This test doesn't apply to Server Density.") config = { "endpoints": { "https://app.datadoghq.com": ['api_key'], "https://app.example.com": ['api_key'] }, "dd_url": "https://app.datadoghq.com", "api_key": 'api_key', "use_sd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY, max_endpoint_errors=100) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints(config['endpoints']) MetricTransaction({}, {}) # 2 endpoints = 2 transactions self.assertEqual(len(trManager._transactions), 2) self.assertEqual(trManager._transactions[0]._endpoint, 'https://app.datadoghq.com') self.assertEqual(trManager._transactions[1]._endpoint, 'https://app.example.com')
def testCustomEndpoint(self): MetricTransaction._endpoints = [] config = { "endpoints": {"https://foo.bar.com": ["foo"]}, "dd_url": "https://foo.bar.com", "api_key": "foo", "use_dd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY, max_endpoint_errors=100) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints(config['endpoints']) transaction = MetricTransaction(None, {}, "msgtype") endpoints = [] for endpoint in transaction._endpoints: for api_key in transaction._endpoints[endpoint]: endpoints.append(transaction.get_url(endpoint, api_key)) expected = ['https://foo.bar.com/intake/msgtype?api_key=foo'] self.assertEqual(endpoints, expected, (endpoints, expected))
def test_endpoint_error(self): trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_endpoint_errors=2) step = 10 oneTrSize = (MAX_QUEUE_SIZE / step) - 1 for i in xrange(step): trManager.append(memTransaction(oneTrSize, trManager)) trManager.flush() # There should be exactly step transaction in the list, # and only 2 of them with a flush count of 1 self.assertEqual(len(trManager._transactions), step) flush_count = 0 for tr in trManager._transactions: flush_count += tr._flush_count self.assertEqual(flush_count, 2) # If we retry to flush, two OTHER transactions should be tried trManager.flush() self.assertEqual(len(trManager._transactions), step) flush_count = 0 for tr in trManager._transactions: flush_count += tr._flush_count self.assertIn(tr._flush_count, [0, 1]) self.assertEqual(flush_count, 4) # Finally when it's possible to flush, everything should go smoothly for tr in trManager._transactions: tr.is_flushable = True trManager.flush() self.assertEqual(len(trManager._transactions), 0)
def __init__(self, port, agent_config, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agent_config self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints(MonAPI(agent_config['Api'])) self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agent_config.get( 'skip_ssl_validation', False) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info( "Skipping SSL hostname validation, useful when using a transparent proxy" ) if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog(watchdog_timeout, max_mem_mb=agent_config.get( 'limit_memory_consumption', None))
def new_revision( session: Session, tm: TransactionManager, content: Content, force_create_new_revision: bool = False, ) -> Content: """ Prepare context to update a Content. It will add a new updatable revision to the content. :param session: Database _session :param tm: TransactionManager :param content: Content instance to update :param force_create_new_revision: Decide if new_rev should or should not be forced. :return: """ with session.no_autoflush: try: if force_create_new_revision \ or inspect(content.revision).has_identity: content.new_revision() RevisionsIntegrity.add_to_updatable(content.revision) yield content except Exception as e: # INFO - GM - 14-11-2018 - rollback session and renew # transaction when error happened # This avoid bad _session data like new "temporary" revision # to be add when problem happen. session.rollback() tm.abort() tm.begin() raise e finally: RevisionsIntegrity.remove_from_updatable(content.revision)
def test_multiple_endpoints(self): config = { "endpoints": { "https://app.datadoghq.com": ['api_key'], "https://app.example.com": ['api_key'] }, "dd_url": "https://app.datadoghq.com", "api_key": 'api_key', "use_dd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY, max_endpoint_errors=100) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints(config['endpoints']) MetricTransaction({}, {}) # 2 endpoints = 2 transactions self.assertEqual(len(trManager._transactions), 2) self.assertEqual(trManager._transactions[0]._endpoint, 'https://app.datadoghq.com') self.assertEqual(trManager._transactions[1]._endpoint, 'https://app.example.com')
def test_proxy(self): config = { "endpoints": {"https://app.datadoghq.com": ["foo"]}, "proxy_settings": { "host": "localhost", "port": PROXY_PORT, "user": None, "password": None } } app = Application() app.skip_ssl_validation = True app._agentConfig = config trManager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop CustomAgentTransaction.set_tr_manager(trManager) app.use_simple_http_client = False # We need proxy capabilities app.agent_dns_caching = False # _test is the instance of this class. It is needed to call the method stop() and deal with the asynchronous # calls as described here : http://www.tornadoweb.org/en/stable/testing.html CustomAgentTransaction._test = self CustomAgentTransaction.set_application(app) CustomAgentTransaction.set_endpoints(config['endpoints']) CustomAgentTransaction('body', {}, "") # Create and flush the transaction self.wait() del CustomAgentTransaction._test access_log = self.docker_client.exec_start( self.docker_client.exec_create(CONTAINER_NAME, 'cat /var/log/squid/access.log')['Id']) self.assertTrue("CONNECT" in access_log) # There should be an entry in the proxy access log self.assertEquals(len(trManager._endpoints_errors), 1) # There should be an error since we gave a bogus api_key
def testCustomEndpoint(self): MetricTransaction._endpoints = [] config = { "sd_url": "https://foo.bar.com", "agent_key": "foo", "use_dd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints() transaction = MetricTransaction(None, {}, "msgtype") endpoints = [transaction.get_url(e) for e in transaction._endpoints] # Direct metric submission is not being enabled. #expected = ['https://foo.bar.com/intake/msgtype?agent_key=foo'] expected = [] self.assertEqual(endpoints, expected, (endpoints, expected))
def testCustomEndpoint(self): MetricTransaction._endpoints = [] config = { "dd_url": "https://foo.bar.com", "api_key": "foo", "use_dd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints() transaction = MetricTransaction(None, {}, "msgtype") endpoints = [transaction.get_url(e) for e in transaction._endpoints] expected = ['https://foo.bar.com/intake/msgtype?api_key=foo'] self.assertEqual(endpoints, expected, (endpoints, expected))
def checkResolve(self, resolvable=True): db = DB(self._storage) t1 = TransactionManager() c1 = db.open(t1) o1 = c1.root()['p'] = (PCounter if resolvable else PCounter2)() o1.inc() t1.commit() t2 = TransactionManager() c2 = db.open(t2) o2 = c2.root()['p'] o2.inc(2) t2.commit() o1.inc(3) try: t1.commit() except ConflictError as err: self.assertIn(".PCounter2,", str(err)) self.assertEqual(o1._value, 3) else: self.assertTrue(resolvable, "Expected ConflictError") self.assertEqual(o1._value, 6) t2.begin() self.assertEqual(o2._value, o1._value) db.close()
def testCustomEndpoint(self): MetricTransaction._endpoints = [] config = { "endpoints": { "https://foo.bar.com": ["foo"] }, "dd_url": "https://foo.bar.com", "api_key": "foo", "use_dd": True } app = Application() app.skip_ssl_validation = False app.agent_dns_caching = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY, max_endpoint_errors=100) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints(config['endpoints']) transaction = MetricTransaction(None, {}, "msgtype") endpoints = [] for endpoint in transaction._endpoints: for api_key in transaction._endpoints[endpoint]: endpoints.append(transaction.get_url(endpoint, api_key)) expected = ['https://foo.bar.com/intake/msgtype?api_key=foo'] self.assertEqual(endpoints, expected, (endpoints, expected))
def exec_eager(self, *args, **kwargs): """Run transaction aware task in eager mode.""" # We are run in a post-commit hook, so there is no transaction manager available tm = TransactionManager() # Do not attempt any transaction retries in eager mode tm.retry_attempt_count = 1 self.request.update(request=self.make_faux_request(tm=tm)) return self.run(*args, **kwargs)
def test_zbigarray_vs_cache_invalidation(): root = testdb.dbopen() conn = root._p_jar db = conn.db() conn.close() del root, conn tm1 = TransactionManager() tm2 = TransactionManager() conn1 = db.open(transaction_manager=tm1) root1 = conn1.root() # setup zarray root1['zarray3'] = a1 = ZBigArray((10,), uint8) tm1.commit() # set zarray initial data a1[0:1] = [1] # XXX -> [0] = 1 after BigArray can tm1.commit() # read zarray in conn2 conn2 = db.open(transaction_manager=tm2) root2 = conn2.root() a2 = root2['zarray3'] assert a2[0:1] == [1] # read data in conn2 + make sure read correctly # XXX -> [0] == 1 after BigArray can # now zarray content is both in ZODB.Connection cache and in _ZBigFileH # cache for each conn1 and conn2. Modify data in conn1 and make sure it # fully propagate to conn2. a1[0:1] = [2] # XXX -> [0] = 2 after BigArray can tm1.commit() # still should be read as old value in conn2 assert a2[0:1] == [1] # and even after virtmem pages reclaim # ( verifies that _p_invalidate() in ZBlk.loadblkdata() does not lead to # reloading data as updated ) ram_reclaim_all() assert a2[0:1] == [1] tm2.commit() # transaction boundary for t2 # data from tm1 should propagate -> ZODB -> ram pages for _ZBigFileH in conn2 assert a2[0] == 2 conn2.close() del conn2, root2 dbclose(root1)
def testEndpoints(self): """Tests that the logic behind the agent version specific endpoints is ok. Also tests that these endpoints actually exist. """ MetricTransaction._endpoints = [] config = { "dd_url": "https://app.datadoghq.com", "api_key": "foo", "use_dd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints() transaction = MetricTransaction(None, {}) endpoints = [transaction.get_url(e) for e in transaction._endpoints] expected = [ 'https://{0}-app.agent.datadoghq.com/intake?api_key=foo'.format( get_version().replace(".", "-")) ] self.assertEqual(endpoints, expected, (endpoints, expected)) for url in endpoints: r = requests.post(url, data=json.dumps({"foo": "bar"}), headers={'Content-Type': "application/json"}) r.raise_for_status() transaction = APIMetricTransaction(None, {}) endpoints = [transaction.get_url(e) for e in transaction._endpoints] expected = [ 'https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key=foo'. format(get_version().replace(".", "-")) ] self.assertEqual(endpoints, expected, (endpoints, expected)) for url in endpoints: r = requests.post(url, data=json.dumps({"foo": "bar"}), headers={'Content-Type': "application/json"}) r.raise_for_status()
def __init__(self, port, agentConfig, watchdog=True): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints() self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog(watchdog_timeout)
def test_no_parallelism(self): step = 2 trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_parallelism=1, max_endpoint_errors=100) for i in xrange(step): trManager.append(SleepingTransaction(trManager, delay=1)) trManager.flush() # Flushes should be sequential for i in xrange(step): self.assertEqual(trManager._running_flushes, 1) self.assertEqual(trManager._finished_flushes, i) self.assertEqual(len(trManager._trs_to_flush), step - (i + 1)) time.sleep(1)
def exec_eager(self, *args, **kwargs): """Run transaction aware task in eager mode.""" # We are run in a post-commit hook, so there is no transaction manager available tm = TransactionManager() # Do not attempt any transaction retries in eager mode tm.retry_attempt_count = 1 self.request.update(request=self.make_faux_request( transaction_manager=tm)) with tm: # This doesn't do transaction retry attempts, but should be good enough for eager return self.run(*args, **kwargs)
def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} self._dns_cache = None AgentTransaction.set_application(self) AgentTransaction.set_endpoints(agentConfig['endpoints']) if agentConfig['endpoints'] == {}: log.warning( u"No valid endpoint found. Forwarder will drop all incoming payloads." ) AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout']) max_parallelism = self.NO_PARALLELISM # Multiple endpoints => enable parallelism if len(agentConfig['endpoints']) > 1: max_parallelism = self.DEFAULT_PARALLELISM self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY, max_parallelism=max_parallelism) AgentTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or _is_affirmative( agentConfig.get('skip_ssl_validation')) self.agent_dns_caching = _is_affirmative( agentConfig.get('dns_caching')) self.agent_dns_ttl = int(agentConfig.get('dns_ttl', DEFAULT_DNS_TTL)) if self.agent_dns_caching: self._dns_cache = DNSCache(ttl=self.agent_dns_ttl) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info( "Skipping SSL hostname validation, useful when using a transparent proxy" ) # Monitor activity if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000 self._watchdog = Watchdog.create( watchdog_timeout, max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD)
def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} AgentTransaction.set_application(self) AgentTransaction.set_endpoints(agentConfig['endpoints']) AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout']) max_parallelism = self.NO_PARALLELISM # Multiple endpoints => enable parallelism if len(agentConfig['endpoints']) > 1: max_parallelism = self.DEFAULT_PARALLELISM self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY, max_parallelism=max_parallelism) AgentTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agentConfig.get('skip_ssl_validation', False) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info("Skipping SSL hostname validation, useful when using a transparent proxy") # Monitor activity if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000 self._watchdog = Watchdog( watchdog_timeout, max_mem_mb=agentConfig.get('limit_memory_consumption', None), max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD )
def __init__(self, port, agentConfig, watchmonitor=True, skip_ssl_validation=False, use_simple_http_client=False): self.ip = get_ip(agentConfig, log) self._port = int(port) self._agentConfig = agentConfig self._metrics = {} AgentTransaction.set_application(self) AgentTransaction.set_endpoints() self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) AgentTransaction.set_tr_manager(self._tr_manager) self._watchmonitor = None self.skip_ssl_validation = skip_ssl_validation or agentConfig.get( 'skip_ssl_validation', False) self.use_simple_http_client = use_simple_http_client self._send_controler = 0 # control self.postAgentInfoToServer run once a minute if self.skip_ssl_validation: log.info( "Skipping SSL hostname validation, useful when using a transparent proxy" ) if watchmonitor: watchmonitor_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHmonitor_INTERVAL_MULTIPLIER / 1000 self._watchmonitor = Watchmonitor(watchmonitor_timeout, max_mem_mb=agentConfig.get( 'limit_memory_consumption', None))
def make_routable_request(dbsession: Optional[Session], registry: Registry, path="/") -> IRequest: """Creates a dummy request that has route_url and other routing methods. As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL. See also :func:`make_dummy_request`. :param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``. """ base_url = registry.get("websauna.site_url", None) # TODO: Honour request_factory here request = Request.blank(path, base_url=base_url) # apply_request_extensions()? request.registry = registry request.user = None if dbsession: request.dbsession = dbsession else: tm = TransactionManager() dbsession = create_dbsession(request.registry, tm) request.dbsession = dbsession request.tm = request.transaction_manager = tm def terminate_session(request): # Close db session at the end of the request and return the db connection back to the pool dbsession.close() request.add_finished_callback(terminate_session) return request
def test_zbigarray_invalidate_shape(): root = testdb.dbopen() conn = root._p_jar db = conn.db() conn.close() del root, conn print tm1 = TransactionManager() tm2 = TransactionManager() conn1 = db.open(transaction_manager=tm1) root1 = conn1.root() # setup zarray root1['zarray4'] = a1 = ZBigArray((10,), uint8) tm1.commit() # set zarray initial data a1[0:1] = [1] # XXX -> [0] = 1 after BigArray can tm1.commit() # read zarray in conn2 conn2 = db.open(transaction_manager=tm2) root2 = conn2.root() a2 = root2['zarray4'] assert a2[0:1] == [1] # read data in conn2 + make sure read correctly # XXX -> [0] == 1 after BigArray can # append to a1 which changes both RAM pages and a1.shape assert a1.shape == (10,) a1.append([123]) assert a1.shape == (11,) assert a1[10:11] == [123] # XXX -> [10] = 123 after BigArray can tm1.commit() tm2.commit() # just transaction boundary for t2 # data from tm1 should propagate to tm assert a2.shape == (11,) assert a2[10:11] == [123] # XXX -> [10] = 123 after BigArray can conn2.close() del conn2, root2, a2 dbclose(root1)
def __init__(self, port, agentConfig): self._port = port self._agentConfig = agentConfig self._metrics = {} self._watchdog = Watchdog(TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER) MetricTransaction.set_application(self) self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager)
def exec_eager(self, *args, **kwargs): """Run transaction aware task in eager mode.""" # We are run in a post-commit hook, so there is no transaction manager available tm = TransactionManager() self.request.update(request=self.make_faux_request( transaction_manager=tm)) return self.run(*args, **kwargs)
def __init__(self): if not hasattr(self, '_conf'): raise Exception('Unconfigured Context') self.log = self._conf.log self.log.debug('Initializing') self._constructed_attrs = OrderedDict() self._active = True self.tx_manager = TransactionManager() for callback in self._conf._create_callbacks: callback(self)
def test_parallelism(self): step = 4 trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_parallelism=step, max_endpoint_errors=100) for i in xrange(step): trManager.append(SleepingTransaction(trManager)) trManager.flush() self.assertEqual(trManager._running_flushes, step) self.assertEqual(trManager._finished_flushes, 0) # If _trs_to_flush != None, it means that it's still running as it should be self.assertEqual(trManager._trs_to_flush, []) time.sleep(1) # It should be finished self.assertEqual(trManager._running_flushes, 0) self.assertEqual(trManager._finished_flushes, step) self.assertIs(trManager._trs_to_flush, None)
def _create_session(transaction_manager: TransactionManager, engine: Engine) -> Session: """Create a new database session with Zope transaction manager attached. The attached transaction manager takes care of committing the transaction at the end of the request. """ dbsession = Session(bind=engine) transaction_manager.retry_attempt_count = 3 # TODO: Hardcoded for now zope.sqlalchemy.register(dbsession, transaction_manager=transaction_manager) dbsession.transaction_manager = transaction_manager return dbsession
class Replica(ServiceBase): transaction_manager = TransactionManager() @rpc(String) def set_server(ctx, server): logging.info(' set_server()') Replica.transaction_manager.set_server(server) @rpc(String) def add_replica(ctx, replica): logging.info(' add_replica()') Replica.transaction_manager.add_replica(replica) @srpc(String, _returns=Boolean) def tpc_vote_replica(msg): logging.info(' tpc_vote_replica()') return Replica.transaction_manager.tpc_vote_replica(msg) @srpc(String, _returns=Boolean) def tpc_commit_replica(msg,): logging.info(' tpc_commit_replica()') return Replica.transaction_manager.tpc_commit_replica(msg) @srpc(String, _returns=Boolean) def tpc_abort_replica(msg): logging.info(' tpc_abort_replica()') return Replica.transaction_manager.tpc_abort_replica(msg) @rpc(String) def tpc_ack(ctx, msg): logging.info(' tpc_ack()') Replica.transaction_manager.tpc_ack(msg) @rpc(String, String) def put(ctx, key, value): logging.info(' put()') return Replica.transaction_manager.put(key, value) @srpc(String, _returns=Boolean) def delete(key): logging.info(' delete()') return Replica.transaction_manager.delete(key) @srpc(String, _returns=String) def get(key): logging.info(' get()') return Replica.transaction_manager.get(key) @srpc(String, _returns=String) def get_value_replica(key): logging.info(' get_value_replica()') return Replica.transaction_manager.get_value_replica(key)
def __init__(self, cache_backend=None, keyhandler=None, keygen=None): self.__dict__ = self.__shared_state self.prefix = settings.MIDDLEWARE_KEY_PREFIX if keyhandler: self.kh_class = keyhandler if keygen: self.kg_class = keygen if not cache_backend and not hasattr(self, 'cache_backend'): cache_backend = settings._get_backend() if not keygen and not hasattr(self, 'kg_class'): self.kg_class = KeyGen if keyhandler is None and not hasattr(self, 'kh_class'): self.kh_class = KeyHandler if cache_backend: self.cache_backend = TransactionManager(cache_backend, self.kg_class) self.keyhandler = self.kh_class(self.cache_backend, self.kg_class, self.prefix) self._patched = getattr(self, '_patched', False)
def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} AgentTransaction.set_application(self) AgentTransaction.set_endpoints(agentConfig['endpoints']) AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout']) max_parallelism = self.NO_PARALLELISM # Multiple endpoints => enable parallelism if len(agentConfig['endpoints']) > 1: max_parallelism = self.DEFAULT_PARALLELISM self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY, max_parallelism=max_parallelism) AgentTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agentConfig.get( 'skip_ssl_validation', False) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info( "Skipping SSL hostname validation, useful when using a transparent proxy" ) # Monitor activity if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000 self._watchdog = Watchdog( watchdog_timeout, max_mem_mb=agentConfig.get('limit_memory_consumption', None), max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD)
def test_drop_repeated_error(self): trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0), max_endpoint_errors=1) # Fail it once oneTrSize = 10 trManager.append(memTransaction(oneTrSize, trManager)) # It should still be there after flush trManager.flush() self.assertEqual(len(trManager._transactions), 1) #Try again, now it should be gone trManager.flush() self.assertEqual(len(trManager._transactions), 0)
def update_schema_manager(event): # Use a local transaction manager to sidestep any issues # with an active transaction or explicit mode. txm = TransactionManager(explicit=True) txm.begin() with contextlib.closing(event.database.open(txm)) as conn: state = conn.root().get(PersistentWebhookSchemaManager.key, State()) txm.abort() schema = get_schema_manager() schema.compareSubscriptionsAndComputeGeneration(state)
def make_routable_request(dbsession: t.Optional[Session] = None, registry: t.Optional[Registry] = None, path='/') -> IRequest: """Creates a dummy request that has route_url and other routing methods. As this request does not get HTTP hostname and such stuff from WSGI environment, a configuration variable ``websauna.site_url`` is passed as the base URL. See also :func:`make_dummy_request`. TODO: Split this to two different functions: one for existing dbsession and one for where dbsession is connected. :param dbsession: Use existing dbsession or set to ``None`` to generate a new dbsession and transaction manager. None that this TM is not the thread local transaction manager in ``transaction.mananger``. :param registry: Configuration registry :param path: Path being requested. :return: Current request. """ base_url = registry.settings.get("websauna.site_url", None) # TODO: Honour request_factory here request = Request.blank(path, base_url=base_url) request.registry = registry request.user = None request.view_name = '' # This will create request.tm, others apply_request_extensions(request) if dbsession: # Use the provided dbsession for this request request.dbsession = dbsession if hasattr(dbsession, "transaction_manager"): request.tm = request.transaction_manager = dbsession.transaction_manager else: # Create a new dbsession and transaction manager for this request tm = TransactionManager() dbsession = create_dbsession(request.registry, tm) request.dbsession = dbsession request.tm = request.transaction_manager = tm def terminate_session(request): # Close db session at the end of the request and return the db connection back to the pool dbsession.close() request.add_finished_callback(terminate_session) return request
def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints() self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agentConfig.get('skip_ssl_validation', False) if self.skip_ssl_validation: log.info("Skipping SSL hostname validation, useful when using a transparent proxy") if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog(watchdog_timeout, max_mem_mb=agentConfig.get('limit_memory_consumption', None))
def testThrottling(self): """Test throttling while flushing""" # No throttling, no delay for replay trManager = TransactionManager(timedelta(seconds = 0), MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop # Add 3 transactions, make sure no memory limit is in the way oneTrSize = MAX_QUEUE_SIZE / 10 for i in xrange(3): tr = memTransaction(oneTrSize, trManager) trManager.append(tr) # Try to flush them, time it before = datetime.now() trManager.flush() after = datetime.now() self.assertTrue( (after-before) > 3 * THROTTLING_DELAY)
def test_notify_transaction_late_comers(self): # If a datamanager registers for synchonization after a # transaction has started, we should call newTransaction so it # can do necessry setup. import mock from .. import TransactionManager manager = TransactionManager() sync1 = mock.MagicMock() manager.registerSynch(sync1) sync1.newTransaction.assert_not_called() t = manager.begin() sync1.newTransaction.assert_called_with(t) sync2 = mock.MagicMock() manager.registerSynch(sync2) sync2.newTransaction.assert_called_with(t) # for, um, completeness t.commit() for s in sync1, sync2: s.beforeCompletion.assert_called_with(t) s.afterCompletion.assert_called_with(t)
def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} self._dns_cache = None AgentTransaction.set_application(self) AgentTransaction.set_endpoints(agentConfig['endpoints']) if agentConfig['endpoints'] == {}: log.warning(u"No valid endpoint found. Forwarder will drop all incoming payloads.") AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout']) max_parallelism = self.NO_PARALLELISM # Multiple endpoints => enable parallelism if len(agentConfig['endpoints']) > 1: max_parallelism = self.DEFAULT_PARALLELISM self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY, max_parallelism=max_parallelism) AgentTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or _is_affirmative(agentConfig.get('skip_ssl_validation')) self.agent_dns_caching = _is_affirmative(agentConfig.get('dns_caching')) self.agent_dns_ttl = int(agentConfig.get('dns_ttl', DEFAULT_DNS_TTL)) if self.agent_dns_caching: self._dns_cache = DNSCache(ttl=self.agent_dns_ttl) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info("Skipping SSL hostname validation, useful when using a transparent proxy") # Monitor activity if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000 self._watchdog = Watchdog.create(watchdog_timeout, max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD)
class QueryCacheBackend(object): """This class is the engine behind the query cache. It reads the queries going through the django Query and returns from the cache using the generation keys, or on a miss from the database and caches the results. Each time a model is updated the table keys for that model are re-created, invalidating all cached querysets for that model. There are different QueryCacheBackend's for different versions of django; call ``johnny.cache.get_backend`` to automatically get the proper class. """ __shared_state = {} def __init__(self, cache_backend=None, keyhandler=None, keygen=None): self.__dict__ = self.__shared_state self.prefix = settings.MIDDLEWARE_KEY_PREFIX if keyhandler: self.kh_class = keyhandler if keygen: self.kg_class = keygen if not cache_backend and not hasattr(self, 'cache_backend'): cache_backend = settings._get_backend() if not keygen and not hasattr(self, 'kg_class'): self.kg_class = KeyGen if keyhandler is None and not hasattr(self, 'kh_class'): self.kh_class = KeyHandler if cache_backend: self.cache_backend = TransactionManager(cache_backend, self.kg_class) self.keyhandler = self.kh_class(self.cache_backend, self.kg_class, self.prefix) self._patched = getattr(self, '_patched', False) def _monkey_select(self, original): from django.db.models.sql.constants import MULTI from django.db.models.sql.datastructures import EmptyResultSet @wraps(original, assigned=available_attrs(original)) def newfun(cls, *args, **kwargs): if args: result_type = args[0] else: result_type = kwargs.get('result_type', MULTI) if any([isinstance(cls, c) for c in self._write_compilers]): return original(cls, *args, **kwargs) try: sql, params = cls.as_sql() if not sql: raise EmptyResultSet except EmptyResultSet: if result_type == MULTI: # this was moved in 1.2 to compiler return empty_iter() else: return db = getattr(cls, 'using', 'default') key, val = None, NotInCache() # check the blacklist for any of the involved tables; if it's not # there, then look for the value in the cache. tables = get_tables_for_query(cls.query) # if the tables are blacklisted, send a qc_skip signal blacklisted = disallowed_table(*tables) if blacklisted: signals.qc_skip.send(sender=cls, tables=tables, query=(sql, params, cls.ordering_aliases), key=key) if tables and not blacklisted: gen_key = self.keyhandler.get_generation(*tables, **{'db': db}) key = self.keyhandler.sql_key(gen_key, sql, params, cls.get_ordering(), result_type, db) val = self.cache_backend.get(key, NotInCache(), db) if not isinstance(val, NotInCache): if val == no_result_sentinel: val = [] signals.qc_hit.send(sender=cls, tables=tables, query=(sql, params, cls.ordering_aliases), size=len(val), key=key) return val if not blacklisted: signals.qc_miss.send(sender=cls, tables=tables, query=(sql, params, cls.ordering_aliases), key=key) val = original(cls, *args, **kwargs) if hasattr(val, '__iter__'): #Can't permanently cache lazy iterables without creating #a cacheable data structure. Note that this makes them #no longer lazy... #todo - create a smart iterable wrapper val = list(val) if key is not None: if not val: self.cache_backend.set(key, no_result_sentinel, settings.MIDDLEWARE_SECONDS, db) else: self.cache_backend.set(key, val, settings.MIDDLEWARE_SECONDS, db) return val return newfun def _monkey_write(self, original): @wraps(original, assigned=available_attrs(original)) def newfun(cls, *args, **kwargs): db = getattr(cls, 'using', 'default') from django.db.models.sql import compiler # we have to do this before we check the tables, since the tables # are actually being set in the original function ret = original(cls, *args, **kwargs) if isinstance(cls, compiler.SQLInsertCompiler): #Inserts are a special case where cls.tables #are not populated. tables = [cls.query.model._meta.db_table] else: #if cls.query.tables != list(cls.query.table_map): # pass #tables = list(cls.query.table_map) tables = cls.query.tables for table in tables: if not disallowed_table(table): self.keyhandler.invalidate_table(table, db) return ret return newfun def patch(self): """ monkey patches django.db.models.sql.compiler.SQL*Compiler series """ from django.db.models.sql import compiler self._read_compilers = ( compiler.SQLCompiler, compiler.SQLAggregateCompiler, compiler.SQLDateCompiler, ) self._write_compilers = ( compiler.SQLInsertCompiler, compiler.SQLDeleteCompiler, compiler.SQLUpdateCompiler, ) if not self._patched: self._original = {} for reader in self._read_compilers: self._original[reader] = reader.execute_sql reader.execute_sql = self._monkey_select(reader.execute_sql) for updater in self._write_compilers: self._original[updater] = updater.execute_sql updater.execute_sql = self._monkey_write(updater.execute_sql) self._patched = True self.cache_backend.patch() self._handle_signals() def unpatch(self): """un-applies this patch.""" if not self._patched: return for func in self._read_compilers + self._write_compilers: func.execute_sql = self._original[func] self.cache_backend.unpatch() self._patched = False def invalidate_m2m(self, instance, **kwargs): if self._patched: table = resolve_table(instance) if not disallowed_table(table): self.keyhandler.invalidate_table(instance) def invalidate(self, instance, **kwargs): if self._patched: table = resolve_table(instance) if not disallowed_table(table): self.keyhandler.invalidate_table(table) tables = set() tables.add(table) try: instance._meta._related_objects_cache except AttributeError: instance._meta._fill_related_objects_cache() for obj in instance._meta._related_objects_cache.keys(): obj_table = obj.model._meta.db_table if obj_table not in tables: tables.add(obj_table) if not disallowed_table(obj_table): self.keyhandler.invalidate_table(obj_table) def _handle_signals(self): post_save.connect(self.invalidate, sender=None) post_delete.connect(self.invalidate, sender=None) # FIXME: only needed in 1.1? signals.qc_m2m_change.connect(self.invalidate_m2m, sender=None) def flush_query_cache(self): from django.db import connection tables = connection.introspection.table_names() #seen_models = connection.introspection.installed_models(tables) for table in tables: # we want this to just work, so invalidate even things in blacklist self.keyhandler.invalidate_table(table)
class Context: """ Base class for Contexts of a ConfiguredCtxModule. Do not use this class directly, use the :attr:`Context <.ConfiguredCtxModule.Context>` member of a ConfiguredCtxModule instead. Every Context object needs to be destroyed manually by calling its :meth:`.destroy` method. Although this method will be called in the destructor of this class, that might already be too late. This is the reason why the preferred way of using this class is within a `with` statement: >>> with ctx_conf.Context() as ctx: ... ctx.logout_user() ... """ def __init__(self): if not hasattr(self, '_conf'): raise Exception('Unconfigured Context') self.log = self._conf.log self.log.debug('Initializing') self._constructed_attrs = OrderedDict() self._active = True self.tx_manager = TransactionManager() for callback in self._conf._create_callbacks: callback(self) def __del__(self): if self._active: self.destroy() def __enter__(self): return self def __exit__(self, type, value, traceback): self.destroy(value) def __hasattr__(self, attr): if self._active: return attr in self._conf.registrations return attr in self.__dict__ def __getattr__(self, attr): if '_active' in self.__dict__ and '_conf' in self.__dict__ and \ self._active and attr in self._conf.registrations: value = self._conf.registrations[attr].constructor(self) if self._conf.registrations[attr].cached: self._constructed_attrs[attr] = value self.__dict__[attr] = value else: self._constructed_attrs[attr] = None self.log.debug('Creating member %s' % attr) return value raise AttributeError(attr) def __setattr__(self, attr, value): try: # call registered contructor, if there is one, so the destructor # gets called with this new value getattr(self, attr) except AttributeError: pass self.__dict__[attr] = value def __delattr__(self, attr): if attr in self._conf.registrations: if attr in self._constructed_attrs: self.__delattr(attr, None) elif attr in self.__dict__: del self.__dict__[attr] else: del self.__dict__[attr] def __delattr(self, attr, exception): """ Deletes a previously constructed *attr*. Its destructor will receive the given *exception*. Note: this function assumes that the *attr* is indeed a registered context member. It will behave unexpectedly when called with an *attr* that has no registration. """ constructor_value = self._constructed_attrs[attr] del self._constructed_attrs[attr] self.log.debug('Deleting member %s' % attr) destructor = self._conf.registrations[attr].destructor if destructor: self.log.debug('Calling destructor of %s' % attr) if self._conf.registrations[attr].cached: destructor(self, constructor_value, None) else: destructor(self, None) try: del self.__dict__[attr] except KeyError: # destructor might have deleted self.attr already pass def destroy(self, exception=None): """ Cleans up this context and makes it unusable. After calling this function, this object will lose all its magic and behave like an empty class. The optional *exception*, that is the cause of this method call, will be passed to the destructors of every :term:`context member`. """ if not self._active: return self.log.debug('Destroying') self._active = False for attr in reversed(list(self._constructed_attrs.keys())): self.__delattr(attr, exception) for callback in self._conf._destroy_callbacks: callback(self, exception) tx = self.tx_manager.get() if exception or tx.isDoomed(): tx.abort() else: tx.commit()
class Application(tornado.web.Application): NO_PARALLELISM = 1 DEFAULT_PARALLELISM = 5 def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} self._dns_cache = None AgentTransaction.set_application(self) AgentTransaction.set_endpoints(agentConfig['endpoints']) if agentConfig['endpoints'] == {}: log.warning( u"No valid endpoint found. Forwarder will drop all incoming payloads." ) AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout']) max_parallelism = self.NO_PARALLELISM # Multiple endpoints => enable parallelism if len(agentConfig['endpoints']) > 1: max_parallelism = self.DEFAULT_PARALLELISM self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY, max_parallelism=max_parallelism) AgentTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or _is_affirmative( agentConfig.get('skip_ssl_validation')) self.agent_dns_caching = _is_affirmative( agentConfig.get('dns_caching')) self.agent_dns_ttl = int(agentConfig.get('dns_ttl', DEFAULT_DNS_TTL)) if self.agent_dns_caching: self._dns_cache = DNSCache(ttl=self.agent_dns_ttl) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info( "Skipping SSL hostname validation, useful when using a transparent proxy" ) # Monitor activity if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000 self._watchdog = Watchdog.create( watchdog_timeout, max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD) def get_from_dns_cache(self, url): if not self.agent_dns_caching: log.debug('Caching disabled, not resolving.') return url location = urlparse(url) resolve = self._dns_cache.resolve(location.netloc) return "{scheme}://{ip}".format(scheme=location.scheme, ip=resolve) def log_request(self, handler): """ Override the tornado logging method. If everything goes well, log level is DEBUG. Otherwise it's WARNING or ERROR depending on the response code. """ if handler.get_status() < 400: log_method = log.debug elif handler.get_status() < 500: log_method = log.warning else: log_method = log.error request_time = 1000.0 * handler.request.request_time() log_method(u"%d %s %.2fms", handler.get_status(), handler._request_summary(), request_time) def appendMetric(self, prefix, name, host, device, ts, value): if prefix in self._metrics: metrics = self._metrics[prefix] else: metrics = {} self._metrics[prefix] = metrics if name in metrics: metrics[name].append([host, device, ts, value]) else: metrics[name] = [[host, device, ts, value]] def _postMetrics(self): if len(self._metrics) > 0: self._metrics['uuid'] = get_uuid() self._metrics['internalHostname'] = get_hostname(self._agentConfig) self._metrics['apiKey'] = self._agentConfig['api_key'] MetricTransaction(json.dumps(self._metrics), headers={'Content-Type': 'application/json'}) self._metrics = {} def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/intake/metrics?", MetricsAgentInputHandler), (r"/intake/metadata?", MetadataAgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/api/v1/check_run/?", ApiCheckRunHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) try: # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address=self._agentConfig['bind_host']) except gaierror: log.warning( "localhost seems undefined in your host file, using 127.0.0.1 instead" ) http_server.listen(self._port, address="127.0.0.1") except socket_error as e: if "Errno 99" in str(e): log.warning( "IPv6 doesn't seem to be fully supported. Falling back to IPv4" ) http_server.listen(self._port, address="127.0.0.1") else: raise except socket_error as e: log.exception( "Socket error %s. Is another application listening on the same port ? Exiting", e) sys.exit(1) except Exception as e: log.exception("Uncaught exception. Forwarder is exiting.") sys.exit(1) log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = tornado.ioloop.IOLoop.current() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address="localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped") def stop(self): self.mloop.stop()
def testMemoryLimit(self): """Test memory limit as well as simple flush""" # No throttling, no delay for replay trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, timedelta(seconds=0)) step = 10 oneTrSize = (MAX_QUEUE_SIZE / step) - 1 for i in xrange(step): tr = memTransaction(oneTrSize, trManager) trManager.append(tr) trManager.flush() # There should be exactly step transaction in the list, with # a flush count of 1 self.assertEqual(len(trManager._transactions), step) for tr in trManager._transactions: self.assertEqual(tr._flush_count, 1) # Try to add one more tr = memTransaction(oneTrSize + 10, trManager) trManager.append(tr) # At this point, transaction one (the oldest) should have been removed from the list self.assertEqual(len(trManager._transactions), step) for tr in trManager._transactions: self.assertNotEqual(tr._id, 1) trManager.flush() self.assertEqual(len(trManager._transactions), step) # Check and allow transactions to be flushed for tr in trManager._transactions: tr.is_flushable = True # Last transaction has been flushed only once if tr._id == step + 1: self.assertEqual(tr._flush_count, 1) else: self.assertEqual(tr._flush_count, 2) trManager.flush() self.assertEqual(len(trManager._transactions), 0)
class Application(tornado.web.Application): def __init__(self, port, agentConfig, watchdog=True): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints() self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog(watchdog_timeout) def appendMetric(self, prefix, name, host, device, ts, value): if self._metrics.has_key(prefix): metrics = self._metrics[prefix] else: metrics = {} self._metrics[prefix] = metrics if metrics.has_key(name): metrics[name].append([host, device, ts, value]) else: metrics[name] = [[host, device, ts, value]] def _postMetrics(self): if len(self._metrics) > 0: self._metrics['uuid'] = get_uuid() self._metrics['internalHostname'] = gethostname(self._agentConfig) self._metrics['apiKey'] = self._agentConfig['api_key'] MetricTransaction(self._metrics, {}) self._metrics = {} def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) # set the root logger to warn so tornado is less chatty logging.getLogger().setLevel(logging.WARNING) # but keep the forwarder logger at the original level forwarder_logger = logging.getLogger('forwarder') log_config = get_logging_config() forwarder_logger.setLevel(log_config['log_level'] or logging.INFO) # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address = "localhost") except gaierror: log.warning("Warning localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address = "127.0.0.1") log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = tornado.ioloop.IOLoop.instance() def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,TRANSACTION_FLUSH_INTERVAL, io_loop = self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, gethostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address = "localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped") def stop(self): self.mloop.stop()
class Application(tornado.web.Application): def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints() self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agentConfig.get('skip_ssl_validation', False) if self.skip_ssl_validation: log.info("Skipping SSL hostname validation, useful when using a transparent proxy") if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog(watchdog_timeout, max_mem_mb=agentConfig.get('limit_memory_consumption', None)) def log_request(self, handler): """ Override the tornado logging method. If everything goes well, log level is DEBUG. Otherwise it's WARNING or ERROR depending on the response code. """ if handler.get_status() < 400: log_method = log.debug elif handler.get_status() < 500: log_method = log.warning else: log_method = log.error request_time = 1000.0 * handler.request.request_time() log_method("%d %s %.2fms", handler.get_status(), handler._request_summary(), request_time) def appendMetric(self, prefix, name, host, device, ts, value): if self._metrics.has_key(prefix): metrics = self._metrics[prefix] else: metrics = {} self._metrics[prefix] = metrics if metrics.has_key(name): metrics[name].append([host, device, ts, value]) else: metrics[name] = [[host, device, ts, value]] def _postMetrics(self): if len(self._metrics) > 0: self._metrics['uuid'] = get_uuid() self._metrics['internalHostname'] = get_hostname(self._agentConfig) self._metrics['apiKey'] = self._agentConfig['api_key'] MetricTransaction(json.dumps(self._metrics), headers={'Content-Type': 'application/json'}) self._metrics = {} def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address = "localhost") except gaierror: log.warning("Warning localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address = "127.0.0.1") log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = get_tornado_ioloop() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,TRANSACTION_FLUSH_INTERVAL, io_loop = self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address = "localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped") def stop(self): self.mloop.stop()
class Forwarder(tornado.web.Application): def __init__(self, port, agent_config, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agent_config self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints(MonAPI(agent_config['Api'])) self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agent_config.get( 'skip_ssl_validation', False) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info("Skipping SSL hostname validation, useful when using a transparent proxy") if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog( watchdog_timeout, max_mem_mb=agent_config.get('limit_memory_consumption', None)) def _post_metrics(self): if len(self._metrics) > 0: MetricTransaction(self._metrics, headers={'Content-Type': 'application/json'}) self._metrics = {} # todo why is the tornado logging method overridden? Perhaps ditch this. def log_request(self, handler): """ Override the tornado logging method. If everything goes well, log level is DEBUG. Otherwise it's WARNING or ERROR depending on the response code. """ if handler.get_status() < 400: log_method = log.debug elif handler.get_status() < 500: log_method = log.warning else: log_method = log.error request_time = 1000.0 * handler.request.request_time() log_method("%d %s %.2fms", handler.get_status(), handler._request_summary(), request_time) def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/api/v1/series/?", AgentInputHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) try: # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address="localhost") except gaierror: log.warning( "localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address="127.0.0.1") except socket_error as e: if "Errno 99" in str(e): log.warning("IPv6 doesn't seem to be fully supported. Falling back to IPv4") http_server.listen(self._port, address="127.0.0.1") else: raise except socket_error as e: log.exception( "Socket error %s. Is another application listening on the same port ? Exiting", e) sys.exit(1) except Exception: log.exception("Uncaught exception. Forwarder is exiting.") sys.exit(1) log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = get_tornado_ioloop() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._post_metrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback( flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop) # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped") def stop(self): self.mloop.stop()
class Application(tornado.web.Application): def __init__(self, port, agentConfig, watchdog=True): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} MetricTransaction.set_application(self) MetricTransaction.set_endpoints() self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY) MetricTransaction.set_tr_manager(self._tr_manager) self._watchdog = None if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER self._watchdog = Watchdog(watchdog_timeout) def appendMetric(self, prefix, name, host, device, ts, value): if self._metrics.has_key(prefix): metrics = self._metrics[prefix] else: metrics = {} self._metrics[prefix] = metrics if metrics.has_key(name): metrics[name].append([host, device, ts, value]) else: metrics[name] = [[host, device, ts, value]] def _postMetrics(self): if len(self._metrics) > 0: self._metrics["uuid"] = getUuid() self._metrics["internalHostname"] = gethostname(self._agentConfig) self._metrics["apiKey"] = self._agentConfig["api_key"] MetricTransaction(self._metrics, {}) self._metrics = {} def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/status/?", StatusHandler), ] settings = dict(cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=True) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) http_server.listen(self._port) logging.info("Listening on port %d" % self._port) # Register callbacks self.mloop = tornado.ioloop.IOLoop.instance() def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: logging.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, gethostname(self._agentConfig), io_loop=self.mloop) gs.listen(gport) # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() def stop(self): self.mloop.stop()
def evolve(self): """Perform a requested evolution This method needs to use the component architecture, so we'll set it up: >>> from zope.component.testing import setUp, tearDown >>> setUp() We also need a test request: >>> from zope.publisher.browser import TestRequest >>> request = TestRequest() We also need to give it a publication with a database: >>> class Publication(object): ... pass >>> request.setPublication(Publication()) >>> from ZODB.tests.util import DB >>> db = DB() >>> request.publication.db = db We need to define some schema managers. We'll define two using the demo package: >>> from zope.generations.generations import SchemaManager >>> from zope import component as ztapi >>> app1 = SchemaManager(0, 1, 'zope.generations.demo') >>> ztapi.provideUtility(app1, ISchemaManager, 'foo.app1') >>> app2 = SchemaManager(0, 0, 'zope.generations.demo') >>> ztapi.provideUtility(app2, ISchemaManager, 'foo.app2') And we need to record some data for them in the database. >>> from zope.generations.generations import evolve >>> evolve(db) This sets up the data and actually evolves app1: >>> conn = db.open() >>> conn.root()[generations_key]['foo.app1'] 1 >>> conn.root()[generations_key]['foo.app2'] 0 To evolve a data base schema, the user clicks on a submit button. If they click on the button for add1, a item will be added to the request, which we simulate: >>> request.form['evolve-app-foo.app1'] = 'evolve' We'll also increase the generation of app1: >>> app1.generation = 2 Now we can create our view: >>> view = Managers(None, request) Now, if we call its `evolve` method, it should see that the app1 evolve button was pressed and evolve app1 to the next generation. >>> status = view.evolve() >>> conn.sync() >>> conn.root()[generations_key]['foo.app1'] 2 The demo evolver just writes the generation to a database key: >>> from zope.generations.demo import key >>> conn.root()[key] ('installed', 'installed', 2) Note that, because the demo package has an install script, we have entries for that script. Which the returned status should indicate: >>> status['app'] u'foo.app1' >>> status['to'] 2 Now, given that the database is at the maximum generation for app1, we can't evolve it further. Calling evolve again won't evolve anything: >>> status = view.evolve() >>> conn.sync() >>> conn.root()[generations_key]['foo.app1'] 2 >>> conn.root()[key] ('installed', 'installed', 2) as the status will indicate by returning a 'to' generation of 0: >>> status['app'] u'foo.app1' >>> status['to'] 0 If the request doesn't have the key: >>> request.form.clear() Then calling evolve does nothing: >>> view.evolve() >>> conn.sync() >>> conn.root()[generations_key]['foo.app1'] 2 >>> conn.root()[key] ('installed', 'installed', 2) We'd better clean upp: >>> db.close() >>> tearDown() """ self.managers = managers = dict( zope.component.getUtilitiesFor(ISchemaManager)) db = self._getdb() transaction_manager = TransactionManager() conn = db.open(transaction_manager=transaction_manager) transaction_manager.begin() try: generations = conn.root().get(generations_key, ()) request = self.request for key in generations: generation = generations[key] rkey = request_key_format % key if rkey in request: manager = managers[key] if generation >= manager.generation: return {'app': key, 'to': 0} context = Context() context.connection = conn generation += 1 manager.evolve(context, generation) generations[key] = generation transaction_manager.commit() return {'app': key, 'to': generation} finally: transaction_manager.abort() conn.close()
def applications(self): """Get information about database-generation status This method needs to use the component architecture, so we'll set it up: >>> from zope.component.testing import setUp, tearDown >>> setUp() We also need a test request: >>> from zope.publisher.browser import TestRequest >>> request = TestRequest() We also need to give it a publication with a database: >>> class Publication(object): ... pass >>> request.setPublication(Publication()) >>> from ZODB.tests.util import DB >>> db = DB() >>> request.publication.db = db We need to define some schema managers. We'll define two using the demo package: >>> from zope.generations.generations import SchemaManager >>> from zope import component as ztapi >>> app1 = SchemaManager(0, 1, 'zope.generations.demo') >>> ztapi.provideUtility(app1, ISchemaManager, 'foo.app1') >>> app2 = SchemaManager(0, 0, 'zope.generations.demo') >>> ztapi.provideUtility(app2, ISchemaManager, 'foo.app2') And we need to record some data for them in the database. >>> from zope.generations.generations import evolve >>> evolve(db) This sets up the data and actually evolves app1: >>> conn = db.open() >>> conn.root()[generations_key]['foo.app1'] 1 >>> conn.root()[generations_key]['foo.app2'] 0 Now, let's increment app1's generation: >>> app1.generation += 1 so we can evolve it. Now we can create our view: >>> view = Managers(None, request) We call its applications method to get data about application generations. We are required to call evolve first: >>> view.evolve() >>> data = list(view.applications()) >>> data.sort(key=lambda d1: d1['id']) >>> for info in data: ... print(info['id']) ... print(info['min'], info['max'], info['generation']) ... print('evolve?', info['evolve'] or None) foo.app1 0 2 1 evolve? evolve-app-foo.app1 foo.app2 0 0 0 evolve? None We'd better clean up: >>> db.close() >>> tearDown() """ result = [] db = self._getdb() transaction_manager = TransactionManager() conn = db.open(transaction_manager=transaction_manager) transaction_manager.begin() try: managers = self.managers generations = conn.root().get(generations_key, ()) for key, generation in generations.items(): manager = managers.get(key) if manager is None: # pragma: no cover continue result.append({ 'id': key, 'min': manager.minimum_generation, 'max': manager.generation, 'generation': generation, 'evolve': (generation < manager.generation and request_key_format % key or '' ), }) return result finally: transaction_manager.abort() conn.close()
class Application(tornado.web.Application): NO_PARALLELISM = 1 DEFAULT_PARALLELISM = 5 def __init__(self, port, agentConfig, watchdog=True, skip_ssl_validation=False, use_simple_http_client=False): self._port = int(port) self._agentConfig = agentConfig self._metrics = {} AgentTransaction.set_application(self) AgentTransaction.set_endpoints(agentConfig['endpoints']) AgentTransaction.set_request_timeout(agentConfig['forwarder_timeout']) max_parallelism = self.NO_PARALLELISM # Multiple endpoints => enable parallelism if len(agentConfig['endpoints']) > 1: max_parallelism = self.DEFAULT_PARALLELISM self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY, MAX_QUEUE_SIZE, THROTTLING_DELAY, max_parallelism=max_parallelism) AgentTransaction.set_tr_manager(self._tr_manager) self._watchdog = None self.skip_ssl_validation = skip_ssl_validation or agentConfig.get('skip_ssl_validation', False) self.use_simple_http_client = use_simple_http_client if self.skip_ssl_validation: log.info("Skipping SSL hostname validation, useful when using a transparent proxy") # Monitor activity if watchdog: watchdog_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHDOG_INTERVAL_MULTIPLIER / 1000 self._watchdog = Watchdog( watchdog_timeout, max_mem_mb=agentConfig.get('limit_memory_consumption', None), max_resets=WATCHDOG_HIGH_ACTIVITY_THRESHOLD ) def log_request(self, handler): """ Override the tornado logging method. If everything goes well, log level is DEBUG. Otherwise it's WARNING or ERROR depending on the response code. """ if handler.get_status() < 400: log_method = log.debug elif handler.get_status() < 500: log_method = log.warning else: log_method = log.error request_time = 1000.0 * handler.request.request_time() log_method( u"%d %s %.2fms", handler.get_status(), handler._request_summary(), request_time ) def appendMetric(self, prefix, name, host, device, ts, value): if prefix in self._metrics: metrics = self._metrics[prefix] else: metrics = {} self._metrics[prefix] = metrics if name in metrics: metrics[name].append([host, device, ts, value]) else: metrics[name] = [[host, device, ts, value]] def _postMetrics(self): if len(self._metrics) > 0: self._metrics['uuid'] = get_uuid() self._metrics['internalHostname'] = get_hostname(self._agentConfig) self._metrics['apiKey'] = self._agentConfig['api_key'] MetricTransaction(json.dumps(self._metrics), headers={'Content-Type': 'application/json'}) self._metrics = {} def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/intake/metrics?", MetricsAgentInputHandler), (r"/intake/metadata?", MetadataAgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/api/v1/check_run/?", ApiCheckRunHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) try: # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address=self._agentConfig['bind_host']) except gaierror: log.warning("localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address="127.0.0.1") except socket_error as e: if "Errno 99" in str(e): log.warning("IPv6 doesn't seem to be fully supported. Falling back to IPv4") http_server.listen(self._port, address="127.0.0.1") else: raise except socket_error as e: log.exception("Socket error %s. Is another application listening on the same port ? Exiting", e) sys.exit(1) except Exception as e: log.exception("Uncaught exception. Forwarder is exiting.") sys.exit(1) log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = get_tornado_ioloop() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address="localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped") def stop(self): self.mloop.stop()
def testEndpoints(self): """ Tests that the logic behind the agent version specific endpoints is ok. Also tests that these endpoints actually exist. """ raise SkipTest("This test doesn't apply to Server Density.") MetricTransaction._endpoints = [] api_key = "a" * 32 config = { "sd_url": "https://agent.serverdensity.io", "api_key": api_key, "use_dd": True } app = Application() app.skip_ssl_validation = False app._agentConfig = config app.use_simple_http_client = True trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE, THROTTLING_DELAY) trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop MetricTransaction._trManager = trManager MetricTransaction.set_application(app) MetricTransaction.set_endpoints() transaction = MetricTransaction(None, {}, "") endpoints = [transaction.get_url(e) for e in transaction._endpoints] expected = ['https://{0}-app.agent.datadoghq.com/intake/?api_key={1}'.format( get_version().replace(".", "-"), api_key)] self.assertEqual(endpoints, expected, (endpoints, expected)) for url in endpoints: r = requests.post(url, data=json.dumps({"foo": "bar"}), headers={'Content-Type': "application/json"}) r.raise_for_status() # API Metric Transaction transaction = APIMetricTransaction(None, {}) endpoints = [transaction.get_url(e) for e in transaction._endpoints] expected = ['https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key={1}'.format( get_version().replace(".", "-"), api_key)] self.assertEqual(endpoints, expected, (endpoints, expected)) for url in endpoints: r = requests.post(url, data=json.dumps({"foo": "bar"}), headers={'Content-Type': "application/json"}) r.raise_for_status() # API Service Check Transaction APIServiceCheckTransaction._trManager = trManager APIServiceCheckTransaction.set_application(app) APIServiceCheckTransaction.set_endpoints() transaction = APIServiceCheckTransaction(None, {}) endpoints = [transaction.get_url(e) for e in transaction._endpoints] expected = ['https://{0}-app.agent.datadoghq.com/api/v1/check_run/?api_key={1}'.format( get_version().replace(".", "-"), api_key)] self.assertEqual(endpoints, expected, (endpoints, expected)) for url in endpoints: r = requests.post(url, data=json.dumps({'check': 'test', 'status': 0}), headers={'Content-Type': "application/json"}) r.raise_for_status()
def test_explicit_mode(self): from .. import TransactionManager from ..interfaces import AlreadyInTransaction, NoTransaction tm = TransactionManager() self.assertFalse(tm.explicit) tm = TransactionManager(explicit=True) self.assertTrue(tm.explicit) for name in 'get', 'commit', 'abort', 'doom', 'isDoomed', 'savepoint': with self.assertRaises(NoTransaction): getattr(tm, name)() t = tm.begin() with self.assertRaises(AlreadyInTransaction): tm.begin() self.assertTrue(t is tm.get()) self.assertFalse(tm.isDoomed()) tm.doom() self.assertTrue(tm.isDoomed()) tm.abort() for name in 'get', 'commit', 'abort', 'doom', 'isDoomed', 'savepoint': with self.assertRaises(NoTransaction): getattr(tm, name)() t = tm.begin() self.assertFalse(tm.isDoomed()) with self.assertRaises(AlreadyInTransaction): tm.begin() tm.savepoint() tm.commit()
def __init__(self, *args, **kwargs): self.made_seekable = 0 self.tm = TransactionManager() super(DummyRequest, self).__init__(self, *args, **kwargs)