def test_worker_process_transactions(m):
stats = Stats()
input_queue = queue.Queue(2)
retry_queue = queue.Queue(2)
w = Worker(input_queue, retry_queue, stats)
t_success = Transaction("data", "https://datadog.com", "/success", None)
t_error = Transaction("data", "https://datadog.com", "/error", None)
m.post("https://datadog.com/success", status_code=200)
m.post("https://datadog.com/error", status_code=402)
input_queue.put(t_success)
input_queue.put(t_error)
# process 2 transactions
w._process_transactions()
w._process_transactions()
assert input_queue.empty()
assert not retry_queue.empty()
assert t_success.nb_try == 1
assert t_error.nb_try == 1
assert t_error.next_flush is not None
retry_item = retry_queue.get()
assert t_error == retry_item
def test_process_error(m):
t = Transaction("data", "https://datadog.com", "/v1/series", {"test": "21"})
headers = {"test": "21"}
nb_try = 1
for err_code in [400, 403, 404, 413]:
m.post("https://datadog.com/v1/series",
status_code=err_code,
headers=headers,
additional_matcher=lambda r: r.text == "data")
assert t.process()
assert t.nb_try == nb_try
nb_try += 1
m.post("https://datadog.com/v1/series",
status_code=402,
headers=headers,
additional_matcher=lambda r: r.text == "data")
assert not t.process()
assert t.nb_try == nb_try
def test_transaction_creation():
start = time.time()
t = Transaction("data", "https://datadog.com", "/v1/series", {"DD": "true", "Content-Type": "application/json"})
assert t.payload == "data"
assert t.domain == "https://datadog.com"
assert t.endpoint == "/v1/series"
assert t.headers == {"DD": "true", "Content-Type": "application/json"}
assert t.nb_try == 0
assert t.timeout == 20 # default value from config
# test created_at value
assert t.created_at >= start
assert t.created_at <= time.time()
def test_retry_worker_process_transaction():
stats = Stats()
input_queue = queue.Queue(2)
retry_queue = queue.Queue(2)
w = RetryWorker(input_queue, retry_queue, stats, flush_interval=1)
# test pulling 1 transaction without flushing
t1 = Transaction("data", "https://datadog.com", "/success", None)
t1.next_flush = time.time()
retry_queue.put(t1)
w.GET_TIMEOUT = 10
base_flush_time = time.time() + 10
flush_time = w._process_transactions(base_flush_time)
end = time.time()
assert flush_time == base_flush_time
assert retry_queue.qsize() == 0
assert len(w.transactions) == 1
assert w.transactions[0] == t1
# now test with flush
w.GET_TIMEOUT = 0.1
start = time.time()
flush_time = w._process_transactions(0)
end = time.time()
assert len(w.transactions) == 0
try:
t = input_queue.get(True, 1)
except queue.Empty:
raise Exception("input_queue should not be empty")
assert t == t1
assert flush_time >= start + 1
assert flush_time <= end + 1
def test_retry_worker_flush():
input_queue = queue.Queue(1)
retry_queue = queue.Queue(1)
w = RetryWorker(input_queue, retry_queue)
t_ready = Transaction("data", "https://datadog.com", "/success", None)
t_ready.next_flush = time.time() - 10
w.transactions.append(t_ready)
t_not_ready = Transaction("data", "https://datadog.com", "/success", None)
t_not_ready.next_flush = time.time() + 1000
w.transactions.append(t_not_ready)
w._flush_transactions()
assert len(w.transactions) == 1
assert w.transactions[0] == t_not_ready
try:
t = input_queue.get_nowait()
except Exception:
# we should not fail to get a transaction
raise Exception("input_queue should not be empty")
assert t == t_ready
def test_process_success(m):
m.post("https://datadog.com/v1/series", additional_matcher=lambda r: r.text == "data")
t = Transaction("data", "https://datadog.com", "/v1/series", None)
assert t.process()
assert t.nb_try == 1
def test_reschedule():
t = Transaction("data", "https://datadog.com", "/", None)
# on the first try "reschedule" should set next_flush to "now"
before = time.time()
t.reschedule()
after = time.time()
assert t.next_flush is not None
assert t.next_flush >= before
assert t.next_flush <= after
t = Transaction("data", "https://datadog.com", "/", None)
t.nb_try = 2
before = time.time()
t.reschedule()
after = time.time()
assert t.next_flush is not None
assert t.next_flush >= before + 2 * Transaction.RETRY_INTERVAL
assert t.next_flush <= after + 2 * Transaction.RETRY_INTERVAL
t = Transaction("data", "https://datadog.com", "/", None)
t.nb_try = 10000
before = time.time()
t.reschedule()
after = time.time()
assert t.next_flush is not None
assert t.next_flush >= before + Transaction.MAX_RETRY_INTERVAL
assert t.next_flush <= after + Transaction.MAX_RETRY_INTERVAL
def test_get_endpoint():
t = Transaction("data", "https://datadog.com", "/v1/series", None)
assert t.get_endpoint() == "/v1/series"
t = Transaction("data", "https://datadog.com", "/series?api_key=abcdefghijklmnopqrstuvwxyz012345", None)
assert t.get_endpoint() == "/series?api_key=***************************12345"
t = Transaction("data", "https://datadog.com", "/series?test=21&api_key=abcdefghijklmnopqrstuvwxyz012345", None)
assert t.get_endpoint() == "/series?test=21&api_key=***************************12345"
t = Transaction("data", "https://datadog.com", "/series?api_key=test1234", None)
assert t.get_endpoint() == "/series?api_key=***************************t1234"
t = Transaction("data", "https://datadog.com", "/series?api_key=test", None)
assert t.get_endpoint() == "/series?api_key=test"