def test_state_only(self):
target_stitch.OUR_SESSION = FakeSession(mock_in_order_all_200)
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
}
}
}))
#will flush here, because TargetStitch.time_last_batch_sent was set to 0 in setUp
self.target_stitch.consume(self.queue)
finish_requests()
emitted_state = list(
map(json.loads,
self.out.getvalue().strip().split('\n')))
self.assertEqual(len(emitted_state), 1)
self.assertEqual(emitted_state[0],
{'bookmarks': {
'chicken_stream': {
'id': 1
}
}})
def test_request_to_big_batch_for_large_record(self):
target_stitch.OUR_SESSION = FakeSession(mock_in_order_all_200)
self.target_stitch.max_batch_records = 4
self.target_stitch.handlers[0].max_batch_records = 4
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "M" * 5000000
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 3,
"name": "Harrsion"
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 4,
"name": "Cathy"
}
}))
#will flush here after 4 records
self.target_stitch.consume(self.queue)
finish_requests()
self.assertEqual(target_stitch.OUR_SESSION.urls, [
target_stitch.CONFIG["big_batch_url"],
target_stitch.CONFIG["small_batch_url"]
])
self.assertEqual(len(target_stitch.OUR_SESSION.messages_sent[0]), 1)
self.assertEqual(len(target_stitch.OUR_SESSION.messages_sent[1]), 3)
def test_activate_version_finishes_pending_requests(self):
target_stitch.OUR_SESSION = FakeSession(mock_out_of_order_all_200)
#request 2 would ordinarily complete first because the mock_out_of_order_all_200, but because
#request 2 contains an ACTIVATE_VERSION, it will not even be sent until request 1 completes
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"version": 1,
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
}
}
}))
#will flush here after 2 records
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
'version': 1,
"record": {
"id": 2,
"name": "Paul"
}
}))
self.queue.append(
json.dumps({
"type": "ACTIVATE_VERSION",
'stream': 'chicken_stream',
'version': 1
}))
#will flush here after 2 records
self.target_stitch.consume(self.queue)
finish_requests()
self.assertEqual(self.first_flush_error, None, self.first_flush_error)
self.assertEqual(self.second_flush_error, None,
self.second_flush_error)
def test_unparseable_json_response(self):
target_stitch.OUR_SESSION = FakeSession(
mock_unparsable_response_body_200)
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
#will flush here after 2 records
target_stitch.check_send_exception = fake_check_send_exception
self.target_stitch.consume(self.queue)
target_stitch.check_send_exception = self.og_check_send_exception
try:
finish_requests()
except Exception as ex:
our_exception = ex
self.assertIsNotNone(our_exception)
def test_requests_out_of_order_first_errors(self):
target_stitch.OUR_SESSION = FakeSession(mock_out_of_order_first_errors)
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
#will flush here after 2 records
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 2
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 3,
"name": "Harrsion"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 3
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 4,
"name": "Cathy"
}
}))
#will flush here after 2 records
#consume() can encounter an exception via check_send_exception in send()
#if SEND_EXCEPTION has already been set by the coroutine it can blow up.
target_stitch.check_send_exception = fake_check_send_exception
self.target_stitch.consume(self.queue)
target_stitch.check_send_exception = self.og_check_send_exception
our_exception = None
try:
finish_requests()
except Exception as ex:
our_exception = ex
self.assertIsNotNone(our_exception)
self.assertTrue(isinstance(our_exception, TargetStitchException))
#no state is emitted
self.assertEqual(self.out.getvalue(), '')
def test_requests_out_of_order(self):
target_stitch.OUR_SESSION = FakeSession(mock_out_of_order_all_200)
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
#will flush here after 2 records
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 2
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 3,
"name": "Harrsion"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 3
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 4,
"name": "Cathy"
}
}))
#will flush here after 2 records
self.target_stitch.consume(self.queue)
finish_requests()
emitted_state = list(
map(json.loads,
self.out.getvalue().strip().split('\n')))
self.assertEqual(len(emitted_state), 2)
self.assertEqual(emitted_state[0],
{'bookmarks': {
'chicken_stream': {
'id': 1
}
}})
self.assertEqual(emitted_state[1],
{'bookmarks': {
'chicken_stream': {
'id': 3
}
}})
def test_will_not_output_empty_state(self):
target_stitch.OUR_SESSION = FakeSession(mock_in_order_all_200)
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
},
'currently_syncing': 'chicken_stream'
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
#will flush here after 2 records, state will reset to None
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 3,
"name": "Kyle"
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 4,
"name": "Alice"
}
}))
#will flush here after 2 records, but will NOT write blank state
self.target_stitch.consume(self.queue)
finish_requests()
emitted_state = list(
map(json.loads,
self.out.getvalue().strip().split('\n')))
self.assertEqual(len(emitted_state), 1)
self.assertEqual(
emitted_state[0], {
"bookmarks": {
"chicken_stream": {
"id": 1
}
},
'currently_syncing': 'chicken_stream'
})
def test_trailing_state_after_final_message(self):
target_stitch.OUR_SESSION = FakeSession(mock_in_order_all_200)
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
},
'currently_syncing': 'chicken_stream'
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
#will flush here after 2 records
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 2
}
},
'currently_syncing': None
}
}))
self.target_stitch.consume(self.queue)
finish_requests()
emitted_state = list(
map(json.loads,
self.out.getvalue().strip().split('\n')))
self.assertEqual(len(emitted_state), 2)
self.assertEqual(
emitted_state[0], {
"bookmarks": {
"chicken_stream": {
"id": 1
}
},
'currently_syncing': 'chicken_stream'
})
self.assertEqual(emitted_state[1], {
"bookmarks": {
"chicken_stream": {
"id": 2
}
},
'currently_syncing': None
})
def test_serialize_floats(self):
floats = [
'-9999999999999999.9999999999999999999999',
'-7187498962233394.3739812942138415666763',
'9273972760690975.2044306442955715221042',
'29515565286974.1188802122612813004366',
'9176089101347578.2596296292040288441238',
'-8416853039392703.306423225471199148379',
'1285266411314091.3002668125515694162268',
'6051872750342125.3812886238958681227336',
'-1132031605459408.5571559429308939781468',
'-6387836755056303.0038029604189860431045', '4526059300505414'
]
target_stitch.OUR_SESSION = FakeSession(mock_in_order_all_200)
for float_val in floats:
self.queue.append(
simplejson.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"my_float": Decimal(float_val)
}
}))
self.queue.append(
simplejson.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"my_float": Decimal(float_val)
}
}
}
}))
self.target_stitch.consume(self.queue)
finish_requests()
output_record_floats = []
for batch in target_stitch.OUR_SESSION.bodies_sent:
output_record_floats.extend([
str(x['data']['my_float'])
for x in simplejson.loads(batch, use_decimal=True)['messages']
])
self.assertEqual(floats, output_record_floats)
emitted_state = list(
map(lambda x: simplejson.loads(x, use_decimal=True),
self.out.getvalue().strip().split('\n')))
self.assertEqual(len(emitted_state), 6)
self.assertEqual(emitted_state[0], {
'bookmarks': {
'chicken_stream': {
'my_float': Decimal(floats[0])
}
}
})
self.assertEqual(emitted_state[1], {
'bookmarks': {
'chicken_stream': {
'my_float': Decimal(floats[2])
}
}
})
self.assertEqual(emitted_state[2], {
'bookmarks': {
'chicken_stream': {
'my_float': Decimal(floats[4])
}
}
})
self.assertEqual(emitted_state[3], {
'bookmarks': {
'chicken_stream': {
'my_float': Decimal(floats[6])
}
}
})
self.assertEqual(emitted_state[4], {
'bookmarks': {
'chicken_stream': {
'my_float': Decimal(floats[8])
}
}
})
self.assertEqual(emitted_state[5], {
'bookmarks': {
'chicken_stream': {
'my_float': Decimal(floats[10])
}
}
})
def test_requests_out_of_order_second_errors(self):
target_stitch.OUR_SESSION = FakeSession(
mock_out_of_order_second_errors)
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 1,
"name": "Mike"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 1
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 2,
"name": "Paul"
}
}))
#will flush here after 2 records
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 2
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 3,
"name": "Harrsion"
}
}))
self.queue.append(
json.dumps({
"type": "STATE",
"value": {
"bookmarks": {
"chicken_stream": {
"id": 3
}
}
}
}))
self.queue.append(
json.dumps({
"type": "RECORD",
"stream": "chicken_stream",
"record": {
"id": 4,
"name": "Cathy"
}
}))
#will flush here after 2 records
#consume() can encounter an exception via check_send_exception in send()
#if SEND_EXCEPTION has already been set by the coroutine it can blow up.
target_stitch.check_send_exception = fake_check_send_exception
self.target_stitch.consume(self.queue)
target_stitch.check_send_exception = self.og_check_send_exception
our_exception = None
try:
finish_requests()
except Exception as ex:
our_exception = ex
#the 2nd request returns immediately with a 400, triggering a TargetStitchException.
#at this point, it is game over and it does NOT matter when or with what status the 1st request comples
self.assertIsNotNone(our_exception)
self.assertTrue(isinstance(our_exception, TargetStitchException))
emitted_state = self.out.getvalue().strip().split('\n')
self.assertEqual(1, len(emitted_state))
self.assertEqual('', emitted_state[0])
