def setUp(self):
self.tape_cassette = InMemoryTapeCassette()
self.tape_recorder = TapeRecorder(self.tape_cassette)
self.tape_recorder.enable_recording()
class TestEqualizer(unittest.TestCase):
def setUp(self):
self.tape_cassette = InMemoryTapeCassette()
self.tape_recorder = TapeRecorder(self.tape_cassette)
self.tape_recorder.enable_recording()
def tearDown(self):
self.tape_cassette.close()
@parameterized.expand([("compare_in_dedicated_process", True),
("playback_same_process", False),
])
def test_equal_comparison(self, name, compare_in_dedicated_process):
class Operation(object):
def __init__(self, value=None, multiply_input=1):
self._value = value
self.multiply_input = multiply_input
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self):
return self.input * self.multiply_input
Operation(3).execute()
Operation(4).execute()
Operation(5).execute()
playback_counter = [0]
def playback_function(recording):
playback_counter[0] += 1
if playback_counter[0] == 2:
operation = Operation(multiply_input=2)
elif playback_counter[0] == 3:
operation = Operation(multiply_input=100)
else:
operation = Operation()
return operation.execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
end_date = datetime.utcnow() + timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(start_date=start_date, end_date=end_date),
)
runner = Equalizer(playable_recordings, player, result_extractor=return_value_result_extractor,
comparator=exact_comparator,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=compare_in_dedicated_process
))
with patch.object(Equalizer, '_play_and_compare_recording',
wraps=runner._play_and_compare_recording) as wrapped:
comparison = list(runner.run_comparison())
if compare_in_dedicated_process:
wrapped.assert_not_called()
else:
wrapped.assert_called()
self.assertEqual(EqualityStatus.Equal, comparison[0].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Different, comparison[1].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Failed, comparison[2].comparator_status.equality_status)
self.assertEqual(3, comparison[0].expected)
self.assertEqual(4, comparison[1].expected)
self.assertEqual(5, comparison[2].expected)
self.assertEqual(3, comparison[0].actual)
self.assertEqual(8, comparison[1].actual)
self.assertEqual(500, comparison[2].actual)
mock.assert_called_with(Operation.__name__, start_date=start_date, end_date=end_date, limit=None, metadata=None)
self.assertGreaterEqual(comparison[0].playback.playback_duration, 0)
self.assertGreaterEqual(comparison[0].playback.recorded_duration, 0)
@parameterized.expand([("compare_in_dedicated_process", True),
("playback_same_process", False),
])
def test_with_exception_comparison(self, name, compare_in_dedicated_process):
class Operation(object):
def __init__(self, value=None, raise_error=None):
self._value = value
self.raise_error = raise_error
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self, value=None):
if self.raise_error:
raise Exception("error")
if value is not None:
return value
return self.input
Operation(3).execute()
Operation(4).execute()
with suppress(Exception):
Operation(raise_error=True).execute()
playback_counter = [0]
def playback_function(recording):
playback_counter[0] += 1
if playback_counter[0] == 2:
operation = Operation(raise_error=True)
elif playback_counter[0] == 3:
return Operation().execute(5)
else:
operation = Operation()
return operation.execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids):
start_date = datetime.utcnow() - timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(start_date=start_date),
)
runner = Equalizer(playable_recordings, player, result_extractor=return_value_result_extractor,
comparator=exact_comparator, compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=compare_in_dedicated_process
))
comparison = list(runner.run_comparison())
self.assertEqual(EqualityStatus.Equal, comparison[0].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Different, comparison[1].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Different, comparison[2].comparator_status.equality_status)
self.assertEqual(3, comparison[0].expected)
self.assertEqual(4, comparison[1].expected)
self.assertIsInstance(comparison[2].expected, Exception)
self.assertTrue(comparison[1].actual_is_exception)
self.assertFalse(comparison[1].expected_is_exception)
self.assertEqual(3, comparison[0].actual)
self.assertIsInstance(comparison[1].actual, Exception)
self.assertEqual(5, comparison[2].actual)
self.assertFalse(comparison[2].actual_is_exception)
self.assertTrue(comparison[2].expected_is_exception)
@parameterized.expand([("compare_in_dedicated_process", True),
("playback_same_process", False),
])
def test_comparison_with_meta_and_limit_filtering(self, name, compare_in_dedicated_process):
class Operation(object):
def __init__(self, value=None):
self._value = value
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation(metadata_extractor=lambda op_self, meta=None: {'meta': meta})
def execute(self, meta=None):
return self.input
Operation(3).execute('a')
Operation(4).execute('b')
Operation(5).execute('b')
def playback_function(recording):
return Operation().execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(
start_date=start_date,
metadata={'meta': 'b'},
limit=1),
)
runner = Equalizer(playable_recordings, player, result_extractor=return_value_result_extractor,
comparator=exact_comparator,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=compare_in_dedicated_process
))
comparison = list(runner.run_comparison())
self.assertEqual(1, len(comparison))
self.assertEqual(EqualityStatus.Equal, comparison[0].comparator_status.equality_status)
mock.assert_called_with(Operation.__name__, start_date=start_date, end_date=None,
limit=1, metadata={'meta': 'b'})
self.assertEqual(4, comparison[0].expected)
self.assertEqual(4, comparison[0].actual)
@parameterized.expand([("compare_in_dedicated_process", True),
("playback_same_process", False),
])
def test_equal_comparison_with_message(self, name, compare_in_dedicated_process):
class Operation(object):
def __init__(self, value=None, multiply_input=1):
self._value = value
self.multiply_input = multiply_input
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self):
return self.input * self.multiply_input
Operation(3).execute()
Operation(4).execute()
Operation(5).execute()
playback_counter = [0]
def playback_function(recording):
playback_counter[0] += 1
if playback_counter[0] == 2:
operation = Operation(multiply_input=2)
elif playback_counter[0] == 3:
operation = Operation(multiply_input=100)
else:
operation = Operation()
return operation.execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
end_date = datetime.utcnow() + timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(
start_date=start_date,
end_date=end_date)
)
runner = Equalizer(playable_recordings, player, result_extractor=return_value_result_extractor,
comparator=exact_comparator_with_message,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=compare_in_dedicated_process
))
comparison = list(runner.run_comparison())
self.assertEqual(EqualityStatus.Equal, comparison[0].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Different, comparison[1].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Failed, comparison[2].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Equal.name, comparison[0].comparator_status.message)
self.assertEqual(EqualityStatus.Different.name, comparison[1].comparator_status.message)
self.assertEqual(EqualityStatus.Failed.name, comparison[2].comparator_status.message)
@parameterized.expand([("compare_in_dedicated_process", True),
("playback_same_process", False),
])
def test_equal_comparison_comparator_data_extraction(self, name, compare_in_dedicated_process):
class Operation(object):
def __init__(self, value=None, override_input=None):
self._value = value
self.override_input = override_input
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self):
if self.override_input:
return self.override_input
return self.input
Operation(3).execute()
Operation(4).execute()
Operation(5).execute()
playback_counter = [0]
def playback_function(recording):
playback_counter[0] += 1
if playback_counter[0] == 2:
operation = Operation(override_input=100)
else:
operation = Operation()
return operation.execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
def comparison_data_extractor(recording):
self.assertIsNotNone(recording)
return {'multiplier': 1}
def exact_comparator_with_multiplier(recorded_result, playback_result, multiplier):
if playback_result >= 10:
playback_result *= 0
recorded_result *= 0
else:
playback_result *= multiplier
recorded_result *= multiplier
return ComparatorResult(
EqualityStatus.Equal if recorded_result * multiplier == playback_result * multiplier
else EqualityStatus.Different, str(multiplier))
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
end_date = datetime.utcnow() + timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(
start_date=start_date,
end_date=end_date)
)
runner = Equalizer(playable_recordings, player, result_extractor=return_value_result_extractor,
comparator=exact_comparator_with_multiplier,
comparison_data_extractor=comparison_data_extractor,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=compare_in_dedicated_process
))
comparison = list(runner.run_comparison())
self.assertEqual(EqualityStatus.Equal, comparison[0].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Equal, comparison[1].comparator_status.equality_status)
self.assertEqual(EqualityStatus.Equal, comparison[2].comparator_status.equality_status)
self.assertEqual('1', comparison[0].comparator_status.message)
self.assertEqual('1', comparison[1].comparator_status.message)
self.assertEqual('1', comparison[2].comparator_status.message)
def test_random_sample(self):
class Operation(object):
def __init__(self, value=None):
self._value = value
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self):
return self.input
for i in range(10):
Operation(i).execute()
def playback_function(recording):
return Operation().execute()
random.seed(12)
first_list = None
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(
start_date=start_date,
random_sample=True,
limit=3)
)
first_list = list(playable_recordings)
self.assertEqual(len(first_list), 3)
random.seed(4)
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(
start_date=start_date,
random_sample=True,
limit=3)
)
second_list = list(playable_recordings)
self.assertNotEqual(second_list, first_list)
@parameterized.expand([("compare_in_dedicated_process", True),
("playback_same_process", False),
])
def test_run_with_specific_ids(self, name, compare_in_dedicated_process):
class Operation(object):
def __init__(self, value=None, override_input=None):
self._value = value
self.override_input = override_input
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self):
if self.override_input:
return self.override_input
return self.input
Operation(3).execute()
id1 = self.tape_cassette.get_last_recording_id()
Operation(4).execute()
Operation(5).execute()
id2 = self.tape_cassette.get_last_recording_id()
Operation(6).execute()
playback_counter = [0]
def playback_function(recording):
playback_counter[0] += 1
if playback_counter[0] == 2:
operation = Operation(override_input=100)
else:
operation = Operation()
return operation.execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
runner = Equalizer([id1, id2], player, result_extractor=return_value_result_extractor,
comparator=exact_comparator,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=compare_in_dedicated_process
))
comparison = list(runner.run_comparison())
self.assertEqual(id1, comparison[0].playback.original_recording.id)
self.assertEqual(id2, comparison[1].playback.original_recording.id)
self.assertEqual(2, len(comparison))
def test_equalizer_recycle_process_thread(self):
class Operation(object):
def __init__(self, value=None):
self._value = value
@property
@self.tape_recorder.intercept_input('input')
def input(self):
return self._value
@self.tape_recorder.operation()
def execute(self):
return self.input
for i in range(20):
Operation(i).execute()
def playback_function(recording):
return Operation().execute()
def player(recording_id):
return self.tape_recorder.play(recording_id, playback_function)
with patch.object(InMemoryTapeCassette, 'iter_recording_ids',
wraps=self.tape_cassette.iter_recording_ids) as mock:
start_date = datetime.utcnow() - timedelta(hours=1)
end_date = datetime.utcnow() + timedelta(hours=1)
playable_recordings = find_matching_recording_ids(
self.tape_recorder,
category=Operation.__name__,
lookup_properties=RecordingLookupProperties(start_date=start_date, end_date=end_date),
)
runner = Equalizer(playable_recordings, player, result_extractor=return_value_result_extractor,
comparator=exact_comparator,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True,
compare_in_dedicated_process=True,
compare_process_recycle_rate=2
))
with patch.object(Equalizer, '_create_new_player_process',
wraps=runner._create_new_player_process) as wrapped:
comparison = list(runner.run_comparison())
self.assertEqual(10, wrapped.call_count)
self.assertEqual(20, len(comparison))
for c in comparison:
self.assertEqual(EqualityStatus.Equal, c.comparator_status.equality_status)
class TestTapeRecorder(unittest.TestCase):
def setUp(self):
self.tape_cassette = InMemoryTapeCassette()
self.tape_recorder = TapeRecorder(self.tape_cassette)
self.tape_recorder.enable_recording()
def test_record_and_playback_basic_operation_no_parameters(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def _assert_playback_vs_recording(self, playback_result, result):
"""
:param playback_result: Playback result
:type playback_result: playback.tape_recorder.Playback
:param result: Operation result
:type result: Any
"""
if six.PY3:
self.assertCountEqual(playback_result.recorded_outputs, playback_result.playback_outputs)
else:
self.assertItemsEqual(playback_result.recorded_outputs, playback_result.playback_outputs)
operation_output = next(po for po in playback_result.playback_outputs
if TapeRecorder.OPERATION_OUTPUT_ALIAS in po.key)
self.assertEqual(result, operation_output.value['args'][0])
self.assertGreater(playback_result.playback_duration, 0)
self.assertGreater(playback_result.recorded_duration, 0)
def test_record_and_playback_basic_operation_data_interception_no_arguments(self):
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
return self.get_value()
@self.tape_recorder.intercept_input('input')
def get_value(self):
return self.seed
instance = Operation(5)
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_data_interception_property(self):
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
return self.get_value
@self.tape_recorder.intercept_input('input')
@property
def get_value(self):
return self.seed
instance = Operation(5)
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_data_interception_with_arguments(self):
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
val1 = self.get_value(2, b=3)
val2 = self.get_value(4, b=6)
return val1 + val2
@self.tape_recorder.intercept_input('input')
def get_value(self, a, b=2):
return (a + b) * self.seed
instance = Operation(seed=1)
result = instance.execute()
self.assertEqual(15, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_static_data_interception_with_arguments(self):
class Operation(object):
seed = 0
@self.tape_recorder.operation()
def execute(self):
val1 = self.get_value(2, b=3)
val2 = self.get_value(4, b=6)
return val1 + val2
@staticmethod
@self.tape_recorder.static_intercept_input('input')
def get_value(a, b=2):
return (a + b) * Operation.seed
instance = Operation()
Operation.seed = 1
result = instance.execute()
self.assertEqual(15, result)
Operation.seed = 0
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_data_interception_with_arguments_and_metadata(self):
def operation_metadata_extractor(obj):
return {'type': type(obj)}
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation(metadata_extractor=operation_metadata_extractor)
def execute(self):
val1 = self.get_value(2, b=3)
val2 = self.get_value(4, b=6)
return val1 + val2
@self.tape_recorder.intercept_input('input')
def get_value(self, a, b=2):
return (a + b) * self.seed
instance = Operation(seed=1)
result = instance.execute()
self.assertEqual(15, result)
recording_id = self.tape_cassette.get_last_recording_id()
recording = self.tape_cassette.get_recording(recording_id)
# pickle encode decode doesn't reconstruct actuall class if this is an inner class
operation_decoded = decode(encode(Operation, unpicklable=False))
self.assertDictContainsSubset({'type': operation_decoded, TapeRecorder.OPERATION_CLASS: operation_decoded},
recording.get_metadata())
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_no_parameters_with_output(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
x = 0
x += self.output(4, arg='a')
x += self.output(3, arg='b')
return x
@self.tape_recorder.intercept_output('output_function')
def output(self, value, arg=None):
return value
instance = Operation()
result = instance.execute()
self.assertEqual(7, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
self.assertEqual({'args': [4], 'kwargs': {'arg': 'a'}}, playback_result.playback_outputs[0].value)
self.assertEqual({'args': [3], 'kwargs': {'arg': 'b'}}, playback_result.playback_outputs[1].value)
self.assertNotEqual(playback_result.playback_outputs[0].key, playback_result.playback_outputs[1].key)
self.assertIn('output_function', playback_result.playback_outputs[0].key)
self.assertIn('output_function', playback_result.playback_outputs[1].key)
def test_record_and_playback_basic_operation_no_parameters_with_static_output(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
x = 0
x += self.output(4, arg='a')
x += self.output(3, arg='b')
return x
@staticmethod
@self.tape_recorder.static_intercept_output('output_function')
def output(value, arg=None):
return value
instance = Operation()
result = instance.execute()
self.assertEqual(7, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
self.assertEqual({'args': [4], 'kwargs': {'arg': 'a'}}, playback_result.playback_outputs[0].value)
self.assertEqual({'args': [3], 'kwargs': {'arg': 'b'}}, playback_result.playback_outputs[1].value)
self.assertNotEqual(playback_result.playback_outputs[0].key, playback_result.playback_outputs[1].key)
self.assertIn('output_function', playback_result.playback_outputs[0].key)
self.assertIn('output_function', playback_result.playback_outputs[1].key)
def test_record_and_playback_basic_operation_similar_duration(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
sleep(0.5)
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self.assertLessEqual(abs(playback_result.recorded_duration - playback_result.playback_duration), 0.1)
def test_record_and_playback_basic_operation_class_extraction_in_metadata(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return 5
@classmethod
@self.tape_recorder.class_operation()
def class_execute(cls):
return 5
instance = Operation()
self.assertEqual(5, instance.execute())
recording_id = self.tape_cassette.get_last_recording_id()
recording = self.tape_cassette.get_recording(recording_id)
# pickle encode decode doesn't reconstruct actuall class if this is an inner class
operation_decoded = decode(encode(Operation, unpicklable=False))
self.assertDictContainsSubset({TapeRecorder.OPERATION_CLASS: operation_decoded}, recording.get_metadata())
self.assertEqual(5, instance.class_execute())
recording_id = self.tape_cassette.get_last_recording_id()
recording = self.tape_cassette.get_recording(recording_id)
# Inner classes are not unpicklable back to class upon decode
self.assertDictContainsSubset({TapeRecorder.OPERATION_CLASS: operation_decoded}, recording.get_metadata())
def test_drop_recording_direct_api_call(self):
local_tape_recorder = self.tape_recorder
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
local_tape_recorder.discard_recording()
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, '_save_recording', wraps=self.tape_cassette._save_recording) \
as intercepted_save, \
patch.object(InMemoryTapeCassette, 'abort_recording', wraps=self.tape_cassette.abort_recording) \
as intercepted_abort:
result = instance.execute()
intercepted_save.assert_not_called()
intercepted_abort.assert_called()
self.assertEqual(5, result)
def test_skip_recording_decorator(self):
@self.tape_recorder.recording_params(skipped=True)
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, 'create_new_recording', wraps=self.tape_cassette.create_new_recording) \
as intercepted:
result = instance.execute()
intercepted.assert_not_called()
self.assertEqual(5, result)
def test_input_interception_key_failure(self):
class UnencodeableObject(object):
def __getstate__(self):
raise Exception()
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
param = UnencodeableObject()
value = self.input(param)
return value
@self.tape_recorder.intercept_input('input')
def input(self, param):
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, '_save_recording', wraps=self.tape_cassette._save_recording) \
as intercepted_save, \
patch.object(InMemoryTapeCassette, 'abort_recording', wraps=self.tape_cassette.abort_recording) \
as intercepted_abort:
result = instance.execute()
intercepted_save.assert_not_called()
intercepted_abort.assert_called()
self.assertEqual(5, result)
def test_input_interception_key_failure_during_playback(self):
class UnencodeableObject(object):
def __getstate__(self):
raise Exception()
class Operation(object):
def __init__(self, raise_on_get=False):
self.raise_on_get = raise_on_get
@self.tape_recorder.operation()
def execute(self):
param = UnencodeableObject() if self.raise_on_get else object()
value = self.input(param)
return value
@self.tape_recorder.intercept_input('input')
def input(self, param):
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
with self.assertRaises(InputInterceptionKeyCreationError):
self.tape_recorder.play(recording_id, playback_function=lambda recording: Operation(True).execute())
def test_input_interception_key_missing_during_playback(self):
class Operation(object):
def __init__(self, input_key):
self.input_key = input_key
@self.tape_recorder.operation()
def execute(self):
value = self.input(self.input_key)
return value
@self.tape_recorder.intercept_input('input')
def input(self, param):
return 5
instance = Operation('key1')
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
with self.assertRaises(RecordingKeyError):
self.tape_recorder.play(recording_id, playback_function=lambda recording: Operation('key2').execute())
def test_output_interception_key_missing_during_playback(self):
class Operation(object):
def __init__(self, output_method):
self.output_method = output_method
@self.tape_recorder.operation()
def execute(self):
value = getattr(self, self.output_method)()
return value
@self.tape_recorder.intercept_output('output_function')
def output(self):
return 5
@self.tape_recorder.intercept_output('output_missing')
def output_missing(self):
return 5
instance = Operation('output')
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
with self.assertRaises(RecordingKeyError):
self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation('output_missing').execute())
def test_record_and_playback_basic_operation_metadata_extractor_raise_exception(self):
def operation_metadata_extractor(*args, **kwargs):
raise Exception('Meta exception')
class Operation(object):
@self.tape_recorder.operation(metadata_extractor=operation_metadata_extractor)
def execute(self):
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_save_recording_raise_exception(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return 5
with patch.object(InMemoryTapeCassette, '_save_recording', side_effect=Exception()):
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
def test_record_and_playback_basic_operation_no_parameters_raise_error(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
raise ValueError("Error")
instance = Operation()
with self.assertRaises(ValueError) as e:
instance.execute()
self.assertEqual("Error", str(e.exception))
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
operation_output = next(po for po in playback_result.playback_outputs
if TapeRecorder.OPERATION_OUTPUT_ALIAS in po.key)
self.assertEqual(ValueError, type(operation_output.value['args'][0]))
self.assertEqual("Error", str(operation_output.value['args'][0]))
self.assertEqual(len(playback_result.recorded_outputs), len(playback_result.playback_outputs))
self.assertGreater(playback_result.playback_duration, 0)
self.assertGreater(playback_result.recorded_duration, 0)
self.assertDictContainsSubset({TapeRecorder.EXCEPTION_IN_OPERATION: True},
playback_result.original_recording.get_metadata())
def test_record_and_playback_basic_operation_data_interception_no_arguments_raise_exception(self):
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
try:
self.get_value()
except Exception as ex:
return str(ex)
@self.tape_recorder.intercept_input('input')
def get_value(self):
raise Exception(self.seed)
instance = Operation(5)
result = instance.execute()
self.assertEqual('5', result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_output_interception_no_arguments_raise_exception(self):
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
try:
self.output()
except Exception as ex:
return str(ex)
@self.tape_recorder.intercept_output('output_function')
def output(self):
raise Exception(self.seed)
instance = Operation(5)
result = instance.execute()
self.assertEqual("5", result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_sampling_rate_class_level_decorator(self):
@self.tape_recorder.recording_params(sampling_rate=0.1)
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, 'save_recording', wraps=self.tape_cassette.create_new_recording) \
as intercepted:
calls = 100
for __ in range(calls):
result = instance.execute()
self.assertEqual(5, result)
call_ratio = float(intercepted.call_count)/calls
print('Call ratio {}'.format(call_ratio))
self.assertAlmostEqual(0.1, call_ratio, places=1)
def test_operation_with_input_dict_as_key(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
# We are shuffling here hoping to get different internal order for the dict between calls but still
# see we are consistent on input interception is these are equivalent dicts
items = list(range(100))
shuffle(items)
argument = {'key{}'.format(i): i for i in items}
return self.get_value(argument)
@self.tape_recorder.intercept_input('input')
def get_value(self, json_dict):
return json_dict['key5']
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_operation_with_alias_param_resolver(self):
class ValueCreator(object):
def __init__(self, name, value):
self.name = name
self.value = value
@self.tape_recorder.intercept_input('input.{name}', alias_params_resolver=lambda s: {'name': s.name})
def get_value(self):
return self.value
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
# We check that we are capturing different value interceptions even though we pass same arguments to
# the intercepted method by using a param resolver that will provide unique interception key for each
# invocation
return self.get_value('a', self.seed) + self.get_value('b', self.seed * 2)
@staticmethod
def get_value(name, value):
return ValueCreator(name, value).get_value()
instance = Operation(5)
result = instance.execute()
self.assertEqual(15, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_operation_with_input_interception_data_handler(self):
test_self = self
class XorDataHandler(InputInterceptionDataHandler):
def prepare_input_for_recording(self, interception_key, result, args, kwargs):
test_self.assertIn('value_input', interception_key)
test_self.assertEqual(5, result)
test_self.assertEqual(('x', ), args[1:])
test_self.assertEqual({'b': 'y'}, kwargs)
return 5 ^ 3141
def restore_input_from_recording(self, recorded_data, args, kwargs):
# Applying same xor twice return original result
return recorded_data ^ 3141
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
return self.get_value('x', b='y')
@self.tape_recorder.intercept_input('value_input', data_handler=XorDataHandler())
def get_value(self, a, b=None):
return self.seed
instance = Operation(5)
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
# Check the recording it self contains the modified value
recording = self.tape_cassette.get_recording(recording_id)
key = next(k for k in recording.get_all_keys() if 'value_input' in k)
self.assertEqual(5 ^ 3141, recording.get_data(key)['value'])
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_input_interception_data_handler_failure(self):
class ErrorDataHandler(InputInterceptionDataHandler):
def prepare_input_for_recording(self, interception_key, result, args, kwargs):
raise Exception('error')
def restore_input_from_recording(self, recorded_data, args, kwargs):
pass
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return self.get_value() + self.get_value2()
@self.tape_recorder.intercept_input('value_input', data_handler=ErrorDataHandler())
def get_value(self):
return 5
@self.tape_recorder.intercept_input('value_input2')
def get_value2(self):
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, '_save_recording', wraps=self.tape_cassette._save_recording) \
as intercepted:
result = instance.execute()
intercepted.assert_not_called()
self.assertEqual(10, result)
def test_record_and_playback_basic_operation_data_interception_with_all_arguments_ignored_as_key(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
arg1 = random()
arg2 = random()
val1 = self.get_value(arg1, b=arg2)
return val1
@self.tape_recorder.intercept_input('input', capture_args=[])
def get_value(self, a, b=2):
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_basic_operation_data_interception_with_false_capture_args(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
arg1 = random()
arg2 = random()
val1 = self.get_value(arg1, b=arg2)
return val1
@self.tape_recorder.intercept_input('input', capture_args=False)
def get_value(self, a, b=2):
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_sampling_rate_with_enforced_recording(self):
local_tape_recorder = self.tape_recorder
test_self = self
@self.tape_recorder.recording_params(sampling_rate=0.2)
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
local_tape_recorder.force_sample_recording()
test_self.assertTrue(local_tape_recorder.is_recording_sample_forced)
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, '_save_recording', wraps=self.tape_cassette._save_recording) \
as intercepted:
calls = 10
for __ in range(calls):
result = instance.execute()
self.assertEqual(5, result)
call_ratio = float(intercepted.call_count)/calls
print('Call ratio {}'.format(call_ratio))
self.assertEqual(1, call_ratio)
def test_sampling_rate_with_ignore_enforced_recording(self):
local_tape_recorder = self.tape_recorder
test_self = self
@self.tape_recorder.recording_params(sampling_rate=0.1, ignore_enforced_sampling=True)
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
local_tape_recorder.force_sample_recording()
test_self.assertFalse(local_tape_recorder.is_recording_sample_forced)
return 5
instance = Operation()
with patch.object(InMemoryTapeCassette, '_save_recording', wraps=self.tape_cassette._save_recording) \
as intercepted_save,\
patch.object(InMemoryTapeCassette, 'abort_recording', wraps=self.tape_cassette.abort_recording) \
as intercepted_abort:
calls = 100
for __ in range(calls):
result = instance.execute()
self.assertEqual(5, result)
expected_aborted = calls - intercepted_save.call_count
call_ratio = float(intercepted_save.call_count) / calls
print('Call ratio {}'.format(call_ratio))
self.assertAlmostEqual(0.1, call_ratio, places=1)
self.assertEqual(expected_aborted, intercepted_abort.call_count)
def test_record_and_playback_basic_operation_data_interception_with_specific_arguments_ignored_as_key(self):
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
# This is random to see that capture args is only capturing the requested arg index
arg1 = random()
arg2 = 'b'
arg3 = 'c'
# This is random to see that capture args is only capturing the requested arg kwargs
arg4 = random()
arg5 = 'e'
val1 = self.get_value(arg1, arg2, c=arg3, d=arg4, e=arg5)
return val1
@self.tape_recorder.intercept_input('input', capture_args=[CapturedArg(2, 'b'), CapturedArg(3, 'c'),
CapturedArg(None, 'e'), CapturedArg(None, 'f')])
def get_value(self, a, b, c, d=None, e=None, f=None):
return 5
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
def test_record_and_playback_operation_with_output_interception_data_handler(self):
test_self = self
class XorDataHandler(OutputInterceptionDataHandler):
def prepare_output_for_recording(self, interception_key, args, kwargs):
test_self.assertIn('output_function', interception_key)
test_self.assertEqual((5, ), args)
test_self.assertEqual({'arg': 'a'}, kwargs)
return args[0] ^ 3141
def restore_output_from_recording(self, recorded_data):
# Applying same xor twice return original result
return recorded_data ^ 3141
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return self.output(5, arg='a')
@self.tape_recorder.intercept_output('output_function', data_handler=XorDataHandler())
def output(self, value, arg=None):
return value
instance = Operation()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
# Check the recording it self contains the modified value
recording = self.tape_cassette.get_recording(recording_id)
key = next(k for k in recording.get_all_keys() if 'output_function' in k)
self.assertEqual(5 ^ 3141, recording.get_data(key))
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
self.assertEqual(result, XorDataHandler().restore_output_from_recording(
playback_result.playback_outputs[0].value))
def test_output_interception_data_handler_failure(self):
class ErrorDataHandler(OutputInterceptionDataHandler):
def prepare_output_for_recording(self, interception_key, args, kwargs):
raise Exception('error')
def restore_output_from_recording(self, recorded_data):
pass
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return self.output(5) + self.output2(5)
@self.tape_recorder.intercept_output('output_function', data_handler=ErrorDataHandler())
def output(self, value):
return value
@self.tape_recorder.intercept_output('output_function2')
def output2(self, value):
return value
instance = Operation()
with patch.object(InMemoryTapeCassette, '_save_recording', wraps=self.tape_cassette._save_recording) \
as intercepted:
result = instance.execute()
intercepted.assert_not_called()
self.assertEqual(10, result)
def test_record_and_playback_basic_operation_data_interception_inside_interception(self):
class Operation(object):
def __init__(self, seed=0):
self.seed = seed
@self.tape_recorder.operation()
def execute(self):
return self.get_value()
@self.tape_recorder.intercept_input('input')
def get_value(self):
return self.get_inner_value()
@self.tape_recorder.intercept_input('inner_input')
def get_inner_value(self):
return self.seed
instance = Operation(5)
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
self._assert_playback_vs_recording(playback_result, result)
# Check the recording it self does not contain the inner interception
recording = self.tape_cassette.get_recording(recording_id)
self.assertIsNone(next((k for k in recording.get_all_keys() if 'inner_input' in k), None))
def test_record_and_playback_basic_operation_new_output_added_post_recording(self):
class OperationOld(object):
@self.tape_recorder.operation()
def execute(self):
return self.output(5)
@self.tape_recorder.intercept_output('output_function')
def output(self, value):
return value
class OperationNew(object):
@self.tape_recorder.operation()
def execute(self):
value = self.output(5)
self.output_new(value)
return value
@self.tape_recorder.intercept_output('output_function')
def output(self, value):
return value
@self.tape_recorder.intercept_output('output_new_function', fail_on_no_recorded_result=False)
def output_new(self, value):
return value
instance = OperationOld()
result = instance.execute()
self.assertEqual(5, result)
recording_id = self.tape_cassette.get_last_recording_id()
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: OperationNew().execute())
self.assertEqual({'args': [5], 'kwargs': {}}, playback_result.playback_outputs[0].value)
self.assertEqual({'args': [5], 'kwargs': {}}, playback_result.playback_outputs[1].value)
self.assertNotEqual(playback_result.playback_outputs[0].key, playback_result.playback_outputs[1].key)
self.assertIn('output_function', playback_result.playback_outputs[0].key)
self.assertIn('output_new_function', playback_result.playback_outputs[1].key)
def test_current_recording_id(self):
tape_recorder = self.tape_recorder
class Operation(object):
@self.tape_recorder.operation()
def execute(self):
return tape_recorder.current_recording_id
instance = Operation()
result = instance.execute()
recording_id = self.tape_cassette.get_last_recording_id()
self.assertEqual(recording_id, result)
playback_result = self.tape_recorder.play(recording_id,
playback_function=lambda recording: Operation().execute())
operation_output = next(po for po in playback_result.playback_outputs
if TapeRecorder.OPERATION_OUTPUT_ALIAS in po.key)
self.assertEqual(recording_id, operation_output.value['args'][0])
self.tape_recorder.disable_recording()
instance = Operation()
self.assertIsNone(instance.execute())
class TestPlaybackStudio(unittest.TestCase):
def setUp(self):
self.tape_cassette = InMemoryTapeCassette()
self.tape_recorder = TapeRecorder(self.tape_cassette)
self.tape_recorder.enable_recording()
def tearDown(self):
self.tape_cassette.close()
def test_run_two_categories(self):
self._test_run_two_operations(use_recording_ids=False)
def test_run_specific_recording_ids(self):
self._test_run_two_operations(use_recording_ids=True)
def _test_run_two_operations(self, use_recording_ids):
class A(object):
@self.tape_recorder.operation()
def execute(self):
return 'AAA'
class B(object):
@self.tape_recorder.operation()
def execute(self):
return 'BBB'
A().execute()
A().execute()
B().execute()
if not use_recording_ids:
categories = ['A', 'B', 'C']
recording_ids = None
else:
categories = None
recording_ids = self.tape_cassette.get_all_recording_ids()
class MockEqualizerTuner(EqualizerTuner):
def create_category_tuning(self, category):
if category not in ['A', 'B']:
raise Exception("Unsupported")
def result_extractor(outputs):
return next(
o.value['args'][0] for o in outputs
if TapeRecorder.OPERATION_OUTPUT_ALIAS in o.key)
def comparator(expected, actual, message):
return ComparatorResult(
EqualityStatus.Equal if expected == actual else
EqualityStatus.Different, message)
def comparison_data_extractor(recording):
return {'message': category}
def playback_function(recording):
cls_name = recording.get_metadata()[
TapeRecorder.OPERATION_CLASS]
if 'studio.A' in list(cls_name.values())[0]:
return A().execute()
return B().execute()
return EqualizerTuning(
playback_function=playback_function,
result_extractor=result_extractor,
comparator=comparator,
comparison_data_extractor=comparison_data_extractor)
equalizer_tuner = MockEqualizerTuner()
start_date = 'a'
studio = PlaybackStudio(
categories,
equalizer_tuner,
self.tape_recorder,
lookup_properties=RecordingLookupProperties(start_date),
recording_ids=recording_ids,
compare_execution_config=CompareExecutionConfig(
keep_results_in_comparison=True))
result = studio.play()
a_results = list(result['A'])
b_results = list(result['B'])
self.assertEqual(2, len(a_results))
self.assertTrue(all('AAA' == result.actual for result in a_results))
self.assertTrue(all('AAA' == result.expected for result in a_results))
self.assertTrue(
all(EqualityStatus.Equal ==
result.comparator_status.equality_status
for result in a_results))
self.assertTrue(
all('A' == result.comparator_status.message
for result in a_results))
self.assertEqual(1, len(b_results))
self.assertTrue(all('BBB' == result.actual for result in b_results))
self.assertTrue(all('BBB' == result.expected for result in b_results))
self.assertTrue(
all(EqualityStatus.Equal ==
result.comparator_status.equality_status
for result in b_results))
self.assertTrue(
all('B' == result.comparator_status.message
for result in b_results))