def test_iteratively_disabled(httpserver: HTTPServer):
p = configure_plugin_and_server(
httpserver,
IterativelyOptions(
disabled=True,
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
))
try:
p.load(PLUGIN_OPTIONS_DEV_NO_LOGGER)
p.post_identify(*post_identify_1_args)
p.post_identify(*post_identify_2_args)
p.post_track(*post_track_1_args)
p.post_group(*post_group_args)
p.flush()
p.post_page(*post_page_args)
p.flush()
p.flush()
p.post_track(*post_track_2_args)
finally:
p.shutdown()
time.sleep(0.1)
httpserver.stop()
requests = _get_cleaned_requests(httpserver)
assert requests == []
def test_iteratively_omit_values(httpserver: HTTPServer):
p = configure_plugin_and_server(
httpserver, IterativelyOptions(omit_values=True, flush_queue_size=10))
try:
p.load(PLUGIN_OPTIONS_DEV_NO_LOGGER)
p.post_identify(*post_identify_1_args)
p.post_identify(*post_identify_2_args)
p.post_track(*post_track_1_args)
p.post_group(*post_group_args)
p.post_page(*post_page_args)
p.post_track(*post_track_2_args)
finally:
p.shutdown()
time.sleep(0.1)
httpserver.stop()
requests = _get_cleaned_requests(httpserver)
expected_requests = deepcopy([
post_identify_1_expected_request, post_identify_2_expected_request,
post_track_1_expected_request, post_group_expected_request,
post_page_expected_request, post_track_2_expected_request
])
for req in expected_requests:
strip_property_values(req)
assert requests == [{'objects': expected_requests}]
def test_iteratively_retry(httpserver: HTTPServer):
logger = CustomLogger()
p = IterativelyPlugin(
'My-Key', httpserver.url_for('/track'),
IterativelyOptions(
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
retry_options=IterativelyRetryOptions(
max_retries=3,
delay_initial=timedelta(seconds=0.1),
delay_maximum=timedelta(seconds=0.5)),
))
expected_request = {'objects': [post_track_1_expected_request]}
try:
p.load(
PluginLoadOptions(environment=Environment.DEVELOPMENT,
logger=logger))
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data(status=429)
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data(status=501)
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data()
p.post_track(*post_track_1_args)
p.flush()
time.sleep(2)
requests = _get_cleaned_requests(httpserver)
assert requests == [
expected_request, expected_request, expected_request
]
assert logger.log_lines == []
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data(status=429)
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data(status=501)
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data(status=504)
httpserver.expect_oneshot_request(
re.compile('/track')).respond_with_data()
p.post_track(*post_track_1_args)
p.flush()
time.sleep(2)
requests = _get_cleaned_requests(httpserver)
assert requests == [
expected_request,
expected_request,
expected_request,
expected_request,
expected_request,
expected_request,
]
assert logger.log_lines == [
'Error. Failed to upload events. Maximum attempts exceeded.'
]
finally:
p.shutdown()
httpserver.stop()
def _build_server(tmpdir, host):
ca = trustme.CA()
ca_cert_path = str(tmpdir / "ca.pem")
ca.cert_pem.write_to_path(ca_cert_path)
server_cert = ca.issue_cert(host)
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
server_crt = server_cert.cert_chain_pems[0]
server_key = server_cert.private_key_pem
with server_crt.tempfile() as crt_file, server_key.tempfile() as key_file:
context.load_cert_chain(crt_file, key_file)
server = HTTPServer(ssl_context=context)
# Fake what the client expects from Elasticsearch
server.expect_request("/").respond_with_json(
headers={
"x-elastic-product": "Elasticsearch",
},
response_json={"version": {
"number": "8.0.0",
}},
)
server.start()
yield server, ca, ca_cert_path
server.clear()
if server.is_running():
server.stop()
def test_flush_at_interval(httpserver: HTTPServer):
p = configure_plugin_and_server(
httpserver,
IterativelyOptions(
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
))
try:
p.load(PLUGIN_OPTIONS_DEV_NO_LOGGER)
p.post_page(*post_page_args)
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == []
time.sleep(1.0)
requests = _get_cleaned_requests(httpserver)
assert requests == [
{
'objects': [post_page_expected_request]
},
]
finally:
p.shutdown()
httpserver.stop()
def test_mixpanel(httpserver: HTTPServer):
httpserver.expect_request(re.compile('/(track|engage)')).respond_with_json({'status': 1})
options = MixpanelOptions(
api_host=httpserver.url_for('').rstrip('/'),
)
p = MixpanelPlugin('My-Key', options)
assert p.id() == 'mixpanel'
p.load(PluginLoadOptions(environment=Environment.DEVELOPMENT, logger=Logger.NONE))
p.identify("user-1", Properties(item1='value1', item2=2))
p.alias("user-1", "prev-user-1")
p.track("user-2", Event('event-1', Properties(item1='value1', item2=1)))
p.track("user-2", Event('event-2', Properties(item1='value2', item2=2)))
p.flush()
p.track("user-2", Event('event-3', Properties(item1='value3', item2=3)))
p.track("user-2", Event('event-4', Properties(item1='value4', item2=4)))
p.track("user-1", Event('event-5', Properties(item1='value5', item2=5)))
p.shutdown()
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
httpserver.stop()
assert requests == [
{'event': 'event-1',
'properties': {'distinct_id': 'user-2',
'item1': 'value1',
'item2': 1,
'mp_lib': 'python',
'token': 'My-Key'}},
{'event': 'event-2',
'properties': {'distinct_id': 'user-2',
'item1': 'value2',
'item2': 2,
'mp_lib': 'python',
'token': 'My-Key'}},
{'$distinct_id': 'user-1',
'$set': {'item1': 'value1', 'item2': 2},
'$token': 'My-Key'},
{'event': 'event-3',
'properties': {'distinct_id': 'user-2',
'item1': 'value3',
'item2': 3,
'mp_lib': 'python',
'token': 'My-Key'}},
{'event': 'event-4',
'properties': {'distinct_id': 'user-2',
'item1': 'value4',
'item2': 4,
'mp_lib': 'python',
'token': 'My-Key'}},
{'event': 'event-5',
'properties': {'distinct_id': 'user-1',
'item1': 'value5',
'item2': 5,
'mp_lib': 'python',
'token': 'My-Key'}}
]
async def test_do_get_connection_refused(
httpserver: pytest_httpserver.HTTPServer):
httpserver.expect_request('/index.html').\
respond_with_data('Not Found', content_type='text/html', status=404)
# close HTTP server on purpose: make sure the connection will be refused
httpserver.stop()
try:
async with http.new_client() as session:
response, error = await http.get(session,
httpserver.url_for('/index.html'))
finally:
# start again to avoid side effect
httpserver.start()
assert response is None
assert error == 'Connection refused'
def test_unzip(test_folder, tmp_folder, clean_tmp):
# GIVEN zip file in test_folder
# WHEN downloaded in tmp folder
# THEN should be unzipped and zip file removed
sample = test_folder / "sample.zip"
server = HTTPServer(port=0)
s = server.expect_request("/sample.zip", method="GET")
s.respond_with_data(sample.read_bytes(), mimetype="application/zip")
server.start()
core.unzip(url=server.url_for("/sample.zip"), dest=tmp_folder, chunk_size=1, remove_zip=True)
server.stop()
assert tmp_folder.joinpath("sample.txt").exists()
assert not tmp_folder.joinpath("sample.zip").exists()
# clean the tmp folder
clean_tmp
async def forwarder(client, example_id, example_entry_ids):
"""A http server forwarding REST requests to `client`, bound to app under test."""
httpserver = HTTPServer()
httpserver \
.expect_request(re.compile(r'/about/[0-9a-f]+')) \
.respond_with_json((await client.get(f'/about/{example_id}')).json())
httpserver \
.expect_request(re.compile(r'/list/[0-9a-f]+')) \
.respond_with_json((await client.get(f'/list/{example_id}')).json())
json_responses = [Response((await client.get(f'/json/{example_id}/{id}')).content)
for id in example_entry_ids]
json_responses_iter = iter(json_responses)
httpserver \
.expect_request(re.compile(rf'/json/{example_id}/[0-9a-f]+')) \
.respond_with_handler(lambda _: next(json_responses_iter))
try:
httpserver.start()
yield httpserver
finally:
httpserver.stop()
def test_flush_on_flush(httpserver: HTTPServer):
p = configure_plugin_and_server(
httpserver,
IterativelyOptions(
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
))
try:
p.load(PLUGIN_OPTIONS_DEV_NO_LOGGER)
p.post_group(*post_group_args)
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
# Assert no requests (not flushed)
assert requests == []
p.flush()
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
# Assert requests were flushed
assert requests == [{
'objects': [
post_group_expected_request,
]
}]
# Assert additional flushes do nothing
p.flush()
p.flush()
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
# Assert requests were flushed
assert requests == [{
'objects': [
post_group_expected_request,
]
}]
finally:
p.shutdown()
httpserver.stop()
def test_flush_on_shutdown(httpserver: HTTPServer):
p = configure_plugin_and_server(
httpserver,
IterativelyOptions(
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
))
assert p.id() == 'iteratively'
try:
p.load(PLUGIN_OPTIONS_DEV_NO_LOGGER)
p.post_track(*post_track_2_args)
finally:
p.shutdown()
time.sleep(0.1)
httpserver.stop()
requests = _get_cleaned_requests(httpserver)
assert requests == [
{
'objects': [post_track_2_expected_request]
},
]
def test_rig():
logging.basicConfig(level=logging.DEBUG)
bootstrap = Bootstrap()
http_server = HTTPServer()
http_server.start()
http_server.expect_request("/" + test_id).respond_with_data("OK")
http_server.expect_request("/" + test_id + "/fail").respond_with_data("OK")
url_for_test = http_server.url_for("/" + test_id)
no_char_to_remove = len(test_id)
url_for_test = url_for_test[:-no_char_to_remove]
rig = {}
rig["client"] = testing.TestClient(
bootstrap.create(hc_override=url_for_test))
rig["server"] = http_server
yield rig
http_server.stop()
def test_flush_at_queue_size(httpserver: HTTPServer):
p = configure_plugin_and_server(
httpserver,
IterativelyOptions(
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
))
try:
p.load(PLUGIN_OPTIONS_DEV_NO_LOGGER)
p.post_identify(*post_identify_1_args)
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
# Assert no requests (not flushed @ 1)
assert requests == []
p.post_identify(*post_identify_2_args)
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
# Assert no requests (not flushed @ 2)
assert requests == []
p.post_track(*post_track_1_args)
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
# Assert requests (flushed @ 3)
assert requests == [{
'objects': [
post_identify_1_expected_request,
post_identify_2_expected_request, post_track_1_expected_request
]
}]
finally:
p.shutdown()
httpserver.stop()
def test_amplitude_metadata(httpserver: HTTPServer):
httpserver.expect_request(
re.compile('/(events|identify)')).respond_with_data()
options = AmplitudeOptions(
events_endpoint=httpserver.url_for('/events'),
identification_endpoint=httpserver.url_for('/identify'),
metadata=AmplitudeMetadata(os_name='ubuntu',
city="York",
os_version="111.0"))
p = AmplitudePlugin('My-Key', options)
try:
p.load(
PluginLoadOptions(environment=Environment.DEVELOPMENT,
logger=Logger.NONE))
p.identify("user-1", Properties(item1='value1', item2=2))
metadata = {
"amplitude": AmplitudeMetadata(platform="LinUx",
os_version="123.45")
}
p.track(
"user-2",
Event('event-1',
Properties(item1='value1', item2=1),
metadata=metadata))
metadata = {
"amplitude": AmplitudeMetadata(os_name="win", os_version="987.45")
}
p.track(
"user-1",
Event('event-2',
Properties(item1='value2', item2=2),
metadata=metadata))
p.flush()
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == [
[{
'city': 'York',
'os_name': 'ubuntu',
'os_version': '111.0',
'user_id': 'user-1',
'user_properties': {
'item1': 'value1',
'item2': 2
}
}],
{
'api_key':
'My-Key',
'events': [{
'user_id': 'user-2',
'event_type': 'event-1',
'event_properties': {
'item1': 'value1',
'item2': 1
},
'city': 'York',
'platform': 'LinUx',
'os_name': 'ubuntu',
'os_version': "123.45"
}, {
'user_id': 'user-1',
'event_type': 'event-2',
'event_properties': {
'item1': 'value2',
'item2': 2
},
'city': 'York',
'os_name': 'win',
'os_version': "987.45"
}],
},
]
finally:
p.shutdown()
time.sleep(0.1)
httpserver.stop()
def test_amplitude(httpserver: HTTPServer):
httpserver.expect_request(
re.compile('/(events|identify)')).respond_with_data()
options = AmplitudeOptions(
events_endpoint=httpserver.url_for('/events'),
identification_endpoint=httpserver.url_for('/identify'),
flush_queue_size=3,
flush_interval=timedelta(seconds=1),
)
p = AmplitudePlugin('My-Key', options)
assert p.id() == 'amplitude'
try:
p.load(
PluginLoadOptions(environment=Environment.DEVELOPMENT,
logger=Logger.NONE))
p.identify("user-1", Properties(item1='value1', item2=2))
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == []
p.track("user-2", Event('event-1', Properties(item1='value1',
item2=1)))
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == [
[{
"user_id": "user-1",
"user_properties": {
"item1": "value1",
"item2": 2
}
}],
]
p.track("user-2", Event('event-2', Properties(item1='value2',
item2=2)))
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == [
[{
"user_id": "user-1",
"user_properties": {
"item1": "value1",
"item2": 2
}
}],
]
p.flush()
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == [
[
{
"user_id": "user-1",
"user_properties": {
"item1": "value1",
"item2": 2
}
},
],
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-1',
'event_properties': {
'item1': 'value1',
'item2': 1
}
}, {
'user_id': 'user-2',
'event_type': 'event-2',
'event_properties': {
'item1': 'value2',
'item2': 2
}
}],
'api_key':
'My-Key'
},
]
p.flush()
p.flush()
time.sleep(0.1)
requests = _get_cleaned_requests(httpserver)
assert requests == [
[{
"user_id": "user-1",
"user_properties": {
"item1": "value1",
"item2": 2
}
}],
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-1',
'event_properties': {
'item1': 'value1',
'item2': 1
}
}, {
'user_id': 'user-2',
'event_type': 'event-2',
'event_properties': {
'item1': 'value2',
'item2': 2
}
}],
'api_key':
'My-Key'
},
]
p.track("user-2", Event('event-3', Properties(item1='value3',
item2=3)))
time.sleep(1.1)
requests = _get_cleaned_requests(httpserver)
assert requests == [
[{
"user_id": "user-1",
"user_properties": {
"item1": "value1",
"item2": 2
}
}],
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-1',
'event_properties': {
'item1': 'value1',
'item2': 1
}
}, {
'user_id': 'user-2',
'event_type': 'event-2',
'event_properties': {
'item1': 'value2',
'item2': 2
}
}],
'api_key':
'My-Key'
},
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-3',
'event_properties': {
'item1': 'value3',
'item2': 3
}
}],
'api_key':
'My-Key'
},
]
p.track("user-2", Event('event-4', Properties(item1='value4',
item2=4)))
p.track("user-1", Event('event-5', Properties(item1='value5',
item2=5)))
finally:
p.shutdown()
time.sleep(0.1)
httpserver.stop()
requests = _get_cleaned_requests(httpserver)
assert requests == [
[{
"user_id": "user-1",
"user_properties": {
"item1": "value1",
"item2": 2
}
}],
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-1',
'event_properties': {
'item1': 'value1',
'item2': 1
}
}, {
'user_id': 'user-2',
'event_type': 'event-2',
'event_properties': {
'item1': 'value2',
'item2': 2
}
}],
'api_key':
'My-Key'
},
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-3',
'event_properties': {
'item1': 'value3',
'item2': 3
}
}],
'api_key':
'My-Key'
},
{
'events': [{
'user_id': 'user-2',
'event_type': 'event-4',
'event_properties': {
'item1': 'value4',
'item2': 4
}
}, {
'user_id': 'user-1',
'event_type': 'event-5',
'event_properties': {
'item1': 'value5',
'item2': 5
}
}],
'api_key':
'My-Key'
},
]
#!.venv/bin/python3
import urllib.request
import urllib.error
from pytest_httpserver import HTTPServer
server = HTTPServer(port=4000)
server.expect_request("/foobar").respond_with_json({"foo": "bar"})
server.start()
try:
print(
urllib.request.urlopen(
"http://localhost:4000/foobar?name=John%20Smith&age=123").read())
except urllib.error.HTTPError as err:
print(err)
server.stop()
def test_api_is_up_bad(httpserver: HTTPServer):
assert httpserver.is_running()
httpserver.expect_request("/api/v1.0/system/test").respond_with_json(
{"status": "bad"})
assert api_is_up() == False
httpserver.stop()
def test_segment(httpserver: HTTPServer):
httpserver.expect_request('/v1/batch').respond_with_data()
options = SegmentOptions(host=httpserver.url_for(''), )
p = SegmentPlugin('My-Key', options)
assert p.id() == 'segment'
p.load(
PluginLoadOptions(environment=Environment.DEVELOPMENT,
logger=Logger.NONE))
p.identify("user-1", Properties(item1='value1', item2=2))
p.alias("user-1", "prev-user-1")
p.track("user-2", Event('event-1', Properties(item1='value1', item2=1)))
p.group("user-2", "group-2", Properties(item1='value2', item2=2))
p.flush()
p.page("user-2", "category-2", "page-3", Properties(item1='value3',
item2=3))
p.track("user-2", Event('event-4', Properties(item1='value4', item2=4)))
p.track("user-1", Event('event-5', Properties(item1='value5', item2=5)))
p.flush()
p.shutdown()
time.sleep(0.1)
httpserver.stop()
requests = _get_cleaned_requests(httpserver)
assert requests == [{
'anonymousId': None,
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'integrations': {},
'traits': {
'item1': 'value1',
'item2': 2
},
'type': 'identify',
'userId': 'user-1'
}, {
'anonymousId': None,
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'integrations': {},
'previousId': 'prev-user-1',
'type': 'alias',
'userId': 'user-1'
}, {
'anonymousId': None,
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'event': 'event-1',
'integrations': {},
'properties': {
'item1': 'value1',
'item2': 1
},
'type': 'track',
'userId': 'user-2'
}, {
'anonymousId': None,
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'groupId': 'group-2',
'integrations': {},
'traits': {
'item1': 'value2',
'item2': 2
},
'type': 'group',
'userId': 'user-2'
}, {
'anonymousId': None,
'category': 'category-2',
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'integrations': {},
'name': 'page-3',
'properties': {
'item1': 'value3',
'item2': 3
},
'type': 'page',
'userId': 'user-2'
}, {
'anonymousId': None,
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'event': 'event-4',
'integrations': {},
'properties': {
'item1': 'value4',
'item2': 4
},
'type': 'track',
'userId': 'user-2'
}, {
'anonymousId': None,
'context': {
'library': {
'name': 'analytics-python',
'version': '1.2.9'
}
},
'event': 'event-5',
'integrations': {},
'properties': {
'item1': 'value5',
'item2': 5
},
'type': 'track',
'userId': 'user-1'
}]
def test_snowplow(httpserver: HTTPServer):
httpserver.expect_request('/v1/batch').respond_with_data()
endpoint = httpserver.url_for("").replace("http://", "")
p = SnowplowPlugin("ly.iterative.test", SnowplowOptions(endpoint))
assert p.id() == 'snowplow'
p.load(PluginLoadOptions(environment=Environment.DEVELOPMENT, logger=Logger.NONE))
p.page("user-1", "category-1", "page-1", Properties(item1='value0', item2=0))
p.track("user-2", Event('event-1', Properties(item1='value1', item2=1), id_="event-1", version="0.0.1"))
p.flush()
p.page("user-2", "category-2", "page-3", Properties(item1='value3', item2=3))
p.track("user-2", Event('event-4', Properties(item1='value4', item2=4), id_="event-4", version="0.0.4"))
p.track("user-1", Event('event-5', Properties(item1='value5', item2=5), id_="event-5", version="0.0.5"))
p.flush()
p.shutdown()
time.sleep(0.1)
httpserver.stop()
requests = _get_cleaned_requests(httpserver)
assert requests == [
[
{
'properties': {
'data': {
'data': {'name': 'page-1'},
'schema': 'iglu:com.snowplowanalytics.snowplow/screen_view/jsonschema/1-0-0'},
'schema': 'iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0'
},
'uid': 'user-1'
},
{
'properties': {
'data': {
'data': {'item1': 'value1', 'item2': 1},
'schema': 'iglu:ly.iterative.test/event-1/jsonschema/0-0-1'
},
'schema': 'iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0'
},
'uid': 'user-2'
}
],
[
{
'properties': {
'data': {
'data': {'name': 'page-3'},
'schema': 'iglu:com.snowplowanalytics.snowplow/screen_view/jsonschema/1-0-0'
},
'schema': 'iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0'
},
'uid': 'user-2'
},
{
'properties': {
'data': {
'data': {'item1': 'value4', 'item2': 4},
'schema': 'iglu:ly.iterative.test/event-4/jsonschema/0-0-4'
},
'schema': 'iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0'
},
'uid': 'user-2'
},
{
'properties': {
'data': {
'data': {'item1': 'value5', 'item2': 5},
'schema': 'iglu:ly.iterative.test/event-5/jsonschema/0-0-5'
},
'schema': 'iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0'
},
'uid': 'user-1'
}
]
]