def test_activate__experiment(agent_server, session_obj, experiment_key,
expected_response, expected_status_code):
"""
Test validates:
1. Presence of correct variation in the returned decision for AB experiment
Instead of on single field (variation, enabled), validation is done on the whole
response (that includes variations and enabled fields).
This is to add extra robustness to the test.
Sort the reponses because dictionaries shuffle order.
:param agent_server: starts agent server with default config
:param session_obj: session object
:param experiment_key: experiment_key
:param expected_response: expected_response
:param expected_status_code: expected_status_code
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"experimentKey": experiment_key}
resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE,
'post',
session_obj,
payload=payload,
params=params)
assert json.loads(expected_response) == resp.json()
assert resp.status_code == expected_status_code, resp.text
resp.raise_for_status()
def test_activate__type(session_obj, decision_type, expected_response,
expected_status_code, bypass_validation):
"""
Test cases:
1. Get decisions with "experiment" type
2. Get decisions with "feature" type
3. Get empty list when non-existent decision type -> bug OASIS-6031
:param session_obj: session object
:param decision_type: parameterized decision type
:param expected_response: expected response
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"type": decision_type}
resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE, 'post', session_obj, bypass_validation, payload=payload, params=params)
if decision_type in ['experiment', 'feature']:
sorted_actual = sort_response(
resp.json(), 'experimentKey', 'featureKey')
sorted_expected = sort_response(json.loads(expected_response), 'experimentKey',
'featureKey')
assert sorted_actual == sorted_expected
elif resp.json()['error']:
with pytest.raises(requests.exceptions.HTTPError):
assert resp.json() == expected_response
resp.raise_for_status()
def test_activate__feature(session_obj, feature_key, expected_response,
expected_status_code):
"""
Test validates:
That feature is enabled in the decision for the feature test
Instead of on single field (variation, enabled), validation is done on the whole
response (that includes variations and enabled fields).
This is to add extra robustness to the test.
Sort the reponses because dictionaries shuffle order.
:param session_obj: session object
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"featureKey": feature_key}
resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE, 'post', session_obj, payload=payload, params=params)
if isinstance(resp.json(), dict) and resp.json()['error']:
with pytest.raises(requests.exceptions.HTTPError):
assert resp.json() == json.loads(expected_response)
assert resp.status_code == expected_status_code, resp.text
resp.raise_for_status()
assert json.loads(expected_response) == resp.json()
assert resp.status_code == expected_status_code, resp.text
def test_activate__enabled(session_obj, enabled, experimentKey, featureKey,
expected_response, expected_status_code, bypass_validation):
"""
Filter the activation response to return only enabled decisions.
Value for enabled key needs to be a string: "true" or "false"
- feature_1 feature is enabled - should not appear in response when enabled is set to False
- featur_3 feature is not enabled in the project - should not appear in the project when enabled is True
:param session_obj: session fixture
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {
"experimentKey": experimentKey,
"featureKey": featureKey,
"enabled": enabled
}
resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE, 'post', session_obj, bypass_validation, payload=payload, params=params)
actual_response = sort_response(resp.json(), 'experimentKey', 'featureKey')
expected_response = sort_response(json.loads(expected_response), 'experimentKey',
'featureKey')
assert actual_response == expected_response
assert resp.status_code == expected_status_code
resp.raise_for_status()
def test_batch_400(session_obj):
"""
Invalid JSON, no SDK key in the operations' header.
:param agent_server: starts agent server with default config
:param session_obj: session object
"""
payload = """{
"operations": [{
"body": {
"status": "subscribed",
"email_address": "*****@*****.**"
},
"method": "GET",
"operationID": "1",
"url": "/v1/config",
"params": {},
"headers": {
"X-Optimizely-SDK-Key": "",
"X-Request-Id": "matjaz_1"
}
}]
}"""
resp = create_and_validate_request_and_response(ENDPOINT_BATCH,
'post',
session_obj,
payload=payload)
actual_response = resp.json()
assert 200 == resp.status_code
assert 1 == actual_response['errorCount']
assert 400 == actual_response['response'][0]['status']
assert 'missing required X-Optimizely-SDK-Key header' == actual_response[
'response'][0]['body']['error']
resp.raise_for_status()
def test_decide__feature(session_obj, flag_key, expected_response, expected_status_code):
"""
Test validates:
Correct response when valid and invalid flag key are passed as parameters.
...
:param session_obj:
:param flag_key:
:param expected_response:
:param expected_status_code:
"""
payload = """
{
"userId": "matjaz",
"decideOptions": [
"ENABLED_FLAGS_ONLY",
"INCLUDE_REASONS"
],
"userAttributes": {"attr_1": "hola"}
}
"""
params = {"keys": flag_key}
resp = create_and_validate_request_and_response(ENDPOINT_DECIDE, 'post', session_obj, payload=payload,
params=params)
assert json.loads(expected_response) == resp.json()
assert resp.status_code == expected_status_code, resp.text
resp.raise_for_status()
def test_track(session_obj, event_key, status_code,bypass_validation):
"""
Track event for the given user.
Track sends event and user details to Optimizely’s analytics backend
for the analysis of a feature test or experiment.
:param session_obj: session fixture
:param event_key: parameterized param
:param status_code: parameterized param
"""
# TODO - ADD EVENT TAGS - AND TEST DIFFERENT SCENARIONS WITH EVENT TAGS
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}, "eventTags": {}}'
params = {"eventKey": event_key}
resp = create_and_validate_request_and_response(ENDPOINT_TRACK, 'post', session_obj, bypass_validation, payload=payload, params=params)
assert resp.status_code == status_code, f'Status code should be {status_code}. {resp.text}'
if event_key == "":
with pytest.raises(requests.exceptions.HTTPError):
assert resp.status_code == status_code
assert resp.text == '{"error":"missing required path parameter: eventKey"}\n'
resp.raise_for_status()
if event_key == "invalid_event_key":
assert resp.status_code == status_code
assert resp.text == '{"userId":"matjaz","eventKey":"invalid_event_key","error":"event with key \\"invalid_event_key\\" not found"}\n'
def test_activate__disable_tracking(agent_server, session_obj, experiment,
disableTracking, expected_status_code,
bypass_validation):
"""
Setting to true will disable impression tracking for ab experiments and feature tests.
It's equivalent to previous "get_variation".
Can not test it in acceptance tests. Just testing basic status code.
FS compatibility test suite uses proxy event displatcher where they test this by
validating that event was not sent.
:param agent_server: starts agent server with default config
:param session_obj: session fixture
:param experiment: ab experiment or feature test
:param disableTracking: true or false
:param expected_status_code
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"experimentKey": experiment, "disableTracking": disableTracking}
resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE,
'post',
session_obj,
bypass_validation,
payload=payload,
params=params)
resp.raise_for_status()
assert resp.status_code == expected_status_code
def test_overrides__invalid_arguments(session_obj, userId, experimentKey, variationKey,
expected_status_code, expected_response, bypass_validation):
payload = f'{{"userId": "{userId}", ' \
f'"experimentKey": "{experimentKey}", "variationKey": "{variationKey}"}}'
resp = create_and_validate_request_and_response(ENDPOINT_OVERRIDE, 'post', session_obj, bypass_validation, payload=payload)
assert resp.status_code == expected_status_code, resp.text
assert resp.text == expected_response
def test_config_403(session_override_sdk_key):
"""
Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403.
:param : session_obj
"""
with pytest.raises(requests.exceptions.HTTPError):
resp = create_and_validate_request_and_response(ENDPOINT_CONFIG, 'get', session_override_sdk_key)
assert resp.status_code == 403
assert resp.json()['error'] == 'unable to fetch fresh datafile (consider ' \
'rechecking SDK key), status code: 403 Forbidden'
resp.raise_for_status()
def test_config(session_obj):
"""
Test validates all returned available experiment and features definitions
for this environment.
Note: Test will fail as soon as anything in the response body is modified.
If someone updates any of the fields, the expected_response will need to be updated
as well.
:param session_obj: session object
"""
resp = create_and_validate_request_and_response(ENDPOINT_CONFIG, 'get', session_obj)
assert resp.status_code == 200
resp.raise_for_status()
assert json.loads(expected_config) == resp.json()
def test_activate_with_config(agent_server, session_obj):
"""
Tests experimentKeys, featureKeys, variables and variations because it
validates against the whole response body.
In "activate"
Request payload defines the “who” (user id and attributes)
while the query parameters define the “what” (feature, experiment, etc)
Request parameter is a list of experiment keys or feature keys.
If you want both add both and separate them with comma.
Example:
params = {
"featureKey": <list of feature keys>,
"experimentKey": <list of experiment keys>
}
Need to sort the response (list of dictionaries). And the sorting needs to be primary
and secondary, because we are getting response for two params - experimentKey and
featureKey and they have different responses. experimentKey has experimentKey field
always populated and it has featureKey empty.
Whereas featureKey response has featureKey field populated and experimentKey empty.
When we sort on both then the responses are properly sorted and ready for being
asserted on.
:param agent_server: starts agent server with default config
:param session_obj: session object
"""
# config
resp = session_obj.get(BASE_URL + ENDPOINT_CONFIG)
resp_config = resp.json()
# activate
feat = [key for key in resp_config['featuresMap']]
exp = [key for key in resp_config['experimentsMap']]
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"featureKey": feat, "experimentKey": exp}
resp_activate = create_and_validate_request_and_response(ENDPOINT_ACTIVATE,
'post',
session_obj,
payload=payload,
params=params)
sorted_actual = sort_response(resp_activate.json(), 'experimentKey',
'featureKey')
sorted_expected = sort_response(json.loads(expected_activate_with_config),
'experimentKey', 'featureKey')
assert sorted_actual == sorted_expected
def test_activate_403(session_override_sdk_key):
"""
Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403.
:param : session_obj
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"type": "experiment"}
with pytest.raises(requests.exceptions.HTTPError):
resp = create_and_validate_request_and_response(ENDPOINT_ACTIVATE, 'post', session_override_sdk_key,payload=payload, params=params)
assert resp.status_code == 403
assert resp.json()['error'] == 'unable to fetch fresh datafile (consider ' \
'rechecking SDK key), status code: 403 Forbidden'
resp.raise_for_status()
def test_overrides_403(session_override_sdk_key):
"""
Test that 403 Forbidden is returned. We use invalid SDK key to trigger 403.
:param : session_override_sdk_key
"""
payload = '{"userId": "matjaz",'\
'"experimentKey": "ab_test1", "variationKey": "my_new_variation"}'
with pytest.raises(requests.exceptions.HTTPError):
resp = create_and_validate_request_and_response(ENDPOINT_OVERRIDE, 'post', session_override_sdk_key,
payload=payload)
assert resp.status_code == 403
assert resp.json()['error'] == 'unable to fetch fresh datafile (consider ' \
'rechecking SDK key), status code: 403 Forbidden'
resp.raise_for_status()
def test_batch_valid_reponse(session_obj):
# TODO - parameterize to feed in different values in the payload: valid SDK string, invalid sdk string,
# empty string, integer, boolean, double
"""
Happy path with a single operation
:param agent_server: starts agent server with default config
:param session_obj: session object
"""
payload = """{
"operations": [{
"body": {
"status": "subscribed",
"email_address": "*****@*****.**"
},
"method": "GET",
"operationID": "1",
"url": "/v1/config",
"params": {
},
"headers": {
"X-Optimizely-SDK-Key": "%s",
"X-Request-Id": "matjaz_1"
}
}]
}""" % sdk_key
resp = create_and_validate_request_and_response(ENDPOINT_BATCH,
'post',
session_obj,
payload=payload,
bypass_validation=False)
actual_response = resp.json()
assert 200 == resp.status_code
assert 200 == actual_response['response'][0]['status']
assert 0 == actual_response['errorCount']
assert 'matjaz_1' == actual_response['response'][0]['requestID']
assert '1' == actual_response['response'][0]['operationID']
assert '/v1/config' == actual_response['response'][0]['url']
assert 'experimentsMap' in actual_response['response'][0]['body']
assert 'ab_test1', 'feature_2_test' in actual_response['response'][0][
'body']['experimentsMap']
resp.raise_for_status()
def test_overrides__invalid_arguments(agent_server, session_obj, userId,
experimentKey, variationKey,
expected_status_code, expected_response,
bypass_validation):
"""
:param agent_server: starts agent server with default config
:param : session_override_sdk_key
"""
payload = f'{{"userId": "{userId}", ' \
f'"experimentKey": "{experimentKey}", "variationKey": "{variationKey}"}}'
resp = create_and_validate_request_and_response(ENDPOINT_OVERRIDE,
'post',
session_obj,
bypass_validation,
payload=payload)
assert resp.status_code == expected_status_code, resp.text
assert resp.text == expected_response
def test_datafile_success(session_obj):
"""
Normally a good practice is to have expected response as a string like in other tests.
Here we are exceptionally making expected response a dict for easier comparison.
String was causing some issues with extra white space characters.
:param session_obj: session object
"""
payload = '{"userId": "matjaz", "userAttributes": {"attr_1": "hola"}}'
params = {"featureKey": "feature_1"}
resp = create_and_validate_request_and_response(
ENDPOINT_DATAFILE,
'get',
session_obj,
bypass_validation_request=False,
payload=payload,
params=params)
assert expected_response == resp.json()
assert resp.status_code == 200, resp.text
def test_overrides__invalid_arguments(session_obj, userId, experimentKey,
variationKey, expected_status_code,
expected_response,
bypass_validation_request):
"""
:param session_obj: session object
:param userId: user id
:param experimentKey: experiment key
:param variationKey: variation key
:param expected_status_code: expected status code in response
:param expected_response: body of expected response
:param bypass_validation: option to enable or disable schema validation
"""
payload = f'{{"userId": "{userId}", ' \
f'"experimentKey": "{experimentKey}", "variationKey": "{variationKey}"}}'
resp = create_and_validate_request_and_response(ENDPOINT_OVERRIDE,
'post',
session_obj,
bypass_validation_request,
payload=payload)
assert resp.status_code == expected_status_code, resp.text
assert resp.text == expected_response
def test_decide__flag_key_parameter(session_obj, parameters, expected_response, expected_status_code,
bypass_validation_request,
bypass_validation_response):
"""
Test validates:
That no required parameter, empty param and all parameters return identical response.
Openapi spec specifies 400 for missing flagKey parameter. But We keep 400 status code in the openapi spec
for missing reuired parameter, even though when no flagKey parameter is supplied to the request,
Agent still responds with all decisions and status 200.
That is consistent with the behavior of activate and other api-s
:param session_obj: session obj
:param parameters: sesison obj, params, expected, expected status code
:param expected_response: expected_flag_keys
:param expected_status_code: 200
"""
payload = """
{
"userId": "matjaz",
"decideOptions": [
"ENABLED_FLAGS_ONLY",
"INCLUDE_REASONS"
],
"userAttributes": {"attr_1": "hola"}
}
"""
params = parameters
resp = create_and_validate_request_and_response(ENDPOINT_DECIDE, 'post', session_obj, bypass_validation_request,
bypass_validation_response,
payload=payload,
params=params)
sorted_actual = sort_response(resp.json(), 'flagKey')
sorted_expected = sort_response(json.loads(expected_response), 'flagKey')
assert sorted_actual == sorted_expected
def test_batch_422(session_obj):
"""
Set env variable OPTIMIZELY_SERVER_BATCHREQUESTS_OPERATIONSLIMIT to 3 (already set to 3 for all tests)
Then send 4 operaions, should fail with code 422
:param operations_limit: starts agent server with custome set operations limit env var
:param session_obj: session object
"""
payload = """{
"operations": [
{
"body": {
"status": "subscribed",
"email_address": "*****@*****.**"
},
"method": "GET",
"operationID": "1",
"url": "/v1/config",
"params": {},
"headers": {
"X-Optimizely-SDK-Key": "%s",
"X-Request-Id": "matjaz 1"
}
},
{
"body": {
"status": "subscribed",
"email_address": "*****@*****.**"
},
"method": "GET",
"operationID": "2",
"url": "/v1/config",
"params": {},
"headers": {
"X-Optimizely-SDK-Key": "TkB2xhu8WEAHa4LphN3xZ2"
}
},
{
"body": {
"userId": "user1"
},
"method": "POST",
"operationID": "3",
"url": "/v1/activate",
"params": {
"type": "experiment",
"experimentKey": "ab_test1"
},
"headers": {
"X-Optimizely-SDK-Key": "%s",
"Content-Type": "application/json"
}
},
{
"body": {
"userId": "user2"
},
"method": "POST",
"operationID": "4",
"url": "/v1/activate",
"params": {
"type": "experiment",
"experimentKey": "ab_test1"
},
"headers": {
"X-Optimizely-SDK-Key": "%s",
"Content-Type": "application/json"
}
}]
}""" % (sdk_key, sdk_key, sdk_key)
resp = create_and_validate_request_and_response(ENDPOINT_BATCH,
'post',
session_obj,
payload=payload,
bypass_validation=False)
assert 422 == resp.status_code
assert resp.json()['error'].startswith('too many operations')
def test_batch_valid_response__multiple_operations(session_obj):
"""
Verify that operations with different sdk keys can be sent in a batch.
:param agent_server: starts agent server with default config
:param session_obj: session object
"""
payload = """{
"operations": [
{
"body": {
"status": "subscribed",
"email_address": "*****@*****.**"
},
"method": "GET",
"operationID": "1",
"url": "/v1/config",
"params": {},
"headers": {
"X-Optimizely-SDK-Key": "%s",
"X-Request-Id": "matjaz 1"
}
},
{
"body": {
"status": "subscribed",
"email_address": "*****@*****.**"
},
"method": "GET",
"operationID": "2",
"url": "/v1/config",
"params": {},
"headers": {
"X-Optimizely-SDK-Key": "TkB2xhu8WEAHa4LphN3xZ2"
}
},
{
"body": {
"userId": "user1"
},
"method": "POST",
"operationID": "3",
"url": "/v1/activate",
"params": {
"type": "feature",
"experimentKey": "ab_test1"
},
"headers": {
"X-Optimizely-SDK-Key": "%s",
"Content-Type": "application/json"
}
}]
}""" % (sdk_key, sdk_key)
resp = create_and_validate_request_and_response(ENDPOINT_BATCH,
'post',
session_obj,
payload=payload,
bypass_validation=False)
actual_response = resp.json()
assert 200 == resp.status_code
assert 0 == actual_response['errorCount']
responses = len(actual_response['response'])
assert 3 == responses
for operation in range(responses):
assert 200 == actual_response['response'][operation]['status']
sorted_actual = sort_response(actual_response['response'], 'operationID',
'status')
assert json.loads(
expected_body_of_operationid_2) == sorted_actual[1]['body']
assert '/v1/config' == sorted_actual[0]['url']
assert '/v1/config' == sorted_actual[1]['url']
assert '/v1/activate' == sorted_actual[2]['url']
resp.raise_for_status()
