def test_dest_returns_unparseable_json_response(mock_server):
with open("tests/resources/circrequests/valid_src_response.json") as file:
valid_src_response = file.read()
with open(
"tests/resources/circrequests/unparseable_json_dest_response.json"
) as file:
unparseable_json_dest_response = file.read()
# Set up mock server with required behavior
imposter = Imposter([
Stub(Predicate(path="/src"), Response(body=valid_src_response)),
Stub(Predicate(path="/dest", method="POST"),
Response(body=unparseable_json_dest_response)),
])
# Create a temporary file to use as last success lookup
try:
[temp_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('etc/items_FIRST.json')
with tempfile.TemporaryDirectory() as temp_storage_dir, mock_server(
imposter) as server:
setup_environment(imposter, temp_storage_dir,
temp_success_filename)
start_time = '20200521132905'
args = []
with pytest.raises(JSONDecodeError):
command = Command()
result = command(start_time, args)
assert result.was_successful() is False
files_in_storage_dir = [
f for f in listdir(temp_storage_dir)
if isfile(join(temp_storage_dir, f))
]
expected_file_regex = re.compile('.*\\.dest_response_body\\..*')
# There should be a "dest_circrequests_response_body" file, even
# though the JSON was unparseable
if not any(
expected_file_regex.match(x)
for x in files_in_storage_dir):
pytest.fail(
f"Expected file matching '#{expected_file_regex.pattern}' was not found."
)
# There should have been three requests to the server
assert 2 == len(server.get_actual_requests()[imposter.port])
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(server,
had_request().with_path("/dest").and_method("POST"))
finally:
# Clean up the temporary file
os.close(temp_file_handle)
os.remove(temp_success_filename)
def test_dest_returns_denied_key(mock_server):
with open("tests/resources/circrequests/valid_src_response.json") as file:
valid_src_response = file.read()
with open(
"tests/resources/circrequests/valid_dest_response_denied_key.json"
) as file:
valid_dest_response_denied_key = file.read()
# Set up mock server with required behavior
imposter = Imposter([
Stub(Predicate(path="/src"), Response(body=valid_src_response)),
Stub(Predicate(path="/dest", method="POST"),
Response(body=valid_dest_response_denied_key)),
])
try:
# Create a temporary file to use as last success lookup
[temp_success_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('etc/circrequests_FIRST.json')
# Create a temporary file to use as denied keys file
[temp_denied_keys_file_handle,
temp_denied_keys_filename] = tempfile.mkstemp()
with open(temp_denied_keys_filename, 'w') as f:
f.write('{}')
with tempfile.TemporaryDirectory() as temp_storage_dir, mock_server(
imposter) as server:
setup_environment(imposter, temp_storage_dir,
temp_success_filename, temp_denied_keys_filename)
start_time = '20200521132905'
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(server,
had_request().with_path("/dest").and_method("POST"))
finally:
# Clean up the temporary files
os.close(temp_success_file_handle)
os.remove(temp_success_filename)
os.close(temp_denied_keys_file_handle)
# Verify that "denied keys" file contains denied entry
with open(temp_denied_keys_filename) as file:
denied_keys = json.load(file)
denied_item_time = datetime.datetime.strptime(
start_time, '%Y%m%d%H%M%S').isoformat()
assert denied_keys == {'31430023550355': denied_item_time}
os.remove(temp_denied_keys_filename)
def test_successful_job(mock_server):
with open("tests/resources/items/valid_src_response.json") as file:
valid_src_response = file.read()
with open("tests/resources/items/valid_dest_new_items_response.json"
) as file:
valid_dest_new_items_response = file.read()
with open("tests/resources/items/valid_dest_updated_items_response.json"
) as file:
valid_dest_updated_items_response = file.read()
# Set up mock server with required behavior
imposter = Imposter([
Stub(Predicate(path="/src"), Response(body=valid_src_response)),
Stub(Predicate(path="/dest/new", method="POST"),
Response(body=valid_dest_new_items_response)),
Stub(Predicate(path="/dest/updated", method="POST"),
Response(body=valid_dest_updated_items_response)),
])
# Create a temporary file to use as last success lookup
try:
[temp_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('etc/items_FIRST.json')
with tempfile.TemporaryDirectory() as temp_storage_dir, mock_server(
imposter) as server:
setup_environment(imposter, temp_storage_dir,
temp_success_filename)
start_time = '20200521132905'
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
had_request().with_path("/dest/new").and_method("POST"))
assert_that(
server,
had_request().with_path("/dest/updated").and_method("POST"))
finally:
# Clean up the temporary file
os.close(temp_file_handle)
os.remove(temp_success_filename)
def test_valid_response_from_server(mock_server):
with open("tests/resources/circrequests/valid_src_response.json") as file:
valid_src_response = file.read()
# Set up mock server with required behavior
imposter = Imposter(
Stub(Predicate(path="/holds"), Response(body=valid_src_response)))
with mock_server(imposter) as server:
config = {
'source_url':
f"{imposter.url}/holds",
'storage_dir':
'/tmp',
'last_success_lookup':
'tests/storage/circrequests/circrequests_last_success.txt',
'denied_keys_filepath':
'tests/storage/circrequests/circrequests_denied_keys.json'
}
job_config = CircrequestsJobConfig(config, 'test')
query_source_url = QuerySourceUrl(job_config)
step_result = query_source_url.execute()
assert step_result.was_successful() is True
assert_that(server,
had_request().with_path("/holds").and_method("GET"))
assert valid_src_response == step_result.get_result()
def test_404_response_from_server(mock_server):
# Set up mock server with required behavior
imposter = Imposter(
Stub(Predicate(path="/holds"), Response(status_code=404)))
with mock_server(imposter) as server:
config = {
'source_url':
f"{imposter.url}/holds",
'storage_dir':
'/tmp',
'last_success_lookup':
'tests/storage/circrequests/circrequests_last_success.txt',
'denied_keys_filepath':
'tests/storage/circrequests/circrequests_denied_keys.json'
}
job_config = CircrequestsJobConfig(config, 'test')
query_source_url = QuerySourceUrl(job_config)
step_result = query_source_url.execute()
assert step_result.was_successful() is False
assert_that(server,
had_request().with_path("/holds").and_method("GET"))
assert f"Retrieval of '{imposter.url}/holds' failed with a status code of 404" in \
step_result.get_errors()
def test_send_updated_items_to_dest_valid_response(mock_server):
with open("tests/resources/items/valid_dest_updated_items_response.json") as file:
valid_dest_response = file.read()
# Set up mock server with required behavior
imposter = Imposter(Stub(Predicate(path="/items/updates", method="POST"),
Response(body=valid_dest_response)))
with mock_server(imposter) as server:
config = {
'dest_updates_url': f"{imposter.url}/items/updates",
'storage_dir': '/tmp',
'last_success_lookup': 'tests/storage/items/items_last_success.txt',
'caiasoft_api_key': "SOME_SECRET_KEY"
}
job_config = ItemsJobConfig(config, 'test')
# Override dest_updated_items_request_body_filepath
job_config["dest_updated_items_request_body_filepath"] = \
"tests/resources/items/valid_dest_updated_items_request.json"
with tempfile.TemporaryDirectory() as temp_storage_dir:
job_config["dest_updated_items_response_body_filepath"] = \
temp_storage_dir + "/dest_updated_items_response.json"
send_updated_items_to_dest = SendUpdatedItemsToDest(job_config)
step_result = send_updated_items_to_dest.execute()
assert step_result.was_successful() is True
assert os.path.exists(job_config["dest_updated_items_response_body_filepath"])
assert_that(server, had_request().with_path("/items/updates").and_method("POST"))
assert valid_dest_response == step_result.get_result()
def test_imposter_had_request_matcher(mock_server):
imposter = Imposter(Stub(Predicate(path="/test"), Response(body="sausages")))
with mock_server(imposter):
response = requests.get(f"{imposter.url}/test")
assert_that(response, is_response().with_status_code(200).and_body("sausages"))
assert_that(imposter, had_request().with_path("/test").and_method("GET"))
def test_proxy(mock_server):
imposter = Imposter(Proxy(to="http://example.com"))
with mock_server(imposter) as server:
response = requests.get("{0}/".format(imposter.url))
assert_that(response, is_(response_with(status_code=200, body=has_title("Example Domain"))))
assert_that(server, had_request(path="/", method="GET"))
def test_request_to_mock_server(mock_server):
# Start mock server with required behavior
imposter = Imposter(Stub(Predicate(path="/test"), Response(body="sausages")))
with mock_server(imposter) as server:
# Make request to mock server
response = requests.get(f"{imposter.url}/test")
assert_that(response, is_response().with_status_code(200).and_body("sausages"))
assert_that(server, had_request().with_path("/test").and_method("GET"))
def test_proxy(mock_server):
imposter = Imposter(Stub(responses=Proxy(to="http://example.com")))
with mock_server(imposter):
response = requests.get(imposter.url)
assert_that(
response,
is_response().with_status_code(200).and_body(
has_title("Example Domain")))
assert_that(imposter, had_request().with_path("/").and_method("GET"))
def test_dest_returns_404_error(mock_server):
with open("tests/resources/items/valid_src_response.json") as file:
valid_src_response = file.read()
# Set up mock server with required behavior
imposter = Imposter([
Stub(Predicate(path="/src"), Response(body=valid_src_response)),
Stub(Predicate(path="/dest/new", method="POST"),
Response(status_code=404)),
])
# Create a temporary file to use as last success lookup
try:
[temp_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('etc/items_FIRST.json')
with tempfile.TemporaryDirectory() as temp_storage_dir, mock_server(
imposter) as server:
setup_environment(imposter, temp_storage_dir,
temp_success_filename)
start_time = '20200521132905'
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is False
assert 1 == len(result.get_errors())
# There should be only two requests to the server
assert 2 == len(server.get_actual_requests()[imposter.port])
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
had_request().with_path("/dest/new").and_method("POST"))
finally:
# Clean up the temporary file
os.close(temp_file_handle)
os.remove(temp_success_filename)
def test_request_matcher():
# Given
server = MagicMock()
request = HttpRequestBuilder().with_path("/test").and_method("GET").build()
server.get_actual_requests.return_value = [request, request]
# When
# Then
assert_that(server, had_request().with_path("/test").and_method("GET"))
assert_that(server, had_request().with_times(2).and_path("/test").and_method("GET"))
assert_that(server, not_(had_request().with_path("/somewhereelse").and_method("GET")))
assert_that(
had_request().with_path("/sausages").and_method("PUT"),
has_string("call with method: 'PUT' path: '/sausages'"),
)
assert_that(
had_request().with_times(4).and_query(has_entries(a="b")).and_body("chips"),
has_string(
"<4> call(s) with query parameters: a dictionary containing {'a': 'b'} body: 'chips'"
),
)
assert_that(
had_request().with_path("/sausages").and_method("PUT").and_times(99),
mismatches_with(
server,
all_of(
contains_string(
"found <0> matching requests: <[]>. All requests: <[mbtest.imposters.imposters.HttpRequest"
),
contains_string("path='/test'"),
contains_string("method='GET'"),
),
),
)
def test_request_matcher():
# Given
server = MagicMock()
request = {"path": "/test", "method": "GET"}
server.get_actual_requests.return_value = {"someport": [request, request]}
# When
# Then
assert_that(server, had_request(path="/test", method="GET"))
assert_that(server, had_request(path="/test", method="GET", times=2))
assert_that(server, not_(had_request(path="/somewhereelse", method="GET")))
assert_that(
had_request(path="/sausages", method="PUT"),
has_string("call with method: 'PUT' path: '/sausages'"),
)
assert_that(
had_request(path="/sausages", times=4), has_string("<4> call(s) with path: '/sausages'")
)
assert_that(
had_request(path="/sausages", method="PUT"),
mismatches_with(
server,
all_of(
contains_string("found <0> matching requests: <[]>. All requests: <[{"),
contains_string("'path': '/test'"),
contains_string("'method': 'GET'"),
),
),
)
def test_inject_headers(mock_server):
target_imposter = Imposter(Stub(Predicate(path="/test")))
with mock_server(target_imposter) as server:
proxy_imposter = Imposter(
Stub(responses=Proxy(
to=target_imposter.url,
inject_headers={"X-Clacks-Overhead": "GNU Terry Pratchett"},
)))
server.add_imposters(proxy_imposter)
requests.get(proxy_imposter.url / "test")
assert_that(
target_imposter,
had_request().with_path("/test").and_headers(
has_entry("X-Clacks-Overhead", "GNU Terry Pratchett")),
)
def test_request_to_mock_server(mock_server):
# Start mock server with required behavior
imposter = Imposter(
Stub(Predicate(path="/test"), Response(body="sausages")))
with mock_server(imposter) as server:
# Make request to mock server
response = requests.get("{0}/test".format(imposter.url))
assert_that(
"We got the expected response",
response,
is_(response_with(status_code=200, body="sausages")),
)
assert_that("The mock server recorded the request", server,
had_request(path="/test", method="GET"))
def test_no_diff_job(mock_server):
with open("tests/resources/circrequests/valid_src_response.json") as file:
valid_src_response = file.read()
with open("tests/resources/circrequests/valid_dest_response.json") as file:
valid_dest_response = file.read()
# Set up mock server with required behavior
imposter = Imposter([
Stub(Predicate(path="/src"), Response(body=valid_src_response)),
Stub(Predicate(path="/dest", method="POST"),
Response(body=valid_dest_response)),
])
# Create a temporary file to use as last success lookup
# This will be comparing against the same response
# (tests/resources/circrequests/valid_src_response.json)
# so there will be no difference
try:
[temp_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('tests/resources/circrequests/valid_src_response.json')
with tempfile.TemporaryDirectory() as temp_storage_dir, mock_server(
imposter) as server:
setup_environment(imposter, temp_storage_dir,
temp_success_filename)
start_time = '20200521132905'
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
# There should be only one request to the server (the /src request)
assert 1 == len(server.get_actual_requests()[imposter.port])
assert_that(server,
had_request().with_path("/src").and_method("GET"))
finally:
# Clean up the temporary file
os.close(temp_file_handle)
os.remove(temp_success_filename)
def test_denied_key_wait_interval(mock_server):
with open("tests/resources/circrequests/valid_src_response.json") as file:
valid_src_response = file.read()
with open(
"tests/resources/circrequests/valid_dest_response_denied_key.json"
) as file:
valid_dest_response_denied_key = file.read()
# Set up mock server with required behavior
imposter = Imposter([
Stub(Predicate(path="/src"), Response(body=valid_src_response)),
Stub(Predicate(path="/dest", method="POST"),
Response(body=valid_dest_response_denied_key)),
])
try:
# Create a temporary file to use as last success lookup
[temp_success_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('etc/circrequests_FIRST.json')
# Create a temporary file to use as denied keys file
[temp_denied_keys_file_handle,
temp_denied_keys_filename] = tempfile.mkstemp()
with open(temp_denied_keys_filename, 'w') as f:
f.write('{}')
with tempfile.TemporaryDirectory() as temp_storage_dir:
with mock_server(imposter) as server:
# First request generates a deny
setup_environment(imposter, temp_storage_dir,
temp_success_filename,
temp_denied_keys_filename)
os.environ[
'CIRCREQUESTS_DENIED_ITEMS_WAIT_INTERVAL'] = '2700' # 45 minutes
start_time = '20200701000000' # 12:00am, July 1
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
had_request().with_path("/dest").and_method("POST"))
# Verify that "denied keys" file contains denied entry
with open(temp_denied_keys_filename) as file:
denied_keys = json.load(file)
denied_item_time = datetime.datetime.strptime(
start_time, '%Y%m%d%H%M%S').isoformat()
assert denied_keys == {'31430023550355': denied_item_time}
with mock_server(imposter) as server:
# Second request does not send any items, as only item is in
# denied list and wait time has not expired
start_time = '20200701003000' # 12:30am, July 1
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
is_not(
had_request().with_path("/dest").and_method("POST")))
with mock_server(imposter) as server:
# Third request occurs after wait interval expires, so
# denied items should be sent again
start_time = '20200701010000' # 1:00am, July 1
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
had_request().with_path("/dest").and_method("POST"))
finally:
# Clean up the temporary files
os.close(temp_success_file_handle)
os.remove(temp_success_filename)
os.close(temp_denied_keys_file_handle)
os.remove(temp_denied_keys_filename)
def test_denied_key_wait_interval(mock_server):
with open("tests/resources/circrequests/valid_src_modified_response.1.json"
) as file:
modified_response1 = file.read()
with open("tests/resources/circrequests/valid_src_modified_response.2.json"
) as file:
modified_response2 = file.read()
with open("tests/resources/circrequests/valid_dest_response.json") as file:
valid_dest_response = file.read()
# Set up mock server with required behavior
source_responses = [
Response(body=modified_response1),
Response(body=modified_response2)
]
imposter = Imposter([
Stub(Predicate(path="/src"), source_responses),
Stub(Predicate(path="/dest", method="POST"),
Response(body=valid_dest_response)),
])
try:
# Create a temporary file to use as last success lookup
[temp_success_file_handle, temp_success_filename] = tempfile.mkstemp()
with open(temp_success_filename, 'w') as f:
f.write('etc/circrequests_FIRST.json')
# Create a temporary file to use as denied keys file
[temp_denied_keys_file_handle,
temp_denied_keys_filename] = tempfile.mkstemp()
with open(temp_denied_keys_filename, 'w') as f:
f.write('{}')
with tempfile.TemporaryDirectory() as temp_storage_dir:
with mock_server(imposter) as server:
# First request is a success
setup_environment(imposter, temp_storage_dir,
temp_success_filename,
temp_denied_keys_filename)
start_time = '20200701000000' # 12:00am, July 1
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
# Verify that CaiaSoft submission was made
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
had_request().with_path("/dest").and_method("POST"))
# Second request should make another request, with the modified item
start_time = '20200701003000' # 12:30am, July 1
args = []
command = Command()
result = command(start_time, args)
assert result.was_successful() is True
assert_that(server,
had_request().with_path("/src").and_method("GET"))
assert_that(
server,
had_request().with_path("/dest").and_method("POST"))
finally:
# Clean up the temporary files
os.close(temp_success_file_handle)
os.remove(temp_success_filename)
os.close(temp_denied_keys_file_handle)
os.remove(temp_denied_keys_filename)
