def test_search_package(zmfapi, caplog):
responses.add(
responses.GET,
ZMF_REST_URL + "package/search",
json=ZMF_RESP_SEARCH_000007,
match=[
responses.urlencoded_params_matcher({
"package":
"APP*",
"packageTitle":
"fancy package title"
}),
],
)
responses.add(
responses.GET,
ZMF_REST_URL + "package/search",
json=ZMF_RESP_ERR_NO_INFO,
match=[
responses.urlencoded_params_matcher({
"package":
"APP*",
"packageTitle":
"not exist package"
}),
],
)
assert zmfapi.search_package("APP", "fancy package title") == "APP 000007"
with pytest.raises(SystemExit) as excinfo:
zmfapi.search_package("APP", "not exist package")
assert excinfo.value.code == EXIT_CODE_ZMF_NOK
assert "CMN6504I" in caplog.text
def test_browse_component(zmfapi):
responses.add(
responses.GET,
ZMF_REST_URL + "component/browse",
body=ZMF_RESP_BROWSE_RICK.encode("iso-8859-1"),
content_type="text/plain;charset=ISO-8859-1",
headers={
"Content-Disposition": "attachment;filename=RICK",
"Content-Length": str(len(ZMF_RESP_BROWSE_RICK)),
},
match=[
responses.urlencoded_params_matcher({
"package": "APP 000001",
"component": "RICK",
"componentType": "LST",
}),
],
)
responses.add(
responses.GET,
ZMF_REST_URL + "component/browse",
body=json.dumps(ZMF_RESP_BROWSE_INFO).encode("iso-8859-1"),
content_type="application/json;charset=ISO-8859-1",
match=[
responses.urlencoded_params_matcher({
"package": "APP 000001",
"component": "NOTEXIST",
"componentType": "LST",
}),
],
)
assert (zmfapi.browse_component("APP 000001", "RICK",
"LST") == ZMF_RESP_BROWSE_RICK)
assert zmfapi.browse_component("APP 000001", "NOTEXIST", "LST") is None
def too_soon_response(too_soon_urls):
target_url = too_soon_urls[0]
srcdir = FIXTURES_DIR.joinpath("job-save-too-soon")
submit_resptext = srcdir.joinpath("submit-response.html").read_text()
with responses.RequestsMock() as rsps:
rsps.add(
"POST",
wb.url_for_savepage(target_url),
body=submit_resptext,
status=200,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
rsps.add(
"GET",
wb.url_for_jobstatus(wb.extract_job_id(submit_resptext)),
body=srcdir.joinpath("status-0.json").read_text(),
status=200,
)
yield rsps
def test_snapshot_too_soon():
srcdir = FIXTURES_DIR.joinpath("job-save-too-soon")
target_url = "https://plainlanguage.gov/"
submit_resptext = srcdir.joinpath("submit-response.html").read_text()
responses.add(
"POST",
wb.url_for_savepage(target_url),
body=submit_resptext,
status=200,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
responses.add(
"GET",
wb.url_for_jobstatus(wb.extract_job_id(submit_resptext)),
body=srcdir.joinpath("status-0.json").read_text(),
status=200,
)
answer, meta = wb.snapshot(target_url)
assert answer == meta.snapshot_url
assert meta.subcommand == "snapshot"
assert meta.was_new_snapshot_created() is False
assert (
meta.too_soon()
== "The same snapshot had been made 4 minutes and 18 seconds ago. We only allow new captures of the same URL every 20 minutes."
)
def test_post_responses():
responses.add(
"POST",
"http://example.com",
body="hello",
status=200,
match=[responses.urlencoded_params_matcher({"id": "good"})],
)
requests.post("http://example.com", data={"id": "good"})
def test_get_package(zmfapi):
assert zmfapi.get_package(params=PKG_CONF_INCL_ID) == "APP 000001"
responses.add(
responses.GET,
ZMF_REST_URL + "package/search",
json=ZMF_RESP_SEARCH_000007,
match=[
responses.urlencoded_params_matcher({
"package":
"APP*",
"packageTitle":
"fancy package title"
}),
],
)
assert zmfapi.get_package(params=PKG_CONF_EXCL_ID) == "APP 000007"
responses.reset()
responses.add(
responses.GET,
ZMF_REST_URL + "package/search",
json=ZMF_RESP_ERR_NO_INFO,
match=[
responses.urlencoded_params_matcher({
"package":
"APP*",
"packageTitle":
"fancy package title"
}),
],
)
responses.add(
responses.POST,
ZMF_REST_URL + "package",
json=ZMF_RESP_CREATE_000009,
match=[
responses.urlencoded_params_matcher(PKG_CONF_EXCL_ID),
],
)
assert zmfapi.get_package(params=PKG_CONF_EXCL_ID) == "APP 000009"
def test_delete_package(zmfapi):
responses.add(
responses.DELETE,
ZMF_REST_URL + "package",
json=ZMF_RESP_XXXX_OK,
match=[
responses.urlencoded_params_matcher({
"package": "APP 000000",
"processingOption": "1",
}),
],
)
assert zmfapi.delete_package("APP 000000") is None
def load_responses_fixtures(path) -> None:
"""Load a JSON fixture containing all the API response examples."""
def _load_json_fixtures(path: str) -> List[Dict[str, Any]]:
"""Load JSON fixtures from file."""
cwd: str = os.path.dirname(os.path.realpath(__file__))
fixtures: str = os.path.join(os.path.join(cwd, 'fixtures'), path)
with open(fixtures) as fixture:
return json.load(fixture)
def _get_responses_method(method: str) -> str:
"""Returns the responses method based upon the supplied method.
Args:
method (str): The response method.
Raises:
ValueError: if the specified method is invalid.
"""
method = method.upper()
if method == "GET":
return responses.GET
elif method == "POST":
return responses.POST
elif method == "DELETE":
return responses.DELETE
elif method == "PUT":
return responses.PUT
elif method == "PATCH":
return responses.PATCH
raise ValueError(f"Unable to find method '{method}' in responses")
fixtures: List[Dict[str, Any]] = _load_json_fixtures(path)
for fixture in fixtures:
# Add the matchers for the request parameters or JSON body
matchers: List[Any] = []
if fixture.get('match_json_params'):
matchers.append(
responses.json_params_matcher(fixture['match_json_params']))
if fixture.get('match_urlencoded_params'):
matchers.append(
responses.urlencoded_params_matcher(
fixture['match_urlencoded_params']))
# Register the mocked response
responses.add(_get_responses_method(fixture["method"]),
url=fixture["url"],
json=fixture.get("json"),
status=fixture["status"],
content_type=fixture.get("content_type",
"application/json"),
match=matchers)
def test_get_components(zmfapi):
responses.add(
responses.GET,
ZMF_REST_URL + "component",
json=ZMF_RESP_COMP_OK,
match=[
responses.urlencoded_params_matcher({
"package": "APP 000001",
"component": "APPB0001",
}),
],
)
assert (zmfapi.get_components(
"APP 000001", component="APPB0001") == ZMF_RESP_COMP_OK["result"])
def test_get_load_components(zmfapi):
responses.add(
responses.GET,
ZMF_REST_URL + "component/load",
json=ZMF_RESP_LOAD_COMP_OK,
match=[
responses.urlencoded_params_matcher({
"package": "APP 000001",
"targetComponentType": "LST",
}),
],
)
assert (zmfapi.get_load_components(
"APP 000001", targetType="LST") == ZMF_RESP_LOAD_COMP_OK["result"])
def test_get_package_list(zmfapi):
responses.add(
responses.GET,
ZMF_REST_URL + "component/packagelist",
json=ZMF_RESP_LOAD_COMP_OK,
match=[
responses.urlencoded_params_matcher({
"package": "APP 000001",
"sourceComponentType": "LST",
}),
],
)
assert (zmfapi.get_package_list(
"APP 000001", componentType="LST") == ZMF_RESP_LOAD_COMP_OK["result"])
def test_refresh_token(octoparse):
"""
Test refresh_token
"""
with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
rsps.add(responses.POST, BASE_URL + 'token',
json=TOKEN_ENTITY, status=200,
match=[
responses.urlencoded_params_matcher({"refresh_token": TOKEN_ENTITY['refresh_token'],
"grant_type": "refresh_token"})
])
token = octoparse.refresh_token()
assert token == TOKEN_ENTITY['access_token']
assert True
def octoparse(monkeypatch):
monkeypatch.setenv("OCTOPARSE_USERNAME", OCTOPARSE_USERNAME)
monkeypatch.setenv("OCTOPARSE_PASSWORD", OCTOPARSE_PASSWORD)
with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
rsps.add(responses.POST, BASE_URL + 'token',
json=TOKEN_ENTITY, status=200,
match=[
responses.urlencoded_params_matcher({"username": OCTOPARSE_USERNAME,
"password": OCTOPARSE_PASSWORD,
"grant_type": "password"})
])
yield Octoparse()
os.remove('octoparse_token.pickle')
def test_create_package(zmfapi):
responses.add(
responses.POST,
ZMF_REST_URL + "package",
json=ZMF_RESP_CREATE_000009,
match=[
responses.urlencoded_params_matcher({
"applName": "APP",
"packageTitle": "fancy package title",
"package": "APP 000001",
}),
],
)
assert zmfapi.create_package(params=PKG_CONF_INCL_ID) == "APP 000009"
def run():
responses.add("POST", "http://example1.com")
responses.add("GET", "http://example.com")
responses.add(
"POST",
"http://example.com",
match=[responses.urlencoded_params_matcher({"foo": "bar"})],
)
with pytest.raises(ConnectionError) as excinfo:
requests.post("http://example.com", data={"id": "bad"})
msg = str(excinfo.value)
assert "- POST http://example1.com/ URL does not match" in msg
assert "- GET http://example.com/ Method does not match" in msg
assert "- POST http://example.com/ Parameters do not match" in msg
def test_revert(zmfapi):
responses.add(
responses.PUT,
ZMF_REST_URL + "package/revert",
json=ZMF_RESP_XXXX_OK,
match=[
responses.urlencoded_params_matcher({
"package": "APP 000000",
"jobCard01": "//U000000R JOB 0,'CHANGEMAN',",
"jobCard02": "// CLASS=A,MSGCLASS=A,",
"jobCard03": "// NOTIFY=&SYSUID",
"jobCard04": "//*",
}),
],
)
assert zmfapi.revert("APP 000000") is None
def run():
responses.add(
method=responses.POST,
url="http://example.com/",
body="one",
match=[
responses.json_params_matcher(
{"page": {
"name": "first",
"type": "json"
}})
],
)
responses.add(
method=responses.POST,
url="http://example.com/",
body="two",
match=[
responses.urlencoded_params_matcher({
"page": "second",
"type": "urlencoded"
})
],
)
resp = requests.request(
"POST",
"http://example.com/",
headers={"Content-Type": "x-www-form-urlencoded"},
data={
"page": "second",
"type": "urlencoded"
},
)
assert_response(resp, "two")
resp = requests.request(
"POST",
"http://example.com/",
headers={"Content-Type": "application/json"},
json={"page": {
"name": "first",
"type": "json"
}},
)
assert_response(resp, "one")
def test_demote(zmfapi):
responses.add(
responses.PUT,
ZMF_REST_URL + "package/demote",
json=ZMF_RESP_XXXX_OK,
match=[
responses.urlencoded_params_matcher({
"package": "APP 000000",
"promotionSiteName": "DEV0",
"promotionLevel": "42",
"promotionName": "ALL",
"jobCards01": "//U000000D JOB 0,'CHANGEMAN',",
"jobCards02": "// CLASS=A,MSGCLASS=A,",
"jobCards03": "// NOTIFY=&SYSUID",
"jobCards04": "//*",
}),
],
)
assert zmfapi.demote("APP 000000", "DEV0", 42, "ALL") is None
def save_success_response(success_urls):
srcdir = FIXTURES_DIR.joinpath("job-save-success")
target_url = success_urls[0]
submit_resptext = srcdir.joinpath("submit-response.html").read_text()
expected_job_url = wb.url_for_jobstatus(submit_resptext)
status_paths = iter(
[
srcdir.joinpath("status-0.json"),
srcdir.joinpath("status-1.json"),
srcdir.joinpath("status-9.json"),
srcdir.joinpath("status-10.json"),
]
)
with responses.RequestsMock() as rsps:
rsps.add(
"POST",
wb.url_for_savepage(target_url),
body=submit_resptext,
status=200,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
rsps.add_callback(
"GET",
expected_job_url,
callback=lambda req: (
200,
{},
next(status_paths).read_text(),
), # 2nd arg is a headers dict
)
yield rsps
def test_snapshot_submit_request(session):
target_url = "https://plainlanguage.gov/"
save_url = wb.url_for_savepage(target_url)
resptext = FIXTURES_DIR.joinpath(
"job-save-success/submit-response.html"
).read_text()
responses.add(
"POST",
save_url,
body=resptext,
status=200,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
resp = wb.submit_snapshot_request(session, target_url, headers={})
assert f'<h2 id="spn-title">Saving page {target_url}</h2>' in resp.text
def test_snapshot_submit_request_not_ok(session):
"""not sure when this would happen, when server is down?"""
target_url = "https://plainlanguage.gov/"
save_url = wb.url_for_savepage(target_url)
resptext = FIXTURES_DIR.joinpath(
"job-save-success/submit-response.html"
).read_text()
responses.add(
"POST",
save_url,
body=resptext,
status=503,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
with pytest.raises(ServerStatusError) as err:
resp = wb.submit_snapshot_request(session, target_url, headers={})
assert (
f"Server status was NOT OK; returned 503 for: {save_url}" in err.value.args[0]
)
def run():
responses.add(
method=responses.POST,
url="http://example.com/",
body="one",
match=[responses.json_params_matcher(None)],
)
responses.add(
method=responses.POST,
url="http://example.com/",
body="two",
match=[responses.urlencoded_params_matcher(None)],
)
resp = requests.request("POST", "http://example.com/")
assert_response(resp, "one")
resp = requests.request(
"POST",
"http://example.com/",
headers={"Content-Type": "x-www-form-urlencoded"},
)
assert_response(resp, "two")
def test_snapshot_too_many_for_period():
srcdir = FIXTURES_DIR.joinpath("job-save-too-many-today")
submit_resptext = srcdir.joinpath("submit-response.html").read_text()
target_url = "https://nytimes.com/"
responses.add(
"POST",
wb.url_for_savepage(target_url),
body=submit_resptext,
status=200,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
# mock request for availability URL
responses.add(
"GET",
wb.url_for_availability(target_url),
body=srcdir.joinpath("check-availability.json").read_text(),
)
answer, meta = wb.snapshot(target_url)
assert answer == meta.snapshot_url
assert meta.subcommand == "snapshot"
assert meta.was_new_snapshot_created() == False
assert (
meta.too_many_during_period()
== """This URL has been already captured 10 times today. Please email us at "*****@*****.**" if you would like to discuss this more."""
)
# import pdb; pdb.set_trace()
# server payload is the payload returned by availability API response
assert meta.server_payload["archived_snapshots"]["closest"]["available"] is True
def test_snapshot_successful(success_status_paths):
#### fixture setup (todo: refactor?)
srcdir = FIXTURES_DIR.joinpath("job-save-success")
target_url = "https://plainlanguage.gov/"
save_url = wb.url_for_savepage(target_url)
submit_resptext = srcdir.joinpath("submit-response.html").read_text()
expected_job_id = wb.extract_job_id(submit_resptext)
expected_job_url = wb.url_for_jobstatus(expected_job_id)
#### mock responses
responses.add(
"POST",
save_url,
body=submit_resptext,
status=200,
match=[
responses.urlencoded_params_matcher(
{"url": target_url, "capture_all": "on"}
)
],
)
responses.add_callback(
"GET",
expected_job_url,
callback=lambda req: (
200,
{},
next(success_status_paths).read_text(),
), # 2nd arg is a headers dict
)
answer, meta = wb.snapshot(target_url, user_agent="guy incognito", poll_interval=0)
# make sure snapshot, as expected by the setup, exhausted the success_status_paths iterator
assert next(success_status_paths, False) is False
# test return values
assert type(answer) is str
assert type(meta) is wb.TaskMeta
assert meta.subcommand == "snapshot"
assert meta.target_url == target_url
assert meta.created_at.strftime("%Y-%m-%d %H:%M:%S%z") == "2020-09-01 14:30:55+0000"
data = meta.to_dict()
# test that answer is snapshot url
assert (
answer
== wb.BASE_DOMAIN
+ "/web/"
+ data["server_payload"]["timestamp"]
+ "/"
+ target_url
)
# test data response
assert data["subcommand"] == "snapshot"
assert data["was_new_snapshot_created"] is True
assert data["snapshot_url"] == answer
assert data["request_meta"]["user_agent"] == "guy incognito"
issues = data["issues"]
assert issues["too_soon"] is False
assert issues["too_many_during_period"] is False
jd = data["server_payload"]
assert jd["status"] == "success"
assert jd["timestamp"] in data["snapshot_url"]
# not sure if this is always the case...what happens if there's a redirect?
assert jd["original_url"] == target_url
