def test_search_package(zmfapi, caplog): responses.add( responses.GET, ZMF_REST_URL + "package/search", json=ZMF_RESP_SEARCH_000007, match=[ responses.urlencoded_params_matcher({ "package": "APP*", "packageTitle": "fancy package title" }), ], ) responses.add( responses.GET, ZMF_REST_URL + "package/search", json=ZMF_RESP_ERR_NO_INFO, match=[ responses.urlencoded_params_matcher({ "package": "APP*", "packageTitle": "not exist package" }), ], ) assert zmfapi.search_package("APP", "fancy package title") == "APP 000007" with pytest.raises(SystemExit) as excinfo: zmfapi.search_package("APP", "not exist package") assert excinfo.value.code == EXIT_CODE_ZMF_NOK assert "CMN6504I" in caplog.text
def test_browse_component(zmfapi): responses.add( responses.GET, ZMF_REST_URL + "component/browse", body=ZMF_RESP_BROWSE_RICK.encode("iso-8859-1"), content_type="text/plain;charset=ISO-8859-1", headers={ "Content-Disposition": "attachment;filename=RICK", "Content-Length": str(len(ZMF_RESP_BROWSE_RICK)), }, match=[ responses.urlencoded_params_matcher({ "package": "APP 000001", "component": "RICK", "componentType": "LST", }), ], ) responses.add( responses.GET, ZMF_REST_URL + "component/browse", body=json.dumps(ZMF_RESP_BROWSE_INFO).encode("iso-8859-1"), content_type="application/json;charset=ISO-8859-1", match=[ responses.urlencoded_params_matcher({ "package": "APP 000001", "component": "NOTEXIST", "componentType": "LST", }), ], ) assert (zmfapi.browse_component("APP 000001", "RICK", "LST") == ZMF_RESP_BROWSE_RICK) assert zmfapi.browse_component("APP 000001", "NOTEXIST", "LST") is None
def too_soon_response(too_soon_urls): target_url = too_soon_urls[0] srcdir = FIXTURES_DIR.joinpath("job-save-too-soon") submit_resptext = srcdir.joinpath("submit-response.html").read_text() with responses.RequestsMock() as rsps: rsps.add( "POST", wb.url_for_savepage(target_url), body=submit_resptext, status=200, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) rsps.add( "GET", wb.url_for_jobstatus(wb.extract_job_id(submit_resptext)), body=srcdir.joinpath("status-0.json").read_text(), status=200, ) yield rsps
def test_snapshot_too_soon(): srcdir = FIXTURES_DIR.joinpath("job-save-too-soon") target_url = "https://plainlanguage.gov/" submit_resptext = srcdir.joinpath("submit-response.html").read_text() responses.add( "POST", wb.url_for_savepage(target_url), body=submit_resptext, status=200, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) responses.add( "GET", wb.url_for_jobstatus(wb.extract_job_id(submit_resptext)), body=srcdir.joinpath("status-0.json").read_text(), status=200, ) answer, meta = wb.snapshot(target_url) assert answer == meta.snapshot_url assert meta.subcommand == "snapshot" assert meta.was_new_snapshot_created() is False assert ( meta.too_soon() == "The same snapshot had been made 4 minutes and 18 seconds ago. We only allow new captures of the same URL every 20 minutes." )
def test_post_responses(): responses.add( "POST", "http://example.com", body="hello", status=200, match=[responses.urlencoded_params_matcher({"id": "good"})], ) requests.post("http://example.com", data={"id": "good"})
def test_get_package(zmfapi): assert zmfapi.get_package(params=PKG_CONF_INCL_ID) == "APP 000001" responses.add( responses.GET, ZMF_REST_URL + "package/search", json=ZMF_RESP_SEARCH_000007, match=[ responses.urlencoded_params_matcher({ "package": "APP*", "packageTitle": "fancy package title" }), ], ) assert zmfapi.get_package(params=PKG_CONF_EXCL_ID) == "APP 000007" responses.reset() responses.add( responses.GET, ZMF_REST_URL + "package/search", json=ZMF_RESP_ERR_NO_INFO, match=[ responses.urlencoded_params_matcher({ "package": "APP*", "packageTitle": "fancy package title" }), ], ) responses.add( responses.POST, ZMF_REST_URL + "package", json=ZMF_RESP_CREATE_000009, match=[ responses.urlencoded_params_matcher(PKG_CONF_EXCL_ID), ], ) assert zmfapi.get_package(params=PKG_CONF_EXCL_ID) == "APP 000009"
def test_delete_package(zmfapi): responses.add( responses.DELETE, ZMF_REST_URL + "package", json=ZMF_RESP_XXXX_OK, match=[ responses.urlencoded_params_matcher({ "package": "APP 000000", "processingOption": "1", }), ], ) assert zmfapi.delete_package("APP 000000") is None
def load_responses_fixtures(path) -> None: """Load a JSON fixture containing all the API response examples.""" def _load_json_fixtures(path: str) -> List[Dict[str, Any]]: """Load JSON fixtures from file.""" cwd: str = os.path.dirname(os.path.realpath(__file__)) fixtures: str = os.path.join(os.path.join(cwd, 'fixtures'), path) with open(fixtures) as fixture: return json.load(fixture) def _get_responses_method(method: str) -> str: """Returns the responses method based upon the supplied method. Args: method (str): The response method. Raises: ValueError: if the specified method is invalid. """ method = method.upper() if method == "GET": return responses.GET elif method == "POST": return responses.POST elif method == "DELETE": return responses.DELETE elif method == "PUT": return responses.PUT elif method == "PATCH": return responses.PATCH raise ValueError(f"Unable to find method '{method}' in responses") fixtures: List[Dict[str, Any]] = _load_json_fixtures(path) for fixture in fixtures: # Add the matchers for the request parameters or JSON body matchers: List[Any] = [] if fixture.get('match_json_params'): matchers.append( responses.json_params_matcher(fixture['match_json_params'])) if fixture.get('match_urlencoded_params'): matchers.append( responses.urlencoded_params_matcher( fixture['match_urlencoded_params'])) # Register the mocked response responses.add(_get_responses_method(fixture["method"]), url=fixture["url"], json=fixture.get("json"), status=fixture["status"], content_type=fixture.get("content_type", "application/json"), match=matchers)
def test_get_components(zmfapi): responses.add( responses.GET, ZMF_REST_URL + "component", json=ZMF_RESP_COMP_OK, match=[ responses.urlencoded_params_matcher({ "package": "APP 000001", "component": "APPB0001", }), ], ) assert (zmfapi.get_components( "APP 000001", component="APPB0001") == ZMF_RESP_COMP_OK["result"])
def test_get_load_components(zmfapi): responses.add( responses.GET, ZMF_REST_URL + "component/load", json=ZMF_RESP_LOAD_COMP_OK, match=[ responses.urlencoded_params_matcher({ "package": "APP 000001", "targetComponentType": "LST", }), ], ) assert (zmfapi.get_load_components( "APP 000001", targetType="LST") == ZMF_RESP_LOAD_COMP_OK["result"])
def test_get_package_list(zmfapi): responses.add( responses.GET, ZMF_REST_URL + "component/packagelist", json=ZMF_RESP_LOAD_COMP_OK, match=[ responses.urlencoded_params_matcher({ "package": "APP 000001", "sourceComponentType": "LST", }), ], ) assert (zmfapi.get_package_list( "APP 000001", componentType="LST") == ZMF_RESP_LOAD_COMP_OK["result"])
def test_refresh_token(octoparse): """ Test refresh_token """ with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: rsps.add(responses.POST, BASE_URL + 'token', json=TOKEN_ENTITY, status=200, match=[ responses.urlencoded_params_matcher({"refresh_token": TOKEN_ENTITY['refresh_token'], "grant_type": "refresh_token"}) ]) token = octoparse.refresh_token() assert token == TOKEN_ENTITY['access_token'] assert True
def octoparse(monkeypatch): monkeypatch.setenv("OCTOPARSE_USERNAME", OCTOPARSE_USERNAME) monkeypatch.setenv("OCTOPARSE_PASSWORD", OCTOPARSE_PASSWORD) with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: rsps.add(responses.POST, BASE_URL + 'token', json=TOKEN_ENTITY, status=200, match=[ responses.urlencoded_params_matcher({"username": OCTOPARSE_USERNAME, "password": OCTOPARSE_PASSWORD, "grant_type": "password"}) ]) yield Octoparse() os.remove('octoparse_token.pickle')
def test_create_package(zmfapi): responses.add( responses.POST, ZMF_REST_URL + "package", json=ZMF_RESP_CREATE_000009, match=[ responses.urlencoded_params_matcher({ "applName": "APP", "packageTitle": "fancy package title", "package": "APP 000001", }), ], ) assert zmfapi.create_package(params=PKG_CONF_INCL_ID) == "APP 000009"
def run(): responses.add("POST", "http://example1.com") responses.add("GET", "http://example.com") responses.add( "POST", "http://example.com", match=[responses.urlencoded_params_matcher({"foo": "bar"})], ) with pytest.raises(ConnectionError) as excinfo: requests.post("http://example.com", data={"id": "bad"}) msg = str(excinfo.value) assert "- POST http://example1.com/ URL does not match" in msg assert "- GET http://example.com/ Method does not match" in msg assert "- POST http://example.com/ Parameters do not match" in msg
def test_revert(zmfapi): responses.add( responses.PUT, ZMF_REST_URL + "package/revert", json=ZMF_RESP_XXXX_OK, match=[ responses.urlencoded_params_matcher({ "package": "APP 000000", "jobCard01": "//U000000R JOB 0,'CHANGEMAN',", "jobCard02": "// CLASS=A,MSGCLASS=A,", "jobCard03": "// NOTIFY=&SYSUID", "jobCard04": "//*", }), ], ) assert zmfapi.revert("APP 000000") is None
def run(): responses.add( method=responses.POST, url="http://example.com/", body="one", match=[ responses.json_params_matcher( {"page": { "name": "first", "type": "json" }}) ], ) responses.add( method=responses.POST, url="http://example.com/", body="two", match=[ responses.urlencoded_params_matcher({ "page": "second", "type": "urlencoded" }) ], ) resp = requests.request( "POST", "http://example.com/", headers={"Content-Type": "x-www-form-urlencoded"}, data={ "page": "second", "type": "urlencoded" }, ) assert_response(resp, "two") resp = requests.request( "POST", "http://example.com/", headers={"Content-Type": "application/json"}, json={"page": { "name": "first", "type": "json" }}, ) assert_response(resp, "one")
def test_demote(zmfapi): responses.add( responses.PUT, ZMF_REST_URL + "package/demote", json=ZMF_RESP_XXXX_OK, match=[ responses.urlencoded_params_matcher({ "package": "APP 000000", "promotionSiteName": "DEV0", "promotionLevel": "42", "promotionName": "ALL", "jobCards01": "//U000000D JOB 0,'CHANGEMAN',", "jobCards02": "// CLASS=A,MSGCLASS=A,", "jobCards03": "// NOTIFY=&SYSUID", "jobCards04": "//*", }), ], ) assert zmfapi.demote("APP 000000", "DEV0", 42, "ALL") is None
def save_success_response(success_urls): srcdir = FIXTURES_DIR.joinpath("job-save-success") target_url = success_urls[0] submit_resptext = srcdir.joinpath("submit-response.html").read_text() expected_job_url = wb.url_for_jobstatus(submit_resptext) status_paths = iter( [ srcdir.joinpath("status-0.json"), srcdir.joinpath("status-1.json"), srcdir.joinpath("status-9.json"), srcdir.joinpath("status-10.json"), ] ) with responses.RequestsMock() as rsps: rsps.add( "POST", wb.url_for_savepage(target_url), body=submit_resptext, status=200, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) rsps.add_callback( "GET", expected_job_url, callback=lambda req: ( 200, {}, next(status_paths).read_text(), ), # 2nd arg is a headers dict ) yield rsps
def test_snapshot_submit_request(session): target_url = "https://plainlanguage.gov/" save_url = wb.url_for_savepage(target_url) resptext = FIXTURES_DIR.joinpath( "job-save-success/submit-response.html" ).read_text() responses.add( "POST", save_url, body=resptext, status=200, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) resp = wb.submit_snapshot_request(session, target_url, headers={}) assert f'<h2 id="spn-title">Saving page {target_url}</h2>' in resp.text
def test_snapshot_submit_request_not_ok(session): """not sure when this would happen, when server is down?""" target_url = "https://plainlanguage.gov/" save_url = wb.url_for_savepage(target_url) resptext = FIXTURES_DIR.joinpath( "job-save-success/submit-response.html" ).read_text() responses.add( "POST", save_url, body=resptext, status=503, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) with pytest.raises(ServerStatusError) as err: resp = wb.submit_snapshot_request(session, target_url, headers={}) assert ( f"Server status was NOT OK; returned 503 for: {save_url}" in err.value.args[0] )
def run(): responses.add( method=responses.POST, url="http://example.com/", body="one", match=[responses.json_params_matcher(None)], ) responses.add( method=responses.POST, url="http://example.com/", body="two", match=[responses.urlencoded_params_matcher(None)], ) resp = requests.request("POST", "http://example.com/") assert_response(resp, "one") resp = requests.request( "POST", "http://example.com/", headers={"Content-Type": "x-www-form-urlencoded"}, ) assert_response(resp, "two")
def test_snapshot_too_many_for_period(): srcdir = FIXTURES_DIR.joinpath("job-save-too-many-today") submit_resptext = srcdir.joinpath("submit-response.html").read_text() target_url = "https://nytimes.com/" responses.add( "POST", wb.url_for_savepage(target_url), body=submit_resptext, status=200, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) # mock request for availability URL responses.add( "GET", wb.url_for_availability(target_url), body=srcdir.joinpath("check-availability.json").read_text(), ) answer, meta = wb.snapshot(target_url) assert answer == meta.snapshot_url assert meta.subcommand == "snapshot" assert meta.was_new_snapshot_created() == False assert ( meta.too_many_during_period() == """This URL has been already captured 10 times today. Please email us at "*****@*****.**" if you would like to discuss this more.""" ) # import pdb; pdb.set_trace() # server payload is the payload returned by availability API response assert meta.server_payload["archived_snapshots"]["closest"]["available"] is True
def test_snapshot_successful(success_status_paths): #### fixture setup (todo: refactor?) srcdir = FIXTURES_DIR.joinpath("job-save-success") target_url = "https://plainlanguage.gov/" save_url = wb.url_for_savepage(target_url) submit_resptext = srcdir.joinpath("submit-response.html").read_text() expected_job_id = wb.extract_job_id(submit_resptext) expected_job_url = wb.url_for_jobstatus(expected_job_id) #### mock responses responses.add( "POST", save_url, body=submit_resptext, status=200, match=[ responses.urlencoded_params_matcher( {"url": target_url, "capture_all": "on"} ) ], ) responses.add_callback( "GET", expected_job_url, callback=lambda req: ( 200, {}, next(success_status_paths).read_text(), ), # 2nd arg is a headers dict ) answer, meta = wb.snapshot(target_url, user_agent="guy incognito", poll_interval=0) # make sure snapshot, as expected by the setup, exhausted the success_status_paths iterator assert next(success_status_paths, False) is False # test return values assert type(answer) is str assert type(meta) is wb.TaskMeta assert meta.subcommand == "snapshot" assert meta.target_url == target_url assert meta.created_at.strftime("%Y-%m-%d %H:%M:%S%z") == "2020-09-01 14:30:55+0000" data = meta.to_dict() # test that answer is snapshot url assert ( answer == wb.BASE_DOMAIN + "/web/" + data["server_payload"]["timestamp"] + "/" + target_url ) # test data response assert data["subcommand"] == "snapshot" assert data["was_new_snapshot_created"] is True assert data["snapshot_url"] == answer assert data["request_meta"]["user_agent"] == "guy incognito" issues = data["issues"] assert issues["too_soon"] is False assert issues["too_many_during_period"] is False jd = data["server_payload"] assert jd["status"] == "success" assert jd["timestamp"] in data["snapshot_url"] # not sure if this is always the case...what happens if there's a redirect? assert jd["original_url"] == target_url