def test_list_projects(): mock_client = MagicMock() mock_client.get.return_value = {'projects': ['test', 'test2']} api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_projects() assert rtn == ['test', 'test2'] mock_client.get.assert_called_with('http://localhost/listprojects.json', )
def test_schedule(): mock_client = MagicMock() job_id = 'ce54b67080280d1ec69821bcb6a88393' settings = { 'BOT_NAME': 'Firefox', 'DOWNLOAD_DELAY': 2 } kwargs = {'extra_detail': 'Test'} mock_client.post.return_value = { 'jobid': job_id } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.schedule(PROJECT, SPIDER, settings=settings, **kwargs) assert rtn == job_id args, kwargs = mock_client.post.call_args assert len(args) == 1 assert args[0] == 'http://localhost/schedule.json' assert len(kwargs) == 1 assert 'data' in kwargs data_kw = kwargs['data'] assert 'project' in data_kw assert data_kw['project'] == PROJECT assert 'extra_detail' in data_kw assert data_kw['extra_detail'] == 'Test' assert 'setting' in data_kw assert sorted(data_kw['setting']) == ['BOT_NAME=Firefox', 'DOWNLOAD_DELAY=2'] assert 'spider' in data_kw assert data_kw['spider'] == SPIDER
def test_schedule(): mock_client = MagicMock() job_id = 'ce54b67080280d1ec69821bcb6a88393' settings = { 'BOT_NAME': 'Firefox', 'DOWNLOAD_DELAY': 2 } kwargs = {'extra_detail': 'Test'} mock_client.post.return_value = { 'jobid': job_id } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.schedule(PROJECT, SPIDER, settings=settings, **kwargs) assert rtn == job_id args, kwargs = mock_client.post.call_args assert len(args) == 1 assert args[0] == 'http://localhost/schedule.json' assert len(kwargs) == 2 assert 'data' in kwargs data_kw = kwargs['data'] assert 'project' in data_kw assert data_kw['project'] == PROJECT assert 'extra_detail' in data_kw assert data_kw['extra_detail'] == 'Test' assert 'setting' in data_kw assert sorted(data_kw['setting']) == ['BOT_NAME=Firefox', 'DOWNLOAD_DELAY=2'] assert 'spider' in data_kw assert data_kw['spider'] == SPIDER
def test_build_url_with_non_existant_endpoint_errors(): """ Supplying _build_url with an endpoint that does not exist in the endpoints dictionary should result in a ValueError. """ api = ScrapydAPI(HOST_URL) with pytest.raises(ValueError): api._build_url('does-not-exist')
def test_build_url_with_non_existant_endpoint_errors(): """ Supplying _build_url with an endpoint that does not exist in the endpoints dictionary should result in a ValueError. """ api = ScrapydAPI(HOST_URL) with pytest.raises(ValueError): api._build_url('does-not-exist')
def test_daemon_status(): api = ScrapydAPI(HOST_URL) rtn = api.daemon_status() assert isinstance(rtn, dict) assert 'finished' in rtn assert 'running' in rtn assert 'pending' in rtn assert 'node_name' in rtn assert isinstance(rtn['finished'], int)
def test_delete_project(): mock_client = MagicMock() mock_client.post.return_value = {} api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.delete_project(PROJECT) assert rtn is True mock_client.post.assert_called_with('http://localhost/delproject.json', data={ 'project': PROJECT, })
def test_list_projects(): mock_client = MagicMock() mock_client.get.return_value = { 'projects': ['test', 'test2'] } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_projects() assert rtn == ['test', 'test2'] mock_client.get.assert_called_with( 'http://localhost/listprojects.json', )
def test_delete_version(): mock_client = MagicMock() mock_client.post.return_value = {} api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.delete_version(PROJECT, VERSION) assert rtn is True mock_client.post.assert_called_with('http://localhost/delversion.json', data={ 'project': PROJECT, 'version': VERSION })
def test_delete_project(): mock_client = MagicMock() mock_client.post.return_value = {} api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.delete_project(PROJECT) assert rtn is True mock_client.post.assert_called_with( 'http://localhost/delproject.json', data={ 'project': PROJECT, } )
def test_list_versions(): mock_client = MagicMock() mock_client.get.return_value = {'versions': ['version', 'version2']} api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_versions(PROJECT) assert rtn == ['version', 'version2'] mock_client.get.assert_called_with( 'http://localhost/listversions.json', params={ 'project': PROJECT, }, )
def test_build_url_with_default_endpoints(): """ Absolute URL constructor should form correct URL when the client is relying on the default endpoints. """ api = ScrapydAPI('http://localhost') url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion.json' # Test trailing slash on target. api = ScrapydAPI('http://localhost/') url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion.json'
def test_cancelling_running_job(): mock_client = MagicMock() mock_client.post.return_value = { 'prevstate': 'running', } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.cancel(PROJECT, JOB) assert rtn is 'running' mock_client.post.assert_called_with('http://localhost/cancel.json', data={ 'project': PROJECT, 'job': JOB })
def test_add_version(): mock_client = MagicMock() mock_client.post.return_value = {'spiders': 3} api = ScrapydAPI(HOST_URL, client=mock_client) test_egg = StringIO('Test egg') rtn = api.add_version(PROJECT, VERSION, test_egg) assert rtn == 3 # The number of spiders uploaded. mock_client.post.assert_called_with('http://localhost/addversion.json', data={ 'project': PROJECT, 'version': VERSION }, files={'egg': test_egg})
def test_delete_version(): mock_client = MagicMock() mock_client.post.return_value = {} api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.delete_version(PROJECT, VERSION) assert rtn is True mock_client.post.assert_called_with( 'http://localhost/delversion.json', data={ 'project': PROJECT, 'version': VERSION } )
def test_list_versions(): mock_client = MagicMock() mock_client.get.return_value = { 'versions': ['version', 'version2'] } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_versions(PROJECT) assert rtn == ['version', 'version2'] mock_client.get.assert_called_with( 'http://localhost/listversions.json', params={ 'project': PROJECT, }, )
def test_list_projects(): """ Test the method which handles listing projects on the server. """ mock_client = MagicMock() mock_client.get.return_value = { 'projects': ['test', 'test2'] } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_projects() assert rtn == ['test', 'test2'] mock_client.get.assert_called_with( 'http://localhost/listprojects.json', )
def test_cancelling_pending_job(): mock_client = MagicMock() mock_client.post.return_value = { 'prevstate': 'pending', } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.cancel(PROJECT, JOB) assert rtn is 'pending' mock_client.post.assert_called_with( 'http://localhost/cancel.json', data={ 'project': PROJECT, 'job': JOB } )
def test_cancelling_with_specific_signal(): mock_client = MagicMock() mock_client.post.return_value = { 'prevstate': 'running', } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.cancel(PROJECT, JOB, signal='TERM') assert rtn is 'running' mock_client.post.assert_called_with( 'http://localhost/cancel.json', data={ 'project': PROJECT, 'job': JOB, 'signal': 'TERM' } )
def test_auth_gets_applied_when_client_is_not_supplied(): """ Auth details should get correctly passed to the client when no client is provided. """ api = ScrapydAPI(HOST_URL, auth=AUTH) assert api.client.auth == AUTH
def test_list_spiders(): mock_client = MagicMock() mock_client.get.return_value = { 'spiders': ['spider', 'spider2'] } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_spiders(PROJECT) assert rtn == ['spider', 'spider2'] mock_client.get.assert_called_with( 'http://localhost/listspiders.json', params={ 'project': PROJECT, }, timeout=None )
def test_request_timeout(): """ The client should raise an exception when the server does not respond in time limit. """ api = ScrapydAPI('http://httpbin.org/delay/5', timeout=1) with pytest.raises(Timeout): api.client.get(api.target, timeout=api.timeout)
def test_job_status(): mock_client = MagicMock() mock_client.get.return_value = { 'pending': [{'id': 'abc'}, {'id': 'def'}], 'running': [], 'finished': [{'id': 'ghi'}], } api = ScrapydAPI(HOST_URL, client=mock_client) expected_results = ( ('abc', PENDING), ('def', PENDING), ('ghi', FINISHED), ('xyz', '') ) for job_id, expected_result in expected_results: rtn = api.job_status(PROJECT, job_id) assert rtn == expected_result
def test_cancelling_with_specific_signal(): mock_client = MagicMock() mock_client.post.return_value = { 'prevstate': 'running', } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.cancel(PROJECT, JOB, signal='TERM') assert rtn is 'running' mock_client.post.assert_called_with( 'http://localhost/cancel.json', data={ 'project': PROJECT, 'job': JOB, 'signal': 'TERM' }, timeout=None )
def test_job_status(): mock_client = MagicMock() mock_client.get.return_value = { 'pending': [{'id': 'abc'}, {'id': 'def'}], 'running': [], 'finished': [{'id': 'ghi'}], } api = ScrapydAPI(HOST_URL, client=mock_client) expected_results = ( ('abc', PENDING), ('def', PENDING), ('ghi', FINISHED), ('xyz', '') ) for job_id, expected_result in expected_results: rtn = api.job_status(PROJECT, job_id) assert rtn == expected_result
def test_auth_doesnt_get_applied_when_client_is_supplied(): """ Auth details should not get set on a passed client. Instantiated clients should handle auth themselves. """ mock_client = MagicMock() api = ScrapydAPI(HOST_URL, auth=AUTH, client=mock_client) assert api.client.auth != AUTH
def test_list_spiders(): """ Test the method which handles listing all spiders in a project. """ mock_client = MagicMock() mock_client.get.return_value = { 'spiders': ['spider', 'spider2'] } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_spiders(PROJECT) assert rtn == ['spider', 'spider2'] mock_client.get.assert_called_with( 'http://localhost/listspiders.json', params={ 'project': PROJECT, }, )
def test_cancel(): """ Test the method which handles cancelling a spider job. """ mock_client = MagicMock() mock_client.post.return_value = { 'prevstate': 'running', } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.cancel(PROJECT, JOB) assert rtn is True mock_client.post.assert_called_with( 'http://localhost/cancel.json', data={ 'project': PROJECT, 'job': JOB } )
def test_add_version(): mock_client = MagicMock() mock_client.post.return_value = { 'spiders': 3 } api = ScrapydAPI(HOST_URL, client=mock_client) test_egg = StringIO('Test egg') rtn = api.add_version(PROJECT, VERSION, test_egg) assert rtn == 3 # The number of spiders uploaded. mock_client.post.assert_called_with( 'http://localhost/addversion.json', data={ 'project': PROJECT, 'version': VERSION }, files={ 'egg': test_egg } )
def test_build_url_with_custom_endpoints(): """ The absolute URL constructor should correctly form URL when the client has custom endpoints passed in. """ custom_endpoints = { ADD_VERSION_ENDPOINT: '/addversion-custom.json' } api = ScrapydAPI('http://localhost', endpoints=custom_endpoints) url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion-custom.json' # Test trailing slash on target. api = ScrapydAPI('http://localhost/', endpoints=custom_endpoints) url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion-custom.json' # Test that endpoints that were not overridden by the custom_endpoints # still work as the defaults. url = api._build_url(CANCEL_ENDPOINT) assert url == 'http://localhost/cancel.json'
def test_list_jobs(): mock_client = MagicMock() mock_client.get.return_value = { 'pending': [{'id': 'abc'}, {'id': 'def'}], 'running': [], 'finished': [{'id': 'ghi'}], } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_jobs(PROJECT) assert len(rtn) == 3 assert sorted(rtn.keys()) == ['finished', 'pending', 'running'] assert rtn['pending'] == [{'id': 'abc'}, {'id': 'def'}] assert rtn['finished'] == [{'id': 'ghi'}] assert rtn['running'] == [] mock_client.get.assert_called_with( 'http://localhost/listjobs.json', params={ 'project': PROJECT, } )
def test_list_jobs(): mock_client = MagicMock() mock_client.get.return_value = { 'pending': [{'id': 'abc'}, {'id': 'def'}], 'running': [], 'finished': [{'id': 'ghi'}], } api = ScrapydAPI(HOST_URL, client=mock_client) rtn = api.list_jobs(PROJECT) assert len(rtn) == 3 assert sorted(rtn.keys()) == ['finished', 'pending', 'running'] assert rtn['pending'] == [{'id': 'abc'}, {'id': 'def'}] assert rtn['finished'] == [{'id': 'ghi'}] assert rtn['running'] == [] mock_client.get.assert_called_with( 'http://localhost/listjobs.json', params={ 'project': PROJECT, }, timeout=None )
def test_build_url_with_default_endpoints(): """ Absolute URL constructor should form correct URL when the client is relying on the default endpoints. """ api = ScrapydAPI('http://localhost') url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion.json' # Test trailing slash on target. api = ScrapydAPI('http://localhost/') url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion.json'
def test_build_url_with_custom_endpoints(): """ The absolute URL constructor should correctly form URL when the client has custom endpoints passed in. """ custom_endpoints = {ADD_VERSION_ENDPOINT: '/addversion-custom.json'} api = ScrapydAPI('http://localhost', endpoints=custom_endpoints) url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion-custom.json' # Test trailing slash on target. api = ScrapydAPI('http://localhost/', endpoints=custom_endpoints) url = api._build_url(ADD_VERSION_ENDPOINT) assert url == 'http://localhost/addversion-custom.json' # Test that endpoints that were not overridden by the custom_endpoints # still work as the defaults. url = api._build_url(CANCEL_ENDPOINT) assert url == 'http://localhost/cancel.json'