def test_delete_on_hubstorage_api_does_not_404(self): # NOTE: The current Hubstorage API does not raise 404 errors on deleting resources that do not exist, # Thus the retry policy does not catch 404 errors when retrying deletes (simplify implementation A LOT). # This test checks that this assumption holds. client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=0) project = client.get_project(projectid=self.projectid) # Check frontier delete project.frontier.delete_slot('frontier_non_existing', 'slot_non_existing') # Check metadata delete job = client.push_job(self.projectid, self.spidername) job.metadata[ 'foo'] = 'bar' # Add then delete key, this will trigger an api delete for item foo del job.metadata['foo'] job.metadata.save() # Check collections delete store = project.collections.new_store('foo') store.set({'_key': 'foo'}) store.delete('bar') self.assertTrue( True, "No error have been triggered by calling a delete on resources that do not exist" )
def test_connect_retry(self): c = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2) job = c.push_job(self.projectid, self.spidername, state='running') m = job.metadata self.assertEqual(m.get('state'), u'running', c.auth) m.expire() self.assertEqual(c.session.adapters['http://'].max_retries, 2)
def test_auth(self): # client without global auth set hsc = HubstorageClient(endpoint=self.hsclient.endpoint) self.assertEqual(hsc.auth, None) # check no-auth access try: hsc.push_job(self.projectid, self.spidername) except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') try: hsc.get_project(self.projectid).push_job(self.spidername) except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') try: hsc.get_job((self.projectid, 1, 1)).items.list() except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') try: hsc.get_project(self.projectid).get_job( (self.projectid, 1, 1)).items.list() except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') # create project with auth auth = self.hsclient.auth project = hsc.get_project(self.projectid, auth) self.assertEqual(project.auth, auth) job = project.push_job(self.spidername) samejob = project.get_job(job.key) self.assertEqual(samejob.key, job.key)
def test_auth(self): # client without global auth set hsc = HubstorageClient(endpoint=self.hsclient.endpoint) self.assertEqual(hsc.auth, None) # check no-auth access try: hsc.push_job(self.projectid, self.spidername) except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') try: hsc.get_project(self.projectid).push_job(self.spidername) except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') try: hsc.get_job((self.projectid, 1, 1)).items.list() except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') try: hsc.get_project(self.projectid).get_job((self.projectid, 1, 1)).items.list() except HTTPError as exc: self.assertTrue(exc.response.status_code, 401) else: self.assertTrue(False, '401 not raised') # create project with auth auth = self.hsclient.auth project = hsc.get_project(self.projectid, auth) self.assertEqual(project.auth, auth) job = project.push_job(self.spidername) samejob = project.get_job(job.key) self.assertEqual(samejob.key, job.key)
def test_push_job_does_not_retry(self): # Prepare client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) callback, attempts_count = self.make_request_callback(2, {'key': '1/2/3'}) self.mock_api(POST, callback=callback) # Act job, err = None, None try: job = client.push_job(self.projectid, self.spidername) except HTTPError as e: err = e # Assert self.assertIsNone(job) self.assertIsNotNone(err) self.assertEqual(err.response.status_code, 504) self.assertEqual(attempts_count[0], 1)
def test_push_job_does_not_retry(self): # Prepare client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3) callback, attempts_count = self.make_request_callback( 2, {'key': '1/2/3'}) self.mock_api(POST, callback=callback) # Act job, err = None, None try: job = client.push_job(self.projectid, self.spidername) except HTTPError as e: err = e # Assert self.assertIsNone(job) self.assertIsNotNone(err) self.assertEqual(err.response.status_code, 504) self.assertEqual(attempts_count[0], 1)
def test_delete_on_hubstorage_api_does_not_404(self): # NOTE: The current Hubstorage API does not raise 404 errors on deleting resources that do not exist, # Thus the retry policy does not catch 404 errors when retrying deletes (simplify implementation A LOT). # This test checks that this assumption holds. client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=0) project = client.get_project(projectid=self.projectid) # Check frontier delete project.frontier.delete_slot('frontier_non_existing', 'slot_non_existing') # Check metadata delete job = client.push_job(self.projectid, self.spidername) job.metadata['foo'] = 'bar' # Add then delete key, this will trigger an api delete for item foo del job.metadata['foo'] job.metadata.save() # Check collections delete store = project.collections.new_store('foo') store.set({'_key': 'foo'}) store.delete('bar') self.assertTrue(True, "No error have been triggered by calling a delete on resources that do not exist")