def setUp(self): skip_if_no_boto() s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request self.spider = Spider('foo')
def test_stat(self): skip_if_no_boto() bucket = 'mybucket' key = 'export.csv' uri = f's3://{bucket}/{key}' checksum = '3187896a9657a28163abb31667df64c8' last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) from botocore.stub import Stubber with Stubber(store.s3_client) as stub: stub.add_response( 'head_object', expected_params={ 'Bucket': bucket, 'Key': key, }, service_response={ 'ETag': f'"{checksum}"', 'LastModified': last_modified, }, ) file_stats = yield store.stat_file('', info=None) self.assertEqual( file_stats, { 'checksum': checksum, 'last_modified': last_modified.timestamp(), }, ) stub.assert_no_pending_responses()
def setUp(self): skip_if_no_boto() self.s3reqh = S3DownloadHandler(Settings(), httpdownloadhandler=HttpDownloadHandlerMock, #anon=True, # is implicit ) self.download_request = self.s3reqh.download_request self.spider = Spider('foo')
def setUp(self): skip_if_no_boto() crawler = get_crawler() self.s3reqh = create_instance( objcls=S3DownloadHandler, settings=None, crawler=crawler, httpdownloadhandler=HttpDownloadHandlerMock, # anon=True, # implicit ) self.download_request = self.s3reqh.download_request self.spider = Spider('foo')
def setUp(self): skip_if_no_boto() crawler = get_crawler() s3reqh = create_instance( objcls=S3DownloadHandler, settings=None, crawler=crawler, aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock, ) self.download_request = s3reqh.download_request self.spider = Spider('foo')
def test_persist(self): skip_if_no_boto() bucket = 'mybucket' key = 'export.csv' uri = f's3://{bucket}/{key}' buffer = mock.MagicMock() meta = {'foo': 'bar'} path = '' content_type = 'image/png' store = S3FilesStore(uri) from botocore.stub import Stubber with Stubber(store.s3_client) as stub: stub.add_response( 'put_object', expected_params={ 'ACL': S3FilesStore.POLICY, 'Body': buffer, 'Bucket': bucket, 'CacheControl': S3FilesStore.HEADERS['Cache-Control'], 'ContentType': content_type, 'Key': key, 'Metadata': meta, }, service_response={}, ) yield store.persist_file( path, buffer, info=None, meta=meta, headers={'Content-Type': content_type}, ) stub.assert_no_pending_responses() self.assertEqual( buffer.method_calls, [ mock.call.seek(0), # The call to read does not happen with Stubber ] )