def setUp(self):
skip_if_no_boto()
s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock)
self.download_request = s3reqh.download_request
self.spider = Spider('foo')
def setUp(self):
skip_if_no_boto()
s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock)
self.download_request = s3reqh.download_request
self.spider = Spider('foo')
def test_stat(self):
skip_if_no_boto()
bucket = 'mybucket'
key = 'export.csv'
uri = f's3://{bucket}/{key}'
checksum = '3187896a9657a28163abb31667df64c8'
last_modified = datetime(2019, 12, 1)
store = S3FilesStore(uri)
from botocore.stub import Stubber
with Stubber(store.s3_client) as stub:
stub.add_response(
'head_object',
expected_params={
'Bucket': bucket,
'Key': key,
},
service_response={
'ETag': f'"{checksum}"',
'LastModified': last_modified,
},
)
file_stats = yield store.stat_file('', info=None)
self.assertEqual(
file_stats,
{
'checksum': checksum,
'last_modified': last_modified.timestamp(),
},
)
stub.assert_no_pending_responses()
def setUp(self):
skip_if_no_boto()
self.s3reqh = S3DownloadHandler(Settings(),
httpdownloadhandler=HttpDownloadHandlerMock,
#anon=True, # is implicit
)
self.download_request = self.s3reqh.download_request
self.spider = Spider('foo')
def setUp(self):
skip_if_no_boto()
self.s3reqh = S3DownloadHandler(Settings(),
httpdownloadhandler=HttpDownloadHandlerMock,
#anon=True, # is implicit
)
self.download_request = self.s3reqh.download_request
self.spider = Spider('foo')
def setUp(self):
skip_if_no_boto()
crawler = get_crawler()
self.s3reqh = create_instance(
objcls=S3DownloadHandler,
settings=None,
crawler=crawler,
httpdownloadhandler=HttpDownloadHandlerMock,
# anon=True, # implicit
)
self.download_request = self.s3reqh.download_request
self.spider = Spider('foo')
def setUp(self):
skip_if_no_boto()
crawler = get_crawler()
s3reqh = create_instance(
objcls=S3DownloadHandler,
settings=None,
crawler=crawler,
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock,
)
self.download_request = s3reqh.download_request
self.spider = Spider('foo')
def test_persist(self):
skip_if_no_boto()
bucket = 'mybucket'
key = 'export.csv'
uri = f's3://{bucket}/{key}'
buffer = mock.MagicMock()
meta = {'foo': 'bar'}
path = ''
content_type = 'image/png'
store = S3FilesStore(uri)
from botocore.stub import Stubber
with Stubber(store.s3_client) as stub:
stub.add_response(
'put_object',
expected_params={
'ACL': S3FilesStore.POLICY,
'Body': buffer,
'Bucket': bucket,
'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
'ContentType': content_type,
'Key': key,
'Metadata': meta,
},
service_response={},
)
yield store.persist_file(
path,
buffer,
info=None,
meta=meta,
headers={'Content-Type': content_type},
)
stub.assert_no_pending_responses()
self.assertEqual(
buffer.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
]
)
