def __init__(self,
uri,
access_key=None,
secret_key=None,
acl=None,
*,
feed_options=None):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3',
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
'suppress this warning, remove the overwrite '
'option from your FEEDS setting or set it to True.')
def __init__(self, uri):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
verify=self.AWS_VERIFY)
if not uri.startswith("s3://"):
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split('/', 1)
def __init__(
self,
settings,
*,
crawler=None,
aws_access_key_id=None,
aws_secret_access_key=None,
httpdownloadhandler=HTTPDownloadHandler,
**kw,
):
if not is_botocore_available():
raise NotConfigured("missing botocore library")
if not aws_access_key_id:
aws_access_key_id = settings["AWS_ACCESS_KEY_ID"]
if not aws_secret_access_key:
aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"]
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
# unless 'anon' was set explicitly (True/False).
anon = kw.get("anon")
if anon is None and not aws_access_key_id and not aws_secret_access_key:
kw["anon"] = True
self.anon = kw.get("anon")
self._signer = None
import botocore.auth
import botocore.credentials
kw.pop("anon", None)
if kw:
raise TypeError(f"Unexpected keyword arguments: {kw}")
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
self._signer = SignerCls(
botocore.credentials.Credentials(aws_access_key_id,
aws_secret_access_key))
_http_handler = create_instance(
objcls=httpdownloadhandler,
settings=settings,
crawler=crawler,
)
self._download_http = _http_handler.download_request
def skip_if_no_boto():
if not is_botocore_available():
raise SkipTest('missing botocore library')