def mkdir(self, dest): """Make a directory. This doesn't actually create directories on S3 (because there is no such thing), but it will create the corresponding bucket if it doesn't exist. """ bucket_name, key_name = parse_s3_uri(dest) client = self.make_s3_client() try: client.head_bucket(Bucket=bucket_name) except botocore.exceptions.ClientError as ex: if _client_error_status(ex) != 404: raise self.create_bucket(bucket_name)
def touchz(self, dest): """Make an empty file in the given location. Raises an error if a non-empty file already exists in that location.""" key = self._get_s3_key(dest) data = None try: data = key.get() except botocore.exceptions.ClientError as ex: # okay if key doesn't exist if _client_error_status(ex) != 404: raise if data and data['ContentLength'] != 0: raise OSError('Non-empty file %r already exists!' % (dest, )) key.put(Body=b'')
def _ls(self, path_glob): """Helper method for :py:meth:`ls`; yields tuples of ``(uri, key)`` where *key* is the corresponding boto3 s3.ObjectSummary. """ # clean up the base uri to ensure we have pass boto3 an s3:// URI # (not s3n://) scheme = urlparse(path_glob).scheme # support globs glob_match = GLOB_RE.match(path_glob) # we're going to search for all keys starting with base_uri if glob_match: # cut it off at first wildcard base_uri = glob_match.group(1) else: base_uri = path_glob bucket_name, base_name = parse_s3_uri(base_uri) # allow subdirectories of the path/glob if path_glob and not path_glob.endswith('/'): dir_glob = path_glob + '/*' else: dir_glob = path_glob + '*' try: bucket = self.get_bucket(bucket_name) except botocore.exceptions.ClientError as ex: if _client_error_status(ex) == 404: # treat nonexistent as empty return raise for key in bucket.objects.filter(Prefix=base_name): uri = "%s://%s/%s" % (scheme, bucket_name, key.key) # enforce globbing if not (fnmatch.fnmatchcase(uri, path_glob) or fnmatch.fnmatchcase(uri, dir_glob)): continue yield uri, key
def get_bucket(self, bucket_name): """Get the (:py:mod:`boto3`) bucket, connecting through the appropriate endpoint.""" client = self.make_s3_client() try: region_name = _get_bucket_region(client, bucket_name) except botocore.exceptions.ClientError as ex: # it's possible to have access to a bucket but not access # to its location metadata. This happens on the 'elasticmapreduce' # bucket, for example (see #1170) if _client_error_status(ex) != 403: raise log.warning( 'Could not infer endpoint for bucket %s; ' 'assuming defaults', bucket_name) region_name = None resource = self.make_s3_resource(region_name) return resource.Bucket(bucket_name)