def testProcessPathPrefix(self):
self.assertEqual(('/bucket', None),
common._process_path_prefix('/bucket'))
self.assertEqual(('/bucket', None),
common._process_path_prefix('/bucket/'))
self.assertEqual(('/bucket', 'prefix/'),
common._process_path_prefix('/bucket/prefix/'))
def listbucket(path_prefix,
marker=None,
prefix=None,
max_keys=None,
delimiter=None,
retry_params=None,
_account_id=None):
"""Returns a GCSFileStat iterator over a bucket.
Optional arguments can limit the result to a subset of files under bucket.
This function has two modes:
1. List bucket mode: Lists all files in the bucket without any concept of
hierarchy. GCS doesn't have real directory hierarchies.
2. Directory emulation mode: If you specify the 'delimiter' argument,
it is used as a path separator to emulate a hierarchy of directories.
In this mode, the "path_prefix" argument should end in the delimiter
specified (thus designates a logical directory). The logical directory's
contents, both files and subdirectories, are listed. The names of
subdirectories returned will end with the delimiter. So listbucket
can be called with the subdirectory name to list the subdirectory's
contents.
Args:
path_prefix: A Google Cloud Storage path of format "/bucket" or
"/bucket/prefix". Only objects whose fullpath starts with the
path_prefix will be returned.
marker: Another path prefix. Only objects whose fullpath starts
lexicographically after marker will be returned (exclusive).
prefix: Deprecated. Use path_prefix.
max_keys: The limit on the number of objects to return. int.
For best performance, specify max_keys only if you know how many objects
you want. Otherwise, this method requests large batches and handles
pagination for you.
delimiter: Use to turn on directory mode. str of one or multiple chars
that your bucket uses as its directory separator.
retry_params: An api_utils.RetryParams for this call to GCS. If None,
the default one is used.
_account_id: Internal-use only.
Examples:
For files "/bucket/a",
"/bucket/bar/1"
"/bucket/foo",
"/bucket/foo/1", "/bucket/foo/2/1", "/bucket/foo/3/1",
Regular mode:
listbucket("/bucket/f", marker="/bucket/foo/1")
will match "/bucket/foo/2/1", "/bucket/foo/3/1".
Directory mode:
listbucket("/bucket/", delimiter="/")
will match "/bucket/a, "/bucket/bar/" "/bucket/foo", "/bucket/foo/".
listbucket("/bucket/foo/", delimiter="/")
will match "/bucket/foo/1", "/bucket/foo/2/", "/bucket/foo/3/"
Returns:
Regular mode:
A GCSFileStat iterator over matched files ordered by filename.
The iterator returns GCSFileStat objects. filename, etag, st_size,
st_ctime, and is_dir are set.
Directory emulation mode:
A GCSFileStat iterator over matched files and directories ordered by
name. The iterator returns GCSFileStat objects. For directories,
only the filename and is_dir fields are set.
The last name yielded can be used as next call's marker.
"""
if prefix:
common.validate_bucket_path(path_prefix)
bucket = path_prefix
else:
bucket, prefix = common._process_path_prefix(path_prefix)
if marker and marker.startswith(bucket):
marker = marker[len(bucket) + 1:]
api = storage_api._get_storage_api(retry_params=retry_params,
account_id=_account_id)
options = {}
if marker:
options['marker'] = marker
if max_keys:
options['max-keys'] = max_keys
if prefix:
options['prefix'] = prefix
if delimiter:
options['delimiter'] = delimiter
return _Bucket(api, bucket, options)
def listbucket(path_prefix, marker=None, prefix=None, max_keys=None,
delimiter=None, retry_params=None, _account_id=None):
"""Returns a GCSFileStat iterator over a bucket.
Optional arguments can limit the result to a subset of files under bucket.
This function has two modes:
1. List bucket mode: Lists all files in the bucket without any concept of
hierarchy. GCS doesn't have real directory hierarchies.
2. Directory emulation mode: If you specify the 'delimiter' argument,
it is used as a path separator to emulate a hierarchy of directories.
In this mode, the "path_prefix" argument should end in the delimiter
specified (thus designates a logical directory). The logical directory's
contents, both files and subdirectories, are listed. The names of
subdirectories returned will end with the delimiter. So listbucket
can be called with the subdirectory name to list the subdirectory's
contents.
Args:
path_prefix: A Google Cloud Storage path of format "/bucket" or
"/bucket/prefix". Only objects whose fullpath starts with the
path_prefix will be returned.
marker: Another path prefix. Only objects whose fullpath starts
lexicographically after marker will be returned (exclusive).
prefix: Deprecated. Use path_prefix.
max_keys: The limit on the number of objects to return. int.
For best performance, specify max_keys only if you know how many objects
you want. Otherwise, this method requests large batches and handles
pagination for you.
delimiter: Use to turn on directory mode. str of one or multiple chars
that your bucket uses as its directory separator.
retry_params: An api_utils.RetryParams for this call to GCS. If None,
the default one is used.
_account_id: Internal-use only.
Examples:
For files "/bucket/a",
"/bucket/bar/1"
"/bucket/foo",
"/bucket/foo/1", "/bucket/foo/2/1", "/bucket/foo/3/1",
Regular mode:
listbucket("/bucket/f", marker="/bucket/foo/1")
will match "/bucket/foo/2/1", "/bucket/foo/3/1".
Directory mode:
listbucket("/bucket/", delimiter="/")
will match "/bucket/a, "/bucket/bar/" "/bucket/foo", "/bucket/foo/".
listbucket("/bucket/foo/", delimiter="/")
will match "/bucket/foo/1", "/bucket/foo/2/", "/bucket/foo/3/"
Returns:
Regular mode:
A GCSFileStat iterator over matched files ordered by filename.
The iterator returns GCSFileStat objects. filename, etag, st_size,
st_ctime, and is_dir are set.
Directory emulation mode:
A GCSFileStat iterator over matched files and directories ordered by
name. The iterator returns GCSFileStat objects. For directories,
only the filename and is_dir fields are set.
The last name yielded can be used as next call's marker.
"""
if prefix:
common.validate_bucket_path(path_prefix)
bucket = path_prefix
else:
bucket, prefix = common._process_path_prefix(path_prefix)
if marker and marker.startswith(bucket):
marker = marker[len(bucket) + 1:]
api = storage_api._get_storage_api(retry_params=retry_params,
account_id=_account_id)
options = {}
if marker:
options['marker'] = marker
if max_keys:
options['max-keys'] = max_keys
if prefix:
options['prefix'] = prefix
if delimiter:
options['delimiter'] = delimiter
return _Bucket(api, bucket, options)
def testProcessPathPrefix(self):
self.assertEqual(('/bucket', None), common._process_path_prefix('/bucket'))
self.assertEqual(('/bucket', None), common._process_path_prefix('/bucket/'))
self.assertEqual(('/bucket', 'prefix/'),
common._process_path_prefix('/bucket/prefix/'))
def testProcessPathPrefix(self):
self.assertEqual(("/bucket", None), common._process_path_prefix("/bucket"))
self.assertEqual(("/bucket", None), common._process_path_prefix("/bucket/"))
self.assertEqual(("/bucket", "prefix/"), common._process_path_prefix("/bucket/prefix/"))
