def get_list_of_suffix(bucket_address, prefix, filter_function):
"""Gets the list of suffixes in files in a google storage bucket.
Example: a google storage bucket containing one file
'full-build-linux_20983' will return ['20983'] if prefix is
provided as 'full-build-linux'. Google Storage bucket
containing multiple files will return multiple suffixes.
Args:
bucket_address(String): Bucket URL to examine files from.
prefix(String): The prefix used in creating build file names
filter_function: A function that returns true if the extracted
suffix is in correct format and false otherwise. It allows
only proper suffix to be extracted and returned.
Returns:
(List) list of proper suffixes in the bucket.
"""
file_list = cloud_storage.List(bucket_address)
suffix_list = []
extract_suffix = '.*?%s_(.*?)\.zip' % (prefix)
for file in file_list:
match = re.match(extract_suffix, file)
if match and filter_function(match.groups()[0]):
suffix_list.append(match.groups()[0])
return suffix_list
def testListWithPrefix(self, mock_run_command):
mock_run_command.return_value = '\n'.join([
'gs://bucket/foo/foo-file.txt', 'gs://bucket/foo/foo1/',
'gs://bucket/foo/foo2/'
])
self.assertEqual(cloud_storage.List('bucket', 'foo'),
['/foo/foo-file.txt', '/foo/foo1/', '/foo/foo2/'])
def _GetProtoTraceLinkFromTraceEventsDir(link_prefix):
"""Returns the first proto trace in |link_prefix|/trace/traceEvents/"""
proto_link_prefix = '/'.join([link_prefix, 'trace/traceEvents/**'])
try:
for link in cloud_storage.List(cloud_storage.TELEMETRY_OUTPUT,
proto_link_prefix):
if link.endswith('.pb.gz') or link.endswith('.pb'):
return link[1:] # Strip the initial '/'.
except cloud_storage.NotFoundError, e:
# This directory doesn't exist at all.
raise cloud_storage.NotFoundError('No URLs match the prefix %s: %s' %
(proto_link_prefix, str(e)))
def GetProtoTraceLinkFromTraceEventsDir(link_prefix):
proto_link_prefix = '/'.join([link_prefix, 'trace/traceEvents/**'])
proto_link = None
try:
for link in cloud_storage.List(cloud_storage.TELEMETRY_OUTPUT,
proto_link_prefix):
if link.endswith('.pb.gz') or link.endswith('.pb'):
proto_link = link
break
if proto_link is not None:
return proto_link
raise cloud_storage.NotFoundError(
'Proto trace link not found in cloud storage. Path: %s.' %
proto_link_prefix)
except cloud_storage.NotFoundError, e:
raise cloud_storage.NotFoundError('No URLs match the prefix %s: %s' %
(proto_link_prefix, str(e)))