def testListDirs(self, mock_run_command):
mock_run_command.return_value = '\n'.join([
'gs://bucket/foo-file.txt', '', 'gs://bucket/foo1/',
'gs://bucket/foo2/', 'gs://bucket/foo1/file.txt'
])
self.assertEqual(cloud_storage.ListDirs('bucket', 'foo*'),
['/foo1/', '/foo2/'])
def _FindClosestChromiumSnapshot(base_position, build_dir):
"""Returns the closest chromium snapshot available in cloud storage.
Chromium snapshots are pulled from _CHROMIUM_BUILD_DIR in CHROMIUM_GS_BUCKET.
Continuous chromium snapshots do not always contain the exact release build.
This function queries the storage bucket and find the closest snapshot within
+/-_CHROMIUM_SNAPSHOT_SEARCH_WINDOW to find the closest build.
"""
min_position = base_position - _CHROMIUM_SNAPSHOT_SEARCH_WINDOW
max_position = base_position + _CHROMIUM_SNAPSHOT_SEARCH_WINDOW
# Getting the full list of objects in cloud storage bucket is prohibitively
# slow. It's faster to list objects with a prefix. Assuming we're looking at
# +/- 10 commit positions, for commit position 123456, we want to look at
# positions between 123446 an 123466. We do this by getting all snapshots
# with prefix 12344*, 12345*, and 12346*. This may get a few more snapshots
# that we intended, but that's fine since we take the min distance anyways.
min_position_prefix = min_position / 10
max_position_prefix = max_position / 10
available_positions = []
for position_prefix in range(min_position_prefix, max_position_prefix + 1):
query = '%s/%d*' % (build_dir, position_prefix)
try:
ls_results = cloud_storage.ListDirs(_CHROMIUM_GS_BUCKET, query)
except cloud_storage.NotFoundError:
# It's fine if there is no chromium snapshot available for one prefix.
# We will look at the rest of the prefixes.
continue
for entry in ls_results:
# entry looks like '/Linux_x64/${commit_position}/'.
position = int(entry.split('/')[2])
available_positions.append(position)
if len(available_positions) == 0:
raise ValueError(
'No chromium build found +/-%d commit positions of %d' %
(_CHROMIUM_SNAPSHOT_SEARCH_WINDOW, base_position))
distance_function = lambda position: abs(position - base_position)
min_distance_snapshot = min(available_positions, key=distance_function)
return min_distance_snapshot