def testGetFilesInDirectoryIfChanged(self):
stubs = system_stub.Override(cloud_storage, ['os'])
stubs.os._directory = {
'dir1': ['1file1.sha1', '1file2.txt', '1file3.sha1'],
'dir2': ['2file.txt'],
'dir3': ['3file1.sha1']
}
stubs.os.path.dirs = ['real_dir_path']
def IncrementFilesUpdated(*_):
IncrementFilesUpdated.files_updated += 1
IncrementFilesUpdated.files_updated = 0
orig_get_if_changed = cloud_storage.GetIfChanged
cloud_storage.GetIfChanged = IncrementFilesUpdated
try:
self.assertRaises(ValueError,
cloud_storage.GetFilesInDirectoryIfChanged,
os.path.abspath(os.sep),
cloud_storage.PUBLIC_BUCKET)
self.assertEqual(0, IncrementFilesUpdated.files_updated)
self.assertRaises(ValueError,
cloud_storage.GetFilesInDirectoryIfChanged,
'fake_dir_path', cloud_storage.PUBLIC_BUCKET)
self.assertEqual(0, IncrementFilesUpdated.files_updated)
cloud_storage.GetFilesInDirectoryIfChanged(
'real_dir_path', cloud_storage.PUBLIC_BUCKET)
self.assertEqual(3, IncrementFilesUpdated.files_updated)
finally:
cloud_storage.GetIfChanged = orig_get_if_changed
stubs.Restore()
def testGetFilesInDirectoryIfChanged(self):
self.CreateFiles([
'real_dir_path/dir1/1file1.sha1', 'real_dir_path/dir1/1file2.txt',
'real_dir_path/dir1/1file3.sha1', 'real_dir_path/dir2/2file.txt',
'real_dir_path/dir3/3file1.sha1'
])
def IncrementFilesUpdated(*_):
IncrementFilesUpdated.files_updated += 1
IncrementFilesUpdated.files_updated = 0
orig_get_if_changed = cloud_storage.GetIfChanged
cloud_storage.GetIfChanged = IncrementFilesUpdated
try:
self.assertRaises(ValueError,
cloud_storage.GetFilesInDirectoryIfChanged,
os.path.abspath(os.sep),
cloud_storage.PUBLIC_BUCKET)
self.assertEqual(0, IncrementFilesUpdated.files_updated)
self.assertRaises(ValueError,
cloud_storage.GetFilesInDirectoryIfChanged,
'fake_dir_path', cloud_storage.PUBLIC_BUCKET)
self.assertEqual(0, IncrementFilesUpdated.files_updated)
cloud_storage.GetFilesInDirectoryIfChanged(
'real_dir_path', cloud_storage.PUBLIC_BUCKET)
self.assertEqual(3, IncrementFilesUpdated.files_updated)
finally:
cloud_storage.GetIfChanged = orig_get_if_changed
def _FetchDependenciesIfNeeded(story_set):
""" Download files needed by a user story set. """
# Download files in serving_dirs.
serving_dirs = story_set.serving_dirs
for directory in serving_dirs:
cloud_storage.GetFilesInDirectoryIfChanged(directory, story_set.bucket)
# Download WPR files.
if any(not story.is_local for story in story_set):
story_set.wpr_archive_info.DownloadArchivesIfNeeded()
def testDisableCloudStorageIo(self, unused_lock_mock):
os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1'
dir_path = 'real_dir_path'
self.fs.CreateDirectory(dir_path)
file_path = os.path.join(dir_path, 'file1')
file_path_sha = file_path + '.sha1'
self.CreateFiles([file_path, file_path_sha])
with open(file_path_sha, 'w') as f:
f.write('hash1234')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1',
'remote_path2')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Get('bucket', 'foo', file_path)
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetIfChanged(file_path, 'foo')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetIfHashChanged('bar', file_path, 'bucket',
'hash1234')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Insert('bucket', 'foo', file_path)
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket')
def Run(test, story_set, finder_options, results, max_failures=None):
"""Runs a given test against a given page_set with the given options.
Stop execution for unexpected exceptions such as KeyboardInterrupt.
We "white list" certain exceptions for which the story runner
can continue running the remaining stories.
"""
# Filter page set based on options.
stories = filter(story_module.StoryFilter.IsSelected, story_set)
if (not finder_options.use_live_sites and story_set.bucket and
finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD):
serving_dirs = story_set.serving_dirs
for directory in serving_dirs:
cloud_storage.GetFilesInDirectoryIfChanged(directory,
story_set.bucket)
if not _UpdateAndCheckArchives(
story_set.archive_data_file, story_set.wpr_archive_info,
stories):
return
if not stories:
return
# Effective max failures gives priority to command-line flag value.
effective_max_failures = finder_options.max_failures
if effective_max_failures is None:
effective_max_failures = max_failures
story_groups = StoriesGroupedByStateClass(
stories,
story_set.allow_mixed_story_states)
for group in story_groups:
state = None
try:
for _ in xrange(finder_options.pageset_repeat):
for story in group.stories:
for _ in xrange(finder_options.page_repeat):
if not state:
state = group.shared_state_class(
test, finder_options, story_set)
results.WillRunPage(story)
try:
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunStoryAndProcessErrorIfNeeded(story, results, state, test)
except exceptions.Error:
# Catch all Telemetry errors to give the story a chance to retry.
# The retry is enabled by tearing down the state and creating
# a new state instance in the next iteration.
try:
# If TearDownState raises, do not catch the exception.
# (The Error was saved as a failure value.)
state.TearDownState()
finally:
# Later finally-blocks use state, so ensure it is cleared.
state = None
finally:
has_existing_exception = sys.exc_info() != (None, None, None)
try:
if state:
_CheckThermalThrottling(state.platform)
results.DidRunPage(story)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from result processing:')
if (effective_max_failures is not None and
len(results.failures) > effective_max_failures):
logging.error('Too many failures. Aborting.')
return
finally:
if state:
has_existing_exception = sys.exc_info() != (None, None, None)
try:
state.TearDownState()
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from TearDownState:')