def testDisableCloudStorageIo(self, unused_lock_mock):
os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1'
dir_path = 'real_dir_path'
self.fs.CreateDirectory(dir_path)
file_path = os.path.join(dir_path, 'file1')
file_path_sha = file_path + '.sha1'
def CleanTimeStampFile():
os.remove(file_path + '.fetchts')
self.CreateFiles([file_path, file_path_sha])
with open(file_path_sha, 'w') as f:
f.write('hash1234')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1',
'remote_path2')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Get('bucket', 'foo', file_path)
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetIfChanged(file_path, 'foo')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetIfHashChanged('bar', file_path, 'bucket',
'hash1234')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Insert('bucket', 'foo', file_path)
CleanTimeStampFile()
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket')
def testSwarmingUsesExistingEnv(self, mock_popen):
os.environ['SWARMING_HEADLESS'] = '1'
mock_gsutil = mock_popen()
mock_gsutil.communicate = mock.MagicMock(return_value=('a', 'b'))
mock_gsutil.returncode = None
cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1',
'remote_path2')
mock_popen.assert_called_with(mock.ANY,
stderr=-1,
env=os.environ,
stdout=-1)
def testCopy(self):
orig_run_command = cloud_storage._RunCommand
def AssertCorrectRunCommandArgs(args):
self.assertEqual(expected_args, args)
cloud_storage._RunCommand = AssertCorrectRunCommandArgs
expected_args = [
'cp', 'gs://bucket1/remote_path1', 'gs://bucket2/remote_path2'
]
try:
cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1',
'remote_path2')
finally:
cloud_storage._RunCommand = orig_run_command
def Upload(self, force=False):
"""Upload all pending files and then write the updated config to disk.
Will attempt to copy files existing in the upload location to a backup
location in the same bucket in cloud storage if |force| is True.
Args:
force: True if files should be uploaded to cloud storage even if a
file already exists in the upload location.
Raises:
CloudStorageUploadConflictError: If |force| is False and the potential
upload location of a file already exists.
CloudStorageError: If copying an existing file to the backup location
or uploading the new file fails.
"""
if cloud_storage.Exists(self._cs_bucket, self._cs_remote_path):
if not force:
#pylint: disable=nonstandard-exception
raise exceptions.CloudStorageUploadConflictError(
self._cs_bucket, self._cs_remote_path)
#pylint: enable=nonstandard-exception
logging.debug(
'A file already exists at upload path %s in self.cs_bucket'
' %s', self._cs_remote_path, self._cs_bucket)
try:
cloud_storage.Copy(self._cs_bucket, self._cs_bucket,
self._cs_remote_path, self._cs_backup_path)
self._backed_up = True
except cloud_storage.CloudStorageError:
logging.error(
'Failed to copy existing file %s in cloud storage bucket '
'%s to backup location %s', self._cs_remote_path,
self._cs_bucket, self._cs_backup_path)
raise
try:
cloud_storage.Insert(self._cs_bucket, self._cs_remote_path,
self._local_path)
except cloud_storage.CloudStorageError:
logging.error(
'Failed to upload %s to %s in cloud_storage bucket %s',
self._local_path, self._cs_remote_path, self._cs_bucket)
raise
self._updated = True
def Rollback(self):
"""Attempt to undo the previous call to Upload.
Does nothing if no previous call to Upload was made, or if nothing was
successfully changed.
Returns:
True iff changes were successfully rolled back.
Raises:
CloudStorageError: If copying the backed up file to its original
location or removing the uploaded file fails.
"""
cloud_storage_changed = False
if self._backed_up:
cloud_storage.Copy(self._cs_bucket, self._cs_bucket, self._cs_backup_path,
self._cs_remote_path)
cloud_storage_changed = True
self._cs_backup_path = None
elif self._updated:
cloud_storage.Delete(self._cs_bucket, self._cs_remote_path)
cloud_storage_changed = True
self._updated = False
return cloud_storage_changed
