def testUploadException(self):
self.copy_file_mock.side_effect = calliope_exceptions.BadFileException()
with self.AssertRaisesExceptionMatches(
exceptions.FileUploadError,
r"Failed to upload files ['foo', '/tmp/bar', 'baz.txt'] "
r"to 'gs://foo/bar/'."):
storage_helpers.Upload(['foo', '/tmp/bar', 'baz.txt'], 'gs://foo/bar/')
def testUploadExceptionWithGsutil(self):
properties.VALUES.storage.use_gsutil.Set(True)
self.mock_exec.return_value = 1
with self.AssertRaisesExceptionMatches(
exceptions.FileUploadError,
"Failed to upload files ['foo', '/tmp/bar', 'baz.txt', "
"'gs://foo/bar/'] to 'gs://foo/bar/' using gsutil."):
storage_helpers.Upload(['foo', '/tmp/bar', 'baz.txt'], 'gs://foo/bar/')
def testUploadWithGsutil(self):
properties.VALUES.storage.use_gsutil.Set(True)
expected_command = ['bin/gsutil', 'cp', 'foo', '/tmp/bar',
'baz.txt', 'gs://foo/bar/']
self.mock_exec.return_value = 0
storage_helpers.Upload(['foo', '/tmp/bar', 'baz.txt'], 'gs://foo/bar/')
self.mock_exec.assert_called_once_with(
expected_command, no_exit=True, out_func=mock.ANY, err_func=mock.ANY)
def testUpload(self):
storage_helpers.Upload(
['foo', '/tmp/bar', 'baz.txt'], 'gs://foo/bar/',
storage_client=self.storage_api_client)
self.copy_file_mock.assert_has_calls(
[mock.call('foo', storage_util.ObjectReference(
self.storage_bucket, 'bar/foo')),
mock.call('/tmp/bar', storage_util.ObjectReference(
self.storage_bucket, 'bar/bar')),
mock.call('baz.txt', storage_util.ObjectReference(
self.storage_bucket, 'bar/baz.txt'))])
def ValidateAndStageFiles(self):
"""Validate file URIs and upload them if they are local."""
for file_type, file_or_files in six.iteritems(self.files_by_type):
# TODO(b/36049793): Validate file suffixes.
if not file_or_files:
continue
elif isinstance(file_or_files, six.string_types):
self.files_by_type[file_type] = self._GetStagedFile(file_or_files)
else:
staged_files = [self._GetStagedFile(f) for f in file_or_files]
self.files_by_type[file_type] = staged_files
if self.files_to_stage:
log.info('Staging local files {0} to {1}.'.format(self.files_to_stage,
self._staging_dir))
storage_helpers.Upload(self.files_to_stage, self._staging_dir)
