def test_Exists(self):
stored_keys = set()
def call(cmd):
self.assertTrue(len(cmd) >= 3)
self.assertTrue(cmd[1] in ['cp', 'ls', 'rm'])
if cmd[1] == 'cp':
# Add the key into stored_keys
copy_key = cmd[-1]
stored_keys.add(copy_key)
return 0
elif cmd[1] == 'ls':
query_key = cmd[-1]
if query_key in stored_keys:
return 0
else:
return 1
elif cmd[1] == 'rm':
return 0
write_storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=[],
call=call)
read_storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='',
read_buckets=['mybucket'],
call=call)
self.assertNotEquals(None, write_storage.PutData('data', 'foo_key'))
self.assertTrue(write_storage.Exists('foo_key'))
self.assertFalse(write_storage.Exists('bad_key'))
self.assertTrue(read_storage.Exists('foo_key'))
self.assertFalse(read_storage.Exists('bad_key'))
def test_PutFile(self):
path = 'my/path'
step = [0]
def call(cmd):
if step[0] == 0:
self.assertEqual(['mygsutil', 'cp', '-a', 'public-read'],
cmd[0:4])
self.assertEqual('gs://mybucket/bar', cmd[5].split('.')[0])
elif step[0] == 1:
self.assertEqual(['mygsutil', 'cp', '-a', 'public-read'],
cmd[0:4])
self.assertEqual('gs://mybucket/bar', cmd[4].split('.')[0])
self.assertEqual('gs://mybucket/bar', cmd[5])
elif step[0] == 2:
self.assertEqual(['mygsutil', 'rm'], cmd[0:2])
self.assertEqual('gs://mybucket/bar', cmd[2].split('.')[0])
else:
self.assertTrue(False)
step[0] += 1
return 0
storage = gsd_storage.GSDStorage(write_bucket='mybucket',
read_buckets=[],
gsutil=['mygsutil'],
call=call)
url = storage.PutFile(path, 'bar')
self.assertEquals('https://storage.googleapis.com/mybucket/bar', url)
def InstallMinGWHostCompiler():
"""Install the MinGW host compiler used to build the host tools on Windows.
We could use an ordinary source rule for this, but that would require hashing
hundreds of MB of toolchain files on every build. Instead, check for the
presence of the specially-named file <version>.installed in the install
directory. If it is absent, check for the presence of the zip file
<version>.zip. If it is absent, attempt to download it from Google Storage.
Then extract the zip file and create the install file.
"""
if not os.path.isfile(os.path.join(MINGW_PATH, MINGW_VERSION + '.installed')):
downloader = gsd_storage.GSDStorage([], ['nativeclient-mingw'])
zipfilename = MINGW_VERSION + '.zip'
zipfilepath = os.path.join(NACL_DIR, zipfilename)
# If the zip file is not present, try to download it from Google Storage.
# If that fails, bail out.
if (not os.path.isfile(zipfilepath) and
not downloader.GetSecureFile(zipfilename, zipfilepath)):
print >>sys.stderr, 'Failed to install MinGW tools:'
print >>sys.stderr, 'could not find or download', zipfilename
sys.exit(1)
logging.info('Extracting %s' % zipfilename)
zf = zipfile.ZipFile(zipfilepath)
if os.path.exists(MINGW_PATH):
shutil.rmtree(MINGW_PATH)
zf.extractall(NACL_DIR)
with open(os.path.join(MINGW_PATH, MINGW_VERSION + '.installed'), 'w') as _:
pass
os.environ['MINGW'] = MINGW_PATH
def test_PutData(self):
# Check that command line is as expected.
# Special handling around the destination file,
# as it's a temporary name and unknown to us.
step = [0]
def call(cmd):
if step[0] == 0:
self.assertEqual(['mygsutil', 'cp', '-a', 'public-read'],
cmd[0:4])
self.assertEqual('foo',
file_tools.ReadFile(cmd[4][len('file://'):]))
self.assertEqual('gs://mybucket/bar', cmd[5].split('.')[0])
elif step[0] == 1:
self.assertEqual(['mygsutil', 'cp', '-a', 'public-read'],
cmd[0:4])
self.assertEqual('gs://mybucket/bar', cmd[4].split('.')[0])
self.assertEqual('gs://mybucket/bar', cmd[5])
elif step[0] == 2:
self.assertEqual(['mygsutil', 'rm'], cmd[0:2])
self.assertEqual('gs://mybucket/bar', cmd[2].split('.')[0])
else:
self.assertTrue(False)
step[0] += 1
return 0
storage = gsd_storage.GSDStorage(write_bucket='mybucket',
read_buckets=[],
gsutil=['mygsutil'],
call=call)
url = storage.PutData('foo', 'bar')
self.assertEquals('https://storage.googleapis.com/mybucket/bar', url)
def test_PutNoBucket(self):
# Check that we raise when writing an no bucket is provided.
storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket=None,
read_buckets=[])
self.assertRaises(gsd_storage.GSDStorageError, storage.PutFile, 'foo',
'bar')
self.assertRaises(gsd_storage.GSDStorageError, storage.PutData, 'foo',
'bar')
def test_GetFails(self):
def download(url, target):
raise Exception('fail download %s to %s' % (url, target))
# Make download raise and confirm this gets intercepted.
storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=['mybucket'],
download=download)
self.assertFalse(storage.GetFile('foo', 'bar'))
self.assertEquals(None, storage.GetData('foo'))
def test_GetData(self):
def download(url, target):
self.assertEqual('https://storage.googleapis.com/mybucket/bar',
url)
file_tools.WriteFile('baz', target)
# Mock out download and confirm we download the expected URL.
storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=['mybucket'],
download=download)
self.assertEqual('baz', storage.GetData('bar'))
def CreateStorage(self):
"""Create a storage object for this build.
Returns:
A storage object (GSDStorage).
"""
if self._options.buildbot:
return gsd_storage.GSDStorage(write_bucket='nativeclient-once',
read_buckets=['nativeclient-once'])
elif self._options.trybot:
return gsd_storage.GSDStorage(
write_bucket='nativeclient-once-try',
read_buckets=['nativeclient-once', 'nativeclient-once-try'])
else:
read_buckets = []
if self._options.use_remote_cache:
read_buckets += ['nativeclient-once']
return local_storage_cache.LocalStorageCache(
cache_path=self._options.cache,
storage=gsd_storage.GSDStorage(write_bucket=None,
read_buckets=read_buckets))
def test_PutFails(self):
def call(cmd):
return 1
# Mock out running gsutil, have it fail, and check that it does.
storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=[],
call=call)
self.assertRaises(gsd_storage.GSDStorageError, storage.PutFile, 'foo',
'bar')
self.assertRaises(gsd_storage.GSDStorageError, storage.PutData, 'foo',
'bar')
def test_GetFallback(self):
# Fail reading from the first bucket, and check fallback to the second.
def download(url, target):
if 'badbucket' in url:
raise Exception('fail download %s to %s' % (url, target))
else:
file_tools.WriteFile('bar', target)
storage = gsd_storage.GSDStorage(
gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=['badbucket', 'goodbucket'],
download=download)
self.assertEquals('bar', storage.GetData('foo'))
def test_GetFile(self):
path = 'my/path'
def download(url, target):
self.assertEqual(path, target)
self.assertEqual('https://storage.googleapis.com/mybucket/bar',
url)
# Mock out download and confirm we download the expected URL.
storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=['mybucket'],
download=download)
storage.GetFile('bar', path)
def test_BadWrite(self):
def call(cmd):
return 1
storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=[],
call=call)
dir_storage = directory_storage.DirectoryStorageAdapter(storage)
# Check that storage exceptions come thru on failure.
with working_directory.TemporaryWorkingDirectory() as work_dir:
temp1 = os.path.join(work_dir, 'temp1')
hashing_tools_test.GenerateTestTree('bad_write', temp1)
self.assertRaises(gsd_storage.GSDStorageError,
dir_storage.PutDirectory, temp1, 'bad')
def test_PutFile(self):
# As we control all the paths, check the full command line for PutFile.
path = 'my/path'
def call(cmd):
self.assertEqual([
'mygsutil', 'cp', '-a', 'public-read', 'file://' +
os.path.abspath(path).replace(os.sep, '/'), 'gs://mybucket/bar'
], cmd)
return 0
storage = gsd_storage.GSDStorage(write_bucket='mybucket',
read_buckets=[],
gsutil=['mygsutil'],
call=call)
url = storage.PutFile(path, 'bar')
self.assertEquals('https://storage.googleapis.com/mybucket/bar', url)
def test_FailsWhenWritingFails(self):
# Check that once doesn't eat the storage layer failures for writes.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('FailsWhenWritingFails', work_dir)
def call(cmd, **kwargs):
# Cause gsutil commands to fail.
return 1
bad_storage = gsd_storage.GSDStorage(gsutil=['mygsutil'],
write_bucket='mybucket',
read_buckets=[],
call=call)
o = once.Once(storage=bad_storage, system_summary='test')
self.assertRaises(
gsd_storage.GSDStorageError, o.Run, 'test', self._input_dirs,
self._output_dirs[0],
[command.Copy('%(input0)s/in0', '%(output)s/out')])
def test_PutData(self):
# Check that command line is as expected.
# Special handling around the destination file,
# as its a temporary name and unknown to us.
def call(cmd):
self.assertEqual(['mygsutil', 'cp', '-a', 'public-read'], cmd[0:4])
self.assertEqual('foo',
file_tools.ReadFile(cmd[4][len('file://'):]))
self.assertEqual('gs://mybucket/bar', cmd[5])
return 0
storage = gsd_storage.GSDStorage(write_bucket='mybucket',
read_buckets=[],
gsutil=['mygsutil'],
call=call)
url = storage.PutData('foo', 'bar')
self.assertEquals(
'https://commondatastorage.googleapis.com/mybucket/bar', url)
