def test_File(self):
# Check that one file works.
with working_directory.TemporaryWorkingDirectory() as work_dir:
filename1 = os.path.join(work_dir, 'myfile1')
filename2 = os.path.join(work_dir, 'myfile2')
file_tools.WriteFile('booga', filename1)
file_tools.WriteFile('booga', filename2)
h1 = hashing_tools.StableHashPath(filename1)
h2 = hashing_tools.StableHashPath(filename2)
self.assertEqual(h1, h2)
self.assertEqual(40, len(h1))
def GenerateTestTree(noise, path):
"""Generate an interesting tree for testing.
Args:
noise: A string to inject to vary tree content.
path: Location to emit tree (should not exist).
"""
os.mkdir(path)
b = os.path.join(path, 'b' + noise)
os.mkdir(b)
b1 = os.path.join(path, 'b1' + noise)
os.mkdir(b1)
b2 = os.path.join(path, 'b2' + noise)
file_tools.WriteFile('inside b2' + noise, b2)
c = os.path.join(b, 'c' + noise)
file_tools.WriteFile('inside c' + noise, c)
def GetFile(self, key, path):
data = self.GetData(key)
if data is not None:
file_tools.WriteFile(data, path)
# Use form: fake://<key> for make-believe URLs.
return 'fake://' + key
else:
return None
def test_WriteFileRead(self):
# Check that things written with PutFile can be read back.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
bar = os.path.join(work_dir, 'bar_file')
file_tools.WriteFile('bar', bar)
storage.PutFile(bar, 'foo')
self.CanBeReadBothWays(storage, 'foo',
os.path.join(work_dir, 'out'), 'bar')
def test_WriteOnlyToLocal(self):
# Check that things written hit local storage, not the network.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
storage.PutData('bar', 'foo')
self.assertEquals(None, mem_storage.GetData('foo'))
bar = os.path.join(work_dir, 'bar_file')
file_tools.WriteFile('bar', bar)
storage.PutFile(bar, 'foo')
self.assertEquals(None, mem_storage.GetData('foo'))
def test_InvalidKey(self):
# Check that an invalid key asserts.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
bar = os.path.join(work_dir, 'bar_file')
file_tools.WriteFile('bar', bar)
self.assertRaises(KeyError, storage.PutData, 'bar', 'foo$')
self.assertRaises(KeyError, storage.GetData, 'foo^')
self.assertRaises(KeyError, storage.PutFile, bar, 'foo#')
self.assertRaises(KeyError, storage.GetFile, 'foo!', 'bar')
def GetFile(self, key, path):
"""Read a file from storage.
Args:
key: Key to store file under.
path: Destination filename.
Returns:
URL used on success or None for failure.
"""
ValidateKey(key)
cache_file = os.path.join(self._cache_path, key)
if os.path.exists(cache_file):
data = file_tools.ReadFile(cache_file)
file_tools.WriteFile(data, path)
return 'LOCAL_CACHE_NO_URL_AVAILABLE'
else:
return self._storage.GetFile(key, path)
def PutData(self, data, key, clobber=True):
"""Write data to global storage.
Args:
data: Data to store.
key: Key to store file under.
Raises:
GSDStorageError if the underlying storage fails.
Returns:
URL written to.
"""
handle, path = tempfile.mkstemp(prefix='gdstore', suffix='.tmp')
try:
os.close(handle)
file_tools.WriteFile(data, path)
return self.PutFile(path, key, clobber=clobber)
finally:
os.remove(path)
def GenerateTestData(self, noise, work_dir):
self._input_dirs = {}
self._input_files = []
for i in range(2):
dir_name = os.path.join(work_dir, noise + 'input%d_dir' % i)
os.mkdir(dir_name)
filename = os.path.join(dir_name, 'in%d' % i)
file_tools.WriteFile(noise + 'data%d' % i, filename)
self._input_dirs['input%d' % i] = dir_name
self._input_files.append(filename)
self._output_dirs = []
self._output_files = []
for i in range(2):
dir_name = os.path.join(work_dir, noise + 'output%d_dir' % i)
os.mkdir(dir_name)
filename = os.path.join(dir_name, 'out')
self._output_dirs.append(dir_name)
self._output_files.append(filename)
def download(url, target):
if 'badbucket' in url:
raise Exception('fail download %s to %s' % (url, target))
else:
file_tools.WriteFile('bar', target)
def download(url, target):
self.assertEqual('https://storage.googleapis.com/mybucket/bar',
url)
file_tools.WriteFile('baz', target)
