def get_config_multi(config_sets, path, hashes_only):
"""Returns configs at |path| in all config sets.
Returns empty config list if requester does not have project access.
"""
res = GetConfigMultiResponseMessage()
config_sets = filter(acl.can_read_config_set, config_sets)
configs = storage.get_latest_multi_async(
config_sets, path, hashes_only).get_result()
for config in configs:
if not hashes_only and config.get('content') is None:
logging.error(
'Blob %s referenced from %s:%s:%s was not found',
config['content_hash'],
config['config_set'],
config['revision'],
path)
continue
res.configs.append(res.ConfigEntry(
config_set=config['config_set'],
revision=config['revision'],
content_hash=config['content_hash'],
content=config.get('content'),
))
return res
def test_get_latest_multi_file_missing(self):
self.put_file('foo', 'deadbeef', 'a.cfg', 'fooo')
self.put_file('bar', 'beefdead', 'b.cfg', 'barr')
expected = [
{
'config_set': 'bar',
'content': 'barr',
'content_hash': 'v1:f89094690273aed90f20da47629315b54e494eb8',
'revision': 'beefdead',
'url': 'https://x.com/+/beefdead/b.cfg',
},
]
actual = storage.get_latest_multi_async(
['foo', 'bar'], 'b.cfg').get_result()
self.assertEqual(expected, actual)
def test_get_latest_multi_hashes_only(self):
self.put_file("foo", "deadbeef", "a.cfg", "fooo")
self.put_file("bar", "beefdead", "a.cfg", "barr")
expected = [
{
"config_set": "foo",
"revision": "deadbeef",
"content_hash": "v1:551f4d3376caed56a600e02dfaa733b68898dc2b",
"content": None,
},
{
"config_set": "bar",
"revision": "beefdead",
"content_hash": "v1:f89094690273aed90f20da47629315b54e494eb8",
"content": None,
},
]
actual = storage.get_latest_multi_async(["foo", "bar"], "a.cfg", hashes_only=True).get_result()
self.assertEqual(expected, actual)
def test_get_latest_multi_hashes_only(self):
self.put_file('foo', 'deadbeef', 'a.cfg', 'fooo')
self.put_file('bar', 'beefdead', 'a.cfg', 'barr')
expected = [
{
'config_set': 'foo',
'revision': 'deadbeef',
'content_hash': 'v1:551f4d3376caed56a600e02dfaa733b68898dc2b',
'content': None,
},
{
'config_set': 'bar',
'revision': 'beefdead',
'content_hash': 'v1:f89094690273aed90f20da47629315b54e494eb8',
'content': None,
},
]
actual = storage.get_latest_multi_async(
['foo', 'bar'], 'a.cfg', hashes_only=True).get_result()
self.assertEqual(expected, actual)
def test_get_latest_multi_hashes_only(self):
self.put_file('foo', 'deadbeef', 'a.cfg', 'fooo')
self.put_file('bar', 'beefdead', 'a.cfg', 'barr')
expected = [
{
'config_set': 'foo',
'revision': 'deadbeef',
'content_hash': 'v1:551f4d3376caed56a600e02dfaa733b68898dc2b',
'content': None,
},
{
'config_set': 'bar',
'revision': 'beefdead',
'content_hash': 'v1:f89094690273aed90f20da47629315b54e494eb8',
'content': None,
},
]
actual = storage.get_latest_multi_async(
['foo', 'bar'], 'a.cfg', hashes_only=True).get_result()
self.assertEqual(expected, actual)
def get_config_multi(scope, path, hashes_only):
"""Returns configs at |path| in all config sets.
scope can be 'projects' or 'refs'.
Returns empty config list if requester does not have project access.
"""
assert scope in ('projects', 'refs'), scope
cache_key = (
'%s%s:%s' % (scope, ',hashes_only' if hashes_only else '', path))
configs = memcache.get(cache_key)
if configs is None:
configs = storage.get_latest_multi_async(
get_config_sets_from_scope(scope), path, hashes_only).get_result()
for config in configs:
if not hashes_only and config.get('content') is None:
logging.error(
'Blob %s referenced from %s:%s:%s was not found',
config['content_hash'],
config['config_set'],
config['revision'],
path)
try:
memcache.add(cache_key, configs, time=10*60)
except ValueError:
logging.exception('%s:%s configs are too big for memcache', scope, path)
res = GetConfigMultiResponseMessage()
for config in configs:
if not acl.can_read_config_set(config['config_set']):
continue
if not hashes_only and config.get('content') is None:
continue
res.configs.append(res.ConfigEntry(
config_set=config['config_set'],
revision=config['revision'],
content_hash=config['content_hash'],
content=config.get('content'),
))
return res
def get_config_multi(scope, path, hashes_only):
"""Returns configs at |path| in all config sets.
scope can be 'projects' or 'refs'.
Returns empty config list if requester does not have project access.
"""
assert scope in ('projects', 'refs'), scope
cache_key = ('v2/%s%s:%s' %
(scope, ',hashes_only' if hashes_only else '', path))
configs = memcache.get(cache_key)
if configs is None:
configs = storage.get_latest_multi_async(
get_config_sets_from_scope(scope), path, hashes_only).get_result()
for config in configs:
if not hashes_only and config.get('content') is None:
logging.error('Blob %s referenced from %s:%s:%s was not found',
config['content_hash'], config['config_set'],
config['revision'], path)
try:
memcache.add(cache_key, configs, time=60)
except ValueError:
logging.exception('%s:%s configs are too big for memcache', scope,
path)
res = GetConfigMultiResponseMessage()
for config in configs:
if not acl.can_read_config_set(config['config_set']):
continue
if not hashes_only and config.get('content') is None:
continue
res.configs.append(
res.ConfigEntry(
config_set=config['config_set'],
revision=config['revision'],
content_hash=config['content_hash'],
content=config.get('content'),
url=config.get('url'),
))
return res