def _fetch_configs(paths):
"""Fetches a bunch of config files in parallel and validates them.
Returns:
dict {path -> (Revision tuple, <config>)}.
Raises:
CannotLoadConfigError if some config is missing or invalid.
"""
paths = sorted(paths)
futures = [
config.get_self_config_async(
p,
dest_type=_CONFIG_SCHEMAS[p]['proto_class'],
store_last_good=False) for p in paths
]
configs_url = _get_configs_url()
ndb.Future.wait_all(futures)
out = {}
for path, future in zip(paths, futures):
rev, conf = future.get_result()
if conf is None:
raise CannotLoadConfigError('Config %s is missing' % path)
try:
validation.validate(config.self_config_set(), path, conf)
except ValueError as exc:
raise CannotLoadConfigError(
'Config %s at rev %s failed to pass validation: %s' %
(path, rev, exc))
out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
return out
def _fetch_configs(paths):
"""Fetches a bunch of config files in parallel and validates them.
Returns:
dict {path -> (Revision tuple, <config>)}.
Raises:
CannotLoadConfigError if some config is missing or invalid.
"""
paths = sorted(paths)
configs_url = _get_configs_url()
out = {}
configs = utils.async_apply(
paths, lambda p: config.get_self_config_async(
p,
dest_type=_CONFIG_SCHEMAS[p]['proto_class'],
store_last_good=False))
for path, (rev, conf) in configs:
if conf is None:
default = _CONFIG_SCHEMAS[path].get('default')
if default is None:
raise CannotLoadConfigError('Config %s is missing' % path)
rev, conf = '0' * 40, default
try:
validation.validate(config.self_config_set(), path, conf)
except ValueError as exc:
raise CannotLoadConfigError(
'Config %s at rev %s failed to pass validation: %s' %
(path, rev, exc))
out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
return out
def _fetch_configs(paths):
"""Fetches a bunch of config files in parallel and validates them.
Returns:
dict {path -> (Revision tuple, <config>)}.
Raises:
CannotLoadConfigError if some config is missing or invalid.
"""
paths = sorted(paths)
futures = [
config.get_self_config_async(
p, dest_type=_CONFIG_SCHEMAS[p]['proto_class'], store_last_good=False)
for p in paths
]
configs_url = _get_configs_url()
out = {}
for path, future in zip(paths, futures):
rev, conf = future.get_result()
try:
validation.validate(config.self_config_set(), path, conf)
except ValueError as exc:
raise config.CannotLoadConfigError(
'Config %s at rev %s failed to pass validation: %s' %
(path, rev, exc))
out[path] = (Revision(rev, _gitiles_url(configs_url, rev, path)), conf)
return out
def get_task_template_async(canary, canary_required=True):
"""Gets a tuple (template_revision, template_dict, canary_bool).
Args:
canary (bool): specifies a whether canary template should be returned.
canary_required (bool): controls the behavior if |canary| is True and
the canary template is not found. If False use the non-canary template,
otherwise raise CanaryTemplateNotFound.
Ignored if canary is False.
Returns:
Tuple (template_revision, template_dict, canary):
template_revision (str): revision of the template, e.g. commit hash.
template_dict (dict): parsed template, or None if not found.
May contain $parameters that must be expanded using format_obj().
canary (bool): True if the returned template is a canary template.
"""
text = None
revision = None
if canary:
logging.warning('using canary swarming task template')
revision, text = yield component_config.get_self_config_async(
'swarming_task_template_canary.json', store_last_good=True)
canary = bool(text)
if not text:
if canary_required:
raise CanaryTemplateNotFound(
'canary swarming task template is requested, '
'but the canary template is not found')
logging.warning(
'canary swarming task template is not found. using the default one'
)
if not text:
revision, text = yield component_config.get_self_config_async(
'swarming_task_template.json', store_last_good=True)
template = None
if text:
template = json.loads(text)
template.pop('__comment__', None)
raise ndb.Return(revision, template, canary)
def get_latest_revs_async():
"""Returns a list of all current RealmsCfgRev by querying LUCI Config."""
# In parallel load all project realms (from projects' config sets) and
# internal realms (from the service config set).
#
# Per the config client library API, here
# `configs` is {project_id -> (rev, body, exc)}, where `exc` is always None.
# `internal` is (rev, body) where (None, None) indicates "no such config".
configs, internal = yield (
config.get_project_configs_async(common.cfg_path()),
config.get_self_config_async(common.cfg_path(), store_last_good=False),
)
# Pretend internal realms came from special "@internal" project. Such project
# name is forbidden by LUCI Config, so there should be no confusion.
if common.INTERNAL_PROJECT in configs:
raise ValueError('Unexpected LUCI project %s' % common.INTERNAL_PROJECT)
internal_rev, internal_body = internal
if internal_body:
configs[common.INTERNAL_PROJECT] = (internal_rev, internal_body, None)
# Convert the result to a list of RealmsCfgRev in no particular order.
out = []
for project_id, (rev, body, exc) in configs.items():
# Errors are impossible when when not specifying 2nd parameter of
# get_project_configs_async.
assert body is not None
assert exc is None
out.append(RealmsCfgRev(
project_id=project_id,
config_rev=rev or 'unknown',
config_digest=hashlib.sha256(body).hexdigest(),
config_body=body,
perms_rev=None,
))
raise ndb.Return(out)
def get_settings_async(): # pragma: no cover
_, global_settings = yield config.get_self_config_async(
'settings.cfg', service_config_pb2.SettingsCfg, store_last_good=True)
raise ndb.Return(global_settings or service_config_pb2.SettingsCfg())
