def import_ref(project_id, ref_name):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id "%s"' % project_id)
if not config.validation.is_valid_ref_name(ref_name):
raise ValueError('Invalid ref name "%s"' % ref_name)
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.config_location.storage_type != GITILES_LOCATION_TYPE:
raise Error('project %s is not a Gitiles project' % project_id)
loc = gitiles.Location.parse_resolve(project.config_location.url)
ref = None
for r in projects.get_refs([project_id])[project_id] or ():
if r.name == ref_name:
ref = r
if ref is None:
raise NotFoundError(
('ref "%s" is not found in project %s. '
'Possibly it is not declared in projects/%s:refs.cfg') %
(ref_name, project_id, project_id))
cfg = get_gitiles_config()
loc = loc._replace(
treeish=ref_name,
path=ref.config_path or cfg.ref_config_default_path,
)
_import_config_set('projects/%s/%s' % (project_id, ref_name), loc)
def import_ref(project_id, ref_name):
if not config.validation.is_valid_project_id(project_id):
raise ValueError('Invalid project id "%s"' % project_id)
if not config.validation.is_valid_ref_name(ref_name):
raise ValueError('Invalid ref name "%s"' % ref_name)
project = projects.get_project(project_id)
if project is None:
raise NotFoundError('project %s not found' % project_id)
if project.HasField('gitiles_location'):
loc = _make_gitiles_location(project.gitiles_location)
else:
raise Error('project %s does not have a gitiles location' % project_id)
ref = None
for r in projects.get_refs([project_id])[project_id] or ():
if r.name == ref_name:
ref = r
if ref is None:
raise NotFoundError(
('ref "%s" is not found in project %s. '
'Possibly it is not declared in projects/%s:refs.cfg') %
(ref_name, project_id, project_id))
cfg = get_gitiles_config()
loc = loc._replace(
treeish=ref_name,
path=ref.config_path or cfg.ref_config_default_path,
)
_import_config_set('projects/%s/%s' % (project_id, ref_name), loc)
def import_projects():
"""Imports all project and ref config sets that are stored in Gitiles.
Logs errors, does not raise them.
"""
cfg = get_gitiles_config()
for project in projects.get_projects():
loc = project.config_location
if loc.storage_type != GITILES_LOCATION_TYPE:
continue
try:
location = gitiles.Location.parse_resolve(loc.url)
except ValueError:
logging.exception('Invalid project location: %s',
project.config_location)
continue
except net.AuthError as ex:
logging.error('Could not resolve %s due to permissions: %s',
project.config_location, ex.message)
continue
with _log_import_error('projects/%s' % project.id):
import_project(project.id, location)
# Import refs of the project
for ref in projects.get_refs(project.id):
assert ref.name
assert ref.name.startswith('refs/'), ref.name
ref_location = location._replace(
treeish=ref.name,
path=ref.config_path or cfg.ref_config_default_path,
)
ref_cs = 'projects/%s/%s' % (project.id, ref.name)
with _log_import_error(ref_cs):
_import_config_set(ref_cs, ref_location)
def import_project(project_id, location):
cfg = get_gitiles_config()
# Adjust location
treeish = location.treeish
if not treeish or treeish == 'HEAD':
treeish = cfg.project_config_default_ref
location = location._replace(
treeish=treeish,
path=location.path.strip('/') or cfg.project_config_default_path,
)
# Update project repo info.
repo_url = str(location._replace(treeish=None, path=None))
projects.update_import_info(
project_id, projects.RepositoryType.GITILES, repo_url)
import_config_set('projects/%s' % project_id, location)
# Import refs
for ref in projects.get_refs(project_id):
assert ref.name
assert ref.name.startswith('refs/'), ref.name
ref_location = location._replace(
treeish=ref.name,
path=ref.config_path or cfg.ref_config_default_path,
)
import_config_set(
'projects/%s/%s' % (project_id, ref.name), ref_location)
def get_ref_names(project_id):
"""Returns list of ref names for a project. Caches results."""
assert project_id
refs = projects.get_refs(project_id)
if refs is None:
# Project does not exist
return None
return [ref.name for ref in refs]
def get_ref_names(project_id):
"""Returns list of ref names for a project. Caches results."""
assert project_id
refs = projects.get_refs(project_id)
if refs is None:
# Project does not exist
return None
return [ref.name for ref in refs]
def get_config_sets_from_scope(scope):
"""Yields config sets from 'projects' or 'refs'."""
assert scope in ('projects', 'refs'), scope
projs = get_projects()
refs = None
if scope == 'refs':
refs = projects.get_refs([p.id for p in projs])
for p in projs:
if scope == 'projects':
yield 'projects/%s' % p.id
else:
for ref in refs[p.id] or ():
yield 'projects/%s/%s' % (p.id, ref.name)
def test_get_refs_returns_no_refs(self):
# TODO(crbug/924803): remove ref support from the service entirely.
self.mock_latest_config(
'projects/chromium', '''
refs {
name: "refs/heads/master"
}
refs {
name: "refs/heads/release42"
config_path: "other"
}
''')
self.assertEqual(projects.get_refs(['chromium']), {'chromium': []})
def get_refs(self, request):
"""Gets list of refs of a project."""
has_access = acl.has_projects_access([request.project_id
]).get(request.project_id)
if not has_access:
raise endpoints.NotFoundException()
refs = projects.get_refs([request.project_id]).get(request.project_id)
if refs is None:
# Project not found
raise endpoints.NotFoundException()
res = self.GetRefsResponseMessage()
res.refs = [res.Ref(name=ref.name) for ref in refs]
return res
def _project_and_ref_config_sets():
"""Returns a list of project and ref config sets stored in Gitiles."""
projs = projects.get_projects()
refs = projects.get_refs([p.id for p in projs])
ret = []
for project in projs:
ret.append('projects/%s' % project.id)
# Import refs of the project
for ref in refs[project.id] or []:
assert ref.name
assert ref.name.startswith('refs/'), ref.name
ret.append('projects/%s/%s' % (project.id, ref.name))
return ret
def test_get_refs(self):
storage.get_latest_async.return_value.set_result('''
refs {
name: "refs/heads/master"
}
refs {
name: "refs/heads/release42"
config_path: "other"
}
''')
expected = project_config_pb2.RefsCfg(refs=[
project_config_pb2.RefsCfg.Ref(name='refs/heads/master'),
project_config_pb2.RefsCfg.Ref(name='refs/heads/release42',
config_path='other'),
], )
self.assertEqual(projects.get_refs('chromium'), expected.refs)
def test_get_refs(self):
self.mock_latest_config(
'projects/chromium', '''
refs {
name: "refs/heads/master"
}
refs {
name: "refs/heads/release42"
config_path: "other"
}
''')
expected = project_config_pb2.RefsCfg(refs=[
project_config_pb2.RefsCfg.Ref(name='refs/heads/master'),
project_config_pb2.RefsCfg.Ref(name='refs/heads/release42',
config_path='other'),
], )
self.assertEqual(projects.get_refs(['chromium']),
{'chromium': expected.refs})
def test_get_refs(self):
storage.get_latest_async.return_value.set_result(
"""
refs {
name: "refs/heads/master"
}
refs {
name: "refs/heads/release42"
config_path: "other"
}
"""
)
expected = project_config_pb2.RefsCfg(
refs=[
project_config_pb2.RefsCfg.Ref(name="refs/heads/master"),
project_config_pb2.RefsCfg.Ref(name="refs/heads/release42", config_path="other"),
]
)
self.assertEqual(projects.get_refs("chromium"), expected.refs)
def test_get_refs(self):
storage.get_latest_async.return_value.set_result('''
refs {
name: "refs/heads/master"
}
refs {
name: "refs/heads/release42"
config_path: "other"
}
''')
expected = project_config_pb2.RefsCfg(
refs=[
project_config_pb2.RefsCfg.Ref(
name='refs/heads/master'),
project_config_pb2.RefsCfg.Ref(
name='refs/heads/release42', config_path='other'),
],
)
self.assertEqual(projects.get_refs('chromium'), expected.refs)
def import_projects():
"""Imports all project and ref config sets that are stored in Gitiles.
Logs errors, does not raise them.
"""
cfg = get_gitiles_config()
for project in projects.get_projects():
loc = project.config_location
if loc.storage_type != GITILES_LOCATION_TYPE:
continue
try:
location = gitiles.Location.parse_resolve(loc.url)
except ValueError:
logging.exception('Invalid project location: %s', project.config_location)
continue
except net.AuthError as ex:
logging.error(
'Could not resolve %s due to permissions: %s',
project.config_location, ex.message)
continue
with _log_import_error('projects/%s' % project.id):
import_project(project.id, location)
# Import refs of the project
for ref in projects.get_refs(project.id):
assert ref.name
assert ref.name.startswith('refs/'), ref.name
ref_location = location._replace(
treeish=ref.name,
path=ref.config_path or cfg.ref_config_default_path,
)
ref_cs = 'projects/%s/%s' % (project.id, ref.name)
with _log_import_error(ref_cs):
_import_config_set(ref_cs, ref_location)
def test_get_refs_of_non_existent_project(self):
storage.get_latest_async.return_value.set_result(None)
self.assertEqual(projects.get_refs("chromium"), None)
def test_get_refs_of_non_existent_project(self):
self.mock(projects, '_filter_existing', lambda pids: [])
self.assertEqual(projects.get_refs(['chromium']), {'chromium': None})
def test_get_refs_of_non_existent_project(self):
storage.get_latest_async.return_value.set_result(None)
self.assertEqual(projects.get_refs('chromium'), None)