def list_build_sources_for_namespace(self, namespace):
def repo_view(repo):
return {
'name': repo.name,
'full_name': repo.full_name,
'description': repo.description or '',
'last_updated': timegm(repo.pushed_at.utctimetuple()) if repo.pushed_at else 0,
'url': repo.html_url,
'has_admin_permissions': repo.permissions.admin,
'private': repo.private,
}
gh_client = self._get_client()
usr = gh_client.get_user()
if namespace == usr.login:
repos = [repo_view(repo) for repo in usr.get_repos(type='owner', sort='updated')]
return BuildTriggerHandler.build_sources_response(repos)
try:
org = gh_client.get_organization(namespace)
if org is None:
return []
except GithubException:
return []
repos = [repo_view(repo) for repo in org.get_repos(type='member')]
return BuildTriggerHandler.build_sources_response(repos)
def list_build_sources_for_namespace(self, namespace):
def repo_view(repo):
return {
"name": repo.name,
"full_name": repo.full_name,
"description": repo.description or "",
"last_updated": timegm(repo.pushed_at.utctimetuple()) if repo.pushed_at else 0,
"url": repo.html_url,
"has_admin_permissions": True,
"private": repo.private,
}
gh_client = self._get_client()
usr = gh_client.get_user()
if namespace == usr.login:
repos = [repo_view(repo) for repo in usr.get_repos(type="owner", sort="updated")]
return BuildTriggerHandler.build_sources_response(repos)
try:
org = gh_client.get_organization(namespace)
if org is None:
return []
except GithubException:
return []
repos = [repo_view(repo) for repo in org.get_repos(type="member")]
return BuildTriggerHandler.build_sources_response(repos)
def trigger_view(trigger, can_read=False, can_admin=False, for_build=False):
if trigger and trigger.uuid:
build_trigger = BuildTriggerHandler.get_handler(trigger)
build_source = build_trigger.config.get("build_source")
repo_url = build_trigger.get_repository_url() if build_source else None
can_read = can_read or can_admin
trigger_data = {
"id":
trigger.uuid,
"service":
trigger.service.name,
"is_active":
build_trigger.is_active(),
"build_source":
build_source if can_read else None,
"repository_url":
repo_url if can_read else None,
"config":
build_trigger.config if can_admin else {},
"can_invoke":
can_admin,
"enabled":
trigger.enabled,
"disabled_reason":
trigger.disabled_reason.name if trigger.disabled_reason else None,
}
if not for_build and can_admin and trigger.pull_robot:
trigger_data["pull_robot"] = user_view(trigger.pull_robot)
return trigger_data
return None
def attach_bitbucket_trigger(namespace_name, repo_name):
permission = AdministerRepositoryPermission(namespace_name, repo_name)
if permission.can():
repo = model.repository.get_repository(namespace_name, repo_name)
if not repo:
msg = "Invalid repository: %s/%s" % (namespace_name, repo_name)
abort(404, message=msg)
elif repo.kind.name != "image":
abort(501)
trigger = model.build.create_build_trigger(
repo, BitbucketBuildTrigger.service_name(), None,
current_user.db_user())
try:
oauth_info = BuildTriggerHandler.get_handler(
trigger).get_oauth_url()
except TriggerProviderException:
trigger.delete_instance()
logger.debug("Could not retrieve Bitbucket OAuth URL")
abort(500)
config = {"access_token": oauth_info["access_token"]}
access_token_secret = oauth_info["access_token_secret"]
model.build.update_build_trigger(trigger,
config,
auth_token=access_token_secret)
return redirect(oauth_info["url"])
abort(403)
def post(self, namespace_name, repo_name, trigger_uuid):
"""
Analyze the specified build trigger configuration.
"""
trigger = get_trigger(trigger_uuid)
if trigger.repository.namespace_user.username != namespace_name:
raise NotFound()
if trigger.repository.name != repo_name:
raise NotFound()
new_config_dict = request.get_json()["config"]
handler = BuildTriggerHandler.get_handler(trigger, new_config_dict)
server_hostname = app.config["SERVER_HOSTNAME"]
try:
trigger_analyzer = TriggerAnalyzer(
handler,
namespace_name,
server_hostname,
new_config_dict,
AdministerOrganizationPermission(namespace_name).can(),
)
return trigger_analyzer.analyze_trigger()
except TriggerException as rre:
return {
"status": "error",
"message": "Could not analyze the repository: %s" % rre.message,
}
except NotImplementedError:
return {
"status": "notimplemented",
}
def post(self, namespace_name, repo_name, trigger_uuid):
"""
Manually start a build from the specified trigger.
"""
trigger = get_trigger(trigger_uuid)
if not trigger.enabled:
raise InvalidRequest("Trigger is not enabled.")
handler = BuildTriggerHandler.get_handler(trigger)
if not handler.is_active():
raise InvalidRequest("Trigger is not active.")
try:
repo = model.repository.get_repository(namespace_name, repo_name)
pull_robot_name = model.build.get_pull_robot_name(trigger)
run_parameters = request.get_json()
prepared = handler.manual_start(run_parameters=run_parameters)
build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
except TriggerException as tse:
raise InvalidRequest(tse.message)
except MaximumBuildsQueuedException:
abort(429, message="Maximum queued build rate exceeded.")
except BuildTriggerDisabledException:
abort(400, message="Build trigger is disabled")
resp = build_status_view(build_request)
repo_string = "%s/%s" % (namespace_name, repo_name)
headers = {
"Location": api.url_for(
RepositoryBuildStatus, repository=repo_string, build_uuid=build_request.uuid
),
}
return resp, 201, headers
def post(self, namespace_name, repo_name, trigger_uuid):
"""
List the subdirectories available for the specified build trigger and source.
"""
trigger = get_trigger(trigger_uuid)
user_permission = UserAdminPermission(trigger.connected_user.username)
if user_permission.can():
new_config_dict = request.get_json()
handler = BuildTriggerHandler.get_handler(trigger, new_config_dict)
try:
subdirs = handler.list_build_subdirs()
context_map = {}
for file in subdirs:
context_map = handler.get_parent_directory_mappings(file, context_map)
return {
"dockerfile_paths": ["/" + subdir for subdir in subdirs],
"contextMap": context_map,
"status": "success",
}
except EmptyRepositoryException as exc:
return {
"status": "success",
"contextMap": {},
"dockerfile_paths": [],
}
except TriggerException as exc:
return {
"status": "error",
"message": exc.message,
}
else:
raise Unauthorized()
def list_build_source_namespaces(self):
bitbucket_client = self._get_authorized_client()
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
if not result:
raise RepositoryReadException('Could not read repository list: ' +
err_msg)
namespaces = {}
for repo in data:
owner = repo['owner']
if owner in namespaces:
namespaces[owner]['score'] = namespaces[owner]['score'] + 1
else:
namespaces[owner] = {
'personal':
owner == self.config.get('nickname',
self.config.get('username')),
'id':
owner,
'title':
owner,
'avatar_url':
repo['logo'],
'url':
'https://bitbucket.org/%s' % (owner),
'score':
1,
}
return BuildTriggerHandler.build_namespaces_response(namespaces)
def put(self, namespace_name, repo_name, trigger_uuid):
"""
Updates the specified build trigger.
"""
trigger = get_trigger(trigger_uuid)
handler = BuildTriggerHandler.get_handler(trigger)
if not handler.is_active():
raise InvalidRequest("Cannot update an unactivated trigger")
enable = request.get_json()["enabled"]
model.build.toggle_build_trigger(trigger, enable)
log_action(
"toggle_repo_trigger",
namespace_name,
{
"repo": repo_name,
"trigger_id": trigger_uuid,
"service": trigger.service.name,
"enabled": enable,
},
repo=model.repository.get_repository(namespace_name, repo_name),
)
return trigger_view(trigger)
def list_build_source_namespaces(self):
bitbucket_client = self._get_authorized_client()
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
if not result:
raise RepositoryReadException("Could not read repository list: " +
err_msg)
namespaces = {}
for repo in data:
owner = repo["owner"]
if owner in namespaces:
namespaces[owner]["score"] = namespaces[owner]["score"] + 1
else:
namespaces[owner] = {
"personal":
owner == self.config.get("nickname",
self.config.get("username")),
"id":
owner,
"title":
owner,
"avatar_url":
repo["logo"],
"url":
"https://bitbucket.org/%s" % (owner),
"score":
1,
}
return BuildTriggerHandler.build_namespaces_response(namespaces)
def list_build_sources_for_namespace(self, namespace):
def repo_view(repo):
last_modified = dateutil.parser.parse(repo['utc_last_updated'])
return {
'name':
repo['slug'],
'full_name':
'%s/%s' % (repo['owner'], repo['slug']),
'description':
repo['description'] or '',
'last_updated':
timegm(last_modified.utctimetuple()),
'url':
'https://bitbucket.org/%s/%s' % (repo['owner'], repo['slug']),
'has_admin_permissions':
repo['read_only'] is False,
'private':
repo['is_private'],
}
bitbucket_client = self._get_authorized_client()
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
if not result:
raise RepositoryReadException('Could not read repository list: ' +
err_msg)
repos = [
repo_view(repo) for repo in data if repo['owner'] == namespace
]
return BuildTriggerHandler.build_sources_response(repos)
def list_build_source_namespaces(self):
gl_client = self._get_authorized_client()
current_user = gl_client.user
if not current_user:
raise RepositoryReadException("Unable to get current user")
namespaces = {}
for namespace in _paginated_iterator(gl_client.namespaces.list,
RepositoryReadException):
namespace_id = namespace.get_id()
if namespace_id in namespaces:
namespaces[namespace_id][
"score"] = namespaces[namespace_id]["score"] + 1
else:
owner = namespace.attributes["name"]
namespaces[namespace_id] = {
"personal": namespace.attributes["kind"] == "user",
"id": str(namespace_id),
"title": namespace.attributes["name"],
"avatar_url": namespace.attributes.get("avatar_url"),
"score": 1,
"url": namespace.attributes.get("web_url") or "",
}
return BuildTriggerHandler.build_namespaces_response(namespaces)
def list_build_source_namespaces(self):
gh_client = self._get_client()
usr = gh_client.get_user()
# Build the full set of namespaces for the user, starting with their own.
namespaces = {}
namespaces[usr.login] = {
"personal": True,
"id": usr.login,
"title": usr.name or usr.login,
"avatar_url": usr.avatar_url,
"url": usr.html_url,
"score": usr.plan.private_repos if usr.plan else 0,
}
for org in usr.get_orgs():
organization = org.login if org.login else org.name
# NOTE: We don't load the organization's html_url nor its plan, because doing
# so requires loading *each organization* via its own API call in this tight
# loop, which was massively slowing down the load time for users when setting
# up triggers.
namespaces[organization] = {
"personal": False,
"id": organization,
"title": organization,
"avatar_url": org.avatar_url,
"url": "",
"score": 0,
}
return BuildTriggerHandler.build_namespaces_response(namespaces)
def trigger_view(trigger, can_read=False, can_admin=False, for_build=False):
if trigger and trigger.uuid:
build_trigger = BuildTriggerHandler.get_handler(trigger)
build_source = build_trigger.config.get('build_source')
repo_url = build_trigger.get_repository_url() if build_source else None
can_read = can_read or can_admin
trigger_data = {
'id':
trigger.uuid,
'service':
trigger.service.name,
'is_active':
build_trigger.is_active(),
'build_source':
build_source if can_read else None,
'repository_url':
repo_url if can_read else None,
'config':
build_trigger.config if can_admin else {},
'can_invoke':
can_admin,
'enabled':
trigger.enabled,
'disabled_reason':
trigger.disabled_reason.name if trigger.disabled_reason else None,
}
if not for_build and can_admin and trigger.pull_robot:
trigger_data['pull_robot'] = user_view(trigger.pull_robot)
return trigger_data
return None
def list_build_source_namespaces(self):
gl_client = self._get_authorized_client()
current_user = gl_client.user
if not current_user:
raise RepositoryReadException('Unable to get current user')
namespaces = {}
for namespace in _paginated_iterator(gl_client.namespaces.list,
RepositoryReadException):
namespace_id = namespace.get_id()
if namespace_id in namespaces:
namespaces[namespace_id][
'score'] = namespaces[namespace_id]['score'] + 1
else:
owner = namespace.attributes['name']
namespaces[namespace_id] = {
'personal': namespace.attributes['kind'] == 'user',
'id': str(namespace_id),
'title': namespace.attributes['name'],
'avatar_url': namespace.attributes.get('avatar_url'),
'score': 1,
'url': namespace.attributes.get('web_url') or '',
}
return BuildTriggerHandler.build_namespaces_response(namespaces)
def list_build_sources_for_namespace(self, namespace):
def repo_view(repo):
last_modified = dateutil.parser.parse(repo["utc_last_updated"])
return {
"name":
repo["slug"],
"full_name":
"%s/%s" % (repo["owner"], repo["slug"]),
"description":
repo["description"] or "",
"last_updated":
timegm(last_modified.utctimetuple()),
"url":
"https://bitbucket.org/%s/%s" % (repo["owner"], repo["slug"]),
"has_admin_permissions":
repo["read_only"] is False,
"private":
repo["is_private"],
}
bitbucket_client = self._get_authorized_client()
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
if not result:
raise RepositoryReadException("Could not read repository list: " +
err_msg)
repos = [
repo_view(repo) for repo in data if repo["owner"] == namespace
]
return BuildTriggerHandler.build_sources_response(repos)
def attach_bitbucket_build_trigger(trigger_uuid):
trigger = model.build.get_build_trigger(trigger_uuid)
if not trigger or trigger.service.name != BitbucketBuildTrigger.service_name(
):
abort(404)
if trigger.connected_user != current_user.db_user():
abort(404)
verifier = request.args.get('oauth_verifier')
handler = BuildTriggerHandler.get_handler(trigger)
result = handler.exchange_verifier(verifier)
if not result:
trigger.delete_instance()
return 'Token has expired'
namespace = trigger.repository.namespace_user.username
repository = trigger.repository.name
repo_path = '%s/%s' % (namespace, repository)
full_url = url_for('web.buildtrigger',
path=repo_path,
trigger=trigger.uuid)
logger.debug('Redirecting to full url: %s', full_url)
return redirect(full_url)
def delete(self, namespace_name, repo_name, trigger_uuid):
"""
Delete the specified build trigger.
"""
trigger = get_trigger(trigger_uuid)
handler = BuildTriggerHandler.get_handler(trigger)
if handler.is_active():
try:
handler.deactivate()
except TriggerException as ex:
# We are just going to eat this error
logger.warning("Trigger deactivation problem: %s", ex)
log_action(
"delete_repo_trigger",
namespace_name,
{"repo": repo_name, "trigger_id": trigger_uuid, "service": trigger.service.name},
repo=model.repository.get_repository(namespace_name, repo_name),
)
trigger.delete_instance(recursive=True)
if trigger.write_token is not None:
trigger.write_token.delete_instance()
return "No Content", 204
def test_subdir_path_map_no_previous(input, output):
actual_mapping = BuildTriggerHandler.get_parent_directory_mappings(input)
for key in actual_mapping:
value = actual_mapping[key]
actual_mapping[key] = value.sort()
for key in output:
value = output[key]
output[key] = value.sort()
assert actual_mapping == output
def test_subdir_path_map(new_path, original_dictionary, output):
actual_mapping = BuildTriggerHandler.get_parent_directory_mappings(
new_path, original_dictionary)
for key in actual_mapping:
value = actual_mapping[key]
actual_mapping[key] = value.sort()
for key in output:
value = output[key]
output[key] = value.sort()
assert actual_mapping == output
def get(self, namespace_name, repo_name, trigger_uuid):
""" List the build sources for the trigger configuration thus far. """
trigger = get_trigger(trigger_uuid)
user_permission = UserAdminPermission(trigger.connected_user.username)
if user_permission.can():
handler = BuildTriggerHandler.get_handler(trigger)
try:
return {"namespaces": handler.list_build_source_namespaces()}
except TriggerException as rre:
raise InvalidRequest(rre.message)
else:
raise Unauthorized()
def put(self, namespace_name, repo_name, trigger_uuid):
""" Updates the specified build trigger. """
trigger = get_trigger(trigger_uuid)
handler = BuildTriggerHandler.get_handler(trigger)
if not handler.is_active():
raise InvalidRequest('Cannot update an unactivated trigger')
enable = request.get_json()['enabled']
model.build.toggle_build_trigger(trigger, enable)
log_action('toggle_repo_trigger', namespace_name,
{'repo': repo_name, 'trigger_id': trigger_uuid,
'service': trigger.service.name, 'enabled': enable},
repo=model.repository.get_repository(namespace_name, repo_name))
return trigger_view(trigger)
def post(self, namespace_name, repo_name, trigger_uuid, field_name):
""" List the field values for a custom run field. """
trigger = get_trigger(trigger_uuid)
config = request.get_json() or None
if AdministerRepositoryPermission(namespace_name, repo_name).can():
handler = BuildTriggerHandler.get_handler(trigger, config)
values = handler.list_field_values(field_name,
limit=FIELD_VALUE_LIMIT)
if values is None:
raise NotFound()
return {"values": values}
else:
raise Unauthorized()
def post(self, namespace_name, repo_name, trigger_uuid):
"""
List the build sources for the trigger configuration thus far.
"""
namespace = request.get_json().get("namespace")
if namespace is None:
raise InvalidRequest()
trigger = get_trigger(trigger_uuid)
user_permission = UserAdminPermission(trigger.connected_user.username)
if user_permission.can():
handler = BuildTriggerHandler.get_handler(trigger)
try:
return {"sources": handler.list_build_sources_for_namespace(namespace)}
except TriggerException as rre:
raise InvalidRequest(str(rre)) from rre
else:
raise Unauthorized()
def to_dict(self):
if not self.uuid:
return None
build_trigger = BuildTriggerHandler.get_handler(self)
build_source = build_trigger.config.get('build_source')
repo_url = build_trigger.get_repository_url() if build_source else None
can_read = self.can_read or self.can_admin
trigger_data = {
'id': self.uuid,
'service': self.service_name,
'is_active': build_trigger.is_active(),
'build_source': build_source if can_read else None,
'repository_url': repo_url if can_read else None,
'config': build_trigger.config if self.can_admin else {},
'can_invoke': self.can_admin,
}
if not self.for_build and self.can_admin and self.pull_robot:
trigger_data['pull_robot'] = user_view(self.pull_robot)
return trigger_data
def to_dict(self):
if not self.uuid:
return None
build_trigger = BuildTriggerHandler.get_handler(self)
build_source = build_trigger.config.get("build_source")
repo_url = build_trigger.get_repository_url() if build_source else None
can_read = self.can_read or self.can_admin
trigger_data = {
"id": self.uuid,
"service": self.service_name,
"is_active": build_trigger.is_active(),
"build_source": build_source if can_read else None,
"repository_url": repo_url if can_read else None,
"config": build_trigger.config if self.can_admin else {},
"can_invoke": self.can_admin,
}
if not self.for_build and self.can_admin and self.pull_robot:
trigger_data["pull_robot"] = user_view(self.pull_robot)
return trigger_data
def post(self, namespace_name, repo_name, trigger_uuid):
"""
Activate the specified build trigger.
"""
trigger = get_trigger(trigger_uuid)
handler = BuildTriggerHandler.get_handler(trigger)
if handler.is_active():
raise InvalidRequest("Trigger config is not sufficient for activation.")
user_permission = UserAdminPermission(trigger.connected_user.username)
if user_permission.can():
# Update the pull robot (if any).
pull_robot_name = request.get_json().get("pull_robot", None)
if pull_robot_name:
try:
pull_robot = model.user.lookup_robot(pull_robot_name)
except model.InvalidRobotException:
raise NotFound()
# Make sure the user has administer permissions for the robot's namespace.
(robot_namespace, _) = parse_robot_username(pull_robot_name)
if not AdministerOrganizationPermission(robot_namespace).can():
raise Unauthorized()
# Make sure the namespace matches that of the trigger.
if robot_namespace != namespace_name:
raise Unauthorized()
# Set the pull robot.
trigger.pull_robot = pull_robot
# Update the config.
new_config_dict = request.get_json()["config"]
write_token_name = "Build Trigger: %s" % trigger.service.name
write_token = model.token.create_delegate_token(
namespace_name, repo_name, write_token_name, "write"
)
try:
path = url_for("webhooks.build_trigger_webhook", trigger_uuid=trigger.uuid)
authed_url = _prepare_webhook_url(
app.config["PREFERRED_URL_SCHEME"],
"$token",
write_token.get_code(),
app.config["SERVER_HOSTNAME"],
path,
)
handler = BuildTriggerHandler.get_handler(trigger, new_config_dict)
final_config, private_config = handler.activate(authed_url)
if "private_key" in private_config:
trigger.secure_private_key = DecryptedValue(private_config["private_key"])
except TriggerException as exc:
write_token.delete_instance()
raise request_error(message=exc.message)
# Save the updated config.
update_build_trigger(trigger, final_config, write_token=write_token)
# Log the trigger setup.
repo = model.repository.get_repository(namespace_name, repo_name)
log_action(
"setup_repo_trigger",
namespace_name,
{
"repo": repo_name,
"namespace": namespace_name,
"trigger_id": trigger.uuid,
"service": trigger.service.name,
"pull_robot": trigger.pull_robot.username if trigger.pull_robot else None,
"config": final_config,
},
repo=repo,
)
return trigger_view(trigger, can_admin=True)
else:
raise Unauthorized()
def test_determine_tags(config, metadata, expected_tags):
tags = BuildTriggerHandler._determine_tags(config, metadata)
assert tags == set(expected_tags)
def test_path_is_dockerfile(input, output):
assert BuildTriggerHandler.filename_is_dockerfile(input) == output
def list_build_sources_for_namespace(self, namespace_id):
if not namespace_id:
return []
def repo_view(repo):
# Because *anything* can be None in GitLab API!
permissions = repo.attributes.get("permissions") or {}
group_access = permissions.get("group_access") or {}
project_access = permissions.get("project_access") or {}
missing_group_access = permissions.get("group_access") is None
missing_project_access = permissions.get("project_access") is None
access_level = max(
group_access.get("access_level") or 0, project_access.get("access_level") or 0
)
has_admin_permission = _ACCESS_LEVEL_MAP.get(access_level, ("", False))[1]
if missing_group_access or missing_project_access:
# Default to has permission if we cannot check the permissions. This will allow our users
# to select the repository and then GitLab's own checks will ensure that the webhook is
# added only if allowed.
# TODO: Do we want to display this differently in the UI?
has_admin_permission = True
view = {
"name": repo.attributes["path"],
"full_name": repo.attributes["path_with_namespace"],
"description": repo.attributes.get("description") or "",
"url": repo.attributes.get("web_url"),
"has_admin_permissions": has_admin_permission,
"private": repo.attributes.get("visibility") == "private",
}
if repo.attributes.get("last_activity_at"):
try:
last_modified = dateutil.parser.parse(repo.attributes["last_activity_at"])
view["last_updated"] = timegm(last_modified.utctimetuple())
except ValueError:
logger.exception(
"Gitlab gave us an invalid last_activity_at: %s", last_modified
)
return view
gl_client = self._get_authorized_client()
try:
gl_namespace = gl_client.namespaces.get(namespace_id)
except gitlab.GitlabGetError:
return []
namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True)
repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException)
try:
return BuildTriggerHandler.build_sources_response(
[repo_view(repo) for repo in repositories]
)
except gitlab.GitlabGetError:
return []
