def list_build_sources_for_namespace(self, namespace): def repo_view(repo): return { 'name': repo.name, 'full_name': repo.full_name, 'description': repo.description or '', 'last_updated': timegm(repo.pushed_at.utctimetuple()) if repo.pushed_at else 0, 'url': repo.html_url, 'has_admin_permissions': repo.permissions.admin, 'private': repo.private, } gh_client = self._get_client() usr = gh_client.get_user() if namespace == usr.login: repos = [repo_view(repo) for repo in usr.get_repos(type='owner', sort='updated')] return BuildTriggerHandler.build_sources_response(repos) try: org = gh_client.get_organization(namespace) if org is None: return [] except GithubException: return [] repos = [repo_view(repo) for repo in org.get_repos(type='member')] return BuildTriggerHandler.build_sources_response(repos)
def list_build_sources_for_namespace(self, namespace): def repo_view(repo): return { "name": repo.name, "full_name": repo.full_name, "description": repo.description or "", "last_updated": timegm(repo.pushed_at.utctimetuple()) if repo.pushed_at else 0, "url": repo.html_url, "has_admin_permissions": True, "private": repo.private, } gh_client = self._get_client() usr = gh_client.get_user() if namespace == usr.login: repos = [repo_view(repo) for repo in usr.get_repos(type="owner", sort="updated")] return BuildTriggerHandler.build_sources_response(repos) try: org = gh_client.get_organization(namespace) if org is None: return [] except GithubException: return [] repos = [repo_view(repo) for repo in org.get_repos(type="member")] return BuildTriggerHandler.build_sources_response(repos)
def trigger_view(trigger, can_read=False, can_admin=False, for_build=False): if trigger and trigger.uuid: build_trigger = BuildTriggerHandler.get_handler(trigger) build_source = build_trigger.config.get("build_source") repo_url = build_trigger.get_repository_url() if build_source else None can_read = can_read or can_admin trigger_data = { "id": trigger.uuid, "service": trigger.service.name, "is_active": build_trigger.is_active(), "build_source": build_source if can_read else None, "repository_url": repo_url if can_read else None, "config": build_trigger.config if can_admin else {}, "can_invoke": can_admin, "enabled": trigger.enabled, "disabled_reason": trigger.disabled_reason.name if trigger.disabled_reason else None, } if not for_build and can_admin and trigger.pull_robot: trigger_data["pull_robot"] = user_view(trigger.pull_robot) return trigger_data return None
def attach_bitbucket_trigger(namespace_name, repo_name): permission = AdministerRepositoryPermission(namespace_name, repo_name) if permission.can(): repo = model.repository.get_repository(namespace_name, repo_name) if not repo: msg = "Invalid repository: %s/%s" % (namespace_name, repo_name) abort(404, message=msg) elif repo.kind.name != "image": abort(501) trigger = model.build.create_build_trigger( repo, BitbucketBuildTrigger.service_name(), None, current_user.db_user()) try: oauth_info = BuildTriggerHandler.get_handler( trigger).get_oauth_url() except TriggerProviderException: trigger.delete_instance() logger.debug("Could not retrieve Bitbucket OAuth URL") abort(500) config = {"access_token": oauth_info["access_token"]} access_token_secret = oauth_info["access_token_secret"] model.build.update_build_trigger(trigger, config, auth_token=access_token_secret) return redirect(oauth_info["url"]) abort(403)
def post(self, namespace_name, repo_name, trigger_uuid): """ Analyze the specified build trigger configuration. """ trigger = get_trigger(trigger_uuid) if trigger.repository.namespace_user.username != namespace_name: raise NotFound() if trigger.repository.name != repo_name: raise NotFound() new_config_dict = request.get_json()["config"] handler = BuildTriggerHandler.get_handler(trigger, new_config_dict) server_hostname = app.config["SERVER_HOSTNAME"] try: trigger_analyzer = TriggerAnalyzer( handler, namespace_name, server_hostname, new_config_dict, AdministerOrganizationPermission(namespace_name).can(), ) return trigger_analyzer.analyze_trigger() except TriggerException as rre: return { "status": "error", "message": "Could not analyze the repository: %s" % rre.message, } except NotImplementedError: return { "status": "notimplemented", }
def post(self, namespace_name, repo_name, trigger_uuid): """ Manually start a build from the specified trigger. """ trigger = get_trigger(trigger_uuid) if not trigger.enabled: raise InvalidRequest("Trigger is not enabled.") handler = BuildTriggerHandler.get_handler(trigger) if not handler.is_active(): raise InvalidRequest("Trigger is not active.") try: repo = model.repository.get_repository(namespace_name, repo_name) pull_robot_name = model.build.get_pull_robot_name(trigger) run_parameters = request.get_json() prepared = handler.manual_start(run_parameters=run_parameters) build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name) except TriggerException as tse: raise InvalidRequest(tse.message) except MaximumBuildsQueuedException: abort(429, message="Maximum queued build rate exceeded.") except BuildTriggerDisabledException: abort(400, message="Build trigger is disabled") resp = build_status_view(build_request) repo_string = "%s/%s" % (namespace_name, repo_name) headers = { "Location": api.url_for( RepositoryBuildStatus, repository=repo_string, build_uuid=build_request.uuid ), } return resp, 201, headers
def post(self, namespace_name, repo_name, trigger_uuid): """ List the subdirectories available for the specified build trigger and source. """ trigger = get_trigger(trigger_uuid) user_permission = UserAdminPermission(trigger.connected_user.username) if user_permission.can(): new_config_dict = request.get_json() handler = BuildTriggerHandler.get_handler(trigger, new_config_dict) try: subdirs = handler.list_build_subdirs() context_map = {} for file in subdirs: context_map = handler.get_parent_directory_mappings(file, context_map) return { "dockerfile_paths": ["/" + subdir for subdir in subdirs], "contextMap": context_map, "status": "success", } except EmptyRepositoryException as exc: return { "status": "success", "contextMap": {}, "dockerfile_paths": [], } except TriggerException as exc: return { "status": "error", "message": exc.message, } else: raise Unauthorized()
def list_build_source_namespaces(self): bitbucket_client = self._get_authorized_client() (result, data, err_msg) = bitbucket_client.get_visible_repositories() if not result: raise RepositoryReadException('Could not read repository list: ' + err_msg) namespaces = {} for repo in data: owner = repo['owner'] if owner in namespaces: namespaces[owner]['score'] = namespaces[owner]['score'] + 1 else: namespaces[owner] = { 'personal': owner == self.config.get('nickname', self.config.get('username')), 'id': owner, 'title': owner, 'avatar_url': repo['logo'], 'url': 'https://bitbucket.org/%s' % (owner), 'score': 1, } return BuildTriggerHandler.build_namespaces_response(namespaces)
def put(self, namespace_name, repo_name, trigger_uuid): """ Updates the specified build trigger. """ trigger = get_trigger(trigger_uuid) handler = BuildTriggerHandler.get_handler(trigger) if not handler.is_active(): raise InvalidRequest("Cannot update an unactivated trigger") enable = request.get_json()["enabled"] model.build.toggle_build_trigger(trigger, enable) log_action( "toggle_repo_trigger", namespace_name, { "repo": repo_name, "trigger_id": trigger_uuid, "service": trigger.service.name, "enabled": enable, }, repo=model.repository.get_repository(namespace_name, repo_name), ) return trigger_view(trigger)
def list_build_source_namespaces(self): bitbucket_client = self._get_authorized_client() (result, data, err_msg) = bitbucket_client.get_visible_repositories() if not result: raise RepositoryReadException("Could not read repository list: " + err_msg) namespaces = {} for repo in data: owner = repo["owner"] if owner in namespaces: namespaces[owner]["score"] = namespaces[owner]["score"] + 1 else: namespaces[owner] = { "personal": owner == self.config.get("nickname", self.config.get("username")), "id": owner, "title": owner, "avatar_url": repo["logo"], "url": "https://bitbucket.org/%s" % (owner), "score": 1, } return BuildTriggerHandler.build_namespaces_response(namespaces)
def list_build_sources_for_namespace(self, namespace): def repo_view(repo): last_modified = dateutil.parser.parse(repo['utc_last_updated']) return { 'name': repo['slug'], 'full_name': '%s/%s' % (repo['owner'], repo['slug']), 'description': repo['description'] or '', 'last_updated': timegm(last_modified.utctimetuple()), 'url': 'https://bitbucket.org/%s/%s' % (repo['owner'], repo['slug']), 'has_admin_permissions': repo['read_only'] is False, 'private': repo['is_private'], } bitbucket_client = self._get_authorized_client() (result, data, err_msg) = bitbucket_client.get_visible_repositories() if not result: raise RepositoryReadException('Could not read repository list: ' + err_msg) repos = [ repo_view(repo) for repo in data if repo['owner'] == namespace ] return BuildTriggerHandler.build_sources_response(repos)
def list_build_source_namespaces(self): gl_client = self._get_authorized_client() current_user = gl_client.user if not current_user: raise RepositoryReadException("Unable to get current user") namespaces = {} for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException): namespace_id = namespace.get_id() if namespace_id in namespaces: namespaces[namespace_id][ "score"] = namespaces[namespace_id]["score"] + 1 else: owner = namespace.attributes["name"] namespaces[namespace_id] = { "personal": namespace.attributes["kind"] == "user", "id": str(namespace_id), "title": namespace.attributes["name"], "avatar_url": namespace.attributes.get("avatar_url"), "score": 1, "url": namespace.attributes.get("web_url") or "", } return BuildTriggerHandler.build_namespaces_response(namespaces)
def list_build_source_namespaces(self): gh_client = self._get_client() usr = gh_client.get_user() # Build the full set of namespaces for the user, starting with their own. namespaces = {} namespaces[usr.login] = { "personal": True, "id": usr.login, "title": usr.name or usr.login, "avatar_url": usr.avatar_url, "url": usr.html_url, "score": usr.plan.private_repos if usr.plan else 0, } for org in usr.get_orgs(): organization = org.login if org.login else org.name # NOTE: We don't load the organization's html_url nor its plan, because doing # so requires loading *each organization* via its own API call in this tight # loop, which was massively slowing down the load time for users when setting # up triggers. namespaces[organization] = { "personal": False, "id": organization, "title": organization, "avatar_url": org.avatar_url, "url": "", "score": 0, } return BuildTriggerHandler.build_namespaces_response(namespaces)
def trigger_view(trigger, can_read=False, can_admin=False, for_build=False): if trigger and trigger.uuid: build_trigger = BuildTriggerHandler.get_handler(trigger) build_source = build_trigger.config.get('build_source') repo_url = build_trigger.get_repository_url() if build_source else None can_read = can_read or can_admin trigger_data = { 'id': trigger.uuid, 'service': trigger.service.name, 'is_active': build_trigger.is_active(), 'build_source': build_source if can_read else None, 'repository_url': repo_url if can_read else None, 'config': build_trigger.config if can_admin else {}, 'can_invoke': can_admin, 'enabled': trigger.enabled, 'disabled_reason': trigger.disabled_reason.name if trigger.disabled_reason else None, } if not for_build and can_admin and trigger.pull_robot: trigger_data['pull_robot'] = user_view(trigger.pull_robot) return trigger_data return None
def list_build_source_namespaces(self): gl_client = self._get_authorized_client() current_user = gl_client.user if not current_user: raise RepositoryReadException('Unable to get current user') namespaces = {} for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException): namespace_id = namespace.get_id() if namespace_id in namespaces: namespaces[namespace_id][ 'score'] = namespaces[namespace_id]['score'] + 1 else: owner = namespace.attributes['name'] namespaces[namespace_id] = { 'personal': namespace.attributes['kind'] == 'user', 'id': str(namespace_id), 'title': namespace.attributes['name'], 'avatar_url': namespace.attributes.get('avatar_url'), 'score': 1, 'url': namespace.attributes.get('web_url') or '', } return BuildTriggerHandler.build_namespaces_response(namespaces)
def list_build_sources_for_namespace(self, namespace): def repo_view(repo): last_modified = dateutil.parser.parse(repo["utc_last_updated"]) return { "name": repo["slug"], "full_name": "%s/%s" % (repo["owner"], repo["slug"]), "description": repo["description"] or "", "last_updated": timegm(last_modified.utctimetuple()), "url": "https://bitbucket.org/%s/%s" % (repo["owner"], repo["slug"]), "has_admin_permissions": repo["read_only"] is False, "private": repo["is_private"], } bitbucket_client = self._get_authorized_client() (result, data, err_msg) = bitbucket_client.get_visible_repositories() if not result: raise RepositoryReadException("Could not read repository list: " + err_msg) repos = [ repo_view(repo) for repo in data if repo["owner"] == namespace ] return BuildTriggerHandler.build_sources_response(repos)
def attach_bitbucket_build_trigger(trigger_uuid): trigger = model.build.get_build_trigger(trigger_uuid) if not trigger or trigger.service.name != BitbucketBuildTrigger.service_name( ): abort(404) if trigger.connected_user != current_user.db_user(): abort(404) verifier = request.args.get('oauth_verifier') handler = BuildTriggerHandler.get_handler(trigger) result = handler.exchange_verifier(verifier) if not result: trigger.delete_instance() return 'Token has expired' namespace = trigger.repository.namespace_user.username repository = trigger.repository.name repo_path = '%s/%s' % (namespace, repository) full_url = url_for('web.buildtrigger', path=repo_path, trigger=trigger.uuid) logger.debug('Redirecting to full url: %s', full_url) return redirect(full_url)
def delete(self, namespace_name, repo_name, trigger_uuid): """ Delete the specified build trigger. """ trigger = get_trigger(trigger_uuid) handler = BuildTriggerHandler.get_handler(trigger) if handler.is_active(): try: handler.deactivate() except TriggerException as ex: # We are just going to eat this error logger.warning("Trigger deactivation problem: %s", ex) log_action( "delete_repo_trigger", namespace_name, {"repo": repo_name, "trigger_id": trigger_uuid, "service": trigger.service.name}, repo=model.repository.get_repository(namespace_name, repo_name), ) trigger.delete_instance(recursive=True) if trigger.write_token is not None: trigger.write_token.delete_instance() return "No Content", 204
def test_subdir_path_map_no_previous(input, output): actual_mapping = BuildTriggerHandler.get_parent_directory_mappings(input) for key in actual_mapping: value = actual_mapping[key] actual_mapping[key] = value.sort() for key in output: value = output[key] output[key] = value.sort() assert actual_mapping == output
def test_subdir_path_map(new_path, original_dictionary, output): actual_mapping = BuildTriggerHandler.get_parent_directory_mappings( new_path, original_dictionary) for key in actual_mapping: value = actual_mapping[key] actual_mapping[key] = value.sort() for key in output: value = output[key] output[key] = value.sort() assert actual_mapping == output
def get(self, namespace_name, repo_name, trigger_uuid): """ List the build sources for the trigger configuration thus far. """ trigger = get_trigger(trigger_uuid) user_permission = UserAdminPermission(trigger.connected_user.username) if user_permission.can(): handler = BuildTriggerHandler.get_handler(trigger) try: return {"namespaces": handler.list_build_source_namespaces()} except TriggerException as rre: raise InvalidRequest(rre.message) else: raise Unauthorized()
def put(self, namespace_name, repo_name, trigger_uuid): """ Updates the specified build trigger. """ trigger = get_trigger(trigger_uuid) handler = BuildTriggerHandler.get_handler(trigger) if not handler.is_active(): raise InvalidRequest('Cannot update an unactivated trigger') enable = request.get_json()['enabled'] model.build.toggle_build_trigger(trigger, enable) log_action('toggle_repo_trigger', namespace_name, {'repo': repo_name, 'trigger_id': trigger_uuid, 'service': trigger.service.name, 'enabled': enable}, repo=model.repository.get_repository(namespace_name, repo_name)) return trigger_view(trigger)
def post(self, namespace_name, repo_name, trigger_uuid, field_name): """ List the field values for a custom run field. """ trigger = get_trigger(trigger_uuid) config = request.get_json() or None if AdministerRepositoryPermission(namespace_name, repo_name).can(): handler = BuildTriggerHandler.get_handler(trigger, config) values = handler.list_field_values(field_name, limit=FIELD_VALUE_LIMIT) if values is None: raise NotFound() return {"values": values} else: raise Unauthorized()
def post(self, namespace_name, repo_name, trigger_uuid): """ List the build sources for the trigger configuration thus far. """ namespace = request.get_json().get("namespace") if namespace is None: raise InvalidRequest() trigger = get_trigger(trigger_uuid) user_permission = UserAdminPermission(trigger.connected_user.username) if user_permission.can(): handler = BuildTriggerHandler.get_handler(trigger) try: return {"sources": handler.list_build_sources_for_namespace(namespace)} except TriggerException as rre: raise InvalidRequest(str(rre)) from rre else: raise Unauthorized()
def to_dict(self): if not self.uuid: return None build_trigger = BuildTriggerHandler.get_handler(self) build_source = build_trigger.config.get('build_source') repo_url = build_trigger.get_repository_url() if build_source else None can_read = self.can_read or self.can_admin trigger_data = { 'id': self.uuid, 'service': self.service_name, 'is_active': build_trigger.is_active(), 'build_source': build_source if can_read else None, 'repository_url': repo_url if can_read else None, 'config': build_trigger.config if self.can_admin else {}, 'can_invoke': self.can_admin, } if not self.for_build and self.can_admin and self.pull_robot: trigger_data['pull_robot'] = user_view(self.pull_robot) return trigger_data
def to_dict(self): if not self.uuid: return None build_trigger = BuildTriggerHandler.get_handler(self) build_source = build_trigger.config.get("build_source") repo_url = build_trigger.get_repository_url() if build_source else None can_read = self.can_read or self.can_admin trigger_data = { "id": self.uuid, "service": self.service_name, "is_active": build_trigger.is_active(), "build_source": build_source if can_read else None, "repository_url": repo_url if can_read else None, "config": build_trigger.config if self.can_admin else {}, "can_invoke": self.can_admin, } if not self.for_build and self.can_admin and self.pull_robot: trigger_data["pull_robot"] = user_view(self.pull_robot) return trigger_data
def post(self, namespace_name, repo_name, trigger_uuid): """ Activate the specified build trigger. """ trigger = get_trigger(trigger_uuid) handler = BuildTriggerHandler.get_handler(trigger) if handler.is_active(): raise InvalidRequest("Trigger config is not sufficient for activation.") user_permission = UserAdminPermission(trigger.connected_user.username) if user_permission.can(): # Update the pull robot (if any). pull_robot_name = request.get_json().get("pull_robot", None) if pull_robot_name: try: pull_robot = model.user.lookup_robot(pull_robot_name) except model.InvalidRobotException: raise NotFound() # Make sure the user has administer permissions for the robot's namespace. (robot_namespace, _) = parse_robot_username(pull_robot_name) if not AdministerOrganizationPermission(robot_namespace).can(): raise Unauthorized() # Make sure the namespace matches that of the trigger. if robot_namespace != namespace_name: raise Unauthorized() # Set the pull robot. trigger.pull_robot = pull_robot # Update the config. new_config_dict = request.get_json()["config"] write_token_name = "Build Trigger: %s" % trigger.service.name write_token = model.token.create_delegate_token( namespace_name, repo_name, write_token_name, "write" ) try: path = url_for("webhooks.build_trigger_webhook", trigger_uuid=trigger.uuid) authed_url = _prepare_webhook_url( app.config["PREFERRED_URL_SCHEME"], "$token", write_token.get_code(), app.config["SERVER_HOSTNAME"], path, ) handler = BuildTriggerHandler.get_handler(trigger, new_config_dict) final_config, private_config = handler.activate(authed_url) if "private_key" in private_config: trigger.secure_private_key = DecryptedValue(private_config["private_key"]) except TriggerException as exc: write_token.delete_instance() raise request_error(message=exc.message) # Save the updated config. update_build_trigger(trigger, final_config, write_token=write_token) # Log the trigger setup. repo = model.repository.get_repository(namespace_name, repo_name) log_action( "setup_repo_trigger", namespace_name, { "repo": repo_name, "namespace": namespace_name, "trigger_id": trigger.uuid, "service": trigger.service.name, "pull_robot": trigger.pull_robot.username if trigger.pull_robot else None, "config": final_config, }, repo=repo, ) return trigger_view(trigger, can_admin=True) else: raise Unauthorized()
def test_determine_tags(config, metadata, expected_tags): tags = BuildTriggerHandler._determine_tags(config, metadata) assert tags == set(expected_tags)
def test_path_is_dockerfile(input, output): assert BuildTriggerHandler.filename_is_dockerfile(input) == output
def list_build_sources_for_namespace(self, namespace_id): if not namespace_id: return [] def repo_view(repo): # Because *anything* can be None in GitLab API! permissions = repo.attributes.get("permissions") or {} group_access = permissions.get("group_access") or {} project_access = permissions.get("project_access") or {} missing_group_access = permissions.get("group_access") is None missing_project_access = permissions.get("project_access") is None access_level = max( group_access.get("access_level") or 0, project_access.get("access_level") or 0 ) has_admin_permission = _ACCESS_LEVEL_MAP.get(access_level, ("", False))[1] if missing_group_access or missing_project_access: # Default to has permission if we cannot check the permissions. This will allow our users # to select the repository and then GitLab's own checks will ensure that the webhook is # added only if allowed. # TODO: Do we want to display this differently in the UI? has_admin_permission = True view = { "name": repo.attributes["path"], "full_name": repo.attributes["path_with_namespace"], "description": repo.attributes.get("description") or "", "url": repo.attributes.get("web_url"), "has_admin_permissions": has_admin_permission, "private": repo.attributes.get("visibility") == "private", } if repo.attributes.get("last_activity_at"): try: last_modified = dateutil.parser.parse(repo.attributes["last_activity_at"]) view["last_updated"] = timegm(last_modified.utctimetuple()) except ValueError: logger.exception( "Gitlab gave us an invalid last_activity_at: %s", last_modified ) return view gl_client = self._get_authorized_client() try: gl_namespace = gl_client.namespaces.get(namespace_id) except gitlab.GitlabGetError: return [] namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True) repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException) try: return BuildTriggerHandler.build_sources_response( [repo_view(repo) for repo in repositories] ) except gitlab.GitlabGetError: return []