def run(self): metadata = get_build_json().get("metadata", {}) kwargs = {} # FIXME: remove `openshift_uri` once osbs-client is released osbs_conf = Configuration(conf_file=None, openshift_uri=self.url, openshift_url=self.url, use_auth=self.use_auth, verify_ssl=self.verify_ssl, build_json_dir=self.build_json_dir, namespace=metadata.get('namespace', None)) osbs = OSBS(osbs_conf, osbs_conf) imagestream = None try: imagestream = osbs.get_image_stream(self.imagestream) except OsbsResponseException: if self.insecure_registry is not None: kwargs['insecure_registry'] = self.insecure_registry self.log.info("Creating ImageStream %s for %s", self.imagestream, self.docker_image_repo) imagestream = osbs.create_image_stream(self.imagestream, self.docker_image_repo, **kwargs) self.log.info("Importing new tags for %s", self.imagestream) primaries = None try: primaries = self.workflow.build_result.annotations['repositories'][ 'primary'] except (TypeError, KeyError): self.log.exception( 'Unable to read primary repositories annotations') if not primaries: raise RuntimeError('Could not find primary images in workflow') failures = False for s in primaries: tag_image_name = ImageName.parse(s) tag = tag_image_name.tag try: osbs.ensure_image_stream_tag(imagestream.json(), tag) self.log.info("Imported ImageStreamTag: (%s)", tag) except OsbsResponseException: failures = True self.log.info("Could not import ImageStreamTag: (%s)", tag) if failures: raise RuntimeError( "Failed to import ImageStreamTag(s). Check logs") osbs.import_image(self.imagestream)
class MockCreator(object): def __init__(self): parser = argparse.ArgumentParser( description="osbs test harness mock JSON creator") parser.add_argument( "user", action='store', help="name of user to use for Basic Authentication in OSBS") parser.add_argument("--config", action='store', metavar="PATH", help="path to configuration file", default=DEFAULT_CONFIGURATION_FILE) parser.add_argument( "--instance", "-i", action='store', metavar="SECTION_NAME", help="section within config for requested instance", default="stage") parser.add_argument( "--password", action='store', help="password to use for Basic Authentication in OSBS") parser.add_argument("--mock-dir", metavar="DIR", action="store", default=DEFAULT_DIR, help="mock JSON responses are stored in DIR") parser.add_argument( "--imagestream", metavar="IMAGESTREAM", action="store", default=DEFAULT_IMAGESTREAM_FILE, help="Image name for image stream import. Defaults to " + DEFAULT_IMAGESTREAM_FILE) parser.add_argument( "--image_server", metavar="IMAGESERVER", action="store", default=DEFAULT_IMAGESTREAM_SERVER, help="Server for image stream import. Defaults to " + DEFAULT_IMAGESTREAM_SERVER) parser.add_argument( "--image_tags", metavar="IMAGETAGS", action="store", nargs=3, default=DEFAULT_IMAGESTREAM_TAGS, help="Image stream tags as 3 space separated values.") parser.add_argument("--os-version", metavar="OS_VER", action="store", default=OS_VERSION, help="OpenShift version of the mock JSONs") args = parser.parse_args() self.user = args.user self.password = args.password mock_path = args.mock_dir self.mock_dir = "/".join([mock_path, args.os_version]) find_or_make_dir(self.mock_dir) self.capture_dir = tempfile.mkdtemp() args.git_url = "https://github.com/TomasTomecek/docker-hello-world.git" args.git_branch = "master" args.git_commit = "HEAD" os_conf = Configuration(conf_file=args.config, conf_section=args.instance, cli_args=args) build_conf = Configuration(conf_file=args.config, conf_section=args.instance, cli_args=args) set_logging(level=logging.INFO) self.osbs = OSBS(os_conf, build_conf) setup_json_capture(self.osbs, os_conf, self.capture_dir) self.imagestream_file = args.imagestream self.imagestream_server = args.image_server self.imagestream_tags = args.image_tags self.rh_pattern = re.template("redhat.com") self.ex_pattern = "(\S+\.)*redhat.com" # noqa:W605 def clean_data(self, out_data): if isinstance(out_data, dict): cleaned_data = {} for key, data in out_data.items(): cleaned_data[key] = self.clean_data(data) return cleaned_data elif isinstance(out_data, list): cleaned_data = [] for data in out_data: cleaned_data.append(self.clean_data(data)) return cleaned_data elif isinstance(out_data, str): if re.search(self.rh_pattern, out_data): return re.sub(self.ex_pattern, "example.com", out_data) else: return out_data else: return out_data def comp_write(self, out_name, out_data): cleaned_data = self.clean_data(out_data) out_path = "/".join([self.mock_dir, out_name]) with open(out_path, "w") as outf: try: json.dump(cleaned_data, outf, indent=4) except (ValueError, TypeError): outf.write(json.dumps(cleaned_data, indent=4)) def create_mock_builds_list(self): kwargs = {} # get build list self.osbs.list_builds(**kwargs) # find 'get-namespaces_osbs-stage_builds_-000.json' file and parse it into # 'builds_list.json', 'builds_list_empty.json', 'builds_list_one.json' all_builds = "get-namespaces_osbs-stage_builds_-000.json" all_builds_path = "/".join([self.capture_dir, all_builds]) with open(all_builds_path, "r") as infile: builds_data = json.load(infile) builds_items = copy.copy(builds_data["items"]) builds_data["items"] = [] self.comp_write("builds_list_empty.json", builds_data) if not builds_items: return builds_data["items"].append(builds_items[0]) self.comp_write("builds_list_one.json", builds_data) if len(builds_items) < 2: return builds_data["items"].append(builds_items[1]) self.comp_write("builds_list.json", builds_data) os.remove(all_builds_path) def create_pods_list(self, build_id): self.osbs.get_pod_for_build(build_id) pods_pre = "get-namespaces_osbs-stage_pods_?labelSelector=openshift.io%2Fbuild.name%3D" for i in range(0, 4): try: pods_fname = pods_pre + build_id + "-00{}.json".format(i) pods_inpath = "/".join([self.capture_dir, pods_fname]) os.stat(pods_inpath) break except OSError: continue with open(pods_inpath, "r") as infile: pods_data = json.load(infile) image = "buildroot:latest" pods_items = pods_data["items"] or [] for pod in pods_items: pod_containers = pod["status"]["containerStatuses"] or [] for container in pod_containers: container["imageID"] = "docker-pullable://" + image container["image"] = image self.comp_write("pods.json", pods_data) os.remove(pods_inpath) def create_mock_get_user(self): self.osbs.get_user() user_list = "get-users_~_-000.json" user_list_path = "/".join([self.capture_dir, user_list]) with open(user_list_path, "r") as infile: user_data = json.load(infile) user_data["groups"] = [] user_data["identities"] = None if "fullName" in user_data: del user_data["fullName"] user_data["metadata"]["name"] = "test" user_data["metadata"][ "selfLink"] = "/apis/user.openshift.io/v1/users/test" self.comp_write("get_user.json", user_data) os.remove(user_list_path) def create_a_mock_build(self, func, build_name, out_tag, build_args): try: build = func(**build_args) build_id = build.get_build_name() build = self.osbs.wait_for_build_to_get_scheduled(build_id) self.osbs.watch_builds() build = self.osbs.wait_for_build_to_finish(build_id) self.osbs.get_build_logs(build_id) except (subprocess.CalledProcessError, OsbsException): pass watch_data = canonize_data(copy.deepcopy(build.json), build_name, "Complete") watch_obj = {"object": watch_data, "type": "MODIFIED"} out_name = "watch_build_test-" + out_tag + "build-123.json" self.comp_write(out_name, watch_obj) build_fname = "get-namespaces_osbs-stage_builds_" + build_id + "_-000.json" build_path = "/".join([self.capture_dir, build_fname]) with open(build_path, "r") as infile: build_data = canonize_data(json.load(infile), build_name, "Complete") out_name = "build_test-" + out_tag + "build-123.json" self.comp_write(out_name, build_data) os.remove(build_path) return out_name def create_mock_build(self): build_kwargs = { 'git_uri': self.osbs.build_conf.get_git_uri(), 'git_ref': self.osbs.build_conf.get_git_ref(), 'git_branch': self.osbs.build_conf.get_git_branch(), 'user': self.osbs.build_conf.get_user(), 'release': TEST_BUILD, 'platform': "x86_64", 'arrangement_version': DEFAULT_ARRANGEMENT_VERSION, 'scratch': True, } build_name = self.create_a_mock_build(self.osbs.create_worker_build, TEST_BUILD, "", build_kwargs) build_path = "/".join([self.mock_dir, build_name]) if os.stat(build_path): with open(build_path, "r") as infile: build_data = json.load(infile) build_data["kind"] = "BuildConfig" self.comp_write( "created_build_config_test-build-config-123.json", build_data) del build_kwargs['platform'] self.create_a_mock_build(self.osbs.create_orchestrator_build, TEST_ORCHESTRATOR_BUILD, "orchestrator-", build_kwargs) def create_mock_imagestream(self): imagestream_repo = "/".join( [self.imagestream_server, self.imagestream_file]) try: imagestream = self.osbs.create_image_stream( TEST_IMAGESTREAM, imagestream_repo) except OsbsResponseException: print("imagestream {} already exists.".format(TEST_IMAGESTREAM)) print("run `oc delete imagestream {}` and try again".format( TEST_IMAGESTREAM)) exit(0) self.osbs.import_image_tags(TEST_IMAGESTREAM, self.imagestream_tags, imagestream_repo) for tag in self.imagestream_tags: self.osbs.ensure_image_stream_tag(imagestream.json(), tag) imagestreamimport_fname = "post-namespaces_osbs-stage_imagestreamimports_-000.json" imagestreamimport_path = "/".join( [self.capture_dir, imagestreamimport_fname]) imagestreamimport_data = [] imagestream_data = [] with open(imagestreamimport_path, "r") as infile: imagestreamimport_data = json.load(infile) self.osbs.get_image_stream(TEST_IMAGESTREAM) imagestream_fname = "get-namespaces_osbs-stage_imagestreams_test_imagestream-001.json" imagestream_path = "/".join([self.capture_dir, imagestream_fname]) with open(imagestream_path, "r") as infile: imagestream_data = json.load(infile) # I'm not really happy with setting this value, but the imagestream tests in test_conf.py # fail without it imagestream_data["spec"].setdefault( "dockerImageRepository", imagestream_data["status"].get("dockerImageRepository")) # override the tags to the values expected by the tasks for i in range(0, 3): tag_line = "7.2.username-{}".format(66 + i) tag_name = "example.com:8888/username/rhel7:" + tag_line spec_tag = imagestream_data["spec"]["tags"][i] spec_tag["from"]["name"] = tag_name spec_tag["name"] = tag_line status_tag = imagestream_data["status"]["tags"][i] status_tag["tag"] = tag_line import_status = imagestreamimport_data["status"] import_status["images"][i]["tag"] = tag_line import_status["import"]["spec"]["tags"][i]["name"] = tag_line import_status["import"]["status"]["tags"][i]["tag"] = tag_line self.comp_write("imagestreamimport.json", imagestreamimport_data) self.comp_write("imagestream.json", imagestream_data) def create_mock_build_other(self): build_kwargs = { 'git_uri': self.osbs.build_conf.get_git_uri(), 'git_ref': self.osbs.build_conf.get_git_ref(), 'git_branch': self.osbs.build_conf.get_git_branch(), 'user': self.osbs.build_conf.get_user(), 'release': TEST_BUILD, 'platform': "x86_64", 'arrangement_version': DEFAULT_ARRANGEMENT_VERSION, 'scratch': True, } build_id = "" try: build = self.osbs.create_worker_build(**build_kwargs) build_id = build.get_build_name() self.osbs.wait_for_build_to_get_scheduled(build_id) self.create_pods_list(build_id) self.osbs.cancel_build(build_id) self.osbs.wait_for_build_to_finish(build_id) self.osbs.get_build_logs(build_id) except OsbsException: self.create_pods_list(build_id) instant_fname = "post-namespaces_osbs-stage_builds_-000.json" instant_path = "/".join([self.capture_dir, instant_fname]) with open(instant_path, "r") as infile: instant_data = canonize_data(json.load(infile)) self.comp_write("instantiated_test-build-config-123.json", instant_data) os.remove(instant_path) cancel_args = [ { "suffix": "_-000-001.json", "version": "get", "phase": None }, { "suffix": "_-000-000.json", "version": "put", "phase": "Cancelled" }, ] for data in cancel_args: build_fname = "get-watch_namespaces_osbs-stage_builds_" + build_id + data[ "suffix"] build_path = "/".join([self.capture_dir, build_fname]) with open(build_path, "r") as infile: cancel_obj = json.load(infile) cancel_data = canonize_data( copy.deepcopy(cancel_obj["object"]), TEST_CANCELLED_BUILD, data["phase"]) self.comp_write( "build_test-build-cancel-123_" + data["version"] + ".json", cancel_data) os.remove(build_path) def create_mock_static_files(self): # these aren't JSON, so just write them out out_path = "/".join( [self.mock_dir, "build_test-orchestrator-build-123_logs.txt"]) with open(out_path, "w") as outf: outf.write(ORCH_BUILD_LOG) out_path = "/".join([self.mock_dir, "build_test-build-123_logs.txt"]) with open(out_path, "wb") as outf: outf.write(BASE_BUILD_LOG) self.comp_write("create_config_map.json", MOCK_CONFIG_MAP)
class ImportImagePlugin(PostBuildPlugin): """ Import image tags from external docker registry into Origin, creating an ImageStream if one does not already exist. """ key = 'import_image' is_allowed_to_fail = False def __init__(self, tasker, workflow, imagestream, docker_image_repo, url, build_json_dir, verify_ssl=True, use_auth=True, insecure_registry=None): """ constructor :param tasker: DockerTasker instance :param workflow: DockerBuildWorkflow instance :param imagestream: str, name of ImageStream :param docker_image_repo: str, image repository to import tags from :param url: str, URL to OSv3 instance :param build_json_dir: str, path to directory with input json :param verify_ssl: bool, verify SSL certificate? :param use_auth: bool, initiate authentication with openshift? :param insecure_registry: bool, whether the Docker registry uses plain HTTP """ # call parent constructor super(ImportImagePlugin, self).__init__(tasker, workflow) self.imagestream_name = imagestream self.docker_image_repo = docker_image_repo self.url = url self.build_json_dir = build_json_dir self.verify_ssl = verify_ssl self.use_auth = use_auth self.insecure_registry = insecure_registry self.osbs = None self.imagestream = None def run(self): self.setup_osbs_api() self.get_or_create_imagestream() self.process_tags() self.osbs.import_image(self.imagestream_name) def setup_osbs_api(self): metadata = get_build_json().get("metadata", {}) osbs_conf = Configuration(conf_file=None, openshift_url=self.url, use_auth=self.use_auth, verify_ssl=self.verify_ssl, build_json_dir=self.build_json_dir, namespace=metadata.get('namespace', None)) self.osbs = OSBS(osbs_conf, osbs_conf) def get_or_create_imagestream(self): try: self.imagestream = self.osbs.get_image_stream(self.imagestream_name) except OsbsResponseException: kwargs = {} if self.insecure_registry is not None: kwargs['insecure_registry'] = self.insecure_registry self.log.info('Creating ImageStream %s for %s', self.imagestream_name, self.docker_image_repo) self.imagestream = self.osbs.create_image_stream(self.imagestream_name, self.docker_image_repo, **kwargs) def process_tags(self): self.log.info('Importing new tags for %s', self.imagestream_name) failures = False for tag in self.get_trackable_tags(): try: self.osbs.ensure_image_stream_tag(self.imagestream.json(), tag) self.log.info('Imported ImageStreamTag: (%s)', tag) except OsbsResponseException: failures = True self.log.info('Could not import ImageStreamTag: (%s)', tag) if failures: raise RuntimeError('Failed to import ImageStreamTag(s). Check logs') def get_trackable_tags(self): primary_images = get_primary_images(self.workflow) if not primary_images: raise RuntimeError('Could not find primary images in workflow') tags = [] for primary_image in primary_images: tag = primary_image.tag if '-' in tag: self.log.info('Skipping non-transient tag, %s', tag) continue tags.append(tag) return tags
class ImportImagePlugin(PostBuildPlugin): """ Import image tags from external docker registry into Origin, creating an ImageStream if one does not already exist. """ key = 'import_image' is_allowed_to_fail = False def __init__(self, tasker, workflow, imagestream, docker_image_repo, url, build_json_dir, verify_ssl=True, use_auth=True, insecure_registry=None): """ constructor :param tasker: DockerTasker instance :param workflow: DockerBuildWorkflow instance :param imagestream: str, name of ImageStream :param docker_image_repo: str, image repository to import tags from :param url: str, URL to OSv3 instance :param build_json_dir: str, path to directory with input json :param verify_ssl: bool, verify SSL certificate? :param use_auth: bool, initiate authentication with openshift? :param insecure_registry: bool, whether the Docker registry uses plain HTTP """ # call parent constructor super(ImportImagePlugin, self).__init__(tasker, workflow) self.imagestream_name = imagestream self.docker_image_repo = docker_image_repo self.url = url self.build_json_dir = build_json_dir self.verify_ssl = verify_ssl self.use_auth = use_auth self.insecure_registry = insecure_registry self.osbs = None self.imagestream = None def run(self): self.setup_osbs_api() self.get_or_create_imagestream() self.process_tags() self.osbs.import_image(self.imagestream_name) def setup_osbs_api(self): metadata = get_build_json().get("metadata", {}) osbs_conf = Configuration(conf_file=None, openshift_url=self.url, use_auth=self.use_auth, verify_ssl=self.verify_ssl, build_json_dir=self.build_json_dir, namespace=metadata.get('namespace', None)) self.osbs = OSBS(osbs_conf, osbs_conf) def get_or_create_imagestream(self): try: self.imagestream = self.osbs.get_image_stream( self.imagestream_name) except OsbsResponseException: kwargs = {} if self.insecure_registry is not None: kwargs['insecure_registry'] = self.insecure_registry self.log.info('Creating ImageStream %s for %s', self.imagestream_name, self.docker_image_repo) self.imagestream = self.osbs.create_image_stream( self.imagestream_name, self.docker_image_repo, **kwargs) def process_tags(self): self.log.info('Importing new tags for %s', self.imagestream_name) failures = False for tag in self.get_trackable_tags(): try: self.osbs.ensure_image_stream_tag(self.imagestream.json(), tag) self.log.info('Imported ImageStreamTag: (%s)', tag) except OsbsResponseException: failures = True self.log.info('Could not import ImageStreamTag: (%s)', tag) if failures: raise RuntimeError( 'Failed to import ImageStreamTag(s). Check logs') def get_trackable_tags(self): primary_images = get_primary_images(self.workflow) if not primary_images: raise RuntimeError('Could not find primary images in workflow') tags = [] for primary_image in primary_images: tag = primary_image.tag if '-' in tag: self.log.info('Skipping non-transient tag, %s', tag) continue tags.append(tag) return tags