def _process_request(self):
        self.log.debug("request_json: %s", self.request_json)

        # if request_hash is available check the database directly
        if "request_hash" in self.request_json:
            self.request = self.database.check_build_request_hash(
                self.request_json["request_hash"])

            if not self.request:
                self.response_status = HTTPStatus.NOT_FOUND
                return self.respond()
            else:
                return self.return_status()
        else:
            # required params for a build request
            missing_params = self.check_missing_params(
                ["distro", "version", "target", "subtarget", "board"])
            if missing_params:
                return self.respond()

        self.request_json["profile"] = self.request_json[
            "board"]  # TODO fix this workaround

        if "defaults" in self.request_json:
            # check if the uci file exceeds the max file size. this should be
            # done as the uci-defaults are at least temporary stored in the
            # database to be passed to a worker
            if getsizeof(self.request_json["defaults"]) > self.config.get(
                    "max_defaults_size"):
                self.response_json[
                    "error"] = "attached defaults exceed max size"
                self.response_status = 420  # this error code is the best I could find
                self.respond()

        # create image object to get the request_hash
        image = Image(self.request_json)
        image.set_packages_hash()
        request_hash = get_hash(" ".join(image.as_array("packages_hash")), 12)
        request_database = self.database.check_build_request_hash(request_hash)

        # if found return instantly the status
        if request_database:
            self.log.debug("found image in database: %s",
                           request_database["status"])
            self.request = request_database
            return self.return_status()
        else:
            self.request["request_hash"] = request_hash

        self.request["packages_hash"] = image.params[
            "packages_hash"]  # TODO make this better

        # if not perform various checks to see if the request is acutally valid
        # check for valid distro and version
        bad_request = self.check_bad_request()
        if bad_request:
            return bad_request

        # check for valid target and subtarget
        bad_target = self.check_bad_target()
        if bad_target:
            return bad_target

        # check for existing packages
        bad_packages = self.check_bad_packages()
        if bad_packages:
            return bad_packages

        # add package_hash to database
        self.database.insert_packages_hash(self.request["packages_hash"],
                                           self.request["packages"])

        # now some heavy guess work is done to figure out the profile
        # eventually this could be simplified if upstream unifirm the profiles/boards
        if "board" in self.request_json:
            self.log.debug("board in request, search for %s",
                           self.request_json["board"])
            self.request["profile"] = self.database.check_profile(
                self.request["distro"], self.request["version"],
                self.request["target"], self.request["subtarget"],
                self.request_json["board"])

        if not self.request["profile"]:
            if "model" in self.request_json:
                self.log.debug("model in request, search for %s",
                               self.request_json["model"])
                self.request["profile"] = self.database.check_model(
                    self.request["distro"], self.request["version"],
                    self.request["target"], self.request["subtarget"],
                    self.request_json["model"])
                self.log.debug("model search found profile %s",
                               self.request["profile"])

        if not self.request["profile"]:
            if self.database.check_profile(self.request["distro"],
                                           self.request["version"],
                                           self.request["target"],
                                           self.request["subtarget"],
                                           "Generic"):
                self.request["profile"] = "Generic"
            elif self.database.check_profile(self.request["distro"],
                                             self.request["version"],
                                             self.request["target"],
                                             self.request["subtarget"],
                                             "generic"):
                self.request["profile"] = "generic"
            else:
                self.response_json[
                    "error"] = "unknown device, please check model and board params"
                self.response_status = HTTPStatus.PRECONDITION_FAILED  # 412
                return self.respond()

        self.request["defaults_hash"] = image.params["defaults_hash"]
        # check if a default uci config is attached to the request
        if image.params["defaults_hash"] != "":
            self.database.insert_defaults(image.params["defaults_hash"],
                                          self.request_json["defaults"])

        # all checks passed, eventually add to queue!
        self.request.pop("packages")
        self.log.debug("add build job %s", self.request)
        self.database.add_build_job(self.request)
        return self.return_queued()
class Worker(threading.Thread):
    def __init__(self, location, job, queue):
        self.location = location
        self.queue = queue
        self.job = job
        threading.Thread.__init__(self)
        self.log = logging.getLogger(__name__)
        self.log.info("log initialized")
        self.config = Config()
        self.log.info("config initialized")
        self.database = Database(self.config)
        self.log.info("database initialized")

    def setup_meta(self):
        self.log.debug("setup meta")
        os.makedirs(self.location, exist_ok=True)
        if not os.path.exists(self.location + "/meta"):
            cmdline = "git clone https://github.com/aparcar/meta-imagebuilder.git ."
            proc = subprocess.Popen(
                cmdline.split(" "),
                cwd=self.location,
                stdout=subprocess.PIPE,
                shell=False,
            )

            _, errors = proc.communicate()
            return_code = proc.returncode

            if return_code != 0:
                self.log.error("failed to setup meta ImageBuilder")
                exit()

        self.log.info("meta ImageBuilder successfully setup")

    def write_log(self, path, stdout=None, stderr=None):
        with open(path, "a") as log_file:
            log_file.write("### BUILD COMMAND:\n\n")
            for key, value in self.params.items():
                log_file.write("{}={}\n".format(key.upper(), str(value)))
            log_file.write("sh meta\n")
            if stdout:
                log_file.write("\n\n### STDOUT:\n\n" + stdout)
            if stderr:
                log_file.write("\n\n### STDERR:\n\n" + stderr)

    # build image
    def build(self):
        self.log.debug("create and parse manifest")

        # fail path in case of erros
        fail_log_path = self.config.get_folder(
            "download_folder") + "/faillogs/faillog-{}.txt".format(
                self.params["request_hash"])

        self.image = Image(self.params)

        # first determine the resulting manifest hash
        return_code, manifest_content, errors = self.run_meta("manifest")

        if return_code == 0:
            self.image.params["manifest_hash"] = get_hash(manifest_content, 15)

            manifest_pattern = r"(.+) - (.+)\n"
            manifest_packages = dict(
                re.findall(manifest_pattern, manifest_content))
            self.database.add_manifest_packages(
                self.image.params["manifest_hash"], manifest_packages)
            self.log.info("successfully parsed manifest")
        else:
            self.log.error("couldn't determine manifest")
            self.write_log(fail_log_path, stderr=errors)
            self.database.set_image_requests_status(
                self.params["request_hash"], "manifest_fail")
            return False

        # set directory where image is stored on server
        self.image.set_image_dir()
        self.log.debug("dir %s", self.image.params["dir"])

        # calculate hash based on resulted manifest
        self.image.params["image_hash"] = get_hash(
            " ".join(self.image.as_array("manifest_hash")), 15)

        # set log path in case of success
        success_log_path = self.image.params[
            "dir"] + "/buildlog-{}.txt".format(self.params["image_hash"])

        # set build_status ahead, if stuff goes wrong it will be changed
        self.build_status = "created"

        # check if image already exists
        if not self.image.created():
            self.log.info("build image")
            with tempfile.TemporaryDirectory(
                    dir=self.config.get_folder("tempdir")) as build_dir:
                # now actually build the image with manifest hash as EXTRA_IMAGE_NAME
                self.params["worker"] = self.location
                self.params["BIN_DIR"] = build_dir
                self.params["j"] = str(os.cpu_count())
                self.params["EXTRA_IMAGE_NAME"] = self.params["manifest_hash"]
                # if uci defaults are added, at least at parts of the hash to time image name
                if self.params["defaults_hash"]:
                    defaults_dir = build_dir + "/files/etc/uci-defaults/"
                    # create folder to store uci defaults
                    os.makedirs(defaults_dir)
                    # request defaults content from database
                    defaults_content = self.database.get_defaults(
                        self.params["defaults_hash"])
                    with open(defaults_dir + "99-server-defaults",
                              "w") as defaults_file:
                        defaults_file.write(
                            defaults_content
                        )  # TODO check if special encoding is required

                    # tell ImageBuilder to integrate files
                    self.params["FILES"] = build_dir + "/files/"
                    self.params["EXTRA_IMAGE_NAME"] += "-" + self.params[
                        "defaults_hash"][:6]

                # download is already performed for manifest creation
                self.params["NO_DOWNLOAD"] = "1"

                build_start = time.time()
                return_code, buildlog, errors = self.run_meta("image")
                self.image.params["build_seconds"] = int(time.time() -
                                                         build_start)

                if return_code == 0:
                    # create folder in advance
                    os.makedirs(self.image.params["dir"], exist_ok=True)

                    self.log.debug(os.listdir(build_dir))

                    for filename in os.listdir(build_dir):
                        if os.path.exists(self.image.params["dir"] + "/" +
                                          filename):
                            break
                        shutil.move(build_dir + "/" + filename,
                                    self.image.params["dir"])

                    # possible sysupgrade names, ordered by likeliness
                    possible_sysupgrade_files = [
                        "*-squashfs-sysupgrade.bin",
                        "*-squashfs-sysupgrade.tar", "*-squashfs.trx",
                        "*-squashfs.chk", "*-squashfs.bin",
                        "*-squashfs-sdcard.img.gz", "*-combined-squashfs*",
                        "*.img.gz"
                    ]

                    sysupgrade = None

                    for sysupgrade_file in possible_sysupgrade_files:
                        sysupgrade = glob.glob(self.image.params["dir"] + "/" +
                                               sysupgrade_file)
                        if sysupgrade:
                            break

                    if not sysupgrade:
                        self.log.debug("sysupgrade not found")
                        if buildlog.find("too big") != -1:
                            self.log.warning("created image was to big")
                            self.database.set_image_requests_status(
                                self.params["request_hash"], "imagesize_fail")
                            self.write_log(fail_log_path, buildlog, errors)
                            return False
                        else:
                            self.build_status = "no_sysupgrade"
                            self.image.params["sysupgrade"] = ""
                    else:
                        self.image.params["sysupgrade"] = os.path.basename(
                            sysupgrade[0])

                    self.write_log(success_log_path, buildlog)
                    self.database.add_image(self.image.get_params())
                    self.log.info("build successfull")
                else:
                    self.log.info("build failed")
                    self.database.set_image_requests_status(
                        self.params["request_hash"], 'build_fail')
                    self.write_log(fail_log_path, buildlog, errors)
                    return False

        self.log.info("link request %s to image %s",
                      self.params["request_hash"], self.params["image_hash"])
        self.database.done_build_job(self.params["request_hash"],
                                     self.image.params["image_hash"],
                                     self.build_status)
        return True

    def run(self):
        self.setup_meta()

        while True:
            self.params = self.queue.get()
            self.version_config = self.config.version(self.params["distro"],
                                                      self.params["version"])
            if self.job == "image":
                self.build()
            elif self.job == "update":
                self.info()
                self.parse_packages()

    def info(self):
        self.parse_info()
        if os.path.exists(
                os.path.join(self.location, "imagebuilder",
                             self.params["distro"], self.params["version"],
                             self.params["target"], self.params["subtarget"],
                             "target/linux", self.params["target"],
                             "base-files/lib/upgrade/platform.sh")):
            self.log.info("%s target is supported", self.params["target"])
            self.database.insert_supported(self.params)

    def run_meta(self, cmd):
        cmdline = ["sh", "meta", cmd]
        env = os.environ.copy()

        if "parent_version" in self.version_config:
            self.params["IB_VERSION"] = self.version_config["parent_version"]
        if "repos" in self.version_config:
            self.params["REPOS"] = self.version_config["repos"]

        for key, value in self.params.items():
            env[key.upper()] = str(
                value)  # TODO convert meta script to Makefile

        proc = subprocess.Popen(cmdline,
                                cwd=self.location,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=False,
                                env=env)

        output, errors = proc.communicate()
        return_code = proc.returncode
        output = output.decode('utf-8')
        errors = errors.decode('utf-8')

        return (return_code, output, errors)

    def parse_info(self):
        self.log.debug("parse info")

        return_code, output, errors = self.run_meta("info")

        if return_code == 0:
            default_packages_pattern = r"(.*\n)*Default Packages: (.+)\n"
            default_packages = re.match(default_packages_pattern, output,
                                        re.M).group(2)
            logging.debug("default packages: %s", default_packages)

            profiles_pattern = r"(.+):\n    (.+)\n    Packages: (.*)\n"
            profiles = re.findall(profiles_pattern, output)
            if not profiles:
                profiles = []
            self.database.insert_profiles(
                {
                    "distro": self.params["distro"],
                    "version": self.params["version"],
                    "target": self.params["target"],
                    "subtarget": self.params["subtarget"]
                }, default_packages, profiles)
        else:
            logging.error("could not receive profiles")
            return False

    def parse_packages(self):
        self.log.info("receive packages")

        return_code, output, errors = self.run_meta("package_list")

        if return_code == 0:
            packages = re.findall(r"(.+?) - (.+?) - .*\n", output)
            self.log.info("found {} packages".format(len(packages)))
            self.database.insert_packages_available(
                {
                    "distro": self.params["distro"],
                    "version": self.params["version"],
                    "target": self.params["target"],
                    "subtarget": self.params["subtarget"]
                }, packages)
        else:
            self.log.warning("could not receive packages")