def get_pkg_hash(self):
# sort list and remove duplicates
self.packages = sorted(list(set(self.packages)))
package_hash = get_hash(" ".join(self.packages), 12)
self.log.debug("pkg hash %s - %s", package_hash, self.packages)
self.database.insert_hash(package_hash, self.packages)
return package_hash
def set_packages_hash(self):
# sort and deduplicate requested packages
if "packages" in self.params:
self.params["packages"] = sorted(list(set(
self.params["packages"])))
else:
self.params["packages"] = ""
# calculate hash of packages
self.params["packages_hash"] = get_hash(
" ".join(self.params["packages"]), 12)
def __init__(self, params):
self.config = Config()
self.log = logging.getLogger(__name__)
self.log.info("config initialized")
self.database = Database(self.config)
self.log.info("database initialized")
self.params = params
if not "defaults_hash" in self.params:
self.params["defaults_hash"] = ""
if "defaults" in self.params:
if self.params["defaults"] != "":
self.params["defaults_hash"] = get_hash(
self.params["defaults"], 32)
if not self.params["defaults_hash"]:
self.params["defaults_hash"] = ""
def check_build_request(self, request):
request_array = request.as_array()
request_hash = get_hash(" ".join(request_array), 12)
self.log.debug("check_request")
sql = "select image_hash, id, request_hash, status from image_requests where request_hash = ?"
self.c.execute(sql, request_hash)
if self.c.rowcount == 1:
return self.c.fetchone()
else:
self.log.debug("add build job")
sql = """INSERT INTO image_requests
(request_hash, distro, release, target, subtarget, profile, packages_hash)
VALUES (?, ?, ?, ?, ?, ?, ?)"""
self.c.execute(sql, request_hash, *request_array)
self.commit()
return('', 0, request_hash, 'requested')
def build(self):
imagebuilder_path = os.path.abspath(
os.path.join("imagebuilder", self.distro, self.target,
self.subtarget))
self.imagebuilder = ImageBuilder(self.distro, self.release,
self.target, self.subtarget)
self.log.info("use imagebuilder %s", self.imagebuilder.path)
with tempfile.TemporaryDirectory(
dir=get_folder("tempdir")) as self.build_path:
already_created = False
# only add manifest hash if special packages
extra_image_name_array = []
if not self.vanilla:
extra_image_name_array.append(self.request_hash)
cmdline = ['make', 'image', "-j", str(os.cpu_count())]
cmdline.append('PROFILE=%s' % self.profile)
# if self.network_profile:
# cmdline.append('FILES=%s' % self.network_profile_path)
extra_image_name = "-".join(extra_image_name_array)
self.log.debug("extra_image_name %s", extra_image_name)
cmdline.append('EXTRA_IMAGE_NAME=%s' % extra_image_name)
if not self.vanilla:
self.diff_packages()
cmdline.append('PACKAGES=%s' % ' '.join(self.packages))
cmdline.append('BIN_DIR=%s' % self.build_path)
self.log.info("start build: %s", " ".join(cmdline))
env = os.environ.copy()
build_start = datetime.now()
proc = subprocess.Popen(cmdline,
cwd=self.imagebuilder.path,
stdout=subprocess.PIPE,
shell=False,
stderr=subprocess.STDOUT,
env=env)
output, erros = proc.communicate()
build_end = datetime.now()
self.build_seconds = int((build_end - build_start).total_seconds())
self.build_log = output.decode("utf-8")
returnCode = proc.returncode
if returnCode == 0:
self.log.info("build successfull")
self.manifest_hash = hashlib.sha256(
open(
glob.glob(os.path.join(self.build_path,
'*.manifest'))[0],
'rb').read()).hexdigest()[0:15]
self.parse_manifest()
self.image_hash = get_hash(" ".join(self.as_array_build()), 15)
path_array = [
get_folder("downloaddir"), self.distro, self.release,
self.target, self.subtarget, self.profile
]
if not self.vanilla:
path_array.append(self.manifest_hash)
else:
path_array.append("vanilla")
self.store_path = os.path.join(*path_array)
create_folder(self.store_path)
self.log.debug(os.listdir(self.build_path))
for filename in os.listdir(self.build_path):
if filename == "sha256sums":
with open(os.path.join(self.build_path, filename),
'r+') as sums:
content = sums.read()
sums.seek(0)
sums.write(self.filename_rename(content))
sums.truncate()
filename_output = os.path.join(
self.store_path, self.filename_rename(filename))
self.log.info("move file %s", filename_output)
shutil.move(os.path.join(self.build_path, filename),
filename_output)
if sign_file(os.path.join(self.store_path, "sha256sums")):
self.log.info("signed sha256sums")
if not already_created or entry_missing:
sysupgrade_files = [
"*-squashfs-sysupgrade.bin",
"*-squashfs-sysupgrade.tar", "*-squashfs.trx",
"*-squashfs.chk", "*-squashfs.bin",
"*-squashfs-sdcard.img.gz", "*-combined-squashfs*"
]
sysupgrade = None
profile_in_sysupgrade = ""
if self.profile.lower() != "generic":
profile_in_sysupgrade = "*" + self.profile
for sysupgrade_file in sysupgrade_files:
if not sysupgrade:
sysupgrade = glob.glob(
os.path.join(
self.store_path,
profile_in_sysupgrade + sysupgrade_file))
else:
break
if not sysupgrade:
self.log.debug("sysupgrade not found")
if self.build_log.find("too big") != -1:
self.log.warning("created image was to big")
self.store_log(
os.path.join(
get_folder("downloaddir"),
"faillogs/request-{}".format(
self.request_hash)))
self.database.set_image_requests_status(
self.request_hash, 'imagesize_fail')
return False
else:
self.profile_in_name = None
self.subtarget_in_name = None
self.sysupgrade_suffix = ""
self.build_status = "no_sysupgrade"
else:
self.path = sysupgrade[0]
sysupgrade_image = os.path.basename(self.path)
self.subtarget_in_name = self.subtarget in sysupgrade_image
self.profile_in_name = self.profile in sysupgrade_image
# ath25/generic/generic results in lede-17.01.4-ath25-generic-squashfs-sysupgrade...
if (self.profile == self.subtarget and "{}-{}".format(
self.subtarget, self.profile)
not in sysupgrade_image):
self.subtarget_in_name = False
name_array = [self.distro]
# snapshot build are no release
if self.release != "snapshot":
name_array.append(self.release)
if not self.vanilla:
name_array.append(self.manifest_hash)
name_array.append(self.target)
if self.subtarget_in_name:
name_array.append(self.subtarget)
if self.profile_in_name:
name_array.append(self.profile)
self.name = "-".join(name_array)
self.sysupgrade_suffix = sysupgrade_image.replace(
self.name + "-", "")
self.build_status = "created"
self.store_log(
os.path.join(self.store_path,
"build-{}".format(self.image_hash)))
self.log.debug("add image: {} {} {} {} {}".format(
self.image_hash, self.as_array_build(),
self.sysupgrade_suffix, self.subtarget_in_name,
self.profile_in_name, self.vanilla,
self.build_seconds))
self.database.add_image(self.image_hash,
self.as_array_build(),
self.sysupgrade_suffix,
self.subtarget_in_name,
self.profile_in_name, self.vanilla,
self.build_seconds)
self.database.done_build_job(self.request_hash,
self.image_hash,
self.build_status)
return True
else:
self.log.info("build failed")
self.database.set_image_requests_status(
self.request_hash, 'build_fail')
self.store_log(
os.path.join(
get_folder("downloaddir"),
"faillogs/request-{}".format(self.request_hash)))
return False
def _request(self):
if "request_hash" in self.request_json:
check_result = self.database.check_upgrade_check_hash(
self.request_json["request_hash"])
if check_result:
self.response_json = check_result
else:
self.response_status = HTTPStatus.NOT_FOUND
return self.respond(True)
else:
for needed_value in ["distro", "version", "target", "subtarget"]:
if not needed_value in self.request_json:
self.response_status = HTTPStatus.BAD_REQUEST
return self.respond()
bad_request = self.check_bad_request()
if bad_request:
return bad_request
# check target for old version
bad_target = self.check_bad_target()
if bad_target:
return bad_target
bad_packages = self.check_bad_packages()
if bad_packages:
return bad_packages
self.installed_release = self.release
if self.installed_release == "snapshot":
self.release = "snapshot"
self.response_json["version"] = "snapshot"
else:
self.release = self.config.get(self.distro).get("latest")
if not self.release == self.installed_release:
self.response_json["version"] = self.release
# check target for new version
bad_target = self.check_bad_target()
if bad_target:
return bad_target
bad_packages = self.check_bad_packages()
if bad_packages:
return bad_packages
if "packages" in self.request_json:
self.log.debug(self.request_json["packages"])
self.packages_installed = OrderedDict(
sorted(self.request_json["packages"].items()))
package_versions = {}
self.response_json["packages"] = OrderedDict()
if "version" in self.response_json:
self.packages_transformed = self.package_transformation(
self.distro, self.installed_release,
self.packages_installed)
package_versions = self.database.packages_versions(
self.distro, self.release, self.target, self.subtarget,
" ".join(self.packages_transformed))
else:
package_versions = self.database.packages_versions(
self.distro, self.release, self.target, self.subtarget,
" ".join(self.packages_installed))
if "upgrade_packages" in self.request_json or "version" in self.response_json:
if self.request_json[
"upgrade_packages"] is 1 or "version" in self.response_json:
for package, version in package_versions:
self.response_json["packages"][package] = version
if package in self.packages_installed.keys():
if self.packages_installed[package] != version:
if not "upgrades" in self.response_json:
self.response_json["upgrades"] = {}
self.response_json["upgrades"][package] = [
version,
self.packages_installed[package]
]
self.response_json["packages"] = OrderedDict(
sorted(self.response_json["packages"].items()))
if "version" in self.response_json or "upgrades" in self.response_json:
self.response_status = HTTPStatus.OK # 200
else:
self.response_status = HTTPStatus.NO_CONTENT # 204
self.request_manifest_hash = get_hash(str(self.packages_installed),
15)
self.database.add_manifest_packages(self.request_manifest_hash,
self.packages_installed)
request_hash = get_hash(
" ".join([
self.distro, self.release, self.target, self.subtarget,
self.request_manifest_hash
]), 16)
if "version" in self.request_json:
self.response_manifest_hash = get_hash(
str(self.response_json["packages"]), 15)
self.database.add_manifest_packages(
self.response_manifest_hash,
self.response_json["packages"])
self.database.insert_upgrade_check(request_hash, self.distro,
self.installed_release,
self.target, self.subtarget,
self.request_manifest_hash,
self.release,
self.response_manifest_hash)
else:
self.database.insert_upgrade_check(request_hash, self.distro,
self.installed_release,
self.target, self.subtarget,
self.request_manifest_hash,
self.release,
self.request_manifest_hash)
self.response_json["request_hash"] = request_hash
return self.respond()
def _process_request(self):
self.log.debug("request_json: %s", self.request_json)
# if request_hash is available check the database directly
if "request_hash" in self.request_json:
self.request = self.database.check_build_request_hash(
self.request_json["request_hash"])
if not self.request:
self.response_status = HTTPStatus.NOT_FOUND
return self.respond()
else:
return self.return_status()
else:
# required params for a build request
missing_params = self.check_missing_params(
["distro", "version", "target", "subtarget", "board"])
if missing_params:
return self.respond()
self.request_json["profile"] = self.request_json[
"board"] # TODO fix this workaround
if "defaults" in self.request_json:
# check if the uci file exceeds the max file size. this should be
# done as the uci-defaults are at least temporary stored in the
# database to be passed to a worker
if getsizeof(self.request_json["defaults"]) > self.config.get(
"max_defaults_size"):
self.response_json[
"error"] = "attached defaults exceed max size"
self.response_status = 420 # this error code is the best I could find
self.respond()
# create image object to get the request_hash
image = Image(self.request_json)
image.set_packages_hash()
request_hash = get_hash(" ".join(image.as_array("packages_hash")), 12)
request_database = self.database.check_build_request_hash(request_hash)
# if found return instantly the status
if request_database:
self.log.debug("found image in database: %s",
request_database["status"])
self.request = request_database
return self.return_status()
else:
self.request["request_hash"] = request_hash
self.request["packages_hash"] = image.params[
"packages_hash"] # TODO make this better
# if not perform various checks to see if the request is acutally valid
# check for valid distro and version
bad_request = self.check_bad_request()
if bad_request:
return bad_request
# check for valid target and subtarget
bad_target = self.check_bad_target()
if bad_target:
return bad_target
# check for existing packages
bad_packages = self.check_bad_packages()
if bad_packages:
return bad_packages
# add package_hash to database
self.database.insert_packages_hash(self.request["packages_hash"],
self.request["packages"])
# now some heavy guess work is done to figure out the profile
# eventually this could be simplified if upstream unifirm the profiles/boards
if "board" in self.request_json:
self.log.debug("board in request, search for %s",
self.request_json["board"])
self.request["profile"] = self.database.check_profile(
self.request["distro"], self.request["version"],
self.request["target"], self.request["subtarget"],
self.request_json["board"])
if not self.request["profile"]:
if "model" in self.request_json:
self.log.debug("model in request, search for %s",
self.request_json["model"])
self.request["profile"] = self.database.check_model(
self.request["distro"], self.request["version"],
self.request["target"], self.request["subtarget"],
self.request_json["model"])
self.log.debug("model search found profile %s",
self.request["profile"])
if not self.request["profile"]:
if self.database.check_profile(self.request["distro"],
self.request["version"],
self.request["target"],
self.request["subtarget"],
"Generic"):
self.request["profile"] = "Generic"
elif self.database.check_profile(self.request["distro"],
self.request["version"],
self.request["target"],
self.request["subtarget"],
"generic"):
self.request["profile"] = "generic"
else:
self.response_json[
"error"] = "unknown device, please check model and board params"
self.response_status = HTTPStatus.PRECONDITION_FAILED # 412
return self.respond()
self.request["defaults_hash"] = image.params["defaults_hash"]
# check if a default uci config is attached to the request
if image.params["defaults_hash"] != "":
self.database.insert_defaults(image.params["defaults_hash"],
self.request_json["defaults"])
# all checks passed, eventually add to queue!
self.request.pop("packages")
self.log.debug("add build job %s", self.request)
self.database.add_build_job(self.request)
return self.return_queued()
def set_request_hash(self):
self.request_hash = get_hash(" ".join(self.as_array()), 12)
def build(self):
self.log.debug("create and parse manifest")
# fail path in case of erros
fail_log_path = self.config.get_folder(
"download_folder") + "/faillogs/faillog-{}.txt".format(
self.params["request_hash"])
self.image = Image(self.params)
# first determine the resulting manifest hash
return_code, manifest_content, errors = self.run_meta("manifest")
if return_code == 0:
self.image.params["manifest_hash"] = get_hash(manifest_content, 15)
manifest_pattern = r"(.+) - (.+)\n"
manifest_packages = dict(
re.findall(manifest_pattern, manifest_content))
self.database.add_manifest_packages(
self.image.params["manifest_hash"], manifest_packages)
self.log.info("successfully parsed manifest")
else:
self.log.error("couldn't determine manifest")
self.write_log(fail_log_path, stderr=errors)
self.database.set_image_requests_status(
self.params["request_hash"], "manifest_fail")
return False
# set directory where image is stored on server
self.image.set_image_dir()
self.log.debug("dir %s", self.image.params["dir"])
# calculate hash based on resulted manifest
self.image.params["image_hash"] = get_hash(
" ".join(self.image.as_array("manifest_hash")), 15)
# set log path in case of success
success_log_path = self.image.params[
"dir"] + "/buildlog-{}.txt".format(self.params["image_hash"])
# set build_status ahead, if stuff goes wrong it will be changed
self.build_status = "created"
# check if image already exists
if not self.image.created():
self.log.info("build image")
with tempfile.TemporaryDirectory(
dir=self.config.get_folder("tempdir")) as build_dir:
# now actually build the image with manifest hash as EXTRA_IMAGE_NAME
self.params["worker"] = self.location
self.params["BIN_DIR"] = build_dir
self.params["j"] = str(os.cpu_count())
self.params["EXTRA_IMAGE_NAME"] = self.params["manifest_hash"]
# if uci defaults are added, at least at parts of the hash to time image name
if self.params["defaults_hash"]:
defaults_dir = build_dir + "/files/etc/uci-defaults/"
# create folder to store uci defaults
os.makedirs(defaults_dir)
# request defaults content from database
defaults_content = self.database.get_defaults(
self.params["defaults_hash"])
with open(defaults_dir + "99-server-defaults",
"w") as defaults_file:
defaults_file.write(
defaults_content
) # TODO check if special encoding is required
# tell ImageBuilder to integrate files
self.params["FILES"] = build_dir + "/files/"
self.params["EXTRA_IMAGE_NAME"] += "-" + self.params[
"defaults_hash"][:6]
# download is already performed for manifest creation
self.params["NO_DOWNLOAD"] = "1"
build_start = time.time()
return_code, buildlog, errors = self.run_meta("image")
self.image.params["build_seconds"] = int(time.time() -
build_start)
if return_code == 0:
# create folder in advance
os.makedirs(self.image.params["dir"], exist_ok=True)
self.log.debug(os.listdir(build_dir))
for filename in os.listdir(build_dir):
if os.path.exists(self.image.params["dir"] + "/" +
filename):
break
shutil.move(build_dir + "/" + filename,
self.image.params["dir"])
# possible sysupgrade names, ordered by likeliness
possible_sysupgrade_files = [
"*-squashfs-sysupgrade.bin",
"*-squashfs-sysupgrade.tar", "*-squashfs.trx",
"*-squashfs.chk", "*-squashfs.bin",
"*-squashfs-sdcard.img.gz", "*-combined-squashfs*",
"*.img.gz"
]
sysupgrade = None
for sysupgrade_file in possible_sysupgrade_files:
sysupgrade = glob.glob(self.image.params["dir"] + "/" +
sysupgrade_file)
if sysupgrade:
break
if not sysupgrade:
self.log.debug("sysupgrade not found")
if buildlog.find("too big") != -1:
self.log.warning("created image was to big")
self.database.set_image_requests_status(
self.params["request_hash"], "imagesize_fail")
self.write_log(fail_log_path, buildlog, errors)
return False
else:
self.build_status = "no_sysupgrade"
self.image.params["sysupgrade"] = ""
else:
self.image.params["sysupgrade"] = os.path.basename(
sysupgrade[0])
self.write_log(success_log_path, buildlog)
self.database.add_image(self.image.get_params())
self.log.info("build successfull")
else:
self.log.info("build failed")
self.database.set_image_requests_status(
self.params["request_hash"], 'build_fail')
self.write_log(fail_log_path, buildlog, errors)
return False
self.log.info("link request %s to image %s",
self.params["request_hash"], self.params["image_hash"])
self.database.done_build_job(self.params["request_hash"],
self.image.params["image_hash"],
self.build_status)
return True