def makeVCSRelease(id, branch):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
if release.package is None:
raise TaskError("No package attached to release")
url = urlparse(release.package.repo)
urlmaker = None
if url.netloc == "github.com":
urlmaker = GithubURLMaker(url)
else:
raise TaskError("Unsupported repo")
if not urlmaker.isValid():
raise TaskError("Invalid github repo URL")
commitsURL = urlmaker.getCommitsURL(branch)
contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
commits = json.loads(contents)
if len(commits) == 0 or not "sha" in commits[0]:
raise TaskError("No commits found")
release.url = urlmaker.getCommitDownload(commits[0]["sha"])
print(release.url)
release.task_id = None
db.session.commit()
return release.url
def makeVCSRelease(self, id, branch):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
with clone_repo(release.package.repo, ref=branch, recursive=True) as repo:
postReleaseCheckUpdate(self, release, repo.working_tree_dir)
filename = randomString(10) + ".zip"
destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
assert(not os.path.isfile(destPath))
archiver = GitArchiver(prefix=release.package.name, force_sub=True, main_repo_abspath=repo.working_tree_dir)
archiver.create(destPath)
assert(os.path.isfile(destPath))
release.url = "/uploads/" + filename
release.task_id = None
release.commit_hash = repo.head.object.hexsha
release.approve(release.package.author)
db.session.commit()
return release.url
def importRepoScreenshot(id):
package = Package.query.get(id)
if package is None:
raise Exception("Unexpected none package")
# Get URL Maker
url = urlparse(package.repo)
urlmaker = None
if url.netloc == "github.com":
urlmaker = GithubURLMaker(url)
else:
raise TaskError("Unsupported repo")
if not urlmaker.isValid():
raise TaskError("Error! Url maker not valid")
try:
filename = randomString(10) + ".png"
imagePath = os.path.join("app/public/uploads", filename)
print(imagePath)
urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath)
ss = PackageScreenshot()
ss.package = package
ss.title = "screenshot.png"
ss.url = "/uploads/" + filename
db.session.add(ss)
db.session.commit()
return "/uploads/" + filename
except HTTPError:
print("screenshot.png does not exist")
return None
def checkZipRelease(self, id, path):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
temp = getTempDir()
try:
with ZipFile(path, 'r') as zip_ref:
zip_ref.extractall(temp)
try:
tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
author=release.package.author.username, name=release.package.name)
except MinetestCheckError as err:
if "Fails validation" not in release.title:
release.title += " (Fails validation)"
release.task_id = self.request.id
release.approved = False
db.session.commit()
raise TaskError(str(err))
release.task_id = None
release.approve(release.package.author)
db.session.commit()
finally:
shutil.rmtree(temp)
def check_update_config(self, package_id):
package: Package = Package.query.get(package_id)
if package is None:
raise TaskError("No such package!")
elif package.update_config is None:
raise TaskError("No update config attached to package")
err = None
try:
check_update_config_impl(package)
except GitCommandError as e:
# This is needed to stop the backtrace being weird
err = e.stderr
except gitdb.exc.BadName as e:
err = "Unable to find the reference " + (package.update_config.ref or "?") + "\n" + e.stderr
except TaskError as e:
err = e.value
if err:
err = err.replace("stderr: ", "") \
.replace("Cloning into '/tmp/", "Cloning into '") \
.strip()
msg = "Error: {}.\n\nTask ID: {}\n\n[Change update configuration]({})" \
.format(err, self.request.id, package.getURL("packages.update_config"))
post_bot_message(package, "Failed to check git repository", msg)
db.session.commit()
return
def makeVCSRelease(id, branch):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
urlmaker = None
url = urlparse(release.package.repo)
if url.netloc == "github.com":
return makeVCSReleaseFromGithub(id, branch, release, url)
else:
gitDir, repo = cloneRepo(release.package.repo,
ref=branch,
recursive=True)
try:
filename = randomString(10) + ".zip"
destPath = os.path.join("app/public/uploads", filename)
with open(destPath, "wb") as fp:
repo.archive(fp, format="zip")
release.url = "/uploads/" + filename
release.task_id = None
release.commit_hash = repo.head.object.hexsha
release.approve(release.package.author)
print(release.url)
db.session.commit()
return release.url
finally:
shutil.rmtree(gitDir)
def makeVCSRelease(self, id, branch):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
postReleaseCheckUpdate(self, release, gitDir)
try:
filename = randomString(10) + ".zip"
destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
assert (not os.path.isfile(destPath))
archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
archiver.create(destPath)
assert (os.path.isfile(destPath))
release.url = "/uploads/" + filename
release.task_id = None
release.commit_hash = repo.head.object.hexsha
release.approve(release.package.author)
db.session.commit()
updateMetaFromRelease.delay(release.id, destPath)
return release.url
finally:
shutil.rmtree(gitDir)
def getMeta(urlstr, author):
url = urlparse(urlstr)
urlmaker = None
if url.netloc == "github.com":
urlmaker = GithubURLMaker(url)
else:
raise TaskError("Unsupported repo")
if not urlmaker.isValid():
raise TaskError("Error! Url maker not valid")
result = {}
result["repo"] = urlmaker.getRepoURL()
result["issueTracker"] = urlmaker.getIssueTrackerURL()
try:
contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
conf = parseConf(contents)
for key in ["name", "description", "title"]:
try:
result[key] = conf[key]
except KeyError:
pass
except HTTPError:
print("mod.conf does not exist")
if "name" in result:
result["title"] = result["name"].replace("_", " ").title()
if not "description" in result:
try:
contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
result["description"] = contents.strip()
except HTTPError:
print("description.txt does not exist!")
if "description" in result:
desc = result["description"]
idx = desc.find(".") + 1
cutIdx = min(len(desc), 200 if idx < 5 else idx)
result["short_description"] = desc[:cutIdx]
info = findModInfo(author, result.get("name"), result["repo"])
if info is not None:
result["forumId"] = info.get("topicId")
return result
def checkZipRelease(self, id, path):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
with get_temp_dir() as temp:
with ZipFile(path, 'r') as zip_ref:
zip_ref.extractall(temp)
postReleaseCheckUpdate(self, release, temp)
release.task_id = None
release.approve(release.package.author)
db.session.commit()
def cloneRepo(urlstr, ref=None, recursive=False):
gitDir = getTempDir()
err = None
try:
gitUrl = generateGitURL(urlstr)
print("Cloning from " + gitUrl)
if ref is None:
repo = git.Repo.clone_from(gitUrl, gitDir, \
progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
else:
repo = git.Repo.init(gitDir)
origin = repo.create_remote("origin", url=gitUrl)
assert origin.exists()
origin.fetch()
origin.pull(ref)
for submodule in repo.submodules:
submodule.update(init=True)
return gitDir, repo
except GitCommandError as e:
# This is needed to stop the backtrace being weird
err = e.stderr
except gitdb.exc.BadName as e:
err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
raise TaskError(err.replace("stderr: ", "") \
.replace("Cloning into '" + gitDir + "'...", "") \
.strip())
def __init__(self, baseDir, author=None, repo=None, name=None):
print("Scanning " + baseDir)
self.baseDir = baseDir
self.author = author
self.name = name
self.repo = repo
self.meta = None
self.children = []
# Detect type
type = None
is_modpack = False
if os.path.isfile(baseDir + "/game.conf"):
type = PackageType.GAME
elif os.path.isfile(baseDir + "/init.lua"):
type = PackageType.MOD
elif os.path.isfile(baseDir + "/modpack.txt") or \
os.path.isfile(baseDir + "/modpack.conf"):
type = PackageType.MOD
is_modpack = True
elif os.path.isdir(baseDir + "/mods"):
type = PackageType.GAME
elif os.listdir(baseDir) == []:
# probably a submodule
return
else:
raise TaskError("Unable to detect package type!")
self.type = type
self.readMetaFiles()
if self.type == PackageType.GAME:
self.addChildrenFromModDir(baseDir + "/mods")
elif is_modpack:
self.addChildrenFromModDir(baseDir)
def getMeta(urlstr, author):
gitDir, _ = cloneRepo(urlstr, recursive=True)
try:
tree = build_tree(gitDir, author=author, repo=urlstr)
except MinetestCheckError as err:
raise TaskError(str(err))
shutil.rmtree(gitDir)
result = {}
result["name"] = tree.name
result["provides"] = tree.getModNames()
result["type"] = tree.type.name
for key in ["depends", "optional_depends"]:
result[key] = tree.fold("meta", key)
for key in [
"title", "repo", "issueTracker", "forumId", "description",
"short_description"
]:
result[key] = tree.get(key)
for mod in result["provides"]:
result["depends"].discard(mod)
result["optional_depends"].discard(mod)
for key, value in result.items():
if isinstance(value, set):
result[key] = list(value)
return result
def makeVCSReleaseFromGithub(id, branch, release, url):
urlmaker = GithubURLMaker(url)
if not urlmaker.isValid():
raise TaskError("Invalid github repo URL")
commitsURL = urlmaker.getCommitsURL(branch)
contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
commits = json.loads(contents)
if len(commits) == 0 or not "sha" in commits[0]:
raise TaskError("No commits found")
release.url = urlmaker.getCommitDownload(commits[0]["sha"])
print(release.url)
release.task_id = None
db.session.commit()
return release.url
def makeVCSRelease(id, branch):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
tree = None
try:
tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
author=release.package.author.username, name=release.package.name)
except MinetestCheckError as err:
raise TaskError(str(err))
try:
filename = randomString(10) + ".zip"
destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
assert (not os.path.isfile(destPath))
archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
archiver.create(destPath)
assert (os.path.isfile(destPath))
release.url = "/uploads/" + filename
release.task_id = None
release.commit_hash = repo.head.object.hexsha
if tree.meta.get("min_minetest_version"):
release.min_rel = MinetestRelease.get(
tree.meta["min_minetest_version"], None)
if tree.meta.get("max_minetest_version"):
release.max_rel = MinetestRelease.get(
tree.meta["max_minetest_version"], None)
release.approve(release.package.author)
db.session.commit()
return release.url
finally:
shutil.rmtree(gitDir)
def getMeta(urlstr, author):
with clone_repo(urlstr, recursive=True) as repo:
try:
tree = build_tree(repo.working_tree_dir,
author=author,
repo=urlstr)
except MinetestCheckError as err:
raise TaskError(str(err))
result = {"name": tree.name, "type": tree.type.name}
for key in [
"title", "repo", "issueTracker", "forumId", "description",
"short_description"
]:
result[key] = tree.get(key)
result["forums"] = result.get("forumId")
readme_path = tree.getReadMePath()
if readme_path:
with open(readme_path, "r") as f:
result["long_description"] = f.read()
try:
with open(os.path.join(tree.baseDir, ".cdb.json"), "r") as f:
data = json.loads(f.read())
for key, value in data.items():
result[key] = value
except LogicError as e:
raise TaskError(e.message)
except IOError:
pass
for alias, to in ALIASES.items():
if alias in result:
result[to] = result[alias]
for key, value in result.items():
if isinstance(value, set):
result[key] = list(value)
return result
def postReleaseCheckUpdate(self, release, path):
try:
tree = build_tree(path, expected_type=ContentType[release.package.type.name],
author=release.package.author.username, name=release.package.name)
cache = {}
def getMetaPackages(names):
return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
provides = tree.getModNames()
package = release.package
package.provides.clear()
package.provides.extend(getMetaPackages(tree.getModNames()))
# Delete all meta package dependencies
package.dependencies.filter(Dependency.meta_package != None).delete()
# Get raw dependencies
depends = tree.fold("meta", "depends")
optional_depends = tree.fold("meta", "optional_depends")
# Filter out provides
for mod in provides:
depends.discard(mod)
optional_depends.discard(mod)
# Add dependencies
for meta in getMetaPackages(depends):
db.session.add(Dependency(package, meta=meta, optional=False))
for meta in getMetaPackages(optional_depends):
db.session.add(Dependency(package, meta=meta, optional=True))
# Update min/max
if tree.meta.get("min_minetest_version"):
release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
if tree.meta.get("max_minetest_version"):
release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
return tree
except MinetestCheckError as err:
db.session.rollback()
if "Fails validation" not in release.title:
release.title += " (Fails validation)"
release.task_id = self.request.id
release.approved = False
db.session.commit()
raise TaskError(str(err))
def outer(self, *args, **kwargs):
lock_id = "global_db_lock"
print("Obtaining lock...")
with memcache_lock("lock_id", self.app.oid) as acquired:
print(
'in memcache_lock and lock_id is {} self.app.oid is {} and acquired is {}'
.format(lock_id, self.app.oid, acquired))
if acquired:
return fun(self, *args, **kwargs)
raise TaskError("Unable to perform task")
def updateMetaFromRelease(self, id, path):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
print("updateMetaFromRelease: {} for {}/{}" \
.format(id, release.package.author.display_name, release.package.name))
temp = getTempDir()
try:
with ZipFile(path, 'r') as zip_ref:
zip_ref.extractall(temp)
postReleaseCheckUpdate(self, release, temp)
db.session.commit()
finally:
shutil.rmtree(temp)
def importForeignDownloads(self, id):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
elif release.package is None:
raise TaskError("No package attached to release")
elif not release.url.startswith("http"):
return
try:
ext = getExtension(release.url)
filename = randomString(10) + "." + ext
filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
urllib.request.urlretrieve(release.url, filepath)
release.url = "/uploads/" + filename
db.session.commit()
except urllib.error.URLError:
db.session.rollback()
release.task_id = self.request.id
release.approved = False
db.session.commit()
def makeVCSReleaseFromGithub(id, branch, release, url):
urlmaker = GithubURLMaker(url)
if not urlmaker.isValid():
raise TaskError("Invalid github repo URL")
commitsURL = urlmaker.getCommitsURL(branch)
try:
contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
commits = json.loads(contents)
except HTTPError:
raise TaskError(
"Unable to get commits for Github repository. Either the repository or reference doesn't exist."
)
if len(commits) == 0 or not "sha" in commits[0]:
raise TaskError("No commits found")
release.url = urlmaker.getCommitDownload(commits[0]["sha"])
release.task_id = None
release.commit_hash = commits[0]["sha"]
release.approve(release.package.author)
db.session.commit()
return release.url
def clone_repo(urlstr, ref=None, recursive=False):
gitDir = os.path.join(tempfile.gettempdir(), randomString(10))
err = None
try:
gitUrl = generateGitURL(urlstr)
print("Cloning from " + gitUrl)
if ref is None:
repo = git.Repo.clone_from(gitUrl,
gitDir,
progress=None,
env=None,
depth=1,
recursive=recursive,
kill_after_timeout=15)
else:
assert ref != ""
repo = git.Repo.init(gitDir)
origin = repo.create_remote("origin", url=gitUrl)
assert origin.exists()
origin.fetch()
repo.git.checkout(ref)
if recursive:
for submodule in repo.submodules:
submodule.update(init=True)
yield repo
shutil.rmtree(gitDir)
return
except GitCommandError as e:
# This is needed to stop the backtrace being weird
err = e.stderr
except gitdb.exc.BadName as e:
err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
raise TaskError(err.replace("stderr: ", "") \
.replace("Cloning into '" + gitDir + "'...", "") \
.strip())
def cloneRepo(urlstr, ref=None, recursive=False):
gitDir = tempfile.gettempdir() + "/" + randomString(10)
err = None
try:
gitUrl = generateGitURL(urlstr)
print("Cloning from " + gitUrl)
repo = git.Repo.clone_from(gitUrl, gitDir, \
progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
if ref is not None:
repo.create_head("myhead", ref).checkout()
return gitDir, repo
except GitCommandError as e:
# This is needed to stop the backtrace being weird
err = e.stderr
raise TaskError(err.replace("stderr: ", "") \
.replace("Cloning into '" + gitDir + "'...", "") \
.strip())
def getMeta(urlstr, author):
with clone_repo(urlstr, recursive=True) as repo:
try:
tree = build_tree(repo.working_tree_dir, author=author, repo=urlstr)
except MinetestCheckError as err:
raise TaskError(str(err))
result = {"name": tree.name, "provides": tree.getModNames(), "type": tree.type.name}
for key in ["depends", "optional_depends"]:
result[key] = tree.fold("meta", key)
for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
result[key] = tree.get(key)
for mod in result["provides"]:
result["depends"].discard(mod)
result["optional_depends"].discard(mod)
for key, value in result.items():
if isinstance(value, set):
result[key] = list(value)
return result
def postReleaseCheckUpdate(self, release: PackageRelease, path):
try:
tree = build_tree(path, expected_type=ContentType[release.package.type.name],
author=release.package.author.username, name=release.package.name)
if tree.name is not None and release.package.name != tree.name:
raise MinetestCheckError(f"Expected {tree.relative} to have technical name {release.package.name}, instead has name {tree.name}")
cache = {}
def getMetaPackages(names):
return [ MetaPackage.GetOrCreate(x, cache) for x in names ]
provides = tree.getModNames()
package = release.package
package.provides.clear()
package.provides.extend(getMetaPackages(tree.getModNames()))
# Delete all meta package dependencies
package.dependencies.filter(Dependency.meta_package != None).delete()
# Get raw dependencies
depends = tree.fold("meta", "depends")
optional_depends = tree.fold("meta", "optional_depends")
# Filter out provides
for mod in provides:
depends.discard(mod)
optional_depends.discard(mod)
# Raise error on unresolved game dependencies
if package.type == PackageType.GAME and len(depends) > 0:
deps = ", ".join(depends)
raise MinetestCheckError("Game has unresolved hard dependencies: " + deps)
# Add dependencies
for meta in getMetaPackages(depends):
db.session.add(Dependency(package, meta=meta, optional=False))
for meta in getMetaPackages(optional_depends):
db.session.add(Dependency(package, meta=meta, optional=True))
# Update min/max
if tree.meta.get("min_minetest_version"):
release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None)
if tree.meta.get("max_minetest_version"):
release.max_rel = MinetestRelease.get(tree.meta["max_minetest_version"], None)
try:
with open(os.path.join(tree.baseDir, ".cdb.json"), "r") as f:
data = json.loads(f.read())
do_edit_package(package.author, package, False, False, data, "Post release hook")
except LogicError as e:
raise TaskError(e.message)
except IOError:
pass
return tree
except MinetestCheckError as err:
db.session.rollback()
msg = f"{err}\n\nTask ID: {self.request.id}\n\nRelease: [View Release]({release.getEditURL()})"
post_bot_message(release.package, f"Release {release.title} validation failed", msg)
if "Fails validation" not in release.title:
release.title += " (Fails validation)"
release.task_id = self.request.id
release.approved = False
db.session.commit()
raise TaskError(str(err))
def check_update_config_impl(package):
config = package.update_config
if config.trigger == PackageUpdateTrigger.COMMIT:
tag = None
commit = get_latest_commit(package.repo, package.update_config.ref)
elif config.trigger == PackageUpdateTrigger.TAG:
tag, commit = get_latest_tag(package.repo)
else:
raise TaskError("Unknown update trigger")
if commit is None:
return
if config.last_commit == commit:
if tag and config.last_tag != tag:
config.last_tag = tag
db.session.commit()
return
if not config.last_commit:
config.last_commit = commit
config.last_tag = tag
db.session.commit()
return
if package.releases.filter_by(commit_hash=commit).count() > 0:
return
if config.make_release:
rel = PackageRelease()
rel.package = package
rel.title = tag if tag else datetime.datetime.utcnow().strftime("%Y-%m-%d")
rel.url = ""
rel.task_id = uuid()
db.session.add(rel)
msg = "Created release {} (Git Update Detection)".format(rel.title)
addSystemAuditLog(AuditSeverity.NORMAL, msg, package.getURL("packages.view"), package)
db.session.commit()
makeVCSRelease.apply_async((rel.id, commit), task_id=rel.task_id)
elif config.outdated_at is None:
config.set_outdated()
if config.trigger == PackageUpdateTrigger.COMMIT:
msg_last = ""
if config.last_commit:
msg_last = " The last commit was {}".format(config.last_commit[0:5])
msg = "New commit {} found on the Git repo, is the package outdated?{}" \
.format(commit[0:5], msg_last)
else:
msg_last = ""
if config.last_tag:
msg_last = " The last tag was {}".format(config.last_tag)
msg = "New tag {} found on the Git repo.{}" \
.format(tag, msg_last)
for user in package.maintainers:
addSystemNotification(user, NotificationType.BOT,
msg, url_for("todo.view_user", username=user.username, _external=False), package)
config.last_commit = commit
config.last_tag = tag
db.session.commit()
