def admin_page():
if request.method == "POST":
action = request.form["action"]
if action == "importusers":
task = importUsersFromModList.delay()
return redirect(
url_for("check_task", id=task.id, r=url_for("user_list_page")))
elif action == "importmodlist":
task = importKrocksModList.delay()
return redirect(
url_for("check_task",
id=task.id,
r=url_for("todo_topics_page")))
elif action == "importscreenshots":
packages = Package.query \
.filter_by(soft_deleted=False) \
.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
.filter(PackageScreenshot.id==None) \
.all()
for package in packages:
importRepoScreenshot.delay(package.id)
return redirect(url_for("admin_page"))
elif action == "restore":
package = Package.query.get(request.form["package"])
if package is None:
flash("Unknown package", "error")
else:
package.soft_deleted = False
db.session.commit()
return redirect(url_for("admin_page"))
elif action == "importdepends":
task = importAllDependencies.delay()
return redirect(
url_for("check_task", id=task.id, r=url_for("admin_page")))
elif action == "modprovides":
packages = Package.query.filter_by(type=PackageType.MOD).all()
mpackage_cache = {}
for p in packages:
if len(p.provides) == 0:
p.provides.append(
MetaPackage.GetOrCreate(p.name, mpackage_cache))
db.session.commit()
return redirect(url_for("admin_page"))
else:
flash("Unknown action: " + action, "error")
deleted_packages = Package.query.filter_by(soft_deleted=True).all()
return render_template("admin/list.html",
deleted_packages=deleted_packages)
def admin_page():
if request.method == "POST":
action = request.form["action"]
if action == "importusers":
task = importUsersFromModList.delay()
return redirect(
url_for("check_task", id=task.id, r=url_for("user_list_page")))
elif action == "importscreenshots":
packages = Package.query \
.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
.filter(PackageScreenshot.id==None).all()
for package in packages:
importRepoScreenshot.delay(package.id)
return redirect(url_for("admin_page"))
else:
flash("Unknown action: " + action, "error")
return render_template("admin/list.html")
def admin_page():
if request.method == "POST":
action = request.form["action"]
if action == "delstuckreleases":
PackageRelease.query.filter(
PackageRelease.task_id != None).delete()
db.session.commit()
return redirect(url_for("admin.admin_page"))
elif action == "checkreleases":
releases = PackageRelease.query.filter(
PackageRelease.url.like("/uploads/%")).all()
tasks = []
for release in releases:
zippath = release.url.replace("/uploads/",
app.config["UPLOAD_DIR"])
tasks.append(checkZipRelease.s(release.id, zippath))
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view"))
elif action == "reimportpackages":
tasks = []
for package in Package.query.filter(
Package.state != PackageState.DELETED).all():
release = package.releases.first()
if release:
zippath = release.url.replace("/uploads/",
app.config["UPLOAD_DIR"])
tasks.append(updateMetaFromRelease.s(release.id, zippath))
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view"))
elif action == "importforeign":
releases = PackageRelease.query.filter(
PackageRelease.url.like("http%")).all()
tasks = []
for release in releases:
tasks.append(importForeignDownloads.s(release.id))
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view"))
elif action == "importmodlist":
task = importTopicList.delay()
return redirect(
url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
elif action == "checkusers":
task = checkAllForumAccounts.delay()
return redirect(
url_for("tasks.check",
id=task.id,
r=url_for("admin.admin_page")))
elif action == "importscreenshots":
packages = Package.query \
.filter(Package.state!=PackageState.DELETED) \
.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
.filter(PackageScreenshot.id==None) \
.all()
for package in packages:
importRepoScreenshot.delay(package.id)
return redirect(url_for("admin.admin_page"))
elif action == "restore":
package = Package.query.get(request.form["package"])
if package is None:
flash("Unknown package", "danger")
else:
package.state = PackageState.READY_FOR_REVIEW
db.session.commit()
return redirect(url_for("admin.admin_page"))
elif action == "recalcscores":
for p in Package.query.all():
p.recalcScore()
db.session.commit()
return redirect(url_for("admin.admin_page"))
elif action == "cleanuploads":
upload_dir = app.config['UPLOAD_DIR']
(_, _, filenames) = next(os.walk(upload_dir))
existing_uploads = set(filenames)
if len(existing_uploads) != 0:
def getURLsFromDB(column):
results = db.session.query(column).filter(
column != None, column != "").all()
return set([os.path.basename(x[0]) for x in results])
release_urls = getURLsFromDB(PackageRelease.url)
screenshot_urls = getURLsFromDB(PackageScreenshot.url)
db_urls = release_urls.union(screenshot_urls)
unreachable = existing_uploads.difference(db_urls)
import sys
print("On Disk: ", existing_uploads, file=sys.stderr)
print("In DB: ", db_urls, file=sys.stderr)
print("Unreachable: ", unreachable, file=sys.stderr)
for filename in unreachable:
os.remove(os.path.join(upload_dir, filename))
flash(
"Deleted " + str(len(unreachable)) +
" unreachable uploads", "success")
else:
flash("No downloads to create", "danger")
return redirect(url_for("admin.admin_page"))
elif action == "delmetapackages":
query = MetaPackage.query.filter(~MetaPackage.dependencies.any(),
~MetaPackage.packages.any())
count = query.count()
query.delete(synchronize_session=False)
db.session.commit()
flash("Deleted " + str(count) + " unused meta packages", "success")
return redirect(url_for("admin.admin_page"))
else:
flash("Unknown action: " + action, "danger")
deleted_packages = Package.query.filter(
Package.state == PackageState.DELETED).all()
return render_template("admin/list.html",
deleted_packages=deleted_packages)
def create_edit(author=None, name=None):
package = None
form = None
if author is None:
form = PackageForm(formdata=request.form)
author = request.args.get("author")
if author is None or author == current_user.username:
author = current_user
else:
author = User.query.filter_by(username=author).first()
if author is None:
flash("Unable to find that user", "danger")
return redirect(url_for("packages.create_edit"))
if not author.checkPerm(current_user, Permission.CHANGE_AUTHOR):
flash("Permission denied", "danger")
return redirect(url_for("packages.create_edit"))
else:
package = getPackageByInfo(author, name)
if package is None:
abort(404)
if not package.checkPerm(current_user, Permission.EDIT_PACKAGE):
return redirect(package.getDetailsURL())
author = package.author
form = PackageForm(formdata=request.form, obj=package)
# Initial form class from post data and default data
if request.method == "GET":
if package is None:
form.name.data = request.args.get("bname")
form.title.data = request.args.get("title")
form.repo.data = request.args.get("repo")
form.forums.data = request.args.get("forums")
form.license.data = None
form.media_license.data = None
else:
# form.harddep_str.data = ",".join([str(x) for x in package.getSortedHardDependencies() ])
# form.softdep_str.data = ",".join([str(x) for x in package.getSortedOptionalDependencies() ])
form.tags.data = list(package.tags)
form.content_warnings.data = list(package.content_warnings)
if request.method == "POST" and form.type.data == PackageType.TXP:
form.license.data = form.media_license.data
if form.validate_on_submit():
wasNew = False
if not package:
package = Package.query.filter_by(name=form["name"].data, author_id=author.id).first()
if package is not None:
if package.state == PackageState.READY_FOR_REVIEW:
Package.query.filter_by(name=form["name"].data, author_id=author.id).delete()
else:
flash("Package already exists!", "danger")
return redirect(url_for("packages.create_edit"))
package = Package()
package.author = author
package.maintainers.append(author)
wasNew = True
elif package.name != form.name.data and not package.checkPerm(current_user, Permission.CHANGE_NAME):
flash("Unable to change package name", "danger")
return redirect(url_for("packages.create_edit", author=author, name=name))
else:
msg = "Edited {}".format(package.title)
addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT,
msg, package.getDetailsURL(), package)
severity = AuditSeverity.NORMAL if current_user in package.maintainers else AuditSeverity.EDITOR
addAuditLog(severity, current_user, msg, package.getDetailsURL(), package)
form.populate_obj(package) # copy to row
if package.type == PackageType.TXP:
package.license = package.media_license
# Dependency.query.filter_by(depender=package).delete()
# deps = Dependency.SpecToList(package, form.harddep_str.data, mpackage_cache)
# for dep in deps:
# dep.optional = False
# db.session.add(dep)
# deps = Dependency.SpecToList(package, form.softdep_str.data, mpackage_cache)
# for dep in deps:
# dep.optional = True
# db.session.add(dep)
if wasNew and package.type == PackageType.MOD:
m = MetaPackage.GetOrCreate(package.name, {})
package.provides.append(m)
package.tags.clear()
for tag in form.tags.raw_data:
package.tags.append(Tag.query.get(tag))
package.content_warnings.clear()
for warning in form.content_warnings.raw_data:
package.content_warnings.append(ContentWarning.query.get(warning))
db.session.commit() # save
next_url = package.getDetailsURL()
if wasNew and package.repo is not None:
task = importRepoScreenshot.delay(package.id)
next_url = url_for("tasks.check", id=task.id, r=next_url)
if wasNew and ("WTFPL" in package.license.name or "WTFPL" in package.media_license.name):
next_url = url_for("flatpage", path="help/wtfpl", r=next_url)
return redirect(next_url)
package_query = Package.query.filter_by(state=PackageState.APPROVED)
if package is not None:
package_query = package_query.filter(Package.id != package.id)
enableWizard = name is None and request.method != "POST"
return render_template("packages/create_edit.html", package=package,
form=form, author=author, enable_wizard=enableWizard,
packages=package_query.all(),
mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all())
def create_edit(author=None, name=None):
package = None
if author is None:
form = PackageForm(formdata=request.form)
author = request.args.get("author")
if author is None or author == current_user.username:
author = current_user
else:
author = User.query.filter_by(username=author).first()
if author is None:
flash("Unable to find that user", "danger")
return redirect(url_for("packages.create_edit"))
if not author.checkPerm(current_user, Permission.CHANGE_AUTHOR):
flash("Permission denied", "danger")
return redirect(url_for("packages.create_edit"))
else:
package = getPackageByInfo(author, name)
if package is None:
abort(404)
if not package.checkPerm(current_user, Permission.EDIT_PACKAGE):
return redirect(package.getDetailsURL())
author = package.author
form = PackageForm(formdata=request.form, obj=package)
# Initial form class from post data and default data
if request.method == "GET":
if package is None:
form.name.data = request.args.get("bname")
form.title.data = request.args.get("title")
form.repo.data = request.args.get("repo")
form.forums.data = request.args.get("forums")
form.license.data = None
form.media_license.data = None
else:
form.tags.data = list(package.tags)
form.content_warnings.data = list(package.content_warnings)
if request.method == "POST" and form.type.data == PackageType.TXP:
form.license.data = form.media_license.data
if form.validate_on_submit():
wasNew = False
if not package:
package = Package.query.filter_by(name=form["name"].data,
author_id=author.id).first()
if package is not None:
if package.state == PackageState.READY_FOR_REVIEW:
Package.query.filter_by(name=form["name"].data,
author_id=author.id).delete()
else:
flash("Package already exists!", "danger")
return redirect(url_for("packages.create_edit"))
package = Package()
package.author = author
package.maintainers.append(author)
wasNew = True
try:
do_edit_package(
current_user, package, wasNew, {
"type": form.type.data,
"title": form.title.data,
"name": form.name.data,
"short_desc": form.short_desc.data,
"tags": form.tags.raw_data,
"content_warnings": form.content_warnings.raw_data,
"license": form.license.data,
"media_license": form.media_license.data,
"desc": form.desc.data,
"repo": form.repo.data,
"website": form.website.data,
"issueTracker": form.issueTracker.data,
"forums": form.forums.data,
})
if wasNew and package.repo is not None:
importRepoScreenshot.delay(package.id)
next_url = package.getDetailsURL()
if wasNew and ("WTFPL" in package.license.name
or "WTFPL" in package.media_license.name):
next_url = url_for("flatpage", path="help/wtfpl", r=next_url)
elif wasNew:
next_url = package.getSetupReleasesURL()
return redirect(next_url)
except LogicError as e:
flash(e.message, "danger")
package_query = Package.query.filter_by(state=PackageState.APPROVED)
if package is not None:
package_query = package_query.filter(Package.id != package.id)
enableWizard = name is None and request.method != "POST"
return render_template("packages/create_edit.html",
package=package,
form=form,
author=author,
enable_wizard=enableWizard,
packages=package_query.all(),
mpackages=MetaPackage.query.order_by(
db.asc(MetaPackage.name)).all())
def admin_page():
if request.method == "POST":
action = request.form["action"]
if action == "delstuckreleases":
PackageRelease.query.filter(
PackageRelease.task_id != None).delete()
db.session.commit()
return redirect(url_for("admin.admin_page"))
elif action == "checkreleases":
releases = PackageRelease.query.filter(
PackageRelease.url.like("/uploads/%")).all()
tasks = []
for release in releases:
zippath = release.url.replace("/uploads/",
app.config["UPLOAD_DIR"])
tasks.append(checkZipRelease.s(release.id, zippath))
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view_editor"))
elif action == "reimportpackages":
tasks = []
for package in Package.query.filter(
Package.state != PackageState.DELETED).all():
release = package.releases.first()
if release:
zippath = release.url.replace("/uploads/",
app.config["UPLOAD_DIR"])
tasks.append(checkZipRelease.s(release.id, zippath))
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view_editor"))
elif action == "importmodlist":
task = importTopicList.delay()
return redirect(
url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
elif action == "checkusers":
task = checkAllForumAccounts.delay()
return redirect(
url_for("tasks.check",
id=task.id,
r=url_for("admin.admin_page")))
elif action == "importscreenshots":
packages = Package.query \
.filter(Package.state!=PackageState.DELETED) \
.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
.filter(PackageScreenshot.id==None) \
.all()
for package in packages:
importRepoScreenshot.delay(package.id)
return redirect(url_for("admin.admin_page"))
elif action == "restore":
package = Package.query.get(request.form["package"])
if package is None:
flash("Unknown package", "danger")
else:
package.state = PackageState.READY_FOR_REVIEW
db.session.commit()
return redirect(url_for("admin.admin_page"))
elif action == "recalcscores":
for p in Package.query.all():
p.recalcScore()
db.session.commit()
return redirect(url_for("admin.admin_page"))
elif action == "cleanuploads":
upload_dir = app.config['UPLOAD_DIR']
(_, _, filenames) = next(os.walk(upload_dir))
existing_uploads = set(filenames)
if len(existing_uploads) != 0:
def getURLsFromDB(column):
results = db.session.query(column).filter(
column != None, column != "").all()
return set([os.path.basename(x[0]) for x in results])
release_urls = getURLsFromDB(PackageRelease.url)
screenshot_urls = getURLsFromDB(PackageScreenshot.url)
db_urls = release_urls.union(screenshot_urls)
unreachable = existing_uploads.difference(db_urls)
import sys
print("On Disk: ", existing_uploads, file=sys.stderr)
print("In DB: ", db_urls, file=sys.stderr)
print("Unreachable: ", unreachable, file=sys.stderr)
for filename in unreachable:
os.remove(os.path.join(upload_dir, filename))
flash(
"Deleted " + str(len(unreachable)) +
" unreachable uploads", "success")
else:
flash("No downloads to create", "danger")
return redirect(url_for("admin.admin_page"))
elif action == "delmetapackages":
query = MetaPackage.query.filter(~MetaPackage.dependencies.any(),
~MetaPackage.packages.any())
count = query.count()
query.delete(synchronize_session=False)
db.session.commit()
flash("Deleted " + str(count) + " unused meta packages", "success")
return redirect(url_for("admin.admin_page"))
elif action == "delremovedpackages":
query = Package.query.filter_by(state=PackageState.DELETED)
count = query.count()
for pkg in query.all():
pkg.review_thread = None
db.session.delete(pkg)
db.session.commit()
flash("Deleted {} soft deleted packages packages".format(count),
"success")
return redirect(url_for("admin.admin_page"))
elif action == "addupdateconfig":
added = 0
for pkg in Package.query.filter(
Package.repo != None, Package.releases.any(),
Package.update_config == None).all():
pkg.update_config = PackageUpdateConfig()
pkg.update_config.auto_created = True
release: PackageRelease = pkg.releases.first()
if release and release.commit_hash:
pkg.update_config.last_commit = release.commit_hash
db.session.add(pkg.update_config)
added += 1
db.session.commit()
flash("Added {} update configs".format(added), "success")
return redirect(url_for("admin.admin_page"))
elif action == "runupdateconfig":
check_for_updates.delay()
flash("Started update configs", "success")
return redirect(url_for("admin.admin_page"))
elif action == "remindwip":
users = User.query.filter(
User.packages.any(
or_(Package.state == PackageState.WIP,
Package.state == PackageState.CHANGES_NEEDED)))
system_user = get_system_user()
for user in users:
packages = db.session.query(Package.title).filter(
Package.author_id==user.id,
or_(Package.state==PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)) \
.all()
# Who needs translations?
packages = [pkg[0] for pkg in packages]
if len(packages) >= 3:
packages[len(packages) -
1] = "and " + packages[len(packages) - 1]
packages_list = ", ".join(packages)
else:
packages_list = "and ".join(packages)
havent = "haven't" if len(packages) > 1 else "hasn't"
if len(packages_list) + 54 > 100:
packages_list = packages_list[0:(100 - 54 - 1)] + "…"
addNotification(
user, system_user, NotificationType.PACKAGE_APPROVAL,
f"Did you forget? {packages_list} {havent} been submitted for review yet",
url_for('todo.view_user', username=user.username))
db.session.commit()
else:
flash("Unknown action: " + action, "danger")
deleted_packages = Package.query.filter(
Package.state == PackageState.DELETED).all()
return render_template("admin/list.html",
deleted_packages=deleted_packages)
def create_edit_package_page(author=None, name=None):
package = None
form = None
if author is None:
form = PackageForm(formdata=request.form)
author = request.args.get("author")
if author is None or author == current_user.username:
author = current_user
else:
author = User.query.filter_by(username=author).first()
if author is None:
flash("Unable to find that user", "error")
return redirect(url_for("create_edit_package_page"))
if not author.checkPerm(current_user, Permission.CHANGE_AUTHOR):
flash("Permission denied", "error")
return redirect(url_for("create_edit_package_page"))
else:
package = getPackageByInfo(author, name)
if not package.checkPerm(current_user, Permission.EDIT_PACKAGE):
return redirect(package.getDetailsURL())
author = package.author
form = PackageForm(formdata=request.form, obj=package)
# Initial form class from post data and default data
if request.method == "POST" and form.validate():
wasNew = False
if not package:
package = Package.query.filter_by(name=form["name"].data,
author_id=author.id).first()
if package is not None:
flash("Package already exists!", "error")
return redirect(url_for("create_edit_package_page"))
package = Package()
package.author = author
wasNew = True
else:
triggerNotif(package.author, current_user,
"{} edited".format(package.title),
package.getDetailsURL())
form.populate_obj(package) # copy to row
package.tags.clear()
for tag in form.tags.raw_data:
package.tags.append(Tag.query.get(tag))
db.session.commit() # save
if wasNew:
url = urlparse(package.repo)
if url.netloc == "github.com":
task = importRepoScreenshot.delay(package.id)
return redirect(
url_for("check_task",
id=task.id,
r=package.getDetailsURL()))
return redirect(package.getDetailsURL())
enableWizard = name is None and request.method != "POST"
return render_template("packages/create_edit.html", package=package, \
form=form, author=author, enable_wizard=enableWizard)
def admin_page():
if request.method == "POST":
action = request.form["action"]
if action == "importmodlist":
task = importTopicList.delay()
return redirect(
url_for("check_task",
id=task.id,
r=url_for("todo_topics_page")))
elif action == "importscreenshots":
packages = Package.query \
.filter_by(soft_deleted=False) \
.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
.filter(PackageScreenshot.id==None) \
.all()
for package in packages:
importRepoScreenshot.delay(package.id)
return redirect(url_for("admin_page"))
elif action == "restore":
package = Package.query.get(request.form["package"])
if package is None:
flash("Unknown package", "error")
else:
package.soft_deleted = False
db.session.commit()
return redirect(url_for("admin_page"))
elif action == "importdepends":
task = importAllDependencies.delay()
return redirect(
url_for("check_task", id=task.id, r=url_for("admin_page")))
elif action == "modprovides":
packages = Package.query.filter_by(type=PackageType.MOD).all()
mpackage_cache = {}
for p in packages:
if len(p.provides) == 0:
p.provides.append(
MetaPackage.GetOrCreate(p.name, mpackage_cache))
db.session.commit()
return redirect(url_for("admin_page"))
elif action == "recalcscores":
for p in Package.query.all():
p.recalcScore()
db.session.commit()
return redirect(url_for("admin_page"))
elif action == "vcsrelease":
for package in Package.query.filter(
Package.repo.isnot(None)).all():
if package.releases.count() != 0:
continue
rel = PackageRelease()
rel.package = package
rel.title = datetime.date.today().isoformat()
rel.url = ""
rel.task_id = uuid()
rel.approved = True
db.session.add(rel)
db.session.commit()
makeVCSRelease.apply_async((rel.id, "master"),
task_id=rel.task_id)
msg = "{}: Release {} created".format(package.title, rel.title)
triggerNotif(package.author, current_user, msg,
rel.getEditURL())
db.session.commit()
else:
flash("Unknown action: " + action, "error")
deleted_packages = Package.query.filter_by(soft_deleted=True).all()
return render_template("admin/list.html",
deleted_packages=deleted_packages)
