def enqueue_bulk():
if not config.CAPTCHA_SUBMIT or captcha.verify():
urls = request.form.get("urls")
if urls:
urls = urls.split()
if 0 < len(urls) <= 1000: # TODO: Load from config & adjust placeholder/messages?
pool = Pool(processes=6)
pool.map(func=check_url, iterable=urls)
pool.close()
flash("Submitted websites to the queue", "success")
return redirect("/submit")
else:
flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit")
else:
flash("Too few or too many urls, please submit 1-10 urls", "danger")
return redirect("/submit")
else:
flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
return redirect("/submit")
def enqueue():
if not config.CAPTCHA_SUBMIT or captcha.verify():
url = os.path.join(request.form.get("url"), "")
message, msg_type = try_enqueue(url)
flash(message, msg_type)
return redirect("/submit")
else:
flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
return redirect("/submit")
def admin_login():
if not config.CAPTCHA_LOGIN or captcha.verify():
username = request.form.get("username")
password = request.form.get("password")
if db.check_login(username, password):
session["username"] = username
flash("Logged in", "success")
return redirect("/dashboard")
flash("Invalid username/password combo", "danger")
return redirect("/admin")
else:
flash("Invalid captcha", "danger")
return redirect("/admin")
def clean_captcha(self):
raw_answer = self.cleaned_data['captcha']
if not verify(raw_answer, self.captcha_answer):
raise forms.ValidationError(
_(u'That verification code is incorrect. Please try again.'))
return ''
def search():
results = 0
q = request.args.get("q") if "q" in request.args else ""
sort_order = request.args.get("sort_order") if "sort_order" in request.args else "score"
page = request.args.get("p") if "p" in request.args else "0"
page = int(page) if page.isdigit() else 0
per_page = request.args.get("per_page") if "per_page" in request.args else "50"
per_page = int(per_page) if per_page.isdigit() else "50"
per_page = per_page if per_page in config.RESULTS_PER_PAGE else 50
extensions = request.args.get("ext") if "ext" in request.args else None
extensions = [ext.strip().strip(".").lower() for ext in extensions.split(",")] if extensions else []
size_min = request.args.get("size_min") if "size_min" in request.args else "size_min"
size_min = int(size_min) if size_min.isdigit() else 0
size_max = request.args.get("size_max") if "size_max" in request.args else "size_max"
size_max = int(size_max) if size_max.isdigit() else 0
date_min = request.args.get("date_min") if "date_min" in request.args else "date_min"
date_min = int(date_min) if date_min.isdigit() else 0
date_max = request.args.get("date_max") if "date_max" in request.args else "date_max"
date_max = int(date_max) if date_max.isdigit() else 0
match_all = "all" in request.args
field_name = "field_name" in request.args
field_trigram = "field_trigram" in request.args
field_path = "field_path" in request.args
if not field_name and not field_trigram and not field_path:
# If no fields are selected, search in all
field_name = field_path = field_trigram = True
fields = []
if field_path:
fields.append("path")
if field_name:
fields.append("name^5")
if field_trigram:
fields.append("name.nGram^2")
if len(q) >= 3:
blocked = False
hits = None
if not config.CAPTCHA_SEARCH or captcha.verify():
try:
hits = searchEngine.search(q, page, per_page, sort_order,
extensions, size_min, size_max, match_all, fields, date_min, date_max)
hits = db.join_website_on_search_result(hits)
except InvalidQueryException as e:
flash("<strong>Invalid query:</strong> " + str(e), "warning")
blocked = True
except:
flash("Query failed, this could mean that the search server is overloaded or is not reachable. "
"Please try again later", "danger")
results = hits["hits"]["total"]["value"] if not isinstance(hits["hits"]["total"], int) else \
hits["hits"]["total"] if hits else -1
took = hits["took"] if hits else -1
forwarded_for = request.headers["X-Forwarded-For"] if "X-Forwarded-For" in request.headers else None
logger.info("SEARCH '{}' [res={}, t={}, p={}x{}, ext={}] by {}{}"
.format(q, results, took, page, per_page, str(extensions),
request.remote_addr, "_" + forwarded_for if forwarded_for else ""))
db.log_search(request.remote_addr, forwarded_for, q, extensions, page, blocked, results, took)
if blocked:
return redirect("/search")
else:
flash("<strong>Error:</strong> Invalid captcha please try again", "danger")
else:
hits = None
return render_template("search.html",
count=results,
results=hits,
q=q,
p=page, per_page=per_page,
sort_order=sort_order,
results_set=config.RESULTS_PER_PAGE,
extensions=",".join(extensions),
size_min=size_min, size_max=size_max,
match_all=match_all,
field_trigram=field_trigram, field_path=field_path, field_name=field_name,
date_min=date_min, date_max=date_max,
show_captcha=config.CAPTCHA_SEARCH, captcha=captcha)
def clean_captcha(self):
raw_answer = self.cleaned_data['captcha']
if not verify(raw_answer, self.captcha_answer):
raise forms.ValidationError(_(u'That verification code is incorrect. Please try again.'))
return ''
