def view():
with db.connect() as connection:
form = v.ViewForm(request.values)
if form.validate() and db.check_active_url_exists(connection, form.data["url"]):
data = {}
data["url_record"] = db.get_url_record_by_url(connection, form.data["url"])
data["data_records"] = db.get_active_data_records_for_url_id(connection, data["url_record"]["id"], int(os.getenv("ARCHIVE3_VIEW_LIMIT")))
data["data_records"] = list(map(misc.data_record_add_proof_status, data["data_records"]))
data["data_records_json"] = json.dumps(list(map(lambda x: dict(x), data["data_records"])), cls=misc.DatetimeEncoder)
data["cdn_base_url"] = os.getenv("AWS_S3_CDN_BASE_URL")
return render_template("view.html", page_title=misc.page_title("view"), data=data)
return render_template("error.html", page_title=misc.page_title("404"), data={"header": "404", "error": f"""Unable to load URL."""}), 404
def web_proof(request_id):
"""Web endpoint to download a blockchain proof once available."""
with db.connect() as connection:
data_record = db.get_data_record_by_request_id(connection, request_id)
if data_record != None:
proof_available = True if int(
(datetime.datetime.now() -
data_record["timestamp"]).total_seconds()) > int(
os.getenv("RIGIDBIT_PROOF_DELAY")) else False
if proof_available:
headers = {"api_key": os.getenv("RIGIDBIT_API_KEY")}
url = os.getenv(
"RIGIDBIT_BASE_URL") + "/api/trace-block/" + str(
data_record["block_id"])
content = requests.get(url, headers=headers).content
return Response(content,
mimetype="application/json",
headers={
"Content-disposition":
f"attachment; filename={request_id}.json"
})
return render_template("error.html",
page_title=misc.page_title("404"),
data={
"header":
"404",
"error":
f"""Request ID not found: {request_id}"""
}), 404
def recent():
with db.connect() as connection:
limit = int(request.values["limit"]) if "limit" in request.values else int(os.getenv("ARCHIVE3_RECENT_LIMIT"))
data = {}
data["cdn_base_url"] = os.getenv("AWS_S3_CDN_BASE_URL")
data["recent"] = db.get_recent_active_data_records(connection, limit)
return render_template("recent.html", page_title=misc.page_title("recent"), data=data)
def manage():
with db.connect() as connection:
data = {}
form = v.ManageForm()
if form.validate_on_submit():
accepted = json.loads(form.data["accepted"])
for item in accepted:
record = db.get_submission_record(connection, item)
if record is not None and not db.check_url_exists(connection, record["url"]):
data = {"url": record["url"]}
db.create_url_record(connection, data)
misc.log_message(f"""Created url record: {record["url"]}""")
db.delete_submission_record(connection, item)
misc.log_message(f"""Deleted submission record: {item}""")
file_path_screenshot = os.path.join(os.getcwd(), os.getenv("ARCHIVE3_SCREENSHOT_DIR"), "preview-" + str(item) + ".jpg")
if os.path.exists(file_path_screenshot):
os.remove(file_path_screenshot)
misc.log_message(f"""Deleted local preview screenshot: {file_path_screenshot}""")
rejected = json.loads(form.data["rejected"])
for item in rejected:
db.delete_submission_record(connection, item)
misc.log_message(f"""Deleted submission record: {item}""")
file_path_screenshot = os.path.join(os.getcwd(), os.getenv("ARCHIVE3_SCREENSHOT_DIR"), "preview-" + str(item) + ".jpg")
if os.path.exists(file_path_screenshot):
os.remove(file_path_screenshot)
misc.log_message(f"""Deleted local preview screenshot: {file_path_screenshot}""")
connection.commit()
data["pending"] = db.get_pending_submission_records(connection, 100)
return render_template("manage.html", page_title=misc.page_title("manage"), data=data)
def preview():
form = v.PreviewForm(request.values)
if form.validate():
filename = os.path.join(os.getcwd(), os.getenv("ARCHIVE3_SCREENSHOT_DIR"), "preview-" + str(form.data["id"]) + ".jpg")
if os.path.exists(filename):
return send_file(filename)
else:
filename = ss.generate_screenshot({"url": form.data["url"], "width": 1280, "height": 720, "delay": 1})
return send_file(filename)
return render_template("error.html", page_title=misc.page_title("500"), data={"header": "500", "error": f"""Unable to generate preview."""}), 500
def stats():
with db.connect() as connection:
data = \
{
"data_count": db.get_data_record_count(connection)["count"],
"url_count": db.get_url_record_count(connection)["count"],
"pending_submission_count": db.get_pending_submission_record_count(connection)["count"],
"ready_submission_count": db.get_ready_submission_record_count(connection)["count"],
}
return render_template("stats.html", page_title=misc.page_title("stats"), data=data)
def search():
with db.connect() as connection:
form = v.SearchForm(request.values)
if form.validate():
data = {}
data["cdn_base_url"] = os.getenv("AWS_S3_CDN_BASE_URL")
data["search_results"] = db.search_active_url_records(connection, request.values["q"], int(os.getenv("ARCHIVE3_SEARCH_RESULT_LIMIT")))
data["search_term"] = request.values["q"]
else:
data = {}
return render_template("search.html", page_title=misc.page_title("search"), data=data)
def proof(data_id):
with db.connect() as connection:
data_record = db.get_active_data_record(connection, data_id)
if data_record != None:
proof_available = True if int((datetime.datetime.now() - data_record["timestamp"]).total_seconds()) > int(os.getenv("RIGIDBIT_PROOF_DELAY")) else False
if proof_available:
headers = {"api_key": os.getenv("RIGIDBIT_API_KEY")}
url = os.getenv("RIGIDBIT_BASE_URL") + "/api/trace-block/" + str(data_record["block_id"])
content = requests.get(url, headers=headers).content
return Response(content, mimetype="application/json", headers={"Content-disposition": f"attachment; filename={str(data_record['timestamp']).replace(' ', '_')}.json"})
return render_template("error.html", page_title=misc.page_title("404"), data={"header": "404", "error": f"""Proof not found or not yet available."""}), 404
def web_stats():
"""Web endpoint to view site stats."""
with db.connect() as connection:
data = \
{
"count": db.get_data_record_count(connection)["count"],
"count_api_keys": db.get_api_key_count(connection)["count"],
"count_users": db.get_user_count(connection)["count"],
"recent": db.get_recent_data_records(connection, 100),
"api_key_use_counts": db.get_api_key_use_counts(connection)
}
return render_template("web_stats.html",
page_title=misc.page_title("Count"),
data=data)
def submit():
with db.connect() as connection:
data = {}
form = v.SubmitForm()
if form.validate_on_submit():
url = misc.normalize_url(form.data["url"])
if not db.check_url_exists(connection, url) and not db.check_active_submission_exists(connection, url):
data = {"url": url, "ip": request.remote_addr, "processed": "false"}
db.create_submission_record(connection, data)
connection.commit()
data["message"] = "Your URL submission has been received. Thank you."
else:
for key in form.errors:
data["message"] = f"""Error: {key} - {form.errors[key][0]}"""
return render_template("submit.html", page_title=misc.page_title("submit"), data=data)
def api_activate(api_key):
"""API endpoint to activate an API key."""
with db.connect() as connection:
record = db.get_unverified_user_record_by_challenge(
connection, api_key)
if record != None:
db.delete_unverified_user_record(connection, record["id"])
user_record = db.get_user_record_by_email(connection,
record["email"])
if user_record != None:
user_id = user_record["id"]
else:
data = {"email": record["email"], "disabled": False}
user_id = db.create_user_record(connection, data)
data = {"email": record["email"], "api_key": api_key}
db.delete_api_key_record_by_user_id(connection, user_id)
data = {"api_key": api_key, "user_id": user_id}
db.create_api_key_record(connection, data)
connection.commit()
data = {
"api_key": api_key,
"base_url": os.getenv("WWW2PNG_BASE_URL")
}
return render_template("web_api_key_activated.html",
page_title=misc.page_title("api_activate"),
data=data)
else:
data = {
"header":
"ERROR",
"error":
"The API Key you specified is not valid or has already been activated."
}
return render_template("error.html",
page_title=misc.page_title("error"),
data=data), 404
def buried():
with greenstalk.Client(host=os.getenv("GREENSTALK_HOST"), port=os.getenv("GREENSTALK_PORT"), use=os.getenv("GREENSTALK_TUBE_QUEUE")) as queue:
form = v.BuriedForm()
if form.validate_on_submit():
try:
print(form.data)
if form.data["action"] == "delete" and form.data["job_id"] is not None:
queue.delete(form.data["job_id"])
elif form.data["action"] == "kick" and form.data["job_id"] is not None:
queue.kick_job(form.data["job_id"])
except greenstalk.NotFoundError:
return redirect("/buried", code=302)
try:
job = queue.peek_buried()
data = {"job_body": job.body, "job_id": job.id}
except greenstalk.NotFoundError:
data = {}
return render_template("buried.html", page_title=misc.page_title("buried"), data=data)
def web_view(request_id):
"""Web endpoint to view the status and content of a request."""
with db.connect() as connection:
data = db.get_data_record_by_request_id(connection, request_id)
if data != None:
data = misc.data_record_to_web_view(data)
data["base_url"] = os.getenv("WWW2PNG_BASE_URL")
return render_template(
"web_view.html",
page_title=
"WWW2PNG - Webpage Screenshot Service with Blockchain Anchoring",
data=data)
else:
return render_template(
"error.html",
page_title=misc.page_title("404"),
data={
"header": "404",
"error": f"""Request ID not found: {request_id}"""
}), 404
def web_image(request_id):
"""Web endpoint to download an image once available."""
with db.connect() as connection:
data = db.get_data_record_by_request_id(connection, request_id)
if data == None or data["removed"] or data["pruned"] or data["queued"]:
return render_template(
"error.html",
page_title=misc.page_title("404"),
data={
"header": "404",
"error": f"""Request ID not found: {request_id}"""
}), 404
else:
filename = request_id + ".png"
as_attachment = "download" in request.values and request.values[
"download"] == "true"
return send_from_directory(
os.getenv("WWW2PNG_SCREENSHOT_DIR"),
filename,
mimetype=mimetypes.guess_type(filename)[0],
as_attachment=as_attachment)
def web_capture():
"""Web endpoint to capture a new screenshot."""
with db.connect() as connection, greenstalk.Client(
host=os.getenv("GREENSTALK_HOST"),
port=os.getenv("GREENSTALK_PORT"),
use=os.getenv("GREENSTALK_TUBE_QUEUE")) as queue:
db.lock_data_table(connection)
form = v.CaptureForm()
if form.validate_on_submit():
request_id = str(uuid4())
settings = misc.screenshot_settings(request.values)
data = {
"request_id": request_id,
"url": settings["url"],
"block_id": 0,
"user_id": 1,
"queued": "true",
"pruned": "false",
"flagged": "false",
"removed": "false",
"failed": "false"
}
db.create_data_record(connection, data)
connection.commit()
payload = {"request_id": request_id, "settings": settings}
queue.put(json.dumps(payload),
ttr=int(os.getenv("WWW2PNG_PROCESSING_TTR")))
return redirect("/web/view/" + request_id, code=303)
else:
for key in form.errors:
return render_template("error.html",
page_title=misc.page_title("500"),
data={
"header":
"Error",
"error":
f"""{key}: {form.errors[key][0]}"""
}), 400
def api_request():
"""API endpoint to register a user and request a new API key."""
with db.connect() as connection, greenstalk.Client(
host=os.getenv("GREENSTALK_HOST"),
port=os.getenv("GREENSTALK_PORT"),
use=os.getenv("GREENSTALK_TUBE_ACTIONS")) as actions:
form = v.ApiKeyForm()
if form.validate_on_submit():
challenge = str(uuid4())
email = request.values["email"]
data = {"email": email, "challenge": challenge}
db.create_unverified_user_record(connection, data)
connection.commit()
data = {"email": email, "challenge": challenge}
payload = {"action": "send_api_request_email", "data": data}
actions.put(json.dumps(payload),
ttr=int(os.getenv("WWW2PNG_PROCESSING_TTR")))
return render_template("web_api_key_requested.html",
page_title=misc.page_title("api_request"),
data=data)
else:
for key in form.errors:
return ({"error": form.errors[key][0]}, 400)
def contact():
"""Web endpoint for Contact page."""
return render_template("contact.html",
page_title=misc.page_title("contact"))
def root():
"""Web endpoint for the landing page."""
return render_template("index.html", page_title=misc.page_title("default"))
def terms_of_service():
return render_template("terms_of_service.html", page_title=misc.page_title("tos"))
def privacy_policy():
return render_template("privacy_policy.html", page_title=misc.page_title("pp"))
def contact():
return render_template("contact.html", page_title=misc.page_title("contact"))
def api_help():
data = {}
return render_template("faq.html", page_title=misc.page_title("faq"), data=data)
def root():
return render_template("index.html", page_title=misc.page_title("default"))
def privacy_policy():
"""Web endpoint for the Privacy Policy page."""
return render_template("privacy_policy.html",
page_title=misc.page_title("pp"))
def urls():
with db.connect() as connection:
data = {"urls": db.get_url_list(connection)}
return render_template("urls.html", page_title=misc.page_title("urls"), data=data)
def terms_of_service():
"""Web endpoint for the Terms of Service page."""
return render_template("terms_of_service.html",
page_title=misc.page_title("tos"))
def api_help():
"""Web endpoint for API Help page."""
data = {"base_url": os.getenv("WWW2PNG_BASE_URL")}
return render_template("api_help.html",
page_title=misc.page_title("api_help"),
data=data)
