def view(): with db.connect() as connection: form = v.ViewForm(request.values) if form.validate() and db.check_active_url_exists(connection, form.data["url"]): data = {} data["url_record"] = db.get_url_record_by_url(connection, form.data["url"]) data["data_records"] = db.get_active_data_records_for_url_id(connection, data["url_record"]["id"], int(os.getenv("ARCHIVE3_VIEW_LIMIT"))) data["data_records"] = list(map(misc.data_record_add_proof_status, data["data_records"])) data["data_records_json"] = json.dumps(list(map(lambda x: dict(x), data["data_records"])), cls=misc.DatetimeEncoder) data["cdn_base_url"] = os.getenv("AWS_S3_CDN_BASE_URL") return render_template("view.html", page_title=misc.page_title("view"), data=data) return render_template("error.html", page_title=misc.page_title("404"), data={"header": "404", "error": f"""Unable to load URL."""}), 404
def web_proof(request_id): """Web endpoint to download a blockchain proof once available.""" with db.connect() as connection: data_record = db.get_data_record_by_request_id(connection, request_id) if data_record != None: proof_available = True if int( (datetime.datetime.now() - data_record["timestamp"]).total_seconds()) > int( os.getenv("RIGIDBIT_PROOF_DELAY")) else False if proof_available: headers = {"api_key": os.getenv("RIGIDBIT_API_KEY")} url = os.getenv( "RIGIDBIT_BASE_URL") + "/api/trace-block/" + str( data_record["block_id"]) content = requests.get(url, headers=headers).content return Response(content, mimetype="application/json", headers={ "Content-disposition": f"attachment; filename={request_id}.json" }) return render_template("error.html", page_title=misc.page_title("404"), data={ "header": "404", "error": f"""Request ID not found: {request_id}""" }), 404
def recent(): with db.connect() as connection: limit = int(request.values["limit"]) if "limit" in request.values else int(os.getenv("ARCHIVE3_RECENT_LIMIT")) data = {} data["cdn_base_url"] = os.getenv("AWS_S3_CDN_BASE_URL") data["recent"] = db.get_recent_active_data_records(connection, limit) return render_template("recent.html", page_title=misc.page_title("recent"), data=data)
def manage(): with db.connect() as connection: data = {} form = v.ManageForm() if form.validate_on_submit(): accepted = json.loads(form.data["accepted"]) for item in accepted: record = db.get_submission_record(connection, item) if record is not None and not db.check_url_exists(connection, record["url"]): data = {"url": record["url"]} db.create_url_record(connection, data) misc.log_message(f"""Created url record: {record["url"]}""") db.delete_submission_record(connection, item) misc.log_message(f"""Deleted submission record: {item}""") file_path_screenshot = os.path.join(os.getcwd(), os.getenv("ARCHIVE3_SCREENSHOT_DIR"), "preview-" + str(item) + ".jpg") if os.path.exists(file_path_screenshot): os.remove(file_path_screenshot) misc.log_message(f"""Deleted local preview screenshot: {file_path_screenshot}""") rejected = json.loads(form.data["rejected"]) for item in rejected: db.delete_submission_record(connection, item) misc.log_message(f"""Deleted submission record: {item}""") file_path_screenshot = os.path.join(os.getcwd(), os.getenv("ARCHIVE3_SCREENSHOT_DIR"), "preview-" + str(item) + ".jpg") if os.path.exists(file_path_screenshot): os.remove(file_path_screenshot) misc.log_message(f"""Deleted local preview screenshot: {file_path_screenshot}""") connection.commit() data["pending"] = db.get_pending_submission_records(connection, 100) return render_template("manage.html", page_title=misc.page_title("manage"), data=data)
def preview(): form = v.PreviewForm(request.values) if form.validate(): filename = os.path.join(os.getcwd(), os.getenv("ARCHIVE3_SCREENSHOT_DIR"), "preview-" + str(form.data["id"]) + ".jpg") if os.path.exists(filename): return send_file(filename) else: filename = ss.generate_screenshot({"url": form.data["url"], "width": 1280, "height": 720, "delay": 1}) return send_file(filename) return render_template("error.html", page_title=misc.page_title("500"), data={"header": "500", "error": f"""Unable to generate preview."""}), 500
def stats(): with db.connect() as connection: data = \ { "data_count": db.get_data_record_count(connection)["count"], "url_count": db.get_url_record_count(connection)["count"], "pending_submission_count": db.get_pending_submission_record_count(connection)["count"], "ready_submission_count": db.get_ready_submission_record_count(connection)["count"], } return render_template("stats.html", page_title=misc.page_title("stats"), data=data)
def search(): with db.connect() as connection: form = v.SearchForm(request.values) if form.validate(): data = {} data["cdn_base_url"] = os.getenv("AWS_S3_CDN_BASE_URL") data["search_results"] = db.search_active_url_records(connection, request.values["q"], int(os.getenv("ARCHIVE3_SEARCH_RESULT_LIMIT"))) data["search_term"] = request.values["q"] else: data = {} return render_template("search.html", page_title=misc.page_title("search"), data=data)
def proof(data_id): with db.connect() as connection: data_record = db.get_active_data_record(connection, data_id) if data_record != None: proof_available = True if int((datetime.datetime.now() - data_record["timestamp"]).total_seconds()) > int(os.getenv("RIGIDBIT_PROOF_DELAY")) else False if proof_available: headers = {"api_key": os.getenv("RIGIDBIT_API_KEY")} url = os.getenv("RIGIDBIT_BASE_URL") + "/api/trace-block/" + str(data_record["block_id"]) content = requests.get(url, headers=headers).content return Response(content, mimetype="application/json", headers={"Content-disposition": f"attachment; filename={str(data_record['timestamp']).replace(' ', '_')}.json"}) return render_template("error.html", page_title=misc.page_title("404"), data={"header": "404", "error": f"""Proof not found or not yet available."""}), 404
def web_stats(): """Web endpoint to view site stats.""" with db.connect() as connection: data = \ { "count": db.get_data_record_count(connection)["count"], "count_api_keys": db.get_api_key_count(connection)["count"], "count_users": db.get_user_count(connection)["count"], "recent": db.get_recent_data_records(connection, 100), "api_key_use_counts": db.get_api_key_use_counts(connection) } return render_template("web_stats.html", page_title=misc.page_title("Count"), data=data)
def submit(): with db.connect() as connection: data = {} form = v.SubmitForm() if form.validate_on_submit(): url = misc.normalize_url(form.data["url"]) if not db.check_url_exists(connection, url) and not db.check_active_submission_exists(connection, url): data = {"url": url, "ip": request.remote_addr, "processed": "false"} db.create_submission_record(connection, data) connection.commit() data["message"] = "Your URL submission has been received. Thank you." else: for key in form.errors: data["message"] = f"""Error: {key} - {form.errors[key][0]}""" return render_template("submit.html", page_title=misc.page_title("submit"), data=data)
def api_activate(api_key): """API endpoint to activate an API key.""" with db.connect() as connection: record = db.get_unverified_user_record_by_challenge( connection, api_key) if record != None: db.delete_unverified_user_record(connection, record["id"]) user_record = db.get_user_record_by_email(connection, record["email"]) if user_record != None: user_id = user_record["id"] else: data = {"email": record["email"], "disabled": False} user_id = db.create_user_record(connection, data) data = {"email": record["email"], "api_key": api_key} db.delete_api_key_record_by_user_id(connection, user_id) data = {"api_key": api_key, "user_id": user_id} db.create_api_key_record(connection, data) connection.commit() data = { "api_key": api_key, "base_url": os.getenv("WWW2PNG_BASE_URL") } return render_template("web_api_key_activated.html", page_title=misc.page_title("api_activate"), data=data) else: data = { "header": "ERROR", "error": "The API Key you specified is not valid or has already been activated." } return render_template("error.html", page_title=misc.page_title("error"), data=data), 404
def buried(): with greenstalk.Client(host=os.getenv("GREENSTALK_HOST"), port=os.getenv("GREENSTALK_PORT"), use=os.getenv("GREENSTALK_TUBE_QUEUE")) as queue: form = v.BuriedForm() if form.validate_on_submit(): try: print(form.data) if form.data["action"] == "delete" and form.data["job_id"] is not None: queue.delete(form.data["job_id"]) elif form.data["action"] == "kick" and form.data["job_id"] is not None: queue.kick_job(form.data["job_id"]) except greenstalk.NotFoundError: return redirect("/buried", code=302) try: job = queue.peek_buried() data = {"job_body": job.body, "job_id": job.id} except greenstalk.NotFoundError: data = {} return render_template("buried.html", page_title=misc.page_title("buried"), data=data)
def web_view(request_id): """Web endpoint to view the status and content of a request.""" with db.connect() as connection: data = db.get_data_record_by_request_id(connection, request_id) if data != None: data = misc.data_record_to_web_view(data) data["base_url"] = os.getenv("WWW2PNG_BASE_URL") return render_template( "web_view.html", page_title= "WWW2PNG - Webpage Screenshot Service with Blockchain Anchoring", data=data) else: return render_template( "error.html", page_title=misc.page_title("404"), data={ "header": "404", "error": f"""Request ID not found: {request_id}""" }), 404
def web_image(request_id): """Web endpoint to download an image once available.""" with db.connect() as connection: data = db.get_data_record_by_request_id(connection, request_id) if data == None or data["removed"] or data["pruned"] or data["queued"]: return render_template( "error.html", page_title=misc.page_title("404"), data={ "header": "404", "error": f"""Request ID not found: {request_id}""" }), 404 else: filename = request_id + ".png" as_attachment = "download" in request.values and request.values[ "download"] == "true" return send_from_directory( os.getenv("WWW2PNG_SCREENSHOT_DIR"), filename, mimetype=mimetypes.guess_type(filename)[0], as_attachment=as_attachment)
def web_capture(): """Web endpoint to capture a new screenshot.""" with db.connect() as connection, greenstalk.Client( host=os.getenv("GREENSTALK_HOST"), port=os.getenv("GREENSTALK_PORT"), use=os.getenv("GREENSTALK_TUBE_QUEUE")) as queue: db.lock_data_table(connection) form = v.CaptureForm() if form.validate_on_submit(): request_id = str(uuid4()) settings = misc.screenshot_settings(request.values) data = { "request_id": request_id, "url": settings["url"], "block_id": 0, "user_id": 1, "queued": "true", "pruned": "false", "flagged": "false", "removed": "false", "failed": "false" } db.create_data_record(connection, data) connection.commit() payload = {"request_id": request_id, "settings": settings} queue.put(json.dumps(payload), ttr=int(os.getenv("WWW2PNG_PROCESSING_TTR"))) return redirect("/web/view/" + request_id, code=303) else: for key in form.errors: return render_template("error.html", page_title=misc.page_title("500"), data={ "header": "Error", "error": f"""{key}: {form.errors[key][0]}""" }), 400
def api_request(): """API endpoint to register a user and request a new API key.""" with db.connect() as connection, greenstalk.Client( host=os.getenv("GREENSTALK_HOST"), port=os.getenv("GREENSTALK_PORT"), use=os.getenv("GREENSTALK_TUBE_ACTIONS")) as actions: form = v.ApiKeyForm() if form.validate_on_submit(): challenge = str(uuid4()) email = request.values["email"] data = {"email": email, "challenge": challenge} db.create_unverified_user_record(connection, data) connection.commit() data = {"email": email, "challenge": challenge} payload = {"action": "send_api_request_email", "data": data} actions.put(json.dumps(payload), ttr=int(os.getenv("WWW2PNG_PROCESSING_TTR"))) return render_template("web_api_key_requested.html", page_title=misc.page_title("api_request"), data=data) else: for key in form.errors: return ({"error": form.errors[key][0]}, 400)
def contact(): """Web endpoint for Contact page.""" return render_template("contact.html", page_title=misc.page_title("contact"))
def root(): """Web endpoint for the landing page.""" return render_template("index.html", page_title=misc.page_title("default"))
def terms_of_service(): return render_template("terms_of_service.html", page_title=misc.page_title("tos"))
def privacy_policy(): return render_template("privacy_policy.html", page_title=misc.page_title("pp"))
def contact(): return render_template("contact.html", page_title=misc.page_title("contact"))
def api_help(): data = {} return render_template("faq.html", page_title=misc.page_title("faq"), data=data)
def root(): return render_template("index.html", page_title=misc.page_title("default"))
def privacy_policy(): """Web endpoint for the Privacy Policy page.""" return render_template("privacy_policy.html", page_title=misc.page_title("pp"))
def urls(): with db.connect() as connection: data = {"urls": db.get_url_list(connection)} return render_template("urls.html", page_title=misc.page_title("urls"), data=data)
def terms_of_service(): """Web endpoint for the Terms of Service page.""" return render_template("terms_of_service.html", page_title=misc.page_title("tos"))
def api_help(): """Web endpoint for API Help page.""" data = {"base_url": os.getenv("WWW2PNG_BASE_URL")} return render_template("api_help.html", page_title=misc.page_title("api_help"), data=data)