def callback(ch, method, properties, body):
"""
This function will be triggered when an element enter the
pastebin_downloader queue. Paste object will be create in
order to save the content of the Paste in a text file stored
at "tmp/pastebin/<filename>".
Then the filename is sent to the queue "pastebin_parser".
"""
msg = f"Data {body} received from queue {lc.DOWNLOADER_QUEUE}"
if gc.DEBUG:
log_info('downloader', msg)
time.sleep(lc.DOWNLOADING_SLEEP + random.choice(lc.DOWNLOADING_DELAY))
scrape_url = json.loads(body)['scrape_url']
r = requests.get(url=scrape_url)
if r.status_code != 200:
msg = f"Connection error: GET:{scrape_url}, Status:{r.status_code}"
log_info('downloader', msg)
return
file = Paste(metadata=body, data=r.text)
filename = file.save()
ch.basic_ack(delivery_tag=method.delivery_tag)
publish(module='downloader',
channel=ch,
data=filename,
queue=lc.PARSER_QUEUE)
def display_paste(paste_id):
now = datetime.now()
keep_alive = False
try:
paste = Paste.load(paste_id)
# Delete the paste if it expired:
if 'burn_after_reading' in str(paste.expiration):
# burn_after_reading contains the paste creation date
# if this read appends 10 seconds after the creation date
# we don't delete the paste because it means it's the redirection
# to the paste that happens during the paste creation
try:
keep_alive = paste.expiration.split('#')[1]
keep_alive = datetime.strptime(keep_alive,
'%Y-%m-%d %H:%M:%S.%f')
keep_alive = now < keep_alive + timedelta(seconds=10)
except IndexError:
keep_alive = False
if not keep_alive:
paste.delete()
elif paste.expiration < now:
paste.delete()
raise ValueError()
except (TypeError, ValueError):
return error404(ValueError)
context = {'paste': paste, 'keep_alive': keep_alive}
return dmerge(context, GLOBAL_CONTEXT)
def display_paste(paste_id):
now = datetime.now()
keep_alive = False
try:
paste = Paste.load(paste_id)
# Delete the paste if it expired:
if 'burn_after_reading' in str(paste.expiration):
# burn_after_reading contains the paste creation date
# if this read appends 10 seconds after the creation date
# we don't delete the paste because it means it's the redirection
# to the paste that happens during the paste creation
try:
keep_alive = paste.expiration.split('#')[1]
keep_alive = datetime.strptime(keep_alive,
'%Y-%m-%d %H:%M:%S.%f')
keep_alive = now < keep_alive + timedelta(seconds=10)
except IndexError:
keep_alive = False
if not keep_alive:
paste.delete()
elif paste.expiration < now:
paste.delete()
raise ValueError()
except (TypeError, ValueError):
return error404(ValueError)
context = {'paste': paste, 'keep_alive': keep_alive}
return dmerge(context, GLOBAL_CONTEXT)
def create_paste():
try:
body = urlparse.parse_qs(
request.body.read(int(settings.MAX_SIZE * 1.1)))
except ValueError:
return {'status': 'error', 'message': u"Wrong data payload."}
try:
content = unicode(''.join(body['content']), 'utf8')
except (UnicodeDecodeError, KeyError):
return {
'status': 'error',
'message': u"Encoding error: the paste couldn't be saved."
}
if '{"iv":' not in content: # reject silently non encrypted content
return {'status': 'error', 'message': u"Wrong data payload."}
if content:
# check size of the paste. if more than settings return error
# without saving paste. prevent from unusual use of the
# system. need to be improved
if len(content) < settings.MAX_SIZE:
expiration = body.get('expiration', [u'burn_after_reading'])[0]
paste = Paste(expiration=expiration,
content=content,
uuid_length=settings.PASTE_ID_LENGTH)
paste.save()
# display counter
if settings.DISPLAY_COUNTER:
#increment paste counter
paste.increment_counter()
# if refresh time elapsed pick up new counter value
now = datetime.now()
timeout = (GLOBAL_CONTEXT['refresh_counter'] +
timedelta(seconds=settings.REFRESH_COUNTER))
if timeout < now:
GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
GLOBAL_CONTEXT['refresh_counter'] = now
return {'status': 'ok', 'paste': paste.uuid}
return {
'status':
'error',
'message':
u"Serveur error: the paste couldn't be saved. "
u"Please try later."
}
def create_paste():
try:
content = unicode(request.forms.get('content', ''), 'utf8')
except UnicodeDecodeError:
return {'status': 'error',
'message': u"Encoding error: the paste couldn't be saved."}
if '{"iv":' not in content: # reject silently non encrypted content
return ''
if content:
# check size of the paste. if more than settings return error without saving paste.
# prevent from unusual use of the system.
# need to be improved
if len(content) < settings.MAX_SIZE:
expiration = request.forms.get('expiration', u'burn_after_reading')
paste = Paste(expiration=expiration, content=content)
paste.save()
return {'status': 'ok',
'paste': paste.uuid}
return {'status': 'error',
'message': u"Serveur error: the paste couldn't be saved. Please try later."}
def parse_paste_page(self, html, id):
soup = bs(html, 'html.parser')
removed = True if soup.find_all("div", "content_title")[0] \
.text.find("removed") != -1 \
else False
if removed:
return None
info_box = soup.find_all("div", "paste_box_info")[0]
info_line = info_box.find_all("div", "paste_box_line2")[0]
paste_box = soup.find_all("div", "textarea_border")[0]
title = info_box.find_all("div", "paste_box_line1")[0].text
date = info_line.span.get("title")
content = paste_box.textarea.text
author = info_line.a.text \
if info_line.text.lower().find("guest") == -1 \
else "guest"
return Paste(id, author, title, content, date)
def parse_link(url: str) -> Paste:
tree = Downloader.downloadPage(url)
if tree is None:
return None
try:
date = tree.xpath(DataParser.DATE_SELECTOR)[0]
username = tree.xpath(DataParser.USERNAME_SELECTOR)[0]
title = tree.xpath(DataParser.TITLE_SELECTOR)[0]
content = tree.xpath(DataParser.CONTENT_SELECTOR)[0]
except Exception:
print("Failed to process url: ", url)
return None
# todo: validate fetched results
return Paste(author=DataParser.norm_author_title(username),
title=DataParser.norm_author_title(title),
content=DataParser.norm_content(content),
datetime=DataParser.strdate_to_arrow(date),
link=url.split('/')[-1])
def new_paste():
form = PasteForm()
if not hcaptcha.verify():
# TODO: Localize error messages
flash('hCaptchan verifiointi epäonnistui.', 'negative')
# TODO: Complain about other errors even when there is an hCaptcha error
return redirect('/'), 400
if form.validate():
letters = string.ascii_letters + string.digits
paste_id = ''.join(random.choice(letters) for i in range(12))
paste = Paste(id=paste_id, content=form.content.data, \
name=form.name.data, private=form.private.data)
db.session.add(paste)
db.session.commit()
return redirect('/p/' + paste_id + '/')
for field, errors in form.errors.items():
if field == 'csrf_token':
errors[0] = 'Tapahtui CSRF-virhe. Yritä uudelleen.'
for error in errors:
flash(error, 'negative')
return redirect('/')
def create_paste():
try:
body = urlparse.parse_qs(request.body.read(int(settings.MAX_SIZE * 1.1)))
except ValueError:
return {'status': 'error',
'message': u"Wrong data payload."}
try:
content = unicode(''.join(body['content']), 'utf8')
except (UnicodeDecodeError, KeyError):
return {'status': 'error',
'message': u"Encoding error: the paste couldn't be saved."}
if '{"iv":' not in content: # reject silently non encrypted content
return {'status': 'error',
'message': u"Wrong data payload."}
if content:
# check size of the paste. if more than settings return error
# without saving paste. prevent from unusual use of the
# system. need to be improved
if len(content) < settings.MAX_SIZE:
expiration = body.get('expiration', [u'burn_after_reading'])[0]
paste = Paste(expiration=expiration, content=content,
uuid_length=settings.PASTE_ID_LENGTH)
paste.save()
# display counter
if settings.DISPLAY_COUNTER:
#increment paste counter
paste.increment_counter()
# if refresh time elapsed pick up new counter value
now = datetime.now()
timeout = (GLOBAL_CONTEXT['refresh_counter']
+ timedelta(seconds=settings.REFRESH_COUNTER))
if timeout < now:
GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
GLOBAL_CONTEXT['refresh_counter'] = now
return {'status': 'ok',
'paste': paste.uuid}
return {'status': 'error',
'message': u"Serveur error: the paste couldn't be saved. "
u"Please try later."}
def api_paste():
if not app.config["ENABLE_API"]:
abort(403)
letters = string.ascii_letters + string.digits
paste_id = ''.join(random.choice(letters) for i in range(12))
try:
data = json.loads(request.data)
except json.decoder.JSONDecodeError:
return jsonify({"error": "INVALID_JSON"}), 400
for param in ("name", "content", "private"):
if not param in data:
return jsonify({"error": "MISSING_PARAMETER", "parameter": \
param}), 400
name = data["name"]
content = data["content"]
private = data["private"]
if not isinstance(name, str) or not name or name.isspace():
return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": "name"}), \
400
if not isinstance(content, str) or not content or content.isspace():
return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": \
"content"}), 400
if not isinstance(private, bool):
return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": \
"private"}), 400
paste = Paste(id=paste_id, content=data["content"], \
name=data["name"], private=data["private"])
db.session.add(paste)
db.session.commit()
return jsonify({"id": paste_id})
def create_paste():
try:
content = unicode(request.forms.get('content', ''), 'utf8')
except UnicodeDecodeError:
return {'status': 'error',
'message': u"Encoding error: the paste couldn't be saved."}
if '{"iv":' not in content: # reject silently non encrypted content
return ''
if content:
# check size of the paste. if more than settings return error without saving paste.
# prevent from unusual use of the system.
# need to be improved
if len(content) < settings.MAX_SIZE:
expiration = request.forms.get('expiration', u'burn_after_reading')
paste = Paste(expiration=expiration, content=content)
paste.save()
# display counter
if settings.DISPLAY_COUNTER:
#increment paste counter
paste.increment_counter()
# if refresh time elapsed pick up new counter value
now = datetime.now()
timeout = (GLOBAL_CONTEXT['refresh_counter']
+ timedelta(seconds=settings.REFRESH_COUNTER))
if timeout < now:
GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
GLOBAL_CONTEXT['refresh_counter'] = now
return {'status': 'ok',
'paste': paste.uuid}
return {'status': 'error',
'message': u"Serveur error: the paste couldn't be saved. Please try later."}
def create_paste():
try:
body = urlparse.parse_qs(request.body.read(int(settings.MAX_SIZE * 1.1)))
except ValueError:
return {"status": "error", "message": u"Wrong data payload."}
try:
content = unicode("".join(body["content"]), "utf8")
except (UnicodeDecodeError, KeyError):
return {"status": "error", "message": u"Encoding error: the paste couldn't be saved."}
if '{"iv":' not in content: # reject silently non encrypted content
return {"status": "error", "message": u"Wrong data payload."}
if content:
# check size of the paste. if more than settings return error
# without saving paste. prevent from unusual use of the
# system. need to be improved
if len(content) < settings.MAX_SIZE:
expiration = body.get("expiration", [u"burn_after_reading"])[0]
paste = Paste(expiration=expiration, content=content)
paste.save()
# display counter
if settings.DISPLAY_COUNTER:
# increment paste counter
paste.increment_counter()
# if refresh time elapsed pick up new counter value
now = datetime.now()
timeout = GLOBAL_CONTEXT["refresh_counter"] + timedelta(seconds=settings.REFRESH_COUNTER)
if timeout < now:
GLOBAL_CONTEXT["pastes_count"] = Paste.get_pastes_count()
GLOBAL_CONTEXT["refresh_counter"] = now
return {"status": "ok", "paste": paste.uuid}
return {"status": "error", "message": u"Serveur error: the paste couldn't be saved. " u"Please try later."}
def get_paste(id):
p = Paste()
return p.get(id).get_data()
# add project dir and libs dir to the PYTHON PATH to ensure they are
# importable
from utils import settings
import bottle
from bottle import Bottle, run, static_file, view, request
import clize
from paste import Paste
from utils import drop_privileges, dmerge
app = Bottle()
GLOBAL_CONTEXT = {"settings": settings, "pastes_count": Paste.get_pastes_count(), "refresh_counter": datetime.now()}
@app.route("/")
@view("home")
def index():
return GLOBAL_CONTEXT
@app.route("/faq/")
@view("faq")
def faq():
return GLOBAL_CONTEXT
@app.route("/paste/create", method="POST")
def put_paste():
data = request.get_data()
print(data)
p = Paste()
return "pasted to id: %s" % (p.put(data))
# importable
from utils import settings, SettingsValidationError
import bottle
from bottle import (Bottle, run, static_file, view, request)
import clize
from paste import Paste
from utils import drop_privileges, dmerge
app = Bottle()
GLOBAL_CONTEXT = {
'settings': settings,
'pastes_count': Paste.get_pastes_count(),
'refresh_counter': datetime.now()
}
@app.route('/')
@view('home')
def index():
return GLOBAL_CONTEXT
@app.route('/faq/')
@view('faq')
def faq():
return GLOBAL_CONTEXT
# add project dir and libs dir to the PYTHON PATH to ensure they are
# importable
from utils import settings, SettingsValidationError
import bottle
from bottle import (Bottle, run, static_file, view, request)
import clize
from paste import Paste
from utils import drop_privileges, dmerge
app = Bottle()
GLOBAL_CONTEXT = {
'settings': settings,
'pastes_count': Paste.get_pastes_count(),
'refresh_counter': datetime.now()
}
@app.route('/')
@view('home')
def index():
return GLOBAL_CONTEXT
@app.route('/faq/')
@view('faq')
def faq():
return GLOBAL_CONTEXT