def callback(ch, method, properties, body):
"""
This function will be triggered when an element enter the
pastebin_downloader queue. Paste object will be create in
order to save the content of the Paste in a text file stored
at "tmp/pastebin/<filename>".
Then the filename is sent to the queue "pastebin_parser".
"""
msg = f"Data {body} received from queue {lc.DOWNLOADER_QUEUE}"
if gc.DEBUG:
log_info('downloader', msg)
time.sleep(lc.DOWNLOADING_SLEEP + random.choice(lc.DOWNLOADING_DELAY))
scrape_url = json.loads(body)['scrape_url']
r = requests.get(url=scrape_url)
if r.status_code != 200:
msg = f"Connection error: GET:{scrape_url}, Status:{r.status_code}"
log_info('downloader', msg)
return
file = Paste(metadata=body, data=r.text)
filename = file.save()
ch.basic_ack(delivery_tag=method.delivery_tag)
publish(module='downloader',
channel=ch,
data=filename,
queue=lc.PARSER_QUEUE)
def create_paste():
try:
body = urlparse.parse_qs(
request.body.read(int(settings.MAX_SIZE * 1.1)))
except ValueError:
return {'status': 'error', 'message': u"Wrong data payload."}
try:
content = unicode(''.join(body['content']), 'utf8')
except (UnicodeDecodeError, KeyError):
return {
'status': 'error',
'message': u"Encoding error: the paste couldn't be saved."
}
if '{"iv":' not in content: # reject silently non encrypted content
return {'status': 'error', 'message': u"Wrong data payload."}
if content:
# check size of the paste. if more than settings return error
# without saving paste. prevent from unusual use of the
# system. need to be improved
if len(content) < settings.MAX_SIZE:
expiration = body.get('expiration', [u'burn_after_reading'])[0]
paste = Paste(expiration=expiration,
content=content,
uuid_length=settings.PASTE_ID_LENGTH)
paste.save()
# display counter
if settings.DISPLAY_COUNTER:
#increment paste counter
paste.increment_counter()
# if refresh time elapsed pick up new counter value
now = datetime.now()
timeout = (GLOBAL_CONTEXT['refresh_counter'] +
timedelta(seconds=settings.REFRESH_COUNTER))
if timeout < now:
GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
GLOBAL_CONTEXT['refresh_counter'] = now
return {'status': 'ok', 'paste': paste.uuid}
return {
'status':
'error',
'message':
u"Serveur error: the paste couldn't be saved. "
u"Please try later."
}
def parse_paste_page(self, html, id):
soup = bs(html, 'html.parser')
removed = True if soup.find_all("div", "content_title")[0] \
.text.find("removed") != -1 \
else False
if removed:
return None
info_box = soup.find_all("div", "paste_box_info")[0]
info_line = info_box.find_all("div", "paste_box_line2")[0]
paste_box = soup.find_all("div", "textarea_border")[0]
title = info_box.find_all("div", "paste_box_line1")[0].text
date = info_line.span.get("title")
content = paste_box.textarea.text
author = info_line.a.text \
if info_line.text.lower().find("guest") == -1 \
else "guest"
return Paste(id, author, title, content, date)
def parse_link(url: str) -> Paste:
tree = Downloader.downloadPage(url)
if tree is None:
return None
try:
date = tree.xpath(DataParser.DATE_SELECTOR)[0]
username = tree.xpath(DataParser.USERNAME_SELECTOR)[0]
title = tree.xpath(DataParser.TITLE_SELECTOR)[0]
content = tree.xpath(DataParser.CONTENT_SELECTOR)[0]
except Exception:
print("Failed to process url: ", url)
return None
# todo: validate fetched results
return Paste(author=DataParser.norm_author_title(username),
title=DataParser.norm_author_title(title),
content=DataParser.norm_content(content),
datetime=DataParser.strdate_to_arrow(date),
link=url.split('/')[-1])
def new_paste():
form = PasteForm()
if not hcaptcha.verify():
# TODO: Localize error messages
flash('hCaptchan verifiointi epäonnistui.', 'negative')
# TODO: Complain about other errors even when there is an hCaptcha error
return redirect('/'), 400
if form.validate():
letters = string.ascii_letters + string.digits
paste_id = ''.join(random.choice(letters) for i in range(12))
paste = Paste(id=paste_id, content=form.content.data, \
name=form.name.data, private=form.private.data)
db.session.add(paste)
db.session.commit()
return redirect('/p/' + paste_id + '/')
for field, errors in form.errors.items():
if field == 'csrf_token':
errors[0] = 'Tapahtui CSRF-virhe. Yritä uudelleen.'
for error in errors:
flash(error, 'negative')
return redirect('/')
def api_paste():
if not app.config["ENABLE_API"]:
abort(403)
letters = string.ascii_letters + string.digits
paste_id = ''.join(random.choice(letters) for i in range(12))
try:
data = json.loads(request.data)
except json.decoder.JSONDecodeError:
return jsonify({"error": "INVALID_JSON"}), 400
for param in ("name", "content", "private"):
if not param in data:
return jsonify({"error": "MISSING_PARAMETER", "parameter": \
param}), 400
name = data["name"]
content = data["content"]
private = data["private"]
if not isinstance(name, str) or not name or name.isspace():
return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": "name"}), \
400
if not isinstance(content, str) or not content or content.isspace():
return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": \
"content"}), 400
if not isinstance(private, bool):
return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": \
"private"}), 400
paste = Paste(id=paste_id, content=data["content"], \
name=data["name"], private=data["private"])
db.session.add(paste)
db.session.commit()
return jsonify({"id": paste_id})
