Example #1
0
    def callback(ch, method, properties, body):
        """
        This function will be triggered when an element enter the  
        pastebin_downloader queue. Paste object will be create in
        order to save the content of the Paste in a text file stored
        at "tmp/pastebin/<filename>".
        Then the filename is sent to the queue "pastebin_parser".
        """
        msg = f"Data {body} received from queue {lc.DOWNLOADER_QUEUE}"
        if gc.DEBUG:
            log_info('downloader', msg)

        time.sleep(lc.DOWNLOADING_SLEEP + random.choice(lc.DOWNLOADING_DELAY))

        scrape_url = json.loads(body)['scrape_url']
        r = requests.get(url=scrape_url)
        if r.status_code != 200:
            msg = f"Connection error: GET:{scrape_url}, Status:{r.status_code}"
            log_info('downloader', msg)
            return

        file = Paste(metadata=body, data=r.text)

        filename = file.save()
        ch.basic_ack(delivery_tag=method.delivery_tag)

        publish(module='downloader',
                channel=ch,
                data=filename,
                queue=lc.PARSER_QUEUE)
Example #2
0
def display_paste(paste_id):

    now = datetime.now()
    keep_alive = False
    try:
        paste = Paste.load(paste_id)
        # Delete the paste if it expired:
        if 'burn_after_reading' in str(paste.expiration):
            # burn_after_reading contains the paste creation date
            # if this read appends 10 seconds after the creation date
            # we don't delete the paste because it means it's the redirection
            # to the paste that happens during the paste creation
            try:
                keep_alive = paste.expiration.split('#')[1]
                keep_alive = datetime.strptime(keep_alive,
                                               '%Y-%m-%d %H:%M:%S.%f')
                keep_alive = now < keep_alive + timedelta(seconds=10)
            except IndexError:
                keep_alive = False
            if not keep_alive:
                paste.delete()

        elif paste.expiration < now:
            paste.delete()
            raise ValueError()

    except (TypeError, ValueError):
        return error404(ValueError)

    context = {'paste': paste, 'keep_alive': keep_alive}
    return dmerge(context, GLOBAL_CONTEXT)
Example #3
0
def display_paste(paste_id):

    now = datetime.now()
    keep_alive = False
    try:
        paste = Paste.load(paste_id)
        # Delete the paste if it expired:
        if 'burn_after_reading' in str(paste.expiration):
            # burn_after_reading contains the paste creation date
            # if this read appends 10 seconds after the creation date
            # we don't delete the paste because it means it's the redirection
            # to the paste that happens during the paste creation
            try:
                keep_alive = paste.expiration.split('#')[1]
                keep_alive = datetime.strptime(keep_alive,
                                               '%Y-%m-%d %H:%M:%S.%f')
                keep_alive = now < keep_alive + timedelta(seconds=10)
            except IndexError:
                keep_alive = False
            if not keep_alive:
                paste.delete()

        elif paste.expiration < now:
            paste.delete()
            raise ValueError()

    except (TypeError, ValueError):
        return error404(ValueError)

    context = {'paste': paste, 'keep_alive': keep_alive}
    return dmerge(context, GLOBAL_CONTEXT)
Example #4
0
def create_paste():
    try:
        body = urlparse.parse_qs(
            request.body.read(int(settings.MAX_SIZE * 1.1)))
    except ValueError:
        return {'status': 'error', 'message': u"Wrong data payload."}

    try:
        content = unicode(''.join(body['content']), 'utf8')
    except (UnicodeDecodeError, KeyError):
        return {
            'status': 'error',
            'message': u"Encoding error: the paste couldn't be saved."
        }

    if '{"iv":' not in content:  # reject silently non encrypted content
        return {'status': 'error', 'message': u"Wrong data payload."}

    if content:
        # check size of the paste. if more than settings return error
        # without saving paste.  prevent from unusual use of the
        # system.  need to be improved
        if len(content) < settings.MAX_SIZE:
            expiration = body.get('expiration', [u'burn_after_reading'])[0]
            paste = Paste(expiration=expiration,
                          content=content,
                          uuid_length=settings.PASTE_ID_LENGTH)
            paste.save()

            # display counter
            if settings.DISPLAY_COUNTER:

                #increment paste counter
                paste.increment_counter()

                # if refresh time elapsed pick up new counter value
                now = datetime.now()
                timeout = (GLOBAL_CONTEXT['refresh_counter'] +
                           timedelta(seconds=settings.REFRESH_COUNTER))
                if timeout < now:
                    GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
                    GLOBAL_CONTEXT['refresh_counter'] = now

            return {'status': 'ok', 'paste': paste.uuid}

    return {
        'status':
        'error',
        'message':
        u"Serveur error: the paste couldn't be saved. "
        u"Please try later."
    }
Example #5
0
def create_paste():

    try:
        content = unicode(request.forms.get('content', ''), 'utf8')
    except UnicodeDecodeError:
        return {'status': 'error',
                'message': u"Encoding error: the paste couldn't be saved."}

    if '{"iv":' not in content: # reject silently non encrypted content
        return ''

    if content:
        # check size of the paste. if more than settings return error without saving paste.
        # prevent from unusual use of the system.
        # need to be improved
        if len(content) < settings.MAX_SIZE:
            expiration = request.forms.get('expiration', u'burn_after_reading')
            paste = Paste(expiration=expiration, content=content)
            paste.save()
            return {'status': 'ok',
                    'paste': paste.uuid}

    return {'status': 'error',
            'message': u"Serveur error: the paste couldn't be saved. Please try later."}
Example #6
0
    def parse_paste_page(self, html, id):
        soup = bs(html, 'html.parser')
        removed = True if soup.find_all("div", "content_title")[0] \
                              .text.find("removed") != -1 \
            else False
        if removed:
            return None

        info_box = soup.find_all("div", "paste_box_info")[0]
        info_line = info_box.find_all("div", "paste_box_line2")[0]
        paste_box = soup.find_all("div", "textarea_border")[0]
        title = info_box.find_all("div", "paste_box_line1")[0].text
        date = info_line.span.get("title")
        content = paste_box.textarea.text
        author = info_line.a.text \
            if info_line.text.lower().find("guest") == -1 \
            else "guest"
        return Paste(id, author, title, content, date)
Example #7
0
    def parse_link(url: str) -> Paste:
        tree = Downloader.downloadPage(url)
        if tree is None:
            return None

        try:
            date = tree.xpath(DataParser.DATE_SELECTOR)[0]
            username = tree.xpath(DataParser.USERNAME_SELECTOR)[0]
            title = tree.xpath(DataParser.TITLE_SELECTOR)[0]
            content = tree.xpath(DataParser.CONTENT_SELECTOR)[0]
        except Exception:
            print("Failed to process url: ", url)
            return None

        # todo: validate fetched results
        return Paste(author=DataParser.norm_author_title(username),
                     title=DataParser.norm_author_title(title),
                     content=DataParser.norm_content(content),
                     datetime=DataParser.strdate_to_arrow(date),
                     link=url.split('/')[-1])
Example #8
0
def new_paste():
    form = PasteForm()
    if not hcaptcha.verify():
        # TODO: Localize error messages
        flash('hCaptchan verifiointi epäonnistui.', 'negative')
        # TODO: Complain about other errors even when there is an hCaptcha error
        return redirect('/'), 400
    if form.validate():
        letters = string.ascii_letters + string.digits
        paste_id = ''.join(random.choice(letters) for i in range(12))
        paste = Paste(id=paste_id, content=form.content.data, \
            name=form.name.data, private=form.private.data)
        db.session.add(paste)
        db.session.commit()
        return redirect('/p/' + paste_id + '/')
    for field, errors in form.errors.items():
        if field == 'csrf_token':
            errors[0] = 'Tapahtui CSRF-virhe. Yritä uudelleen.'
        for error in errors:
            flash(error, 'negative')
    return redirect('/')
Example #9
0
def create_paste():
    try:
        body = urlparse.parse_qs(request.body.read(int(settings.MAX_SIZE * 1.1)))
    except ValueError:
        return {'status': 'error',
                'message': u"Wrong data payload."}

    try:
        content = unicode(''.join(body['content']), 'utf8')
    except (UnicodeDecodeError, KeyError):
        return {'status': 'error',
                'message': u"Encoding error: the paste couldn't be saved."}

    if '{"iv":' not in content:  # reject silently non encrypted content
        return {'status': 'error',
                'message': u"Wrong data payload."}

    if content:
        # check size of the paste. if more than settings return error
        # without saving paste.  prevent from unusual use of the
        # system.  need to be improved
        if len(content) < settings.MAX_SIZE:
            expiration = body.get('expiration', [u'burn_after_reading'])[0]
            paste = Paste(expiration=expiration, content=content,
                          uuid_length=settings.PASTE_ID_LENGTH)
            paste.save()

            # display counter
            if settings.DISPLAY_COUNTER:

                #increment paste counter
                paste.increment_counter()

                # if refresh time elapsed pick up new counter value
                now = datetime.now()
                timeout = (GLOBAL_CONTEXT['refresh_counter']
                           + timedelta(seconds=settings.REFRESH_COUNTER))
                if timeout < now:
                    GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
                    GLOBAL_CONTEXT['refresh_counter'] = now

            return {'status': 'ok',
                    'paste': paste.uuid}

    return {'status': 'error',
            'message': u"Serveur error: the paste couldn't be saved. "
                       u"Please try later."}
Example #10
0
def api_paste():
    if not app.config["ENABLE_API"]:
        abort(403)

    letters = string.ascii_letters + string.digits
    paste_id = ''.join(random.choice(letters) for i in range(12))

    try:
        data = json.loads(request.data)
    except json.decoder.JSONDecodeError:
        return jsonify({"error": "INVALID_JSON"}), 400

    for param in ("name", "content", "private"):
        if not param in data:
            return jsonify({"error": "MISSING_PARAMETER", "parameter": \
                    param}), 400

    name = data["name"]
    content = data["content"]
    private = data["private"]

    if not isinstance(name, str) or not name or name.isspace():
        return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": "name"}), \
                400

    if not isinstance(content, str) or not content or content.isspace():
        return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": \
                "content"}), 400

    if not isinstance(private, bool):
        return jsonify({"error": "WRONG_TYPE_OR_NULL", "parameter": \
                "private"}), 400

    paste = Paste(id=paste_id, content=data["content"], \
        name=data["name"], private=data["private"])
    db.session.add(paste)
    db.session.commit()
    return jsonify({"id": paste_id})
Example #11
0
def create_paste():

    try:
        content = unicode(request.forms.get('content', ''), 'utf8')
    except UnicodeDecodeError:
        return {'status': 'error',
                'message': u"Encoding error: the paste couldn't be saved."}

    if '{"iv":' not in content: # reject silently non encrypted content
        return ''

    if content:
        # check size of the paste. if more than settings return error without saving paste.
        # prevent from unusual use of the system.
        # need to be improved
        if len(content) < settings.MAX_SIZE:
            expiration = request.forms.get('expiration', u'burn_after_reading')
            paste = Paste(expiration=expiration, content=content)
            paste.save()

            # display counter
            if settings.DISPLAY_COUNTER:

                #increment paste counter
                paste.increment_counter()

                # if refresh time elapsed pick up new counter value
                now = datetime.now()
                timeout = (GLOBAL_CONTEXT['refresh_counter']
                           + timedelta(seconds=settings.REFRESH_COUNTER))
                if timeout < now:
                    GLOBAL_CONTEXT['pastes_count'] = Paste.get_pastes_count()
                    GLOBAL_CONTEXT['refresh_counter'] = now


            return {'status': 'ok',
                    'paste': paste.uuid}

    return {'status': 'error',
            'message': u"Serveur error: the paste couldn't be saved. Please try later."}
Example #12
0
def create_paste():
    try:
        body = urlparse.parse_qs(request.body.read(int(settings.MAX_SIZE * 1.1)))
    except ValueError:
        return {"status": "error", "message": u"Wrong data payload."}

    try:
        content = unicode("".join(body["content"]), "utf8")
    except (UnicodeDecodeError, KeyError):
        return {"status": "error", "message": u"Encoding error: the paste couldn't be saved."}

    if '{"iv":' not in content:  # reject silently non encrypted content
        return {"status": "error", "message": u"Wrong data payload."}

    if content:
        # check size of the paste. if more than settings return error
        # without saving paste.  prevent from unusual use of the
        # system.  need to be improved
        if len(content) < settings.MAX_SIZE:
            expiration = body.get("expiration", [u"burn_after_reading"])[0]
            paste = Paste(expiration=expiration, content=content)
            paste.save()

            # display counter
            if settings.DISPLAY_COUNTER:

                # increment paste counter
                paste.increment_counter()

                # if refresh time elapsed pick up new counter value
                now = datetime.now()
                timeout = GLOBAL_CONTEXT["refresh_counter"] + timedelta(seconds=settings.REFRESH_COUNTER)
                if timeout < now:
                    GLOBAL_CONTEXT["pastes_count"] = Paste.get_pastes_count()
                    GLOBAL_CONTEXT["refresh_counter"] = now

            return {"status": "ok", "paste": paste.uuid}

    return {"status": "error", "message": u"Serveur error: the paste couldn't be saved. " u"Please try later."}
Example #13
0
def get_paste(id):
    p = Paste()
    return p.get(id).get_data()
Example #14
0
# add project dir and libs dir to the PYTHON PATH to ensure they are
# importable
from utils import settings

import bottle
from bottle import Bottle, run, static_file, view, request

import clize

from paste import Paste
from utils import drop_privileges, dmerge


app = Bottle()
GLOBAL_CONTEXT = {"settings": settings, "pastes_count": Paste.get_pastes_count(), "refresh_counter": datetime.now()}


@app.route("/")
@view("home")
def index():
    return GLOBAL_CONTEXT


@app.route("/faq/")
@view("faq")
def faq():
    return GLOBAL_CONTEXT


@app.route("/paste/create", method="POST")
Example #15
0
def put_paste():
    data = request.get_data()
    print(data)
    p = Paste()
    return "pasted to id: %s" % (p.put(data))
Example #16
0
# importable
from utils import settings, SettingsValidationError

import bottle
from bottle import (Bottle, run, static_file, view, request)

import clize

from paste import Paste
from utils import drop_privileges, dmerge


app = Bottle()
GLOBAL_CONTEXT = {
    'settings': settings,
    'pastes_count': Paste.get_pastes_count(),
    'refresh_counter': datetime.now()
}


@app.route('/')
@view('home')
def index():
    return GLOBAL_CONTEXT


@app.route('/faq/')
@view('faq')
def faq():
    return GLOBAL_CONTEXT
Example #17
0
# add project dir and libs dir to the PYTHON PATH to ensure they are
# importable
from utils import settings, SettingsValidationError

import bottle
from bottle import (Bottle, run, static_file, view, request)

import clize

from paste import Paste
from utils import drop_privileges, dmerge

app = Bottle()
GLOBAL_CONTEXT = {
    'settings': settings,
    'pastes_count': Paste.get_pastes_count(),
    'refresh_counter': datetime.now()
}


@app.route('/')
@view('home')
def index():
    return GLOBAL_CONTEXT


@app.route('/faq/')
@view('faq')
def faq():
    return GLOBAL_CONTEXT