Example #1
0
def add_link(alias, outgoing_url, password=None, user_id=None, require_recaptcha=False):
    """
    Add a new link to the database after performing necessary input validation.

    :param alias: The link alias.
    :param outgoing_url: The associated outgoing URL.
    :param password: Plain-text password associated with this link, if applicable.
    :param user_id: ID of the user to associate with this link, if applicable.
    :param require_recaptcha: True to require ReCAPTCHA for accessing this link; False otherwise.
    :return: An instance of models.Link representing the new entry.
    :raises InvalidAliasException: If the alias is invalid.
    :raises ReservedAliasException: If the alias is reserved.
    :raises InvalidURLException: If the outgoing URL is invalid.
    :raises UnavailableAliasException: If the alias already exists in the database.
    """
    if not util.validation.is_alias_valid(alias):
        raise InvalidAliasException('Alias `{alias}` is not URL safe'.format(alias=alias))
    if util.validation.is_alias_reserved(alias):
        raise ReservedAliasException('Alias `{alias}` is reserved'.format(alias=alias))
    if not util.validation.is_url_valid(outgoing_url):
        raise InvalidURLException('URL `{url}` is not a valid URL'.format(url=outgoing_url))
    if models.Link.query.filter_by(alias=alias).scalar():
        raise UnavailableAliasException('Alias `{alias}` already exists'.format(alias=alias))

    new_link = models.Link(
        alias=alias,
        outgoing_url=outgoing_url,
        password=password,
        user_id=user_id,
        require_recaptcha=require_recaptcha,
    )
    db.session.add(new_link)

    return new_link
Example #2
0
def generate_link(account, path):
    link_code = _generate_link_code()
    expiration = datetime.datetime.now() + datetime.timedelta(days=7)
    models.Link(parent=account.key, expiration=expiration,
                link_code=link_code).put()
    query_string = urllib.urlencode({'linkCode': link_code})
    full_path = posixpath.join(config.URL, path)
    return 'https://%s?%s' % (full_path, query_string)
Example #3
0
    def GET(self,link_id=None,page=1,success=False,csrf_error=False,create=False,cl='en',ct='?',cp=None,cu='http://',*args,**kwargs):
        """Can display 3 kinds of pages (depending on args):
1) If link_id is supplied (url was /link/<n>), shows the link's edit form
2) If create is not False, shows a create form with values from cl,ct,cp,cu
3) Otherwise, shows "editor view" of all links (with edit button per link, and a create button)
"""
        conf = cherrypy.request.app.config['ligiloj']
        try:
            page = max(int(page),1)
        except:
            raise cherrypy.HTTPError(404,"nevalida paĝo nombro :(")
        if not link_id and not create:
            #--- 3rd option: show a list of all links
            query = models.Link().select(models.Link,models.Language,models.Link.id).join(models.Language)
            result = {
                #'debug':[cherrypy.request.script_name,cherrypy.request.path_info],
                'user':cherrypy.serving.user.name,
                'title':'Redaktilo',
                'site_title':conf['site_title'],
                'url_base':cherrypy.request.base,
                'site_root':cherrypy.request.base+cherrypy.request.script_name+'/',
                'here':cherrypy.request.base+cherrypy.request.script_name+cherrypy.request.path_info+'/',
                'success':success,
                'links':query.paginate(page,cherrypy.request.config['paginate_by']).dicts()
            }
            result.update(nextprev(query.count(),page,cherrypy.request.config['paginate_by']))
            return stache.render(stache.load_template('edit_links'),result)
        elif create:
            #--- 2nd option: show a creation form
            try: # I'm sure there's a simpler way :)
                cp = datetime.date.fromtimestamp(time.mktime(time.strptime(cp,'%Y-%m-%d')))
            except:
                cp = datetime.date.today()
            l = models.Link(
                published=cp, title=ct, url=cu,
                language=models.get_language(cl) or models.get_language('eo'))
            title = u'Aldonu ligilon'
            fancy_title =u'<i class="glyphicon glyphicon-plus"></i> Aldonu ligilon'
        else:
            #--- 1st option: show edit form for an existing link
            try: # get link to edit
                l = models.Link.get(id=link_id)
            except ValueError,models.DoesNotExist:
                raise cherrypy.HTTPError(404,"Ligilo ne estas trovita :(")
            title = u'Redaktu ligilon: {0}'.format(l.__unicode__())
            fancy_title =u'<i class="glyphicon glyphicon-edit"></i> Redaktu ligilon'
Example #4
0
def seed():
    Session = sessionmaker(bind=engine)
    session = Session()

    domain = models.Domain(url='https://register.start.bg')
    link = models.Link(url=domain.ulr, domain=domain)
    session.add(domain)
    session.add(link)
    session.commit()
Example #5
0
def add_link(user, collection_id, url):
    collection = read(user, collection_id)

    if not link:
        return collection

    link = models.Link(url=url)
    collection.links.append(link)
    collection.put()
    return collection
Example #6
0
def create_an_link():
    link = models.Link(
        url="dubyadubyadubyaDOTyermomDOTcom",
        stream_id = 0,
        message_id = 0,
        relevance_score = 0,
        timestamp = "timeless"
    )
    session.add(link)
    session.commit()
def seed():
    Session = sessionmaker(bind=engine)
    session = Session()

    domain1 = models.Domain(url='http://register.start.bg/')

    link1 = models.Link(url=domain1.url,
                        domain=domain1)

    session.add(domain1)
    session.add(link1)

    session.commit()
Example #8
0
 def rss(self,language=None,*args,**kwargs):
     if language:
         l = models.get_language(language)
     conf = cherrypy.request.app.config['ligiloj']
     query = models.Link().select(models.Link,models.Language).join(models.Language)
     if language:
         query = query.where(models.Link.language == l)
     cherrypy.response.headers['Content-Type'] = 'application/xml'
     return RSS2(title=u'{0} - {1}'.format(conf['site_title'],language and l.name or conf['global_title_text']),
         link=conf['rss_site_url'],
         description=conf['rss_description'],
         language=language or conf['rss_default_language'],
         items=[RSSItem(title=language and link.title or u"{0}: {1}".format(link.language.name,link.title),
             link=link.url,
             pubDate=link.published.isoformat(),
             guid=Guid(link.url,str(link.id))) for link in query]).to_xml('utf-8')
Example #9
0
 def default(self,*args,**kwargs):
     """The end user method at /[language/][page]"""
     arglist = list(args)
     if not arglist or not arglist[-1].isdigit():
         arglist.append('1')
     if len(arglist)<2:
         arglist.insert(0,None)
     try:
         page = max(1,int(arglist[1]))
     except:
         raise cherrypy.HTTPError(404,"Nevalida nombro :(")
     lang = arglist and arglist[0] or None
     language = None
     if lang is not None:
         language = models.get_language(lang)
         if language is None:
             raise cherrypy.HTTPError(404,"Nekonata lingvo")
     conf = cherrypy.request.app.config['ligiloj']
     page_title = language and language.__unicode__().split(':')[-1].strip() or conf['global_title_text']
     menu,active_language = make_menu(lang)
     query = models.Link().select(models.Link,models.Language).join(models.Language)
     if language:
         query = query.where(models.Link.language == language)
     try:
         user = cherrypy.serving.user and cherrypy.serving.user.name
     except:
         user = None
     result = {
         #'debug':[cherrypy.request.base,cherrypy.request.script_name,cherrypy.request.path_info],
         'user':user,
         'title':u'{0} - {1}'.format(conf['site_title'],page_title),
         'fancy_title':u'{0} <small>{1}</small>'.format(
             conf['site_title_html'],
             language and page_title or conf['global_title_html']),
         'site_root':cherrypy.request.base+cherrypy.request.script_name+'/',
         'lang':lang or 'en',
         'menu':menu,
         'active_language':active_language,
         'is_international':lang is None,
         'count':query.count(),
         'links':query.paginate(page,cherrypy.request.config['paginate_by']).dicts()
     }
     result.update(nextprev(query.count(),page,cherrypy.request.config['paginate_by']))
     skin = kwargs.get('skin','index')
     return stache.render(stache.load_template(skin),result)
Example #10
0
def process_one(stats, dct):
    stats['all_comments'] += 1
    created = int(dct['created_utc'])

    comment_id = db36(dct['id'])
    subreddit_id = db36(dct['subreddit_id'])
    link_id = db36(dct['link_id'])

    comment = models.Comment.get(comment_id)
    if not comment:
        stats['comments'] += 1
        user = models.User.get(dct['author'])
        if not user:
            stats['users'] += 1
            user = models.User(username=dct['author'],
                               first_seen=created,
                               last_seen=created,
                               comments=0)
        user.comments += 1
        user.last_seen = max(user.last_seen, created)

        reddit = models.Reddit.get(subreddit_id)
        if not reddit:

            stats['reddits'] += 1
            reddit = models.Reddit(name=dct['subreddit'],
                                   id=subreddit_id,
                                   first_seen=created,
                                   last_seen=created,
                                   links=0,
                                   comments=0)
        reddit.last_seen = created
        reddit.comments += 1

        link = models.Link.get(link_id)
        if not link:
            stats['links'] += 1
            link = models.Link(id=link_id,
                               subreddit_id=subreddit_id,
                               title=dct['link_title'],
                               first_seen=created,
                               last_seen=created,
                               comments=0)
            reddit.links += 1

        link.last_seen = created
        link.comments += 1

        reddit.save()
        user.save()
        link.save()

    if dct['parent_id'].startswith('t3_'):
        parent_id = None
    else:
        parent_id = db36(dct['parent_id'])

    comment = models.Comment(id=comment_id,
                             subreddit_id=subreddit_id,
                             author=dct['author'],
                             body=dct['body'],
                             created=created,
                             link_id=link_id,
                             parent_id=parent_id,
                             ups=dct['ups'],
                             downs=dct['downs'])
    if parent_id and not comment.get_parent():
        stats['orphans'] += 1
        stats['comments'] -= 1
    else:
        comment.save()
Example #11
0
    def handleArticle(self, current_page):
        #print('ARTICLE:', repr(current_page.title))

        if self.pass_num == 0:
            models.articles.save(
                models.Article(id=current_page['id'],
                               title=current_page['title'],
                               revision_id=current_page['revision_id']))
        elif self.pass_num == 1:
            aid = models.articles.resolve_title(current_page['title'])
            wikitext__ = utils.unescape_html(current_page['text'])

            redirect = wikitext.parse_redirect(wikitext__)
            if redirect:
                redirect_dest_title, dest_frag, redirect_label = redirect
                article = models.articles[aid]
                if redirect_dest_title in models.articles:
                    dest_id = models.articles.resolve_title(
                        redirect_dest_title)
                else:
                    dest_id = None
                    #print('Broken REDIRECT FROM ', repr(current_page['title']), ' TO ', repr(redirect), file=parse_log)
                    #print(repr(redirect_dest_title), file=parse_log)
                    print >> parse_log, repr(redirect_dest_title)
                article.redirect = models.Redirect(dest_id, dest_frag,
                                                   redirect_label)
                models.articles.save(article)
            else:
                #do not parse/allow links if there is a redirect. If there is a redirect, links are meaningless. besides, parse_links does not work properly with redirects.
                for link_dest_title, dest_frag, link_label, snippet in wikitext.parse_links(
                        wikitext__):
                    lid = self.num_links + 1  #0 is an invalid key
                    #assert lid not in models.links
                    if link_dest_title in models.articles:
                        dest_id = models.articles.resolve_title(
                            link_dest_title)
                    else:
                        dest_id = None
                        #print('Broken LINK FROM ', repr(current_page['title']), ' TO ', repr((link_dest_title, dest_frag, link_label)), file=parse_log)
                        #print(repr(link_dest_title), file=parse_log)
                        print >> parse_log, repr(link_dest_title)
                    models.links.save(
                        models.Link(src_id=aid,
                                    dest_id=dest_id,
                                    dest_fragment=dest_frag,
                                    label=link_label,
                                    snippet=snippet,
                                    id=lid))
                    self.num_links += 1
                    '''
                    if lid > len(models.links):
                        ...
                    else:
                        self.num_links += 1 #skip
                    '''

        self.num_articles += 1

        if self.num_articles % 1000 == 0:
            #print(self.num_articles, self.num_links)
            print self.num_articles, self.num_links
def create_link(db: Session, link: schemas.LinkCreate):
    db_link = models.Link(link.full_link)
    db.add(db_link)
    db.commit()
    db.refresh(db_link)
    return db_link
Example #13
0
def start_fetching():
    """
    Starts the fetching proces.
    Requires at least one entry in models.Page.__tablename__
    """
    session = models.Session()

    debug("Started fetching")
    while True:
        page = session.query(models.Page).filter(
            models.Page.parse_status == models.Page.PENDING
        ).first()

        debug("Got %s" % page)
        if page is None:
            debug("Returning...")
            break

        try:
            debug("Fetching... %s" % page.url)
            if dangling_extension(page.url.lower()):
                debug("Dangling extension - skipping")
                page.parse_status = models.Page.COMPLETED
                session.commit()
            else:
                response = urllib2.urlopen(page.url)
        except urllib2.HTTPError as e:
            print "%s: %s" % (e.errno, e.strerror)
            page.parse_status = models.Page.FAILED
            session.commit()
            continue

        page.last_update = datetime.datetime.utcnow()
        page.parse_status = models.Page.COMPLETED
        session.commit()
        debug("Fetch completed.")
        try:
            html = BeautifulSoup(response.read(), "html.parser")
        except Exception as e:
            print e
            continue
        valid_links = {}
        for link in html.find_all('a'):
            if not link.get('href'):
                continue

            href = fullPath(page.url, link.get('href'))

            print "H", href
            if href is None:
                continue

            valid_domain = False
            for domain in conf['domain']:
                if 'http://' + domain in href:
                    valid_domain = True
                if 'https://' + domain in href:
                    valid_domain = True

            print href, valid_domain
            if not valid_domain:
                continue

            if href not in valid_links:
                valid_links[href] = 1
            else:
                valid_links[href] += 1

        debug("Found valid links: %s" % valid_links)
        debug("Processing links")
        for url in valid_links:
            if url.startswith('http'):
                otherUrl = 'https' + url[url.find(':'):]
            else:
                otherUrl = 'http' + url[url.find(':'):]

            destination = session.query(models.Page).filter(or_(
                models.Page.url == url,
                models.Page.url == otherUrl
            )
            ).first()

            debug("Destination: %s" % destination)
            if destination is None:
                destination = models.Page(
                    url=url,
                    parse_status=models.Page.COMPLETED if dangling_extension(url) else models.Page.PENDING
                )
                session.add(destination)
                debug("Destination not found. New one created: %s " % destination)

            link = models.Link(
                from_page=page,
                to_page=destination,
                n=valid_links[url]
            )
            page.outlinks.append(link)
            debug("Created new link: %s" % link)
        session.commit()
Example #14
0
If srsly_delete, will [SRSLY] delete the link"""
        conf = cherrypy.request.app.config['ligiloj']
        if not check_csrf_token(cherrypy.request.params.get('csrf_token','')):
            raise cherrypy.HTTPRedirect('?csrf_error=Vera',303)
        here = cherrypy.request.base+cherrypy.request.script_name+cherrypy.request.path_info
        if link_id: # edit existing link
            try: # get link to save
                l=models.Link.get(id=link_id)
            except ValueError,models.DoesNotExist:
                raise cherrypy.HTTPError(404,"Ligilo ne estas trovita :(")
            if srsly_delete:
                l.delete_instance()
                raise cherrypy.HTTPRedirect('.?success=Vera',303)
            form = models.LinkForm(FakeMultiDict(cherrypy.request.body_params),obj=l)
        else: # create a new link
            l = models.Link()
            form = models.LinkForm(FakeMultiDict(cherrypy.request.params))
        if form.validate():
            form.populate_obj(l)
            l.save()
            raise cherrypy.HTTPRedirect('{0}?success=Vera'.format(link_id and '.' or ''),303)
        return stache.render(stache.load_template('edit_link'),l,
            user=cherrypy.serving.user.name,
            title=u'Redaktu ligilon: {0}'.format(l.__unicode__()), site_title=conf['site_title'],
            site_root=cherrypy.request.base+cherrypy.request.script_name+'/',
            here=here,
            csrf_token=make_csrf_token(),
            form=bootstrapize_form(form))

class LigilojApp(object):
    """The end user app at /[language/][page]"""
def create_link(subcategory_id, state):
    return models.Link(subcategory_id=subcategory_id, state=state).save()