def add_link(alias, outgoing_url, password=None, user_id=None, require_recaptcha=False):
"""
Add a new link to the database after performing necessary input validation.
:param alias: The link alias.
:param outgoing_url: The associated outgoing URL.
:param password: Plain-text password associated with this link, if applicable.
:param user_id: ID of the user to associate with this link, if applicable.
:param require_recaptcha: True to require ReCAPTCHA for accessing this link; False otherwise.
:return: An instance of models.Link representing the new entry.
:raises InvalidAliasException: If the alias is invalid.
:raises ReservedAliasException: If the alias is reserved.
:raises InvalidURLException: If the outgoing URL is invalid.
:raises UnavailableAliasException: If the alias already exists in the database.
"""
if not util.validation.is_alias_valid(alias):
raise InvalidAliasException('Alias `{alias}` is not URL safe'.format(alias=alias))
if util.validation.is_alias_reserved(alias):
raise ReservedAliasException('Alias `{alias}` is reserved'.format(alias=alias))
if not util.validation.is_url_valid(outgoing_url):
raise InvalidURLException('URL `{url}` is not a valid URL'.format(url=outgoing_url))
if models.Link.query.filter_by(alias=alias).scalar():
raise UnavailableAliasException('Alias `{alias}` already exists'.format(alias=alias))
new_link = models.Link(
alias=alias,
outgoing_url=outgoing_url,
password=password,
user_id=user_id,
require_recaptcha=require_recaptcha,
)
db.session.add(new_link)
return new_link
def generate_link(account, path):
link_code = _generate_link_code()
expiration = datetime.datetime.now() + datetime.timedelta(days=7)
models.Link(parent=account.key, expiration=expiration,
link_code=link_code).put()
query_string = urllib.urlencode({'linkCode': link_code})
full_path = posixpath.join(config.URL, path)
return 'https://%s?%s' % (full_path, query_string)
def GET(self,link_id=None,page=1,success=False,csrf_error=False,create=False,cl='en',ct='?',cp=None,cu='http://',*args,**kwargs):
"""Can display 3 kinds of pages (depending on args):
1) If link_id is supplied (url was /link/<n>), shows the link's edit form
2) If create is not False, shows a create form with values from cl,ct,cp,cu
3) Otherwise, shows "editor view" of all links (with edit button per link, and a create button)
"""
conf = cherrypy.request.app.config['ligiloj']
try:
page = max(int(page),1)
except:
raise cherrypy.HTTPError(404,"nevalida paĝo nombro :(")
if not link_id and not create:
#--- 3rd option: show a list of all links
query = models.Link().select(models.Link,models.Language,models.Link.id).join(models.Language)
result = {
#'debug':[cherrypy.request.script_name,cherrypy.request.path_info],
'user':cherrypy.serving.user.name,
'title':'Redaktilo',
'site_title':conf['site_title'],
'url_base':cherrypy.request.base,
'site_root':cherrypy.request.base+cherrypy.request.script_name+'/',
'here':cherrypy.request.base+cherrypy.request.script_name+cherrypy.request.path_info+'/',
'success':success,
'links':query.paginate(page,cherrypy.request.config['paginate_by']).dicts()
}
result.update(nextprev(query.count(),page,cherrypy.request.config['paginate_by']))
return stache.render(stache.load_template('edit_links'),result)
elif create:
#--- 2nd option: show a creation form
try: # I'm sure there's a simpler way :)
cp = datetime.date.fromtimestamp(time.mktime(time.strptime(cp,'%Y-%m-%d')))
except:
cp = datetime.date.today()
l = models.Link(
published=cp, title=ct, url=cu,
language=models.get_language(cl) or models.get_language('eo'))
title = u'Aldonu ligilon'
fancy_title =u'<i class="glyphicon glyphicon-plus"></i> Aldonu ligilon'
else:
#--- 1st option: show edit form for an existing link
try: # get link to edit
l = models.Link.get(id=link_id)
except ValueError,models.DoesNotExist:
raise cherrypy.HTTPError(404,"Ligilo ne estas trovita :(")
title = u'Redaktu ligilon: {0}'.format(l.__unicode__())
fancy_title =u'<i class="glyphicon glyphicon-edit"></i> Redaktu ligilon'
def seed():
Session = sessionmaker(bind=engine)
session = Session()
domain = models.Domain(url='https://register.start.bg')
link = models.Link(url=domain.ulr, domain=domain)
session.add(domain)
session.add(link)
session.commit()
def add_link(user, collection_id, url):
collection = read(user, collection_id)
if not link:
return collection
link = models.Link(url=url)
collection.links.append(link)
collection.put()
return collection
def create_an_link():
link = models.Link(
url="dubyadubyadubyaDOTyermomDOTcom",
stream_id = 0,
message_id = 0,
relevance_score = 0,
timestamp = "timeless"
)
session.add(link)
session.commit()
def seed():
Session = sessionmaker(bind=engine)
session = Session()
domain1 = models.Domain(url='http://register.start.bg/')
link1 = models.Link(url=domain1.url,
domain=domain1)
session.add(domain1)
session.add(link1)
session.commit()
def rss(self,language=None,*args,**kwargs):
if language:
l = models.get_language(language)
conf = cherrypy.request.app.config['ligiloj']
query = models.Link().select(models.Link,models.Language).join(models.Language)
if language:
query = query.where(models.Link.language == l)
cherrypy.response.headers['Content-Type'] = 'application/xml'
return RSS2(title=u'{0} - {1}'.format(conf['site_title'],language and l.name or conf['global_title_text']),
link=conf['rss_site_url'],
description=conf['rss_description'],
language=language or conf['rss_default_language'],
items=[RSSItem(title=language and link.title or u"{0}: {1}".format(link.language.name,link.title),
link=link.url,
pubDate=link.published.isoformat(),
guid=Guid(link.url,str(link.id))) for link in query]).to_xml('utf-8')
def default(self,*args,**kwargs):
"""The end user method at /[language/][page]"""
arglist = list(args)
if not arglist or not arglist[-1].isdigit():
arglist.append('1')
if len(arglist)<2:
arglist.insert(0,None)
try:
page = max(1,int(arglist[1]))
except:
raise cherrypy.HTTPError(404,"Nevalida nombro :(")
lang = arglist and arglist[0] or None
language = None
if lang is not None:
language = models.get_language(lang)
if language is None:
raise cherrypy.HTTPError(404,"Nekonata lingvo")
conf = cherrypy.request.app.config['ligiloj']
page_title = language and language.__unicode__().split(':')[-1].strip() or conf['global_title_text']
menu,active_language = make_menu(lang)
query = models.Link().select(models.Link,models.Language).join(models.Language)
if language:
query = query.where(models.Link.language == language)
try:
user = cherrypy.serving.user and cherrypy.serving.user.name
except:
user = None
result = {
#'debug':[cherrypy.request.base,cherrypy.request.script_name,cherrypy.request.path_info],
'user':user,
'title':u'{0} - {1}'.format(conf['site_title'],page_title),
'fancy_title':u'{0} <small>{1}</small>'.format(
conf['site_title_html'],
language and page_title or conf['global_title_html']),
'site_root':cherrypy.request.base+cherrypy.request.script_name+'/',
'lang':lang or 'en',
'menu':menu,
'active_language':active_language,
'is_international':lang is None,
'count':query.count(),
'links':query.paginate(page,cherrypy.request.config['paginate_by']).dicts()
}
result.update(nextprev(query.count(),page,cherrypy.request.config['paginate_by']))
skin = kwargs.get('skin','index')
return stache.render(stache.load_template(skin),result)
def process_one(stats, dct):
stats['all_comments'] += 1
created = int(dct['created_utc'])
comment_id = db36(dct['id'])
subreddit_id = db36(dct['subreddit_id'])
link_id = db36(dct['link_id'])
comment = models.Comment.get(comment_id)
if not comment:
stats['comments'] += 1
user = models.User.get(dct['author'])
if not user:
stats['users'] += 1
user = models.User(username=dct['author'],
first_seen=created,
last_seen=created,
comments=0)
user.comments += 1
user.last_seen = max(user.last_seen, created)
reddit = models.Reddit.get(subreddit_id)
if not reddit:
stats['reddits'] += 1
reddit = models.Reddit(name=dct['subreddit'],
id=subreddit_id,
first_seen=created,
last_seen=created,
links=0,
comments=0)
reddit.last_seen = created
reddit.comments += 1
link = models.Link.get(link_id)
if not link:
stats['links'] += 1
link = models.Link(id=link_id,
subreddit_id=subreddit_id,
title=dct['link_title'],
first_seen=created,
last_seen=created,
comments=0)
reddit.links += 1
link.last_seen = created
link.comments += 1
reddit.save()
user.save()
link.save()
if dct['parent_id'].startswith('t3_'):
parent_id = None
else:
parent_id = db36(dct['parent_id'])
comment = models.Comment(id=comment_id,
subreddit_id=subreddit_id,
author=dct['author'],
body=dct['body'],
created=created,
link_id=link_id,
parent_id=parent_id,
ups=dct['ups'],
downs=dct['downs'])
if parent_id and not comment.get_parent():
stats['orphans'] += 1
stats['comments'] -= 1
else:
comment.save()
def handleArticle(self, current_page):
#print('ARTICLE:', repr(current_page.title))
if self.pass_num == 0:
models.articles.save(
models.Article(id=current_page['id'],
title=current_page['title'],
revision_id=current_page['revision_id']))
elif self.pass_num == 1:
aid = models.articles.resolve_title(current_page['title'])
wikitext__ = utils.unescape_html(current_page['text'])
redirect = wikitext.parse_redirect(wikitext__)
if redirect:
redirect_dest_title, dest_frag, redirect_label = redirect
article = models.articles[aid]
if redirect_dest_title in models.articles:
dest_id = models.articles.resolve_title(
redirect_dest_title)
else:
dest_id = None
#print('Broken REDIRECT FROM ', repr(current_page['title']), ' TO ', repr(redirect), file=parse_log)
#print(repr(redirect_dest_title), file=parse_log)
print >> parse_log, repr(redirect_dest_title)
article.redirect = models.Redirect(dest_id, dest_frag,
redirect_label)
models.articles.save(article)
else:
#do not parse/allow links if there is a redirect. If there is a redirect, links are meaningless. besides, parse_links does not work properly with redirects.
for link_dest_title, dest_frag, link_label, snippet in wikitext.parse_links(
wikitext__):
lid = self.num_links + 1 #0 is an invalid key
#assert lid not in models.links
if link_dest_title in models.articles:
dest_id = models.articles.resolve_title(
link_dest_title)
else:
dest_id = None
#print('Broken LINK FROM ', repr(current_page['title']), ' TO ', repr((link_dest_title, dest_frag, link_label)), file=parse_log)
#print(repr(link_dest_title), file=parse_log)
print >> parse_log, repr(link_dest_title)
models.links.save(
models.Link(src_id=aid,
dest_id=dest_id,
dest_fragment=dest_frag,
label=link_label,
snippet=snippet,
id=lid))
self.num_links += 1
'''
if lid > len(models.links):
...
else:
self.num_links += 1 #skip
'''
self.num_articles += 1
if self.num_articles % 1000 == 0:
#print(self.num_articles, self.num_links)
print self.num_articles, self.num_links
def create_link(db: Session, link: schemas.LinkCreate):
db_link = models.Link(link.full_link)
db.add(db_link)
db.commit()
db.refresh(db_link)
return db_link
def start_fetching():
"""
Starts the fetching proces.
Requires at least one entry in models.Page.__tablename__
"""
session = models.Session()
debug("Started fetching")
while True:
page = session.query(models.Page).filter(
models.Page.parse_status == models.Page.PENDING
).first()
debug("Got %s" % page)
if page is None:
debug("Returning...")
break
try:
debug("Fetching... %s" % page.url)
if dangling_extension(page.url.lower()):
debug("Dangling extension - skipping")
page.parse_status = models.Page.COMPLETED
session.commit()
else:
response = urllib2.urlopen(page.url)
except urllib2.HTTPError as e:
print "%s: %s" % (e.errno, e.strerror)
page.parse_status = models.Page.FAILED
session.commit()
continue
page.last_update = datetime.datetime.utcnow()
page.parse_status = models.Page.COMPLETED
session.commit()
debug("Fetch completed.")
try:
html = BeautifulSoup(response.read(), "html.parser")
except Exception as e:
print e
continue
valid_links = {}
for link in html.find_all('a'):
if not link.get('href'):
continue
href = fullPath(page.url, link.get('href'))
print "H", href
if href is None:
continue
valid_domain = False
for domain in conf['domain']:
if 'http://' + domain in href:
valid_domain = True
if 'https://' + domain in href:
valid_domain = True
print href, valid_domain
if not valid_domain:
continue
if href not in valid_links:
valid_links[href] = 1
else:
valid_links[href] += 1
debug("Found valid links: %s" % valid_links)
debug("Processing links")
for url in valid_links:
if url.startswith('http'):
otherUrl = 'https' + url[url.find(':'):]
else:
otherUrl = 'http' + url[url.find(':'):]
destination = session.query(models.Page).filter(or_(
models.Page.url == url,
models.Page.url == otherUrl
)
).first()
debug("Destination: %s" % destination)
if destination is None:
destination = models.Page(
url=url,
parse_status=models.Page.COMPLETED if dangling_extension(url) else models.Page.PENDING
)
session.add(destination)
debug("Destination not found. New one created: %s " % destination)
link = models.Link(
from_page=page,
to_page=destination,
n=valid_links[url]
)
page.outlinks.append(link)
debug("Created new link: %s" % link)
session.commit()
If srsly_delete, will [SRSLY] delete the link"""
conf = cherrypy.request.app.config['ligiloj']
if not check_csrf_token(cherrypy.request.params.get('csrf_token','')):
raise cherrypy.HTTPRedirect('?csrf_error=Vera',303)
here = cherrypy.request.base+cherrypy.request.script_name+cherrypy.request.path_info
if link_id: # edit existing link
try: # get link to save
l=models.Link.get(id=link_id)
except ValueError,models.DoesNotExist:
raise cherrypy.HTTPError(404,"Ligilo ne estas trovita :(")
if srsly_delete:
l.delete_instance()
raise cherrypy.HTTPRedirect('.?success=Vera',303)
form = models.LinkForm(FakeMultiDict(cherrypy.request.body_params),obj=l)
else: # create a new link
l = models.Link()
form = models.LinkForm(FakeMultiDict(cherrypy.request.params))
if form.validate():
form.populate_obj(l)
l.save()
raise cherrypy.HTTPRedirect('{0}?success=Vera'.format(link_id and '.' or ''),303)
return stache.render(stache.load_template('edit_link'),l,
user=cherrypy.serving.user.name,
title=u'Redaktu ligilon: {0}'.format(l.__unicode__()), site_title=conf['site_title'],
site_root=cherrypy.request.base+cherrypy.request.script_name+'/',
here=here,
csrf_token=make_csrf_token(),
form=bootstrapize_form(form))
class LigilojApp(object):
"""The end user app at /[language/][page]"""
def create_link(subcategory_id, state):
return models.Link(subcategory_id=subcategory_id, state=state).save()