def boards():
##if empty "/boards" return to main
if len(request.args)==0:
redirect(URL('index'))
id = long(request.args(0))
current_board = cacher.get('board', id) or redirect(URL('index'))
response.title = 'Pinformation :: {0}'.replace('{0}', current_board['name'])
api_url = URL('api', 'boards', args=[current_board['id'], 'pins'])
top_pins = current_board['pins'][:page_size]
dic_pins = cacher.get_multi('pin', top_pins)
pins = [dic_pins[id] for id in top_pins]
action = None
boards = []
if auth.user:
current_user = cacher.get('auth_user', auth.user.id)
boards = cacher.get_multi('board', current_user['boards']).values()
is_following = any(current_board['id'] == id for id in current_user['follow_boards'])
#update the follow button
if is_following:
action = 'unfollow'
else:
action = 'follow'
##perform actions on user
if request.args(1):
action = request.args(1).lower()
if action == 'follow' or action == 'unfollow':
logic.toggle_follow_board(current_user, current_board, action)
redirect(URL('boards', args=[current_board['id']]))
return dict(current_board=current_board, pins=pins, boards=boards, show_pinner=False, action=action, api_url=api_url, categories=get_all_categories())
def GET(id):
id = long(id)
pin = cacher.get('pin', id) or redirect(URL('index'))
article = logic.get_article(pin['article'])
if auth.user:
user = cacher.get('auth_user', auth.user.id)
else:
user = None;
comments = cacher.get_multi('comment', pin['comments']).values()
comments = sorted(comments, key=lambda c: c['created_on'])
return dict(user=user, pin=pin, article=article, comments=comments)
def add_pin(article, board):
db = current.db
pin = db((db.pin.article==article['id'])&(db.pin.board==board['id'])).select().first()
if pin == None:
pin = db.pin.insert(
article=article['id'],
article_title = article['title'],
article_domain = article['domain'],
article_excerpt = article['excerpt'],
board=board['id'],
board_name=board['name'],
user = board['user'],
user_name = board['user_name'],
)
#put it in the cache
cacher.get('pin', pin.id)
#get from db as board might be from cache
db_board = db.board(board['id'])
db_board.pins.insert(0, pin.id)
db_board.pins_created_on.insert(0, pin.created_on)
db_board.update_record()
cacher.delete(board['id'])
#update article from db
db_article = db.article(article['id'])
db_article.pins.insert(0, pin.id)
db_article.pins_created_on.insert(0, pin.created_on)
db_article.update_record()
cacher.delete(article['id'])
#update user pins
result = get_user_pins(db_board.user.id, 0, 500)
ids = []
ts = []
for (i,t,s) in result:
ids.append(i)
ts.append(s)
db_board.user.pins = ids
db_board.user.pins_created_on = ts
db_board.user.update_record()
cacher.delete(db_board.user.id)
_add_pin_to_category(pin)
return pin
return None
def _add_pin_to_category(pin):
cat = cacher.get('category', pin.board.category['id'])
cat['pins'].insert(0, pin.id)
if len(cat['pins'])>1000: cat['pins'].pop()
last_db_update = cat['pins_updated_on']
if last_db_update is None:
last_db_update = datetime.datetime.now() - datetime.timedelta(days=2)
else:
last_db_update = _strptime(last_db_update)
now = datetime.datetime.now()
if now - last_db_update > datetime.timedelta(days=1):
pin.board.category.pins = cat['pins']
pin.board.category.pins_updated_on = now
pin.board.category.update_record()
cat['pins_updated_on'] = _strftime(now)
memcache.set('%u' % cat['id'], cat)
all = _get_all_pins()
all.insert(0, pin.id)
if len(all)>1000: all.pop()
memcache.set('category.all.pins', all)
def search(self, term, sort=None, category=None, page=None):
sort = parser.fill_none(self.source.search["sort_codes"], sort)
category = parser.fill_none(self.source.search["categories"], category)
page = parser.fill_none(self.source.search["page_number"], page)
url = parser.translate_schema(self.source.search["schema"],
{ "search_term" : term,
"page_number" : page,
"sort_code" : sort,
"category" : category })
response = cacher.get(self.session, url, headers=self.source.info['headers'], update_cache=_update_cache)
if response.status_code == requests.codes.ok:
tree = etree.HTML(response.text)
urls = Parser(tree, self.source.search["url"]).extract(first=False)
titles = Parser(tree, self.source.search["title"]).extract(first=False)
magnets = Parser(tree, self.source.search["magnet"]).extract(first=False)
seeders = Parser(tree, self.source.search["seeders"]).extract(first=False)
leechers = Parser(tree, self.source.search["leechers"]).extract(first=False)
torrents = zip(urls, titles, magnets, seeders, leechers)
torrents = map(lambda x: Torrent(x[0], title=x[1], magnet=x[2], seeders=x[3], leechers=x[4]), torrents)
return torrents
else:
raise FetchError(url, response.status_code)
def send_emails():
emails = []
if len(request.vars.email1) > 5 and IS_EMAIL()(request.vars.email1)[1] is None:
emails.append(request.vars.email1)
if len(request.vars.email2) > 5 and IS_EMAIL()(request.vars.email2)[1] is None:
emails.append(request.vars.email2)
if len(request.vars.email3) > 5 and IS_EMAIL()(request.vars.email3)[1] is None:
emails.append(request.vars.email3)
if len(request.vars.email4) > 5 and IS_EMAIL()(request.vars.email4)[1] is None:
emails.append(request.vars.email4)
if len(request.vars.email5) > 5 and IS_EMAIL()(request.vars.email5)[1] is None:
emails.append(request.vars.email5)
if len(emails) == 0:
return 'No invitations sent, please check your email addresses'
else:
msg = request.vars.message
current_user = cacher.get('auth_user', auth.user.id)
message=response.render('layout/email.html',from_user=current_user, personal_msg=msg)
auth.settings.mailer.send(to=emails,subject='I would like to invite you to Pinformation!',message=message)
return 'Invitation successfully sent to %d recipients' % len(emails)
def index():
if auth.user is None:
return all()
current_user = cacher.get('auth_user', auth.user.id)
response.title = 'Pinformation :: {0}'.replace('{0}', current_user['full_name'])
boards = cacher.get_multi('board', current_user['boards']).values()
result = logic.get_follow_boards_pins(auth.user.id, 0, page_size)
api_url = URL('api', 'pins')
# if we want to add myself
# me = logic.get_user_pins(auth.user.id, 0, page_size)
# result.extend(me)
# result = sorted(all, key=lambda (id,t,s): t, reverse=True)
pin_ids = [id for (id,t,s) in result]
dic_pins = cacher.get_multi('pin', pin_ids)
pins = [dic_pins[id] for id in pin_ids]
if len(pins) == 0 and len(boards) == 0:
redirect(URL('default','welcome'))
return dict(show_pinner=True, categories=get_all_categories(), boards=boards, pins=pins, api_url=api_url)
def add_article():
url = request.vars.url
board = cacher.get('board',long(request.vars.board))
article = logic.get_article_by_url(url)
if article is None:
r = Readability()
json = r.content(url)
article = db.article.insert(
url=json['url'],
readability_url=json['short_url'],
title=json['title'],
#content=json['content'],
domain=json['domain'],
author=json['author'],
excerpt=json['excerpt'],
word_count=json['word_count'],
total_pages=json['total_pages'],
date_published=json['date_published'],
next_page_id=json['next_page_id'],
rendered_pages=json['rendered_pages'],
)
pin = logic.add_pin(article, board)
if request.vars.linkedin and request.env.http_host != '127.0.0.1:8080':
logic.share_on_linkedin(session.linkedin, pin)
return 'Success'
def add_repin():
pin = cacher.get('pin', long(request.vars.id))
board = cacher.get('board', long(request.vars.repin_board))
if pin and board:
repin = db((db.pin.article==pin['article'])&(db.pin.board==board['id'])).select().first()
if repin is None:
article = cacher.get('article', pin['article'])
repin = logic.add_pin(article, board)
repin.original_pin = pin['id']
repin.update_record()
cacher.delete(repin.id)
if request.vars.linkedin and request.env.http_host != '127.0.0.1:8080':
logic.share_on_linkedin(session.linkedin, repin)
return 'Success'
def linkedin():
if not session.verified: redirect(URL('default', 'index'))
current_user = cacher.get('auth_user', auth.user.id)
connections = session.linkedin.get_connections(fields=['id', 'first-name', 'last-name','picture-url'])
un = [c.id for c in connections]
users = logic.get_users_by_username(un)
in_users = []
out_users = []
for c in connections:
if c.picture_url is None:
c.picture_url = 'http://static02.linkedin.com/scds/common/u/img/icon/icon_no_photo_60x60.png'
if users.has_key(c.id):
u = users[c.id]
is_following = any(u['id'] == id for id in current_user['follow_users'])
if is_following:
c.action = 'unfollow'
else:
c.action = 'follow'
# move it to the else if we want to eliminate already followed people
c.user_id = u['id']
in_users.append(c)
else:
out_users.append(c)
return dict(in_users=in_users, out_users=out_users)
def get_article(id):
article = cacher.get('article', id)
if not article.has_key('content'):
#fetch content from readability in real time
r = Readability()
json = r.content(article['url'])
article['content'] = json['content'].encode('UTF8', 'replace')
cacher.set(article)
return article
def pins():
##if empty "/pins" return to main
if len(request.args)==0:
redirect(URL('index'))
if auth.user:
user = cacher.get('auth_user', auth.user.id)
else:
user = {}
id = long(request.args(0))
pin = cacher.get('pin', id) or redirect(URL('index'))
article = logic.get_article(pin['article'])
board = cacher.get('board', pin['board'])
comments = cacher.get_multi('comment', pin['comments']).values()
comments = sorted(comments, key=lambda c: c['created_on'])
response.title = 'Pinformation :: {0}'.replace('{0}', pin['article_title'])
return dict(user=user, pin=pin, article=article, comments=comments, board=board, categories=get_all_categories())
def GET(id, param, page=0):
response.view = 'api/pins.html'
page = max(0, int(page)-1)
id = long(id)
user = cacher.get('auth_user', id)
top_pins = user['pins'][min(page*page_size, len(user['pins'])):min((page+1)*page_size, len(user['pins']))]
dic_pins = cacher.get_multi('pin', top_pins)
pins = [dic_pins[id] for id in top_pins]
return _serialize(dict(pins=pins, show_pinner=False))
def people():
ids = logic.search('auth_user', request.vars.q, 0, page_size)
users = cacher.get_multi('auth_user', ids).values()
if auth.user:
current_user = cacher.get('auth_user', auth.user.id)
for u in users:
if any(u['id'] == id for id in current_user['follow_users']):
u['action'] = 'unfollow'
else:
u['action'] = 'follow'
return dict(results=users)
def get_article_by_url(url):
key = 'article.url=%s' % url
id = memcache.get(key)
if id:
return cacher.get('article', id)
db = current.db
article = db(db.article.url == url).select().first()
if article:
memcache.set(key, article['id'])
return article
def GET(id, param, page=0):
response.view = 'api/pins.html'
page = max(0, int(page)-1)
cat = None
try:
cat = cacher.get('category', long(id))
except:
pass
pins = logic.get_pins_by_category(cat, page*page_size, (page+1)*page_size)
return _serialize(dict(pins=pins, show_pinner=True))
def get_user_by_username(username):
key = 'user.username=%s' % username
id = memcache.get(key)
if id:
return cacher.get('auth_user', id)
db = current.db
user = db(db.auth_user.username == username).select().first()
if user:
memcache.set(key, user['id'])
return user
def add_board():
name = request.vars.name
if re.match('(^[\w\s.-]+$)', name): #sanitize the board name
category = cacher.get('category', long(request.vars.category))
user = cacher.get('auth_user', auth.user.id)
if category and user:
boards = cacher.get_multi('board', user['boards']).values()
already_exists = any(name == b['name'] for b in boards)
if not already_exists:
board = db.board.insert(
user=user['id'],
user_name = user['full_name'],
category=category['id'],
name=name
)
board.user.boards.append(board)
board.user.update_record()
cacher.delete(board.user.id)
##add the board to every user that follow this one
##should be moved to async when we have async
try:
users = db(db.auth_user.follow_users.contains(board.user.id)).select()
for u in users:
logic.toggle_follow_board(u, board, 'follow')
except:
pass
return 'Success'
else:
return 'You already have a board with that name'
else:
return 'category does not exist'
return 'please use only alpha numeric characters'
def pre_run(model, *args, **kwargs):
q = """
prefix dcterms: <http://purl.org/dc/terms/>
SELECT ?a ?x
WHERE {?a dcterms:creator ?x. }
"""
result = model.query(q)
author = ""
rc = kwargs.get("res_context", None)
for row in result:
author = str(row['x'])
auth_url = author.split('#')[0]
if auth_url != "":
var = urllib2.urlopen(auth_url)
auth_url = var.geturl()
auth_url = auth_url +"rdf.xml"
f = tempfile.NamedTemporaryFile("w", delete=False)
data = cacher.get(auth_url)
f.write(data)
auth_url = f.name
f.close()
model.parse(auth_url)
bnf_code = row["a"].split("#")[0].split("/")[-1]
bnf_code = bnf_code[2:]
#bnf_code = re.findall(r'\d+', bnf_code)[0]
rc["bnf_code"] = bnf_code
q2 = """
prefix foaf: <http://xmlns.com/foaf/0.1/>
SELECT ?x
WHERE {?a foaf:familyName ?x. }
"""
result = model.query(q2)
for row in result:
rc["author"] = row["x"]
def get_pins_by_category(cat, start, end):
ids = []
if cat == None:
ids = _get_all_pins()
else:
#get updated list from cache - db saves once a day
cat = cacher.get('category', cat['id'])
ids = cat['pins']
if ids and len(ids)>0:
ids = ids[start:end]
dic_pins = cacher.get_multi('pin', ids)
pins = [dic_pins[id] for id in ids]
return pins
return []
def pre_run(model, *args, **kwargs):
q = """
prefix dcterms: <http://purl.org/dc/terms/>
SELECT ?a ?x
WHERE {?a dcterms:creator ?x. }
"""
result = model.query(q)
author = ""
rc = kwargs.get("res_context", None)
for row in result:
author = str(row['x'])
auth_url = author.split('#')[0]
if auth_url != "":
var = urllib2.urlopen(auth_url)
auth_url = var.geturl()
auth_url = auth_url + "rdf.xml"
f = tempfile.NamedTemporaryFile("w", delete=False)
data = cacher.get(auth_url)
f.write(data)
auth_url = f.name
f.close()
model.parse(auth_url)
bnf_code = row["a"].split("#")[0].split("/")[-1]
bnf_code = bnf_code[2:]
#bnf_code = re.findall(r'\d+', bnf_code)[0]
rc["bnf_code"] = bnf_code
q2 = """
prefix foaf: <http://xmlns.com/foaf/0.1/>
SELECT ?x
WHERE {?a foaf:familyName ?x. }
"""
result = model.query(q2)
for row in result:
rc["author"] = row["x"]
def get_torrent(self, torrent):
url = parser.translate_schema(self.source.torrent["schema"],
{ "url" : torrent.url })
response = cacher.get(self.session, url, headers=self.source.info['headers'], update_cache=_update_cache)
if response.status_code == requests.codes.ok:
tree = etree.HTML(response.text)
# torrent.date = Parser(tree, self.source.torrent["date"]).extract()
torrent.description = Parser(tree, self.source.torrent["description"]).extract()
torrent.seeders = Parser(tree, self.source.torrent["seeders"]).extract()
torrent.leechers = Parser(tree, self.source.torrent["leechers"]).extract()
torrent.magnet = Parser(tree, self.source.torrent["magnet"]).extract()
return torrent
else:
raise FetchError(url, response.status_code)
def add_comment():
if len(request.vars.comment) > 0 and re.match('(^[\w\s.-]+$)', request.vars.comment):
pin = db.pin(long(request.vars.pin_id))
user = cacher.get('auth_user', auth.user.id)
if pin:
comment = db.comment.insert(
user=user['id'],
user_name=user['full_name'],
user_picture_url=user['picture_url'],
pin=pin['id'],
content=request.vars.comment
)
pin.comments.append(comment.id)
pin.update_record()
cacher.delete(pin.id)
try:
users = [pin['user']]
comments = cacher.get_multi('auth_user', pin['comments']).values()
for c in comments:
users.append(c['user'])
users = cacher.get_multi('auth_user', users).values()
if len(users) > 0:
usernames = []
for u in users:
if not u['id'] == user['id']:
usernames.append(u['username'])
subject = '{0} commented on "{1}"'.replace('{0}',user['full_name']).replace('{1}', pin['article_title'])
msg_template = '{0}\n\n ---------------------------------------------------------------\nwww.pinformation.co is an innovative way to curate and share information on the web.\nWe help you become better at what you do.'
msg = '{0} commented on "{1}"\n\nClick the link to continue the conversation: http://www.pinformation.co{2}'
msg = msg.replace('{0}', user['full_name']).replace('{1}', pin['article_title']).replace('{2}', URL('default', 'pins', args=[pin['id']]))
msg = msg_template.replace('{0}', msg)
result = session.linkedin.send_message(subject, msg, usernames, False)
except:
pass
return 'Success'
else:
return 'please use only alpha numeric characters'
def all():
if request.vars.category:
cat_name = request.vars.category
category = db(db.category.name==cat_name.lower()).select().first()
if category:
api_url = URL('api', 'categories', args=[category['id'], 'pins'])
else:
api_url = URL('api', 'categories', args=['all', 'pins'])
else:
category = None
api_url = URL('api', 'categories', args=['all', 'pins'])
pins = logic.get_pins_by_category(category, 0, page_size)
boards = []
if auth.user:
current_user = cacher.get('auth_user', auth.user.id)
boards = cacher.get_multi('board', current_user['boards']).values()
return dict(show_pinner=True, current_category=category, categories=get_all_categories(), boards=boards, pins=pins, api_url=api_url)
def GetQuote(author):
results = []
author_id = None
last_page = None
print('Searching for {}...'.format(author))
response = get('https://www.brainyquote.com/authors/{}'.format(author))
QuoteObject = bs4.BeautifulSoup(response, 'html.parser')
info = QuoteObject.select('.bio-under > a')
cats = [] # categories
for i in info:
if 'profession' in i.get('href') or 'nationality' in i.get('href'):
cats.append(i.getText())
quotes = QuoteObject.select('#quotesList a')
for quote in quotes:
if quote.get('title') == 'view quote' and 'b-qt' in quote.get('class'):
results.append(quote)
for line in response.split('\n'):
if 'PG_DM_ID' in line:
author_id = line[10:-2]
if 'LPAGE' in line:
last_page = line[8:-1]
for html_page in range(3, int(last_page) + 1):
payload = {'pg': html_page, 'typ': 'author', 'id': author_id}
response = post('https://www.brainyquote.com/api/inf', payload)
QuoteObject = bs4.BeautifulSoup(response['content'], 'html.parser')
quotes = QuoteObject.select('a')
for quote in quotes:
if quote.get('title') == 'view quote' and 'b-qt' in quote.get(
'class'):
results.append(quote)
return cats, results
for x in os.walk('./flavours/'):
if not x[0] == "." and not len(x[0]) == 11:
itms.append(x[0][11:])
if args.output == "json":
print json.dumps(itms)
else:
print ", ".join(itms)
else:
if args.instance.startswith("http"):
args.mode = "url"
if args.mode == "url":
f = tempfile.NamedTemporaryFile("w", delete=False)
data = cacher.get(args.instance)
#print data.encoding
#print data.text
#if data.status_code != 200:
# raise Exception("Wrong Status ==> Network Error.")
f.write(data)
f.close()
args.instance = f.name
files = [ os.path.join("countries", args.country,"flow.json"), os.path.join("flavours",args.flavor, "local.json"), os.path.join("countries", args.country, "local.json"), os.path.join("mappings", args.country+"-"+args.flavor+".json"), args.globalmap, args.instance]
#print files
files = map(path.isfile, files)
#print files
if all(files):
from reasoner import Reasoner
a = Reasoner(os.path.join("countries",args.country,"flow.json"), local_map = os.path.join("countries", args.country, "local.json"), flavor_map = os.path.join("flavours",args.flavor, "local.json"), global_map = args.globalmap, mapping=os.path.join("mappings", args.country+"-"+args.flavor+".json"),detail=args.detail, output=args.output, language=os.path.join("countries", args.country, "i18n", args.language+".json"))
#!/usr/bin/env python3
import bs4, sys
from cacher import get
baseurl = 'https://www.brainyquote.com/authors/'
verbose = True
for c in range(ord('a'), ord('z') + 1):
for i in range(1, 20):
if i == 1:
i = ''
else:
i = str(i)
html = get(baseurl + chr(c) + i)
if html == False:
if verbose:
sys.stderr.write(
'Reached end of letter {} after {} pages.\n'.format(
chr(c), i))
break
dom = bs4.BeautifulSoup(html, 'html.parser')
thingies = dom.select(
'.table.table-hover.table-bordered > tbody > tr > td > a')
for thingy in thingies:
print(thingy.get('href') + '\t' + thingy.string)
for x in os.walk('./flavours/'):
if not x[0] == "." and not len(x[0]) == 11:
itms.append(x[0][11:])
if args.output == "json":
print json.dumps(itms)
else:
print ", ".join(itms)
else:
if args.instance.startswith("http"):
args.mode = "url"
if args.mode == "url":
f = tempfile.NamedTemporaryFile("w", delete=False)
data = cacher.get(args.instance)
#print data.encoding
#print data.text
#if data.status_code != 200:
# raise Exception("Wrong Status ==> Network Error.")
f.write(data)
f.close()
args.instance = f.name
files = [
os.path.join("countries", args.country, "flow.json"),
os.path.join("flavours", args.flavor, "local.json"),
os.path.join("countries", args.country, "local.json"),
os.path.join("mappings",
args.country + "-" + args.flavor + ".json"),
args.globalmap, args.instance
at getting realpath of files without using os.walk().
The next implementation will be os.walk()
'''
import os
import sys
import numbers
import cacher
from subprocess import Popen as Popen
from subprocess import PIPE as PIPE
cache = '/'.join([sys.argv[3], 'cached_images'])
cached_files = cacher.get(cache)
class File:
# TODO
'''
__init__ should be shortened and a function convert() should be added
'''
def __init__(self, filepath):
self.filename = os.path.basename(filepath)
self.path = filepath
if self.filename not in cached_files:
print("Preparing "+self.filename)
self.cached = False
def get_follow_boards_pins(id, start, end):
user = cacher.get('auth_user', id)
boards = cacher.get_multi('board', user['follow_boards']).values()
return _get_boards_pins(boards, start, end)
