def submit_bm(feed, user, title, url, comment):
bm = Bookmarks()
result = urlfetch.fetch(url=url, follow_redirects=True, allow_truncated=True, deadline=60)
if result.status_code == 200 and result.final_url:
a = result.final_url
elif result.status_code == 500:
pass
else:
a = url
url_candidate = a.lstrip().rstrip().split('?utm_source')[0].split('&feature')[0]
url_parsed = urlparse(url_candidate)
query = parse_qs(url_parsed.query)
name = url_parsed.path.split('/')[-1]
ext = name.split('.')[-1].lower()
bm.title = url_candidate if title == '' or None else title
if url_parsed.netloc == 'www.youtube.com':
bm.url = 'http://www.youtube.com/watch?v=%s' % query["v"][0]
bm.comment = """<embed
width="640" height="360"
src="http://www.youtube.com/v/%s"
type="application/x-shockwave-flash">
</embed>""" % query["v"][0]
elif url_parsed.netloc == 'vimeo.com':
bm.url = 'http://vimeo.com/%s' % name
bm.comment = '''<iframe src="http://player.vimeo.com/video/%s?color=ffffff"
width="640" height="360" frameborder="0" webkitAllowFullScreen mozallowfullscreen
allowFullScreen></iframe>''' % name
elif ext in ['jpg', 'png', 'jpeg', 'gif']:
bm.url = url_candidate
blob_key = upload_to_blobstore(url_candidate, ext)
bm.comment = '<img src="%s" />' % images.get_serving_url(blob_key, size=1600)
else:
bm.comment = comment
bm.url = url_candidate
bm.domain = url_parsed.netloc
bm.user = user
bm.feed = feed
bm.put()
Bookmarks.index_bm(bm.key)
copie = Bookmarks.query(Bookmarks.url == bm.url,
Bookmarks.user == user)
if copie.get():
ndb.delete_multi([cp.key for cp in copie])
ui = UserInfo.get_or_insert(str(user.user_id()), user=user)
if feed is None and ui.mys is True:
deferred.defer(util.send_bm, bm.key, _queue="email")
elif feed is not None and feed.get().notify == 'email':
deferred.defer(util.send_bm, bm.key, _queue="email")
def daily_digest(uik):
ui = uik.get()
delta = datetime.timedelta(days=1)
now = datetime.datetime.now()
period = now - delta
bmq = Bookmarks.query(Bookmarks.user == ui.user,
Bookmarks.trashed == False,
Bookmarks.data > period).order(-Bookmarks.data)
title = '[%s] Daily digest for your activity: %s' % (app_identity.get_application_id(), dtf(now))
template = jinja_environment.get_template('digest.html')
values = {'bmq': bmq, 'title': title}
html = template.render(values)
if bmq.get() != None:
deferred.defer(send_digest, ui.email, html, title)
def feed_digest(feedk):
import datetime, time
timestamp = time.time() - 87000
period = datetime.datetime.fromtimestamp(timestamp)
feed = feedk.get()
bmq = Bookmarks.query(Bookmarks.user == feed.user)
bmq = bmq.filter(Bookmarks.feed == feed.key)
bmq = bmq.filter(Bookmarks.trashed == False)
bmq = bmq.filter(Bookmarks.create > period)
bmq = bmq.order(-Bookmarks.create)
title = '(%s) Daily digest for %s' % (app_identity.get_application_id(), feed.blog)
template = jinja_environment.get_template('digest.html')
values = {'bmq': bmq, 'title': title}
html = template.render(values)
if bmq.get():
deferred.defer(send_digest, feed.user.email(), html, title, _target="worker", _queue="emails")
def feed_digest(feedk):
delta = datetime.timedelta(days=1)
now = datetime.datetime.now()
period = now - delta
feed = feedk.get()
bmq = Bookmarks.query(Bookmarks.user == feed.user,
Bookmarks.feed == feed.key,
Bookmarks.trashed == False,
Bookmarks.data > period).order(-Bookmarks.data)
title = '[%s] Daily digest for %s' % (app_identity.get_application_id(), feed.title)
template = jinja_environment.get_template('digest.html')
values = {'bmq': bmq, 'title': title}
html = template.render(values)
if bmq.get() != None:
deferred.defer(send_digest, feed.user.email(), html, title)
queue = []
for bm in bmq:
bm.trashed = True
queue.append(bm)
ndb.put_multi(queue)
def feed_digest(feedk):
import datetime, time
timestamp = time.time() - 87000
period = datetime.datetime.fromtimestamp(timestamp)
feed = feedk.get()
bmq = Bookmarks.query(Bookmarks.user == feed.user)
bmq = bmq.filter(Bookmarks.feed == feed.key)
bmq = bmq.filter(Bookmarks.trashed == False)
bmq = bmq.filter(Bookmarks.create > period)
bmq = bmq.order(-Bookmarks.create)
title = '(%s) Daily digest for %s' % (app_identity.get_application_id(),
feed.blog)
template = jinja_environment.get_template('digest.html')
values = {'bmq': bmq, 'title': title}
html = template.render(values)
if bmq.get():
deferred.defer(send_digest,
feed.user.email(),
html,
title,
_target="worker",
_queue="emails")
def main_parser(bmk):
import urlparse
bm = bmk.get()
try:
result = urlfetch.fetch(url=bm.original, follow_redirects=True, allow_truncated=True, deadline=600)
if result.status_code == 200 and result.final_url:
a = result.final_url
else:
a = bm.original
except:
a = bm.original
b = a.split('?utm_source')[0]
c = b.split('&feature')[0]
bm.url = c.encode('utf8')
# TAGS
q = Bookmarks.query(Bookmarks.user == bm.user, Bookmarks.original == bm.original)
if q.count() > 1:
tag_list = []
for e in q:
for t in e.tags:
if t not in tag_list:
tag_list.append(t)
bm.tags = tag_list
ekeys = [e.key for e in q]
ekeys.remove(bm.key)
for ekey in ekeys:
e = ekey.get()
e.tags = []
e.trashed = True
e.put()
# COMMENTS
url_parsed = urlparse.urlparse(bm.url)
query = urlparse.parse_qs(url_parsed.query)
if url_parsed.netloc == 'www.youtube.com':
video = query["v"][0]
bm.url = 'http://www.youtube.com/watch?v=%s' % video
bm.comment = '''<iframe width="640" height="360"
src="http://www.youtube.com/embed/%s" frameborder="0"
allowfullscreen></iframe>''' % video
if url_parsed.netloc == 'vimeo.com':
video = url_parsed.path.split('/')[-1]
bm.url = 'http://vimeo.com/%s' % video
bm.comment = '''<iframe src="http://player.vimeo.com/video/%s?color=ffffff"
width="640" height="360" frameborder="0" webkitAllowFullScreen mozallowfullscreen
allowFullScreen></iframe>''' % video
ext = bm.url.split('.')[-1]
if ext == 'jpg' or ext == 'png' or ext == 'jpeg':
bm.comment = '<img src="%s" />' % bm.url
if bm.title == '':
bm.title = url_parsed.path
bm.domain = url_parsed.netloc
bm.put()
try:
if bm.feed.get().notify == 'email':
deferred.defer(utils.send_bm, bm.key, _target="worker", _queue="emails")
except:
if bm.ha_mys:
deferred.defer(utils.send_bm, bm.key, _target="worker", _queue="emails")
def get(self):
bmq = Bookmarks.query(Bookmarks.user == users.get_current_user())
bmq = bmq.filter(Bookmarks.trashed == True)
bmq = bmq.order(-Bookmarks.data).fetch()
ndb.delete_multi([bmk.key for bmk in bmq])
self.redirect(self.request.referer)
def get(self):
for bm in Bookmarks.query():
deferred.defer(add_domain, bm.key, _target="worker", _queue="admin")
def main_parser(bmk):
import urlparse
bm = bmk.get()
try:
result = urlfetch.fetch(url=bm.original,
follow_redirects=True,
allow_truncated=True,
deadline=600)
if result.status_code == 200 and result.final_url:
a = result.final_url
else:
a = bm.original
except:
a = bm.original
b = a.split('?utm_source')[0]
c = b.split('&feature')[0]
bm.url = c.encode('utf8')
# TAGS
q = Bookmarks.query(Bookmarks.user == bm.user,
Bookmarks.original == bm.original)
if q.count() > 1:
tag_list = []
for e in q:
for t in e.tags:
if t not in tag_list:
tag_list.append(t)
bm.tags = tag_list
ekeys = [e.key for e in q]
ekeys.remove(bm.key)
for ekey in ekeys:
e = ekey.get()
e.tags = []
e.trashed = True
e.put()
# COMMENTS
url_parsed = urlparse.urlparse(bm.url)
query = urlparse.parse_qs(url_parsed.query)
if url_parsed.netloc == 'www.youtube.com':
video = query["v"][0]
bm.url = 'http://www.youtube.com/watch?v=%s' % video
bm.comment = '''<iframe width="640" height="360"
src="http://www.youtube.com/embed/%s" frameborder="0"
allowfullscreen></iframe>''' % video
if url_parsed.netloc == 'vimeo.com':
video = url_parsed.path.split('/')[-1]
bm.url = 'http://vimeo.com/%s' % video
bm.comment = '''<iframe src="http://player.vimeo.com/video/%s?color=ffffff"
width="640" height="360" frameborder="0" webkitAllowFullScreen mozallowfullscreen
allowFullScreen></iframe>''' % video
ext = bm.url.split('.')[-1]
if ext == 'jpg' or ext == 'png' or ext == 'jpeg':
bm.comment = '<img src="%s" />' % bm.url
if bm.title == '':
bm.title = url_parsed.path
bm.domain = url_parsed.netloc
bm.put()
try:
if bm.feed.get().notify == 'email':
deferred.defer(utils.send_bm,
bm.key,
_target="worker",
_queue="emails")
except:
if bm.ha_mys:
deferred.defer(utils.send_bm,
bm.key,
_target="worker",
_queue="emails")