def visit_route(link_id):
# get old link and ripple info
link, ripple = find_ripple_link(link_id)
if link == None or ripple == None:
abort(404)
Link.incrementField(link_id, "total_views")
# get the user's link_id associated with thie ripple
user = user_from_cookie(request)
user_link_id = user.get_link(ripple._id)
# if user does not have link with this ripple, make one
if user_link_id == None:
child_index = link.total_children + 1 # the total children + 1 == how many links were created before this
user_link_id = create_link(ripple._id, link_id, None, None,
child_index)
user.set_ripple_link(ripple._id, user_link_id)
else:
# user already has a link, so user won't be redirected to create a new one
# and this increment wont be called twice, one for first load, and one for new load after auto creation
Ripple.incrementField(link.ripple_id, "total_views")
uid = user.save()
# get the link the user has associated with ripple
link = Link.queryById(user_link_id)
resp = make_response({"link": link.dict(), "ripple": ripple.dict()})
resp.set_cookie("uid", uid)
return resp
def fetch_telegram(url, page_name):
print('Start fetch {}: {}'.format(page_name, url))
s = requests.Session()
headers = {
"User-Agent":
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:71.0) Gecko/20100101 Firefox/71.0'
}
resp = s.get(url, headers=headers, verify=False)
soup = BeautifulSoup(resp.text, 'html.parser')
soup = soup.find_all("div", {"class": "tgme_widget_message_bubble"})
for i in range(len(soup)):
info_post = soup[i].find_all("div",
{"class": "tgme_widget_message_info"})
mess_text = soup[i].find_all("div",
{"class": "tgme_widget_message_text"})
# remove posts with only images
if mess_text == []:
continue
content = BeautifulSoup(str(mess_text[0]), 'html.parser')
info_post = BeautifulSoup(str(info_post[0]), 'html.parser')
media = soup[i].find_all("i", {"class": "link_preview_image"})
# Post announcements removed
if content.find_all('a') == []:
continue
get_url = content.find_all('a')[0]
url = get_url.get('href')
origin = info_post.find_all('a')[0].get('href')
content = mess_text[0].decode_contents()
post_time = info_post.find_all('time')[0].get('datetime')
if media != []:
div_style = BeautifulSoup(str(media[0]),
'html.parser').find('i')['style']
style = cssutils.parseStyle(div_style)
url_image = style['background-image']
url_image = url_image[4:-1]
else:
url_image = ''
info = {
"category": LINK.CATEGORY_WEB,
"content": content,
"created_at": datetime.strptime(post_time[:-6],
"%Y-%m-%dT%H:%M:%S"),
"kind": LINK.KIND_LINK,
"media": url_image,
"origin": origin,
"read": LINK.UNREAD,
"status": LINK.STATUS_DONE,
"title": page_name,
"url": url
}
info['url'] = clean_up_url(info['url'])
if Link.query.filter(Link.url == info['url']).first():
continue
else:
Link.insert_from(info)
def update_depth(parent_link_id, depth=1):
link = Link.queryById(parent_link_id)
if link.total_depth < depth:
oldLink = Link.setField(parent_link_id, "total_depth", depth)
if link.parent_id != None:
update_depth(link.parent_id, depth=depth + 1)
else:
Ripple.setField(oldLink.ripple_id, "total_depth", depth)
def create_link(ripple_id, parent_id, user_id, start_location, child_index):
link = Link(ripple_id=ripple_id,
parent_id=parent_id,
user_id=user_id,
start_location=start_location,
child_index=child_index)
increment_descendent_counts(parent_id)
increment_child_counts(parent_id)
update_depth(parent_id)
link_id = link.save()
return link_id
def find_ripple_link(link_id):
link = Link.queryById(link_id)
if link == None:
return None, None
ripple = Ripple.queryById(link.ripple_id)
if ripple == None:
return None, None
return link, ripple
def index():
try:
page = int(request.args.get('page'))
links = Link.query.filter(Link.read == LINK.UNREAD).order_by(
desc(Link.id)).paginate(page=page, per_page=10).items
except Exception as e:
print(e)
page = 1
links = []
return jsonify({'unread_count': Link.get_unread_count(), 'links': links})
def mark_read():
try:
params = request.get_json(force=True)
link = Link.query.get(params['id'])
link.rating = LINK.HELPFUL if params['rating'] else LINK.SPAM
link.read = LINK.READ
db.session.add(link)
db.session.commit()
return jsonify({
'status': 'OK',
'unread_count': Link.get_unread_count()
})
except Exception as e:
return jsonify({'status': 'Error', 'msg': str(e)})
def create_ripple(title, org_ids, user_id, start_location):
ripple = Ripple(title=title,
org_ids=org_ids,
start_location=start_location)
ripple_id = ripple.save()
root_link = Link(ripple_id=ripple_id,
user_id=user_id,
start_location=start_location)
link_id = root_link.save()
ripple.root_id = link_id
ripple.save()
user = user_from_cookie(request)
user.set_ripple_link(ripple_id, link_id)
uid = user.save()
resp = make_response({"ripple_id": ripple_id, "link_id": link_id})
resp.set_cookie("uid", uid)
return resp
def fetch_twitter():
MIN_LIKED = 20
query = app.config['TWITTER_QUERY']
print('Start fetch twitter with query {}'.format(query))
cached_contents = [
content for (content, ) in Link.query.with_entities(
Link.content).order_by(Link.id.desc()).limit(500)
]
for tweet_info in tweepy.Cursor(twitter_api.search,
q=query,
lang='en',
tweet_mode='extended').items(50):
media = ''
links = []
origin_tweet = ''
liked_count = 0
if 'retweeted_status' in dir(tweet_info):
tweet = tweet_info.retweeted_status.full_text
if 'media' in tweet_info.retweeted_status.entities:
for m in tweet_info.retweeted_status.entities['media']:
media = m['media_url']
for u in tweet_info.retweeted_status.entities['urls']:
links.append(u['expanded_url'])
tweet = tweet.replace(u['url'], u['expanded_url'])
origin_tweet = "https://twitter.com/i/web/status/{}".format(
tweet_info.retweeted_status.id)
liked_count = tweet_info.retweeted_status.retweet_count + \
tweet_info.retweeted_status.favorite_count
else:
tweet = tweet_info.full_text
if 'media' in tweet_info.entities:
for m in tweet_info.entities['media']:
media = m['media_url']
for u in tweet_info.entities['urls']:
links.append(u['expanded_url'])
tweet = tweet.replace(u['url'], u['expanded_url'])
origin_tweet = "https://twitter.com/i/web/status/{}".format(
tweet_info.id)
liked_count = tweet_info.retweet_count + tweet_info.favorite_count
tweet_content = tweet.encode('utf-8')
similar_content = False
ignored_content = False
for c in cached_contents:
if SequenceMatcher(None, c, tweet_content).ratio() >= 0.6:
similar_content = True
break
for kw in app.config['IGNORED_KEYWORDS']:
if kw in tweet_content.lower().decode('utf-8'):
ignored_content = True
break
created = False
for idx, item in enumerate(links):
link_info = {}
(title, final_url) = link_expander(item)
final_url = clean_up_url(final_url)
if Link.query.filter(Link.url == final_url).first(
) or similar_content or ignored_content or liked_count < MIN_LIKED:
created = True
pass
else:
link_info['origin'] = "{}#link{}".format(origin_tweet, idx)
if title != "BAD_LINK":
link_info['title'] = title.encode('utf-8')
link_info['url'] = final_url
link_info['status'] = LINK.STATUS_DONE
link_info['media'] = media
link_info['content'] = tweet_content
link_info['read'] = LINK.UNREAD
link_info['kind'] = LINK.KIND_LINK
link_info['category'] = LINK.CATEGORY_WEB
link_info['created_at'] = tweet_info.created_at
Link.insert_from(link_info)
created = True
tweet_url = origin_tweet
same_url = Link.query.filter(Link.url == tweet_url).first() != None
cond = not created and not similar_content and not same_url and not ignored_content and liked_count > MIN_LIKED
print(
"{} -> Created: {}, Similar content: {}, Same URL: {}, Ignored: {}, Liked: {} => {}"
.format(tweet_url, created, similar_content, same_url,
ignored_content, liked_count, cond))
cached_contents.append(tweet_content)
if cond:
link_info = {}
link_info['title'] = 'Tip from Twitter'
link_info['url'] = tweet_url
link_info['origin'] = tweet_url
link_info['status'] = LINK.STATUS_DONE
link_info['media'] = media
link_info['content'] = tweet_content
link_info['read'] = LINK.UNREAD
link_info['kind'] = LINK.KIND_LINK
link_info['category'] = LINK.CATEGORY_WEB
link_info['created_at'] = tweet_info.created_at
Link.insert_from(link_info)
def increment_descendent_counts(parent_link_id):
oldLink = Link.incrementField(parent_link_id, "total_descendants")
if oldLink.parent_id != None:
increment_descendent_counts(oldLink.parent_id)
else:
Ripple.incrementField(oldLink.ripple_id, "total_links")
def increment_child_counts(parent_link_id):
Link.incrementField(parent_link_id, "total_children")
def getRippleId(parent_id):
parent = Link.queryById(parent_id)
if parent == None or parent.ripple_id == None:
return None
return str(parent.ripple_id)