def add(self, word, value):
"""
Add a Node with the given as the prefix key
and the value in the, if the word already
exists the value is unioned with the existing
word's value
:param word: The word to add in
:param value: The value to add for the Node
:return: True if added successfully else None
"""
prefix = Util.get_prefix(self.key, word)
"""
if current node matches the prefix we already have that word
"""
if prefix == word:
if self.value:
self.value = self.value.union(value)
else:
self.value = value
return True
"""
If current node is leaf and we need to add another leaf node,
split the current node
"""
if self.is_leaf:
if prefix:
self.children.append(
Node(self.key[len(prefix):], list(), value=self.value))
self.children.append(Node(word[len(prefix):], list(), value=value))
self.key = prefix
self.is_leaf = False
return True
"""
Check if word can be traced with current internal nodes
"""
if self.children:
for child in self.children:
if not prefix and Util.has_prefix(child.key, word):
return child.add(word, value)
if Util.has_prefix(child.key, word[len(prefix):]):
return child.add(word[len(prefix):], value)
"""
Add new external nodes
"""
if not prefix:
self.children.append(Node(word, list(), value=value))
elif prefix != self.key:
new_node = Node(word[len(prefix):], list(), value=value)
new_child_node = Node(self.key[len(prefix):],
self.children,
is_leaf=False)
self.children = list()
self.children.append(new_child_node)
self.children.append(new_node)
self.key = prefix
else:
self.children.append(Node(word[len(prefix):], list(), value=value))
self.is_leaf = False
return True
def create(cls, session, username):
user = cls()
user.name = username
Util.add_and_refresh(session, user)
dispatcher.send(signal=cls.CREATED_SIGNAL, sender=user)
return user
def create(cls, session, user, post, vote):
inst = cls()
inst.post_id = post.id
inst.user_id = user.id
inst.vote = vote
Util.add_and_refresh(session, inst)
return inst
def update(session, user, praw_post, score):
post = Post.get_or_create(session, praw_post.id)
post.update_from_praw(praw_post)
post.author_id = user.id
Util.update_subreddit(Subreddit, post, session, praw_post.subreddit.display_name)
vote = Vote.get_or_create(session, user, post, score)
vote.vote = score
session.add(vote)
session.add(post)
def update_from_praw(self, p):
self.reddit_id = Util.plain_id(p.id)
self.gilded = bool(p.gilded)
self.edited = float(p.edited)
self.body = p.body
self.ups = p.ups
self.downs = p.downs
self.created = p.created
self.created_utc = p.created_utc
self.scraped_time = Util.now()
def post_links():
session = Session()
data = json.loads(request.data)
for link in data['links']:
post = Post.get_or_create(session, link['reddit_id'])
subreddit = Subreddit.get_or_create(session, link['sub_name'])
user = User.get_or_create(session, link['authorname'])
post.subreddit_id = subreddit.id
post.author_id = user.id
post.domain = link['domain']
post.title = link['title']
post.url = link['url']
post.score = link['score']
post.downs = link['downs']
post.ups = link['ups']
post.is_self = link['is_self']
post.over_18 = link['over_18']
post.thumbnail = link['thumbnail']
post.created = float(link['created'])
post.scraped_time = Util.now()
session.add(post)
session.commit()
session.close()
return jsonify({'received':True})
def get_or_create(cls, session, comment_id):
comment_id = Util.plain_id(comment_id)
comment = session.query(cls).filter(cls.reddit_id == comment_id).first()
if (comment == None):
comment = cls.create(session, comment_id)
return comment
def get_or_create(cls, session, post_id):
post_id = Util.plain_id(post_id)
post = session.query(cls).filter(cls.reddit_id == post_id).first()
if (post == None):
post = cls.create(session, post_id)
return post
def test_4_has_prefix_if_absent(self):
# Arrange
string1 = "ack"
string2 = "hackathon"
# Act
result = Util.has_prefix(string1, string2)
# Assert
self.assertFalse(result, "Correct prefix not found")
def test_1_get_non_empty_prefix(self):
# Arrange
string1 = "hack"
string2 = "hackathon"
expected = "hack"
# Act
actual = Util.get_prefix(string1, string2)
# Assert
self.assertEqual(actual, expected, "Correct non empty prefix not found")
def search(self, word):
"""
Recursively search a word in the from this Node
:param word: The word to search
:return: The node where the word matches after
tracing from the root else None
"""
prefix = Util.get_prefix(self.key, word)
if prefix == word:
return self
if self.is_leaf:
return None
if self.children:
for child in self.children:
if not prefix and Util.has_prefix(child.key, word):
return child.search(word)
if Util.has_prefix(child.key, word[len(prefix):]):
return child.search(word[len(prefix):])
def update_from_praw(self, p):
self.header_img = p.header_img
self.header_title = p.header_title
self.title = p.title
self.display_name = p.display_name
self.name = p.name
self.url = p.url.lower().strip()
self.reddit_id = Util.plain_id(p.id)
self.description = p.description
self.description_html = p.description_html
self.public_description = p.public_description
self.created = p.created
self.created_utc = p.created_utc
self.scraped_time = Util.now()
self.accounts_active = p.accounts_active
self.subscribers = p.subscribers
self.over18 = bool(p.over18)
def main(notify):
session = Session()
gen = r.get_subreddit("all").get_top(limit=3000)
start = session.query(Post).count()
notify("Getting posts, initial count: %d" % start)
count = 0
for post in gen:
count += 1
p = Post.get_or_create(session, post.id)
p.update_from_praw(post)
author_name = Util.patch_author(post.author)
Util.update_user(User, p, session, author_name)
Util.update_subreddit(Subreddit, p, session, post.subreddit.display_name)
session.add(p)
session.commit()
diff = session.query(Post).count() - start
notify("Added %d posts" % diff)
def update_from_json(self, j):
self.reddit_id = j['id']
self.name = j['name']
self.created = j['created']
self.created_utc = j['created_utc']
self.scraped_time = Util.now()
self.link_karma = j['link_karma']
self.comment_karma = j['comment_karma']
self.over_18 = j['over_18']
self.is_gold = j['is_gold']
self.is_mod = j['is_mod']
self.has_verified_email = j['has_verified_email']
def update_from_praw(self, p):
self.reddit_id = p.id
self.name = p.name
self.created = p.created
self.created_utc = p.created_utc
self.scraped_time = Util.now()
self.link_karma = p.link_karma
self.comment_karma = p.comment_karma
self.over_18 = p.over_18
self.is_gold = p.is_gold
self.is_mod = p.is_mod
self.has_verified_email = p.has_verified_email
def update_from_praw(self, p):
self.domain = p.domain
self.url = p.url.lower().strip()
self.permalink = p.permalink
self.selftext_html = p.selftext_html
self.selftext = p.selftext
self.title = p.title
self.thumbnail = p.thumbnail
self.edited = float(p.edited)
self.reddit_id = p.id
self.score = p.score
self.downs = p.downs
self.ups = p.ups
self.over_18 = p.over_18
self.is_self = p.is_self
self.num_comments = p.num_comments
self.created = p.created
self.created_utc = p.created_utc
self.scraped_time = Util.now()
def create(cls, session, comment_id):
comment = cls()
comment.reddit_id = Util.plain_id(comment_id)
Util.add_and_refresh(session, comment)
return comment
def touch(self):
self.scraped_time = Util.now()
def create(cls, session, post_id):
post = cls()
post.reddit_id = Util.plain_id(post_id)
Util.add_and_refresh(session, post)
return post
def create(cls, session, user_id):
meta = cls()
meta.user_id = user_id
Util.add_and_refresh(session, meta)
return meta
def create(cls, session, display_name):
sub = cls()
sub.display_name = display_name
Util.add_and_refresh(session, sub)
return sub
