def addAllToDB(conn, tweets, formatted):
if formatted:
formattedTweets = tweets
else:
formattedTweets = []
for tweet in tweets:
formattedTweets.append(formatTweet(conn, tweet))
for tweet in formattedTweets:
#print("adding user: %s" % tweet['user'])
db.addUser(conn, tweet['user'])
db.addTweetToDB(conn, tweet['tweet'])
def register():
error = None
if request.form['password'] != request.form['password_rt']:
return render_template('register.html', error="Password doesn't match", tab='profile-tab')
user, error = db.addUser(request.form['username'], request.form['email'], request.form['password'])
if user:
session['logged_in'] = True
session['username'] = user.name
flash('You were now registered')
return redirect(url_for('item_page', item_id=session['current_item']))
def transfer():
user_lookup = {}
item_lookup = {}
rel_lookup = {}
global db
db = connect_db()
mon.init('localhost', 27017, 'conversion')
dd = mon.createInitialDb()
user_lookup[0] = mon.getUser(dd['admin'])
lite_items = query_db('SELECT * FROM items')
lite_rels = query_db('SELECT * FROM relations')
lite_comments = query_db('SELECT * FROM comments')
lite_users = query_db('SELECT * FROM users')
lite_votes = query_db('SELECT * FROM votes')
for lu in lite_users:
print lu['username']
if lu['username'] == 'NKessel':
user_lookup[3] = user_lookup[1]
# print lu['date_registered']
dr = datetime.strptime(lu['date_registered'], '%Y-%m-%d %H:%M:%S')
# print str(dr)
u, error = mon.addUser(unicode(lu['username']), unicode(lu['email']), unicode(lu['password']), dr)
print 'inserted users: ' + str(u)
if u:
user_lookup[lu['id']] = u
for li in lite_items:
# print 'uid lookup: ' + str(li['user_id'])+ ' -- ' + str(user_lookup[li['user_id']])
#print li['time_submitted']
ts = datetime.strptime(li['time_submitted'].split('.')[0], '%Y-%m-%d %H:%M:%S')
if not li['body']:
li['body'] = li['title']
li['title'] = ""
i = mon.addItem({
'body': unicode(li['body']),
'tldr': unicode(li['title']),
'user': user_lookup[li['user_id']].name,
"time_submitted": ts,
"upvotes": li['upvotes'],
"downvotes": li['downvotes']
})
print 'inserted items: ' + str(li['id']) + ' -- ' + str(i)
item_lookup[li['id']] = i
for lr in lite_rels:
print lr.get('time_submitted', "no time")
#tl = datetime.strptime(lr['time_linked'].split('.')[0], '%Y-%m-%d %H:%M:%S')
if lr['parent'] == 0:
rp = mon.root._id
else:
rp = item_lookup[lr['parent']]._id
if lr['child'] == 0:
rc = mon.root._id
else:
rc = item_lookup[lr['child']]._id
lb = user_lookup.get(lr['linked_by'], None)
if lb:
lb = lb.name
else:
lb = mon.getUser(item_lookup[lr['child']].user).name
r = mon.addRel(rp, rc, lb)
r.upvotes = lr['upvotes']
r.downvotes = lr['downvotes']
# r.time_linked = tl
r.save()
print 'inserted rels: ' + str(lr['id']) + ' -- ' + str(r)
rel_lookup[lr['id']] = r
for lc in lite_comments:
rel = rel_lookup.get(lc['rel_id'], None)
if rel is None:
continue
rel = mon.getRel(rel._id)
if lc['head_item'] == 0:
cp = mon.root._id
else:
cp = item_lookup[lc['head_item']]._id
rel.comment_parent = cp
rel.save()
item = mon.getItem(rel.child)
item.tags.append(u'comment')
item.save()
for lv in lite_votes:
user = mon.getUser(user_lookup[lv['user_id']].name)
rel = rel_lookup.get(lv['rel_id'], None)
if rel is None:
continue
rel_id = rel._id
if lv['vote_type'] == 'up':
is_upvote = True
else:
is_upvote = False
user.votes.append({
'rel': rel_id,
'is_upvote': is_upvote
})
user.save()
db.close()
def formatTweet(conn, tweet):
userDetails = {}
tweetDetails = {}
userDetails['userID'] = tweet['user']['id']
userDetails['screenName'] = tweet['user']['screen_name']
userDetails['followerCount'] = tweet['user']['friends_count'] # FIX!
tweetDetails['userID'] = str(tweet['user']['id'])
tweetDetails['tweetText'] = tweet['text']
tweetDetails['tweetID'] = tweet['id_str']
tweetDetails['favCount'] = tweet['favorite_count']
tweetDetails['rtCount'] = tweet['retweet_count']
datePosted = datetime.datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y")
#adjust timezone by adding 8 to hours <--maybe make this part a bit more formal?
#print datePosted
datePosted = datePosted + datetime.timedelta(hours=8)
tweetDetails['datePosted'] = datePosted.strftime("%Y-%m-%d %H:%M:%S")
#DATETIME - format: YYYY-MM-DD HH:MI:SS
tweetDetails['tweetLang'] = tweet['lang']
#tweetDetails['expandedURL'] = tweet['expanded_url']
tweetDetails['retweeted'] = tweet['retweeted']
tweetDetails['favorited'] = tweet['favorited']
tweetDetails['userDetails'] = userDetails
#datetime format 'created_at': u'Sat Oct 24 23:22:20 +0000 2015'
coordinates = tweet['coordinates']
# print(coordinates)
if coordinates:
tweetDetails['coordinates'] = tweet['coordinates']
# Check if tweet is a retweet. If yes and not yet in DB then add
if "retweeted_status" in tweet.keys():
retweet = tweet['retweeted_status']
rtID = retweet['id_str']
if not db.checkTweet(conn, rtID):
#tweet doesn't exist in DB yet
#format tweet
formattedTweet = formatTweet(conn, retweet)
db.addUser(conn, formattedTweet['user'])
db.addTweetToDB(conn, formattedTweet['tweet'])
#add to DB
else:
# Update tweet with new retweet and favorite count!
pass
tweetDetails['rtSource'] = rtID
else:
tweetDetails['rtSource'] = None
#Add to passed params
# Check if tweet is a reply
replyTo = tweet['in_reply_to_status_id_str']
tweetDetails['inReplyToStatusID'] = None
if replyTo:
tweetDetails['inReplyToStatusID'] = "uncheckedReplyTweet"
# #if tweet is a reply
# #check if tweet exists in database
# if not db.checkTweet(conn, replyTo):
# #if tweet is not in database
# #fetch tweet from API
# #print("SEARCHING FOR TWEET ID: %s" % replyTo)
# try:
# recipientTweet = twitter.show_status(id=replyTo)
# formattedTweet = formatTweet(conn, recipientTweet)
# db.addUser(conn, formattedTweet['user'])
# db.addTweetToDB(conn, formattedTweet['tweet'])
# except TwythonRateLimitError as e:
# print("[ERROR] TwythonRateLimitError on show_status")
# print(e)
# retryIn = datetime.datetime.fromtimestamp(float(e.retry_after)) - datetime.datetime.now()
# retryIn = float(retryIn.total_seconds())
# print("Time to sleep: %d" % math.ceil(retryIn))
# #raw_input("Press enter to continue to sleep")
# print("Time at start of sleep:")
# print datetime.datetime.now()
# print("End of sleep:")
# print(datetime.datetime.fromtimestamp(float(e.retry_after)))
# retryIn = datetime.datetime.fromtimestamp(float(e.retry_after)) - datetime.datetime.now()
# retryIn = float(retryIn.total_seconds())
# if retryIn > 0:
# time.sleep(math.ceil(retryIn))
# pass
# except Exception as e:
# print("ERROR: Error while populating database.") # File saved as %s\n" % filename)
# print(e)
# pass
# tweetDetails['inReplyToStatusID'] = replyTo
# else:
# # Update tweet with new retweet and favorite count!
# pass
# tweetDetails['inReplyToStatusID'] = replyTo
else:
tweetDetails['inReplyToStatusID'] = None
return {"user":userDetails, "tweet":tweetDetails}
def transfer():
user_lookup = {}
item_lookup = {}
rel_lookup = {}
global db
db = connect_db()
mon.init("localhost", 27017, "conversion")
dd = mon.createInitialDb()
user_lookup[0] = mon.getUser(dd["admin"])
lite_items = query_db("SELECT * FROM items")
lite_rels = query_db("SELECT * FROM relations")
lite_comments = query_db("SELECT * FROM comments")
lite_users = query_db("SELECT * FROM users")
lite_votes = query_db("SELECT * FROM votes")
for lu in lite_users:
print lu["username"]
if lu["username"] == "NKessel":
user_lookup[3] = user_lookup[1]
# print lu['date_registered']
dr = datetime.strptime(lu["date_registered"], "%Y-%m-%d %H:%M:%S")
# print str(dr)
u, error = mon.addUser(unicode(lu["username"]), unicode(lu["email"]), unicode(lu["password"]), dr)
print "inserted users: " + str(u)
if u:
user_lookup[lu["id"]] = u
for li in lite_items:
# print 'uid lookup: ' + str(li['user_id'])+ ' -- ' + str(user_lookup[li['user_id']])
# print li['time_submitted']
ts = datetime.strptime(li["time_submitted"].split(".")[0], "%Y-%m-%d %H:%M:%S")
if not li["body"]:
li["body"] = li["title"]
li["title"] = ""
i = mon.addItem(
{
"body": unicode(li["body"]),
"tldr": unicode(li["title"]),
"user": user_lookup[li["user_id"]].name,
"time_submitted": ts,
"upvotes": li["upvotes"],
"downvotes": li["downvotes"],
}
)
print "inserted items: " + str(li["id"]) + " -- " + str(i)
item_lookup[li["id"]] = i
for lr in lite_rels:
print lr.get("time_submitted", "no time")
# tl = datetime.strptime(lr['time_linked'].split('.')[0], '%Y-%m-%d %H:%M:%S')
if lr["parent"] == 0:
rp = mon.root._id
else:
rp = item_lookup[lr["parent"]]._id
if lr["child"] == 0:
rc = mon.root._id
else:
rc = item_lookup[lr["child"]]._id
lb = user_lookup.get(lr["linked_by"], None)
if lb:
lb = lb.name
else:
lb = mon.getUser(item_lookup[lr["child"]].user).name
r = mon.addRel(rp, rc, lb)
r.upvotes = lr["upvotes"]
r.downvotes = lr["downvotes"]
# r.time_linked = tl
r.save()
print "inserted rels: " + str(lr["id"]) + " -- " + str(r)
rel_lookup[lr["id"]] = r
for lc in lite_comments:
rel = rel_lookup.get(lc["rel_id"], None)
if rel is None:
continue
rel = mon.getRel(rel._id)
if lc["head_item"] == 0:
cp = mon.root._id
else:
cp = item_lookup[lc["head_item"]]._id
rel.comment_parent = cp
rel.save()
item = mon.getItem(rel.child)
item.tags.append(u"comment")
item.save()
for lv in lite_votes:
user = mon.getUser(user_lookup[lv["user_id"]].name)
rel = rel_lookup.get(lv["rel_id"], None)
if rel is None:
continue
rel_id = rel._id
if lv["vote_type"] == "up":
is_upvote = True
else:
is_upvote = False
user.votes.append({"rel": rel_id, "is_upvote": is_upvote})
user.save()
db.close()
