def addAllToDB(conn, tweets, formatted):
	if formatted:
		formattedTweets = tweets
	else:
		formattedTweets = []
		for tweet in tweets:
			formattedTweets.append(formatTweet(conn, tweet))
	for tweet in formattedTweets:
		#print("adding user: %s" % tweet['user'])
		db.addUser(conn, tweet['user'])
		db.addTweetToDB(conn, tweet['tweet'])
示例#2
0
def register():
    error = None
    if request.form['password'] != request.form['password_rt']:
        return render_template('register.html', error="Password doesn't match", tab='profile-tab')

    user, error = db.addUser(request.form['username'], request.form['email'], request.form['password'])
    if user:
        session['logged_in'] = True
        session['username'] = user.name

        flash('You were now registered')
        return redirect(url_for('item_page', item_id=session['current_item']))
示例#3
0
def transfer():
    user_lookup = {}
    item_lookup = {}
    rel_lookup = {}
    global db
    db = connect_db()
    mon.init('localhost', 27017, 'conversion')
    dd = mon.createInitialDb()
    user_lookup[0] = mon.getUser(dd['admin'])

    lite_items = query_db('SELECT * FROM items')
    lite_rels = query_db('SELECT * FROM relations')
    lite_comments = query_db('SELECT * FROM comments')
    lite_users = query_db('SELECT * FROM users')
    lite_votes = query_db('SELECT * FROM votes')

    for lu in lite_users:
        print lu['username']
        if lu['username'] == 'NKessel':
            user_lookup[3] = user_lookup[1]
        # print lu['date_registered']
        dr = datetime.strptime(lu['date_registered'], '%Y-%m-%d %H:%M:%S')
        # print str(dr)
        u, error = mon.addUser(unicode(lu['username']), unicode(lu['email']), unicode(lu['password']), dr)
        print 'inserted users: ' + str(u)
        if u:
            user_lookup[lu['id']] = u

    for li in lite_items:
        # print 'uid lookup: ' + str(li['user_id'])+ ' -- ' + str(user_lookup[li['user_id']]) 

        #print li['time_submitted']
        ts = datetime.strptime(li['time_submitted'].split('.')[0], '%Y-%m-%d %H:%M:%S')
        if not li['body']:
            li['body'] = li['title']
            li['title'] = ""
        i = mon.addItem({
            'body': unicode(li['body']),
            'tldr': unicode(li['title']),
            'user': user_lookup[li['user_id']].name,
            "time_submitted": ts,
            "upvotes": li['upvotes'],
            "downvotes": li['downvotes']
        })
        print 'inserted items: ' + str(li['id']) + ' -- ' + str(i)
        item_lookup[li['id']] = i

    for lr in lite_rels:
        print lr.get('time_submitted', "no time")
        #tl = datetime.strptime(lr['time_linked'].split('.')[0], '%Y-%m-%d %H:%M:%S')

        if lr['parent'] == 0:
            rp = mon.root._id
        else:
            rp = item_lookup[lr['parent']]._id

        if lr['child'] == 0:
            rc = mon.root._id
        else:
            rc = item_lookup[lr['child']]._id

        lb = user_lookup.get(lr['linked_by'], None)
        if lb:
            lb = lb.name
        else:
            lb = mon.getUser(item_lookup[lr['child']].user).name

        r = mon.addRel(rp, rc, lb)
        r.upvotes = lr['upvotes']
        r.downvotes = lr['downvotes']
        # r.time_linked = tl
        r.save()
        print 'inserted rels: ' + str(lr['id']) + ' -- ' + str(r)
        rel_lookup[lr['id']] = r

    for lc in lite_comments:
        rel = rel_lookup.get(lc['rel_id'], None)
        if rel is None:
            continue
        rel = mon.getRel(rel._id)
        if lc['head_item'] == 0:
            cp = mon.root._id
        else:
            cp = item_lookup[lc['head_item']]._id
        rel.comment_parent = cp
        rel.save()

        item = mon.getItem(rel.child)
        item.tags.append(u'comment')
        item.save()

    for lv in lite_votes:
        user = mon.getUser(user_lookup[lv['user_id']].name)
        rel = rel_lookup.get(lv['rel_id'], None)
        if rel is None:
            continue
        rel_id = rel._id

        if lv['vote_type'] == 'up':
            is_upvote = True
        else:
            is_upvote = False

        user.votes.append({
            'rel': rel_id,
            'is_upvote': is_upvote
        })
        user.save()

    db.close()
def formatTweet(conn, tweet):
	userDetails = {}
	tweetDetails = {}
	userDetails['userID'] = tweet['user']['id']
	userDetails['screenName'] = tweet['user']['screen_name']
	userDetails['followerCount'] = tweet['user']['friends_count']	# FIX!
	tweetDetails['userID'] = str(tweet['user']['id'])
	tweetDetails['tweetText'] = tweet['text']
	tweetDetails['tweetID'] = tweet['id_str']
	tweetDetails['favCount'] = tweet['favorite_count']
	tweetDetails['rtCount'] = tweet['retweet_count']
	datePosted = datetime.datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y")
	#adjust timezone by adding 8 to hours <--maybe make this part a bit more formal?
	#print datePosted
	datePosted = datePosted + datetime.timedelta(hours=8)
	tweetDetails['datePosted'] = datePosted.strftime("%Y-%m-%d %H:%M:%S")
	#DATETIME - format: YYYY-MM-DD HH:MI:SS
	tweetDetails['tweetLang'] = tweet['lang']
	#tweetDetails['expandedURL'] = tweet['expanded_url']
	tweetDetails['retweeted'] = tweet['retweeted']
	tweetDetails['favorited'] = tweet['favorited']
	tweetDetails['userDetails'] = userDetails
	#datetime format 'created_at': u'Sat Oct 24 23:22:20 +0000 2015'

	coordinates = tweet['coordinates']
	# print(coordinates)
	if coordinates:
		tweetDetails['coordinates'] = tweet['coordinates']

	# Check if tweet is a retweet. If yes and not yet in DB then add
	if "retweeted_status" in tweet.keys():
		retweet = tweet['retweeted_status']
		rtID = retweet['id_str']
		if not db.checkTweet(conn, rtID):
			#tweet doesn't exist in DB yet
			#format tweet
			formattedTweet = formatTweet(conn, retweet)
			db.addUser(conn, formattedTweet['user'])
			db.addTweetToDB(conn, formattedTweet['tweet'])
			#add to DB
		else:
			# Update tweet with new retweet and favorite count!
			pass
		tweetDetails['rtSource'] = rtID
	else:
		tweetDetails['rtSource'] = None
	#Add to passed params

	# Check if tweet is a reply
	replyTo = tweet['in_reply_to_status_id_str']
	tweetDetails['inReplyToStatusID'] = None
	if replyTo:
		tweetDetails['inReplyToStatusID'] = "uncheckedReplyTweet"
		# #if tweet is a reply
		# #check if tweet exists in database
		# if not db.checkTweet(conn, replyTo):
		# 	#if tweet is not in database
		# 	#fetch tweet from API
		# 	#print("SEARCHING FOR TWEET ID: %s" % replyTo)
		# 	try:
		# 		recipientTweet = twitter.show_status(id=replyTo)
		# 		formattedTweet = formatTweet(conn, recipientTweet)
		# 		db.addUser(conn, formattedTweet['user'])
		# 		db.addTweetToDB(conn, formattedTweet['tweet'])
		# 	except TwythonRateLimitError as e:
		# 		print("[ERROR] TwythonRateLimitError on show_status")
		# 		print(e)
		# 		retryIn = datetime.datetime.fromtimestamp(float(e.retry_after)) - datetime.datetime.now()
		# 		retryIn = float(retryIn.total_seconds())
		# 		print("Time to sleep: %d" % math.ceil(retryIn))
		# 		#raw_input("Press enter to continue to sleep")
		# 		print("Time at start of sleep:")
		# 		print datetime.datetime.now()
		# 		print("End of sleep:")
		# 		print(datetime.datetime.fromtimestamp(float(e.retry_after)))
		# 		retryIn = datetime.datetime.fromtimestamp(float(e.retry_after)) - datetime.datetime.now()
		# 		retryIn = float(retryIn.total_seconds())
		# 		if retryIn > 0:
		# 			time.sleep(math.ceil(retryIn))
		# 		pass
		# 	except Exception as e:
		# 		print("ERROR: Error while populating database.") # File saved as %s\n" % filename)
		# 		print(e)
		# 		pass
		# 	tweetDetails['inReplyToStatusID'] = replyTo
		# else:
		# 	# Update tweet with new retweet and favorite count!
		# 	pass
		# tweetDetails['inReplyToStatusID'] = replyTo
	else:
		tweetDetails['inReplyToStatusID'] = None


	return {"user":userDetails, "tweet":tweetDetails}
示例#5
0
文件: lite_mong.py 项目: yedi/Avalon
def transfer():
    user_lookup = {}
    item_lookup = {}
    rel_lookup = {}
    global db
    db = connect_db()
    mon.init("localhost", 27017, "conversion")
    dd = mon.createInitialDb()
    user_lookup[0] = mon.getUser(dd["admin"])

    lite_items = query_db("SELECT * FROM items")
    lite_rels = query_db("SELECT * FROM relations")
    lite_comments = query_db("SELECT * FROM comments")
    lite_users = query_db("SELECT * FROM users")
    lite_votes = query_db("SELECT * FROM votes")

    for lu in lite_users:
        print lu["username"]
        if lu["username"] == "NKessel":
            user_lookup[3] = user_lookup[1]
        # print lu['date_registered']
        dr = datetime.strptime(lu["date_registered"], "%Y-%m-%d %H:%M:%S")
        # print str(dr)
        u, error = mon.addUser(unicode(lu["username"]), unicode(lu["email"]), unicode(lu["password"]), dr)
        print "inserted users: " + str(u)
        if u:
            user_lookup[lu["id"]] = u

    for li in lite_items:
        # print 'uid lookup: ' + str(li['user_id'])+ ' -- ' + str(user_lookup[li['user_id']])

        # print li['time_submitted']
        ts = datetime.strptime(li["time_submitted"].split(".")[0], "%Y-%m-%d %H:%M:%S")
        if not li["body"]:
            li["body"] = li["title"]
            li["title"] = ""
        i = mon.addItem(
            {
                "body": unicode(li["body"]),
                "tldr": unicode(li["title"]),
                "user": user_lookup[li["user_id"]].name,
                "time_submitted": ts,
                "upvotes": li["upvotes"],
                "downvotes": li["downvotes"],
            }
        )
        print "inserted items: " + str(li["id"]) + " -- " + str(i)
        item_lookup[li["id"]] = i

    for lr in lite_rels:
        print lr.get("time_submitted", "no time")
        # tl = datetime.strptime(lr['time_linked'].split('.')[0], '%Y-%m-%d %H:%M:%S')

        if lr["parent"] == 0:
            rp = mon.root._id
        else:
            rp = item_lookup[lr["parent"]]._id

        if lr["child"] == 0:
            rc = mon.root._id
        else:
            rc = item_lookup[lr["child"]]._id

        lb = user_lookup.get(lr["linked_by"], None)
        if lb:
            lb = lb.name
        else:
            lb = mon.getUser(item_lookup[lr["child"]].user).name

        r = mon.addRel(rp, rc, lb)
        r.upvotes = lr["upvotes"]
        r.downvotes = lr["downvotes"]
        # r.time_linked = tl
        r.save()
        print "inserted rels: " + str(lr["id"]) + " -- " + str(r)
        rel_lookup[lr["id"]] = r

    for lc in lite_comments:
        rel = rel_lookup.get(lc["rel_id"], None)
        if rel is None:
            continue
        rel = mon.getRel(rel._id)
        if lc["head_item"] == 0:
            cp = mon.root._id
        else:
            cp = item_lookup[lc["head_item"]]._id
        rel.comment_parent = cp
        rel.save()

        item = mon.getItem(rel.child)
        item.tags.append(u"comment")
        item.save()

    for lv in lite_votes:
        user = mon.getUser(user_lookup[lv["user_id"]].name)
        rel = rel_lookup.get(lv["rel_id"], None)
        if rel is None:
            continue
        rel_id = rel._id

        if lv["vote_type"] == "up":
            is_upvote = True
        else:
            is_upvote = False

        user.votes.append({"rel": rel_id, "is_upvote": is_upvote})
        user.save()

    db.close()