예제 #1
0
def insertTweet(tweet):
    twitterID = tweet.id_str
    time = tweet.created_at

    #oembed endpoint https://dev.twitter.com/docs/api/1/get/statuses/oembed v1 endpoint
    oembedParams = {'id': twitterID, 'omit_script': 'true'}
    oembedRequest = requests.get('https://api.twitter.com/1/statuses/oembed.json', params=oembedParams)
    embedResponse = json.loads(oembedRequest.text)
    embed_html = embedResponse['html']

    processed = True
    display = True
    num_images = len(tweet.termIDs)

    q = 'INSERT INTO tweet (twitter_id, time, embed_html, processed, display, num_images) VALUES ( %s, %s, %s, %s, %s, %s ) RETURNING tweet_id'
    tweetID = pgutils.getQueryDictionary(q, twitterID, time, embed_html, processed, display, num_images)
    tweetID = tweetID[0]['tweet_id']
    
    #insert terms
    if len(tweet.termIDs) > 0:
        termIDsStr = ','.join(pgCursor.mogrify("(%s,%s)", (tweetID, termID)) for termID in tweet.termIDs)
        q = 'INSERT INTO tweet_has_term (tweet_id, term_id) VALUES ' + termIDsStr
        pgCursor.execute(q)

    #insert users
    if len(tweet.screenNames) > 0:
        twitterUsersStr = ','.join(pgCursor.mogrify("(%s, %s, %s)", [twitterUser['screen_name'], tweetID, twitterUser['relationship']]) for twitterUser in tweet.screenNames)
        q = 'INSERT INTO tweet_has_user (screen_name, tweet_id, relationship) VALUES ' + twitterUsersStr
        pgCursor.execute(q)
예제 #2
0
def searchTweetImages(tweet):
    exists = pgutils.getQueryDictionary('SELECT COUNT(*) as exists FROM tweet WHERE twitter_id=%s', tweet.id_str)
    if exists[0]['exists'] != 0:
        return False
    tweet.termIDs = []
    tweet.screenNames = []
    for type in tweetTypes:
        typeObj = tweetTypes[type]
        success = tweepyutils.fetchImages(typeObj, tweet)
        if not success:
            return False
    return True
예제 #3
0
def insertTermImages(term_id, urls, removePreviousTermImages = False):
    if len(urls) == 0:
        return
    if removePreviousTermImages:
        pgCursor.execute('DELETE FROM image WHERE term_id=%s', [term_id])
    for url in urls:
        q = 'SELECT COUNT(*) as count FROM image WHERE term_id=%s AND url=%s'
        exists = pgutils.getQueryDictionary(q, term_id, url);
        if exists[0]['count'] == 0:
            q = 'INSERT INTO image (url, term_id, retrieved_at) VALUES (%s, %s, now())'
            pgCursor.execute(q, [url, term_id])
    q = 'UPDATE term SET last_queried_at=now() WHERE term_id=%s'
    pgCursor.execute(q, [term_id])
예제 #4
0
    def get(self):
        self.set_header('Content-Type','text/json')
        dreams = pgutils.getQueryDictionary('SELECT * FROM tweet ORDER BY time DESC LIMIT 30')
        for dream in dreams:
            q = 'SELECT term.* FROM term, tweet, tweet_has_term WHERE tweet_has_term.tweet_id = tweet.tweet_id AND tweet_has_term.term_id = term.term_id AND tweet.tweet_id=%s'
            terms = pgutils.getQueryDictionary(q, dream['tweet_id'])
            termObjs = []
            for term in terms:
                q = 'SELECT url FROM image WHERE term_id=%s'
                termImages = pgutils.getQueryDictionary(q, term['term_id'])
                termObjs.append( {"term": term, "images": termImages})
            dream['terms'] = termObjs
            q = 'SELECT tweet_has_user.*, twitter_user.user_json  FROM  tweet_has_user, twitter_user WHERE tweet_id=%s AND tweet_has_user.screen_name=twitter_user.screen_name'
            dreamPeople = pgutils.getQueryDictionary(q, dream['tweet_id'])
            userProperties = [
                'profile_use_background_image', 'default_profile_image', 'profile_background_image_url_https',
                'profile_text_color','profile_background_color','description',
                'profile_image_url_https'
            ]
            people = []
            for person in dreamPeople:
                p = {
                    "screen_name": person['screen_name'],
                    "relationship": person['relationship']
                }
                for prop in userProperties:
                    p[prop] = person['user_json'][prop]
                people.append( p )
            dream['people'] = people

        # convert python datetime to json 
        class DateTimeJSONEncoder(json.JSONEncoder):
            def default(self, obj):
                if isinstance(obj, datetime.datetime):
                    return int(mktime(obj.timetuple()))
                return json.JSONEncoder.default(self, obj)
        rtnJson = dreams
        self.write(json.dumps(rtnJson, cls = DateTimeJSONEncoder))
예제 #5
0
def selectOrInsertTerm(term, term_type):
    termExists = pgutils.getQueryDictionary('SELECT * FROM term WHERE term=%s and term_type_id=%s', term, term_type['term_type_id'])
    termRtn = {}
    if len(termExists) == 0:
        termRtn['expired'] = True
        #insert
        q = 'INSERT INTO term (term, term_type_id) VALUES (%s, %s) RETURNING term_id'
        type_id = term_type['term_type_id']
        id = pgutils.getQueryDictionary(q, term, type_id)
        termRtn['id'] = id[0]['term_id']

    else:
        #check expiration
        last_queried_at = termExists[0]['last_queried_at']
        if last_queried_at == None:
            termRtn['expired'] = True
        else:
            expiredAt = datetime.now() - term_type['expires_after']
            if last_queried_at < expiredAt:
                termRtn['expired'] = True
            else:
                termRtn['expired'] = False
        termRtn['id'] = termExists[0]['term_id']
    return termRtn
예제 #6
0
def insertTwitterUserIfNotPresent(screen_name,term_type):
    userExists = pgutils.getQueryDictionary('SELECT * FROM twitter_user WHERE screen_name=%s', screen_name)
    expired = None
    if len(userExists) == 0:
        expired = True
        #insert
        q = 'INSERT INTO twitter_user (screen_name) VALUES (%s)'
        pgCursor.execute(q, [screen_name])
    else:
        #check expiration
        last_queried_at = userExists[0]['updated_at']
        if last_queried_at == None:
            expired = True
        else:
            expiredAt = datetime.now() - term_type['expires_after']
            if last_queried_at < expiredAt:
                expired = True
            else:
                expired = False
    return expired