Exemple #1
0
def retrieve(force, **args):
    """ this is how we will create new items/tweets """
    username, password = args['account'], None
    if isinstance(username, tuple):
        username, password = username

    search = True
    user = False
    force = False
    search_term = "swarthmore"
    
    if search:
        print "[community.sources.twitter | INFO]: Working with Search"
        url = "http://search.twitter.com/search.json?q=%s" % search_term
        last_id = 0

        if force:
            log.info("Forcing update of all tweets available.")
        else:
            try:
                last_id = Tweet.objects.order_by('-tweet_id')[0].tweet_id
            except Exception, e:
                log.debug('%s', e)
        log.debug("Last id processed: %s", last_id)

        tweets = utils.get_remote_data(url, rformat="json")

        if not tweets:
            log.warning('no tweets returned, twitter possibly overloaded.')
            return
            
        tweets = tweets['results']
        for t in tweets:
            curr_id = t['id']
            if curr_id > last_id:
                log.info("Working with %s.", t['id'])

                tweet_text = t['text']
                tweet_text = re.sub(r'@((?:\w|\.(?=\w))+)',r'<a href="http://www.twitter.com/\1/">\1</a>',tweet_text)
                tweet_text = tweet_text.replace("@","&#64;")
                owner_user = smart_unicode(t['from_user'])
                url = "http://twitter.com/%s/statuses/%s" % (owner_user, t['id'])
                icon = t['profile_image_url'].replace("_normal","_bigger",1)
            
                tweet, created = Tweet.objects.get_or_create(
                        title       = str(curr_id) + " " + tweet_text[:50],
                        description = tweet_text,
                        tweet_id    = curr_id, 
                        timestamp   = utils.parsedate(t['created_at']),
                        source_type = "tweet",
                        owner_user  = owner_user,
                        url         = url,
                        icon        = icon,
                )
            
                tweet.source      = smart_unicode(t['source'])
            
            else:
                log.warning("No more tweets, stopping...")
                break        
Exemple #2
0
def _handle_rss_bookmark(mark, username):
    log.info("working with bookmark => %s" % mark.get('extended'))

    bookmark, created = Bookmark.objects.get_or_create(
        description=mark.get('extended'),
        url=mark.get("href"),
        title=mark.get("description"),
        timestamp=utils.parsedate(mark.get('time')))
Exemple #3
0
def _handle_rss_bookmark(mark, username):
    log.info("working with bookmark => %s" % mark.get('extended'))

    bookmark, created = Bookmark.objects.get_or_create(
        description = mark.get('extended'),
        url = mark.get("href"),
        title = mark.get("description"),
        timestamp = utils.parsedate(mark.get('time'))
    )
Exemple #4
0
        try:
            last_update = Bookmark.objects.filter(owner_user="******"+tag).order_by('-timestamp')[0].timestamp
        except Exception, e:
            log.debug('%s', e)

    if force and password:
        marks = utils.get_remote_data(url, rformat=rformat, username=username, password=password)
    else:
        marks = utils.get_remote_data(url, rformat=rformat)

    if marks:
        for mark in marks:
            if password and force:
                _handle_rss_bookmark(mark, tag)
                continue
            dt = utils.parsedate(mark['dt']) 
            if dt > last_update:
                _handle_bookmark(mark, dt, tag)
            else:
                log.warning("No more bookmarks, stopping...")
                break

def _handle_bookmark(mark, dt, tag):
    log.info("working with bookmark => %s" % mark['d'])

    bookmark, created = Bookmark.objects.get_or_create(
        timestamp   = dt,
        url         = mark['u'],
        title       = mark['d'],
        owner_user  = "******"+tag,
        description = mark['n'],
Exemple #5
0
def retrieve(force, **args):
    """ this is how we will create new items/tweets """
    username, password = args['account'], None
    if isinstance(username, tuple):
        username, password = username

    search = True
    user = False
    force = False
    search_term = "swarthmore"

    if search:
        print "[community.sources.twitter | INFO]: Working with Search"
        url = "http://search.twitter.com/search.json?q=%s" % search_term
        last_id = 0

        if force:
            log.info("Forcing update of all tweets available.")
        else:
            try:
                last_id = Tweet.objects.order_by('-tweet_id')[0].tweet_id
            except Exception, e:
                log.debug('%s', e)
        log.debug("Last id processed: %s", last_id)

        tweets = utils.get_remote_data(url, rformat="json")

        if not tweets:
            log.warning('no tweets returned, twitter possibly overloaded.')
            return

        tweets = tweets['results']
        for t in tweets:
            curr_id = t['id']
            if curr_id > last_id:
                log.info("Working with %s.", t['id'])

                tweet_text = t['text']
                tweet_text = re.sub(
                    r'@((?:\w|\.(?=\w))+)',
                    r'<a href="http://www.twitter.com/\1/">\1</a>', tweet_text)
                tweet_text = tweet_text.replace("@", "&#64;")
                owner_user = smart_unicode(t['from_user'])
                url = "http://twitter.com/%s/statuses/%s" % (owner_user,
                                                             t['id'])
                icon = t['profile_image_url'].replace("_normal", "_bigger", 1)

                tweet, created = Tweet.objects.get_or_create(
                    title=str(curr_id) + " " + tweet_text[:50],
                    description=tweet_text,
                    tweet_id=curr_id,
                    timestamp=utils.parsedate(t['created_at']),
                    source_type="tweet",
                    owner_user=owner_user,
                    url=url,
                    icon=icon,
                )

                tweet.source = smart_unicode(t['source'])

            else:
                log.warning("No more tweets, stopping...")
                break
Exemple #6
0
            tweet_text = t['text']
            tweet_text = re.sub(
                r'@((?:\w|\.(?=\w))+)',
                r'<a href="http://www.twitter.com/\1/">\1</a>', tweet_text)
            tweet_text = tweet_text.replace("@", "&#64;")
            owner_user = smart_unicode(t['user']['screen_name'])
            url = "http://twitter.com/%s/statuses/%s" % (owner_user, t['id'])
            icon = t['user']['profile_image_url'].replace(
                "_normal", "bigger", 1)

            tweet, created = Tweet.objects.get_or_create(
                title=str(curr_id) + " " + tweet_text[:50],
                description=tweet_text,
                tweet_id=curr_id,
                timestamp=utils.parsedate(t['created_at']),
                source_type="tweet",
                owner_user=owner_user,
                url=url,
                icon=icon,
            )

            tweet.source = smart_unicode(t['source'])

        else:
            log.warning("No more tweets, stopping...")
            break


admin.site.register(Tweet, TweetAdmin)
Exemple #7
0
    for t in tweets:
        if t['id'] > last_id:
            log.info("Working with %s.", t['id'])
            
            tweet_text = t['text']
            tweet_text = re.sub(r'@((?:\w|\.(?=\w))+)',r'<a href="http://www.twitter.com/\1/">\1</a>',tweet_text)
            tweet_text = tweet_text.replace("@","&#64;")
            owner_user = smart_unicode(t['user']['screen_name'])
            url = "http://twitter.com/%s/statuses/%s" % (owner_user, t['id'])
            icon = t['user']['profile_image_url'].replace("_normal","bigger",1)
        
            tweet, created = Tweet.objects.get_or_create(
                    title       = str(curr_id) + " " + tweet_text[:50],
                    description = tweet_text,
                    tweet_id    = curr_id, 
                    timestamp   = utils.parsedate(t['created_at']),
                    source_type = "tweet",
                    owner_user  = owner_user,
                    url         = url,
                    icon        = icon,
            )
        
            tweet.source      = smart_unicode(t['source'])
    
        else:
            log.warning("No more tweets, stopping...")
            break

admin.site.register(Tweet, TweetAdmin)
Exemple #8
0
            log.debug('%s', e)

    if force and password:
        marks = utils.get_remote_data(url,
                                      rformat=rformat,
                                      username=username,
                                      password=password)
    else:
        marks = utils.get_remote_data(url, rformat=rformat)

    if marks:
        for mark in marks:
            if password and force:
                _handle_rss_bookmark(mark, tag)
                continue
            dt = utils.parsedate(mark['dt'])
            if dt > last_update:
                _handle_bookmark(mark, dt, tag)
            else:
                log.warning("No more bookmarks, stopping...")
                break


def _handle_bookmark(mark, dt, tag):
    log.info("working with bookmark => %s" % mark['d'])

    bookmark, created = Bookmark.objects.get_or_create(timestamp=dt,
                                                       url=mark['u'],
                                                       title=mark['d'],
                                                       owner_user="******" + tag,
                                                       description=mark['n'],