示例#1
0
def getNewWechatExtInfo():
    '''
    modify by bigzhu at 15/09/13 17:25:43 为了用订阅号给指定用户发消息,改用WechatExt
    '''
    wechat_ext = WechatExt(user_name, password,
                           appid=appid
                           )
    token_cookies = wechat_ext.get_token_cookies()
    pg.insertOrUpdate(pg, 'wechat_dead_line', token_cookies, where='1=1')
    return wechat_ext
def getNewWechatExtInfo():
    '''
    modify by bigzhu at 15/09/13 17:25:43 为了用订阅号给指定用户发消息,改用WechatExt
    '''
    wechat_ext = WechatExt(user_name, password,
                           appid=appid
                           )
    token_cookies = wechat_ext.get_token_cookies()
    pg.insertOrUpdate(pg, 'wechat_dead_line', token_cookies, where='1=1')
    return wechat_ext
示例#3
0
def saveUserCheckNew(blogs):
    '''
    create by bigzhu at 15/09/05 11:57:11
    '''
    user = blogs['blog']
    user_name = user['name']
    where = "name='%s'" % user_name
    result = list(pg.select('tumblr_user', where=where))
    if result:
        if result[0].updated == user['updated']:  # 如果没有更新过,就不用继续了
            print user_name, ' no update'
            # return
            pass
    pg.insertOrUpdate(pg, 'tumblr_user', user, where)
    blogs = blogs['posts']
    saveBlogs(user_name, blogs, offset=20)
示例#4
0
def getUser(user_name, always_check=False):
    '''
    modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
    '''
    users = list(
        pg.select('instagram_user',
                  where="lower(username)=lower('%s')" % user_name))
    if users and not always_check:
        return users[0]
    else:
        user = None
        try:
            for this_user in api.user_search(user_name, 10):
                if this_user.username.lower() == user_name.lower():
                    user = this_user
                    break
        except instagram.bind.InstagramClientError:
            print public_bz.getExpInfoAll()
            return
        # 如果没有这个用户
        if user is None:
            public_db.delNoName('instagram', user_name)
            return
        try:
            user = api.user(user.id)
        except (instagram.bind.InstagramAPIError,
                instagram.bind.InstagramClientError):
            # 通常是没有访问权限
            print public_bz.getExpInfoAll()
            public_db.delNoName('instagram', user_name)
            return

        db_user = storage()
        db_user.id_str = user.id
        db_user.username = user.username
        db_user.full_name = user.full_name
        db_user.profile_picture = user.profile_picture
        db_user.bio = user.bio
        db_user.website = user.website
        db_user.counts = json.dumps(user.counts)
        #pg.insert('instagram_user', **db_user)
        pg.insertOrUpdate(pg, 'instagram_user', db_user,
                          "id_str='%s'" % db_user.id_str)
        #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
        return getUser(user_name)
示例#5
0
def getUser(user_name, always_check=False):
    '''
    modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
    '''
    users = list(pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name))
    if users and not always_check:
        return users[0]
    else:
        user = None
        try:
            for this_user in api.user_search(user_name, 10):
                if this_user.username.lower() == user_name.lower():
                    user = this_user
                    break
        except instagram.bind.InstagramClientError:
            print public_bz.getExpInfoAll()
            return
        # 如果没有这个用户
        if user is None:
            public_db.delNoName('instagram', user_name)
            return
        try:
            user = api.user(user.id)
        except (instagram.bind.InstagramAPIError, instagram.bind.InstagramClientError):
            # 通常是没有访问权限
            print public_bz.getExpInfoAll()
            public_db.delNoName('instagram', user_name)
            return

        db_user = storage()
        db_user.id_str = user.id
        db_user.username = user.username
        db_user.full_name = user.full_name
        db_user.profile_picture = user.profile_picture
        db_user.bio = user.bio
        db_user.website = user.website
        db_user.counts = json.dumps(user.counts)
        #pg.insert('instagram_user', **db_user)
        pg.insertOrUpdate(pg, 'instagram_user', db_user, "id_str='%s'" % db_user.id_str)
        #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
        return getUser(user_name)
示例#6
0
def saveTwitter(tweet):
    '''
    create by bigzhu at 15/07/10 14:39:48
        保存twitter
    '''

    del tweet.user._json
    #del tweet.user._api
    tweet.user.entities = json.dumps(tweet.user.entities)

    del tweet.user.id
    pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str)

    tweet.t_user_id = tweet.user.id_str
    screen_name = tweet.user.screen_name
    del tweet.user

    if hasattr(tweet, 'author'):
        #del tweet.author.id
        pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str)
        tweet.t_author_id = tweet.author.id_str

        del tweet.author

    if hasattr(tweet, '_api'):
        del tweet._api
    if hasattr(tweet, '_json'):
        del tweet._json
    # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
    if hasattr(tweet, 'id'):
        del tweet.id

    if hasattr(tweet, 'entities'):
        tweet.entities = json.dumps(tweet.entities)
    if hasattr(tweet, 'geo'):
        tweet.geo = json.dumps(tweet.geo)
    if hasattr(tweet, 'coordinates'):
        tweet.coordinates = json.dumps(tweet.coordinates)
    if hasattr(tweet, 'extended_entities'):
        tweet.extended_entities = json.dumps(tweet.extended_entities)
    if hasattr(tweet, 'scopes'):
        tweet.scopes = json.dumps(tweet.scopes)

    if hasattr(tweet, 'retweeted_status'):
        saveTwitter(tweet.retweeted_status)
        tweet.retweeted_status = tweet.retweeted_status.id_str
    if hasattr(tweet, 'quoted_status'):
        # print tweet.quoted_status
        del tweet.quoted_status
    #    saveTwitter(tweet.quoted_status)
    #    tweet.quoted_status = tweet.quoted_status.id_str

    # place  是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
    if hasattr(tweet, 'place'):
        del tweet.place

    # for k, v in vars(tweet).items():
    #    print '%s=%s' % (k, v)

    #return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)

    m = public_bz.storage()
    m.id_str = tweet.id_str
    m.m_type = 'twitter'
    m.m_user_id = tweet.t_user_id
    m.created_at = tweet.created_at
    if hasattr(tweet, 'extended_entities'):
        m.extended_entities = tweet.extended_entities
    m.content = None
    m.text = tweet.text
    m.href = 'https://twitter.com/'+screen_name+'/status/'+tweet.id_str

    return pg.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)
示例#7
0
def saveTwitter(tweet):
    '''
    create by bigzhu at 15/07/10 14:39:48
        保存twitter
    '''

    del tweet.user._json
    #del tweet.user._api
    tweet.user.entities = json.dumps(tweet.user.entities)

    del tweet.user.id
    pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user),
                      "id_str='%s'" % tweet.user.id_str)

    tweet.t_user_id = tweet.user.id_str
    screen_name = tweet.user.screen_name
    del tweet.user

    if hasattr(tweet, 'author'):
        #del tweet.author.id
        pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author),
                          "id_str='%s'" % tweet.author.id_str)
        tweet.t_author_id = tweet.author.id_str

        del tweet.author

    if hasattr(tweet, '_api'):
        del tweet._api
    if hasattr(tweet, '_json'):
        del tweet._json
    # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
    if hasattr(tweet, 'id'):
        del tweet.id

    if hasattr(tweet, 'entities'):
        tweet.entities = json.dumps(tweet.entities)
    if hasattr(tweet, 'geo'):
        tweet.geo = json.dumps(tweet.geo)
    if hasattr(tweet, 'coordinates'):
        tweet.coordinates = json.dumps(tweet.coordinates)
    if hasattr(tweet, 'extended_entities'):
        tweet.extended_entities = json.dumps(tweet.extended_entities)
    if hasattr(tweet, 'scopes'):
        tweet.scopes = json.dumps(tweet.scopes)

    if hasattr(tweet, 'retweeted_status'):
        saveTwitter(tweet.retweeted_status)
        tweet.retweeted_status = tweet.retweeted_status.id_str
    if hasattr(tweet, 'quoted_status'):
        # print tweet.quoted_status
        del tweet.quoted_status
    #    saveTwitter(tweet.quoted_status)
    #    tweet.quoted_status = tweet.quoted_status.id_str

    # place  是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
    if hasattr(tweet, 'place'):
        del tweet.place

    # for k, v in vars(tweet).items():
    #    print '%s=%s' % (k, v)

    #return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)

    m = public_bz.storage()
    m.id_str = tweet.id_str
    m.m_type = 'twitter'
    m.m_user_id = tweet.t_user_id
    m.created_at = tweet.created_at
    if hasattr(tweet, 'extended_entities'):
        m.extended_entities = tweet.extended_entities
    m.content = None
    m.text = tweet.text
    m.href = 'https://twitter.com/' + screen_name + '/status/' + tweet.id_str

    return pg.insertIfNotExist(
        pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)