Exemplo n.º 1
0
def saveTwitter(tweet):
    '''
    create by bigzhu at 15/07/10 14:39:48
        保存twitter
    '''

    if hasattr(tweet, 'user'):
        del tweet.user._json
        #del tweet.user._api
        tweet.user.entities = json.dumps(tweet.user.entities)

        del tweet.user.id
        db_bz.insertIfNotExist(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str)
        tweet.t_user_id = tweet.user.id_str
        del tweet.user

    if hasattr(tweet, 'author'):
        #del tweet.author.id
        db_bz.insertIfNotExist(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str)
        tweet.t_author_id = tweet.author.id_str

        del tweet.author

    if hasattr(tweet, '_api'):
        del tweet._api
    if hasattr(tweet, '_json'):
        del tweet._json
    # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
    if hasattr(tweet, 'id'):
        del tweet.id

    if hasattr(tweet, 'entities'):
        tweet.entities = json.dumps(tweet.entities)
    if hasattr(tweet, 'geo'):
        tweet.geo = json.dumps(tweet.geo)
    if hasattr(tweet, 'coordinates'):
        tweet.coordinates = json.dumps(tweet.coordinates)
    if hasattr(tweet, 'extended_entities'):
        tweet.extended_entities = json.dumps(tweet.extended_entities)
    if hasattr(tweet, 'retweeted_status'):
        saveTwitter(tweet.retweeted_status)
        tweet.retweeted_status = tweet.retweeted_status.id_str
    if hasattr(tweet, 'quoted_status'):
        # print tweet.quoted_status
        del tweet.quoted_status
    #    saveTwitter(tweet.quoted_status)
    #    tweet.quoted_status = tweet.quoted_status.id_str

    # place  是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
    if hasattr(tweet, 'place'):
        del tweet.place

    # for k, v in vars(tweet).items():
    #    print '%s=%s' % (k, v)

    return db_bz.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)
Exemplo n.º 2
0
def saveMessage(message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    message.id_str = message.pop('id')

    content = storage()
    content.type = message.type
    content.repo = message.pop('repo')
    content.payload = message.pop('payload')
    message.content = json.dumps(content)

    if message.get('org'):
        message.org = json.dumps(message.org)

    # return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)

    m = public_bz.storage()
    m.id_str = message.id_str
    m.m_type = 'github'
    m.m_user_id = message.actor
    m.created_at = message.created_at
    m.content = message.content
    m.text = None
    m.href = None
    return db_bz.insertIfNotExist(
        pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
Exemplo n.º 3
0
def saveMessage(message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    message.id_str = message.pop('id')

    content = storage()
    content.type = message.type
    content.repo = message.pop('repo')
    content.payload = message.pop('payload')
    message.content = json.dumps(content)

    if message.get('org'):
        message.org = json.dumps(message.org)

    # return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)

    m = public_bz.storage()
    m.id_str = message.id_str
    m.m_type = 'github'
    m.m_user_id = message.actor
    m.created_at = message.created_at
    m.content = message.content
    m.text = None
    m.href = None
    return db_bz.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
Exemplo n.º 4
0
def follow(user_id, god_id, make_sure=True):
    '''
    create by bigzhu at 15/07/15 14:22:51
    modify by bigzhu at 15/07/15 15:00:28 如果不用告警,就不要make_sure
    '''
    id = db_bz.insertIfNotExist(pg, 'follow_who', {'user_id': user_id, 'god_id': god_id}, "user_id=%s and god_id=%s" % (user_id, god_id))
    if id is None and make_sure:
        raise Exception('没有正确的Follow, 似乎已经Follow过了呢')
    def post(self):
        self.set_header("Content-Type", "application/json")
        user_id = self.get_secure_cookie("user_id")
        for card_number in self.request.body.splitlines():
            print "card_number:", card_number
            values = {"user_id": user_id, "card_number": card_number}
            id = db_bz.insertIfNotExist(pg, "available_card_numbers", values, where=" card_number='%s'" % card_number)
            print "insert ", values, "id:", id

        self.write(json.dumps({"error": "0"}, cls=public_bz.ExtEncoder))
Exemplo n.º 6
0
def saveLast(last_time, last_message_id, user_id):
    '''
    create by bigzhu at 15/08/16 16:22:39 保存最后一条的message
    '''
    last_time = int(last_time)
    datetime_last_time = time_bz.timestampToDateTime(last_time, millisecond=True)
    id = db_bz.insertIfNotExist(pg, 'last', {'user_id': user_id, 'last_time': datetime_last_time, 'last_message_id': last_message_id}, "user_id=%s" % user_id)
    if id is None:
        count = pg.update('last', where='last_time< to_timestamp(%s/1000) and user_id=%s' % (last_time, user_id),  last_message_id=last_message_id, last_time=datetime_last_time)
        return count
    return id
Exemplo n.º 7
0
def follow(user_id, god_id, make_sure=True):
    '''
    create by bigzhu at 15/07/15 14:22:51
    modify by bigzhu at 15/07/15 15:00:28 如果不用告警,就不要make_sure
    '''
    id = db_bz.insertIfNotExist(pg, 'follow_who', {
        'user_id': user_id,
        'god_id': god_id
    }, "user_id=%s and god_id=%s" % (user_id, god_id))
    if id is None and make_sure:
        raise Exception('没有正确的Follow, 似乎已经Follow过了呢')
Exemplo n.º 8
0
    def post(self):

        self.set_header("Content-Type", "application/json")
        data = json.loads(self.request.body)

        id = db_bz.insertIfNotExist(pg, 'user_info', data, "user_name='%s'" % data['user_name'])
        if id is None:
            where = "user_name='%s'" % data['user_name']
            self.pg.update("user_info", where=where, **data)
            id = self.pg.select('user_info', where=where)[0].id

        oper.follow(self.current_user, id, make_sure=False)

        self.write(json.dumps({'error': '0'}))
Exemplo n.º 9
0
    def post(self):

        self.set_header("Content-Type", "application/json")
        data = json.loads(self.request.body)

        id = db_bz.insertIfNotExist(pg, 'user_info', data,
                                    "user_name='%s'" % data['user_name'])
        if id is None:
            where = "user_name='%s'" % data['user_name']
            self.pg.update("user_info", where=where, **data)
            id = self.pg.select('user_info', where=where)[0].id

        oper.follow(self.current_user, id, make_sure=False)

        self.write(json.dumps({'error': '0'}))
Exemplo n.º 10
0
def saveMessage(message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    message.id_str = message.pop('id')

    content = storage()
    content.type = message.type
    content.repo = message.pop('repo')
    content.payload = message.pop('payload')
    message.content = json.dumps(content)

    if message.get('org'):
        message.org = json.dumps(message.org)

    return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)
Exemplo n.º 11
0
def saveLast(last_message_id, user_id):
    '''
    create by bigzhu at 15/08/16 16:22:39 保存最后一条的message
    '''
    #last_time = int(last_time)
    #datetime_last_time = time_bz.timestampToDateTime(last_time, millisecond=True)
    id = db_bz.insertIfNotExist(pg, 'last', {
        'user_id': user_id,
        'last_message_id': last_message_id
    }, "user_id=%s" % user_id)
    if id is None:
        count = pg.update('last',
                          where='last_message_id< %s and user_id=%s' %
                          (last_message_id, user_id),
                          last_message_id=last_message_id)
        return count
    return 1
Exemplo n.º 12
0
def saveUser(id, url):
    '''
    create by bigzhu at 15/07/15 21:27:19 保存github信息
    create by bigzhu at 15/07/22 16:17:37 fix bug, not return id
    '''
    if list(pg.select('github_user', where='id=%s' % id)):
        return id
    else:
        r = requests.get(url)
        user = storage(r.json())
        del user.url
        del user.followers_url
        del user.following_url
        del user.gists_url
        del user.starred_url
        del user.subscriptions_url
        del user.organizations_url
        del user.repos_url
        del user.events_url
        del user.received_events_url
        del user.type
        return db_bz.insertIfNotExist(pg, 'github_user', user)
Exemplo n.º 13
0
def saveUser(id, url):
    '''
    create by bigzhu at 15/07/15 21:27:19 保存github信息
    create by bigzhu at 15/07/22 16:17:37 fix bug, not return id
    '''
    if list(pg.select('github_user', where='id=%s' % id)):
        return id
    else:
        r = requests.get(url)
        user = storage(r.json())
        print user
        if hasattr(user, 'url'):
            del user.url
        del user.followers_url
        del user.following_url
        del user.gists_url
        del user.starred_url
        del user.subscriptions_url
        del user.organizations_url
        del user.repos_url
        del user.events_url
        del user.received_events_url
        del user.type
        return db_bz.insertIfNotExist(pg, 'github_user', user)
Exemplo n.º 14
0
def insertIfNotExist(*args, **kwargs):
    return db_bz.insertIfNotExist(*args, **kwargs)
Exemplo n.º 15
0
def getMedia(user_name=None, with_next_url=None, user=None):
    if user_name:
        user = getUser(user_name)
        if user is None:
            return
        # min_id 会查出大于等于这个id的
        try:
            medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id)
        except instagram.bind.InstagramClientError:
            print public_bz.getExpInfoAll()
            public_db.delNoName('instagram', user_name)
            return
        if medias:
            last_id = medias[0].id
            pg.update('instagram_user', where="lower(username)=lower('%s')" % user_name, last_id=last_id)
    else:
        medias, next_ = api.user_recent_media(with_next_url=with_next_url)

    for media in medias:
        db_media = storage()
        if media.caption:
            caption = media.caption.__dict__
            caption['user_id'] = caption['user'].id
            del caption['user']
        else:
            caption = ''
        db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder)
        db_media.comment_count = media.comment_count

        if media.comments:
            media.comments = [d.__dict__ for d in media.comments]
            for comment in media.comments:
                comment['user'] = comment['user'].__dict__
        db_media.comments = json.dumps(media.comments, cls=public_bz.ExtEncoder)
        db_media.created_time = media.created_time
        # 8小时的问题
        db_media.created_time += timedelta(hours=8)
        db_media.filter = media.filter
        db_media.low_resolution = json.dumps(media.images['low_resolution'].__dict__)
        db_media.standard_resolution = json.dumps(media.images['standard_resolution'].__dict__)
        db_media.thumbnail = json.dumps(media.images['thumbnail'].__dict__)
        db_media.id_str = media.id
        db_media.like_count = media.like_count
        # likes里有User对象,暂时不存了
        #db_media.likes = json.dumps(media.likes)
        db_media.link = media.link
        db_media.type = media.type
        db_media.user_id = user.id
        id = db_bz.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str)
        print 'new=', media.id, user.username
        if id is not None and len(medias) <= 2:  # 新增加消息,微信通知只通知2条以内
            openids = public_db.getOpenidsByName('instagram', user.username)
            for data in openids:
                if caption != '':
                    text = caption.get('text')
                else:
                    text = ''
                wechat_oper.sendInstagram(data.openid, text, media.images['standard_resolution'].url, user.username, id)
    # 递归查出
    if next_ != with_next_url:
        getMedia(with_next_url=next_, user=user)
Exemplo n.º 16
0
def insertIfNotExist(*args, **kwargs):
    return db_bz.insertIfNotExist(*args, **kwargs)