Example #1
0
def saveMessage(god_name, github_name, god_id, message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    content = storage()
    content.type = message.type
    content.repo = message.repo
    content.payload = message.payload
    content = json.dumps(content)

    m = public_bz.storage()
    m.god_id = god_id
    m.god_name = god_name
    m.name = github_name
    # m.avatar = message.actor['avatar_url']

    m.id_str = message['id']
    m.m_type = 'github'
    m.created_at = time_bz.unicodeToDateTIme(message.created_at)
    m.created_at += timedelta(hours=8)
    m.content = content
    m.text = None
    m.href = None
    id = pg.insertIfNotExist('message', m,
                             "id_str='%s' and m_type='github'" % m.id_str)
    if id is not None:
        print '%s new github %s' % (m.name, m.id_str)
    return id
Example #2
0
def sendTwitter(openid, tweet, screen_name, id):
    '''
    发送twitter的消息
    '''

    articles = []
    if hasattr(tweet,
               'extended_entities') and tweet.extended_entities['media']:
        for media in tweet.extended_entities['media']:
            article = storage()
            article.picurl = "http://follow.center/ProxyHandler/%s" % media[
                'media_url_https']
            article.url = "http://follow.center/message?t=twitter&id=%s" % id
            articles.append(article)
        if len(articles) == 1:
            articles[0].title = screen_name
        else:
            articles[0].title = screen_name + ': ' + tweet.text
        articles[0].description = tweet.text
    else:
        article = storage()
        article.title = screen_name
        article.url = "http://follow.center/message?t=twitter&id=%s" % id
        article.description = tweet.text
        articles = [article]
    sendArticle(openid, articles)
Example #3
0
def saveMessage(message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    message.id_str = message.pop('id')

    content = storage()
    content.type = message.type
    content.repo = message.pop('repo')
    content.payload = message.pop('payload')
    message.content = json.dumps(content)

    if message.get('org'):
        message.org = json.dumps(message.org)

    # return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)

    m = public_bz.storage()
    m.id_str = message.id_str
    m.m_type = 'github'
    m.m_user_id = message.actor
    m.created_at = message.created_at
    m.content = message.content
    m.text = None
    m.href = None
    return db_bz.insertIfNotExist(
        pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
Example #4
0
def saveMessage(god_name, facebook_name, god_id, message):
    '''
    '''
    message = public_bz.storage(message)
    m = public_bz.storage()
    m.god_id = god_id
    m.god_name = god_name
    m.name = facebook_name

    m.m_type = 'facebook'
    m.id_str = message.id
    m.created_at = message.created_time
    m.content = json.dumps({'description': message.get('description')})
    m.text = message.get('message')
    m.extended_entities = json.dumps({
        'pictrue': message.get('full_picture'),
        'source': message.get('source')
    })
    m.type = message.get('type')
    if m.type == 'video':
        print('facebook 不再同步 video, 有时效, 无法一直看')
        return
    m.href = message.get('link')
    id = pg.insertIfNotExist('message', m,
                             "id_str='%s' and m_type='facebook'" % m.id_str)
    if id is not None:
        print '%s new facebook message %s' % (m.name, m.id_str)
    return id
Example #5
0
def saveMessage(message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    message.id_str = message.pop('id')

    content = storage()
    content.type = message.type
    content.repo = message.pop('repo')
    content.payload = message.pop('payload')
    message.content = json.dumps(content)

    if message.get('org'):
        message.org = json.dumps(message.org)

    # return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)

    m = public_bz.storage()
    m.id_str = message.id_str
    m.m_type = 'github'
    m.m_user_id = message.actor
    m.created_at = message.created_at
    m.content = message.content
    m.text = None
    m.href = None
    return db_bz.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='github'" % message.id_str)
Example #6
0
def getHtmlListByNameLike(search_name):
    '''
    根据名字查找对应html
    返回 list
    {name:'', time: ''}
    '''
    html_list = []
    for html in os.listdir(HTML_PATH):
        html_name = cutName(html)
        if html_name == '':
            continue
        if isNameLike(html_name, search_name):
            pass
        else:
            continue

        try:
            modify_time = getModifyTime(html_name)
        except OSError as e:  # 可能会有非html结尾的文件,忽略
            print e
            continue

        the_html = storage()
        the_html.name = html_name
        the_html.time = modify_time
        html_list.append(the_html)

    # 按时间排序
    html_list = sorted(html_list, key=lambda d: d.time, reverse=True)
    return html_list
Example #7
0
def saveMessage(god_name, twitter_name, god_id, tweet):
    '''
    create by bigzhu at 15/07/10 14:39:48
        保存twitter
    create by bigzhu at 16/03/26 06:05:12 重构,改很多
    modify by bigzhu at 16/03/26 20:33:59 重构
        user 本系统用户信息
        tweet 消息本身
    modify by bigzhu at 17/01/13 15:38:11 去了用不到的
    '''
    m = public_bz.storage()
    m.god_id = god_id
    m.god_name = god_name
    m.name = twitter_name

    m.id_str = tweet.id_str
    m.m_type = 'twitter'
    m.created_at = tweet.created_at
    m.content = None
    m.text = tweet.text
    if hasattr(tweet, 'extended_entities'):
        m.extended_entities = json.dumps(tweet.extended_entities)
        m.type = tweet.extended_entities['media'][0]['type']
    m.href = 'https://twitter.com/' + m.name + '/status/' + m.id_str
    id = pg.insertIfNotExist('message', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)
    if id is not None:
        print '%s new twitter %s' % (m.name, m.id_str)
    return id
Example #8
0
 def post(self):
     '''
     新增文件
     '''
     global md5
     self.set_header("Content-Type", "application/json")
     results = []
     if self.request.files:
         for i in self.request.files:
             fd = self.request.files[i]
             for f in fd:
                 file_name = f.get("filename")
                 file_suffix = file_name[file_name.rfind("."):]
                 file_body = f["body"]
                 md5.update(file_body)
                 file_hash = md5.hexdigest()
                 file_path = "static/uploaded_files/%s_%s_%s" % (self.current_user, int(time.time()), file_name)
                 # hash
                 img = open(file_path, 'w')
                 img.write(file_body)
                 img.close()
                 new_file = storage(file_name=file_name, file_path="/" + file_path, file_hash=file_hash, file_type="file", suffix=file_suffix, seqname='uploaded_files_id_seq')
                 file_id = self.pg.db.insert("uploaded_files", **new_file)
                 r = {
                     'file_id': file_id,
                     'file_name': file_name,
                     'file_path': '/' + file_path,
                     'suffix': file_suffix
                 }
                 results.append(r)
     self.write(json.dumps({'error': '0', 'results': results}))
Example #9
0
    def render(self, comments, key_type, key):
        parm = storage()
        parm.comments = comments
        parm.key_type = key_type
        parm.key = key

        return self.render_string(self.html_name, parm=parm)
Example #10
0
def getUser(user_name):
    '''
    modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
    '''
    users = list(pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name))
    if users:
        return users[0]
    else:
        user = None
        for this_user in api.user_search(user_name, 10):
            if this_user.username.lower() == user_name.lower():
                user = this_user
                break
        #如果没有这个用户
        if user is None:
            public_db.delNoName('instagram', user_name)
        #user = api.user_search(user_name, 1)[0]
        user = api.user(user.id)

        db_user = storage()
        db_user.id = user.id
        db_user.username = user.username
        db_user.full_name = user.full_name
        db_user.profile_picture = user.profile_picture
        db_user.bio = user.bio
        db_user.website = user.website
        db_user.counts = json.dumps(user.counts)
        pg.insert('instagram_user', **db_user)
        #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
        return getUser(user_name)
Example #11
0
def main(god, wait):
    '''
    create by bigzhu at 15/07/15 17:54:08 取github
    modify by bigzhu at 15/07/22 16:20:42 时间同样要加入8小时,否则不正确
    '''
    god_name = god.name
    github_name = god.github['name']
    god_id = god.id

    etag = god.github.get('sync_key')
    github_user = getGithubUser(github_name, god_name)
    if github_user is None:
        return
    headers = {'If-None-Match': etag}
    try:
        r = requests.get('https://api.github.com/users/%s/events' %
                         github_name,
                         headers=headers,
                         params=params)
    except requests.exceptions.ConnectionError:
        print public_bz.getExpInfoAll()
        return
    if r.status_code == 200:
        etag = r.headers['etag']
        limit = r.headers['X-RateLimit-Remaining']
        if limit == '0':
            return
        for message in r.json():
            message = storage(message)
            saveMessage(god_name, github_name, god_id, message)
        # oper.noMessageTooLong('github', github_name)
        saveUser(god_name, github_name, github_user, etag)  # 为了update etag
    if r.status_code == 404:
        # public_db.sendDelApply('github', god_name, github_name, '404')
        god_oper.delNoName('github', god_name)
Example #12
0
def saveMessage(god_name, twitter_name, god_id, blog):
    m = public_bz.storage()
    m.god_id = god_id
    m.god_name = god_name
    m.name = twitter_name

    m.id_str = blog['id']
    m.m_type = 'tumblr'
    m.created_at = time_bz.timestampToDateTime(blog['timestamp'])
    type = blog.get('type')
    m.href = blog.get('short_url')
    m.type = type
    if type == 'text':
        m.title = blog.get('title')
        m.text = blog.get('body')
    elif type == 'photo':
        m.text = blog.get('caption')
        m.extended_entities = json.dumps(blog.get('photos'))
    elif type == 'video':
        m.extended_entities = json.dumps({'video_url': blog.get('video_url')})
    m.content = None

    id = pg.insertIfNotExist('message', m,
                             "id_str='%s' and m_type='tumblr' " % m.id_str)
    if id is None:
        pass
    else:
        print '%s new tumblr message %s' % (m.name, m.id_str)
Example #13
0
def saveBlogs(user_name, blogs, offset):
    if blogs:
        pass
    else:
        return
    for blog in blogs:

        m = public_bz.storage()
        m.id_str = blog['id']
        m.m_type = 'tumblr'
        m.m_user_id = user_name
        m.created_at = time_bz.timestampToDateTime(blog['timestamp'])
        m.extended_entities = json.dumps(blog.get('photos'))
        m.content = None
        m.text = blog.get('caption')
        m.href = blog.get('short_url')
        m.type = blog.get('type')

        result = pg.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='tumblr' " % m.id_str)
        if result is None:  # 有重复记录了,就不再继续了
            print 'have same data'
            return
        else:
            print 'new ', m.id_str, m.type, 'offset:', offset, 'name:', user_name
    # 继续取
    new_offset = offset + 20
    new_blogs = callGetMeidaApi(user_name, offset=new_offset)['response']['posts']
    saveBlogs(user_name, new_blogs, new_offset)
Example #14
0
def saveGraphqlMessage(ins_name, user_name, god_id, message):
    '''
    用 Graphql 取到的数据
    '''
    message = storage(message)

    m = public_bz.storage()
    m.god_name = user_name
    m.name = ins_name
    m.m_type = 'instagram'
    m.id_str = message.id
    m.created_at = time_bz.timestampToDateTime(message.taken_at_timestamp)

    if message.get('edge_media_to_caption').get('edges'):
        m.text = message.get('edge_media_to_caption').get('edges')[0].get(
            'node').get('text')
    else:
        m.text = None
    m.extended_entities = json.dumps({'url': message.display_url})

    m.href = 'https://www.instagram.com/p/%s/' % message.shortcode
    if message.__typename == 'GraphSidecar':  # mutiple image
        edges = getMutipleImage(message.shortcode)
        images = []
        for edge in edges:
            url = edge['node']['display_url']
            images.append({'url': url})
        m.extended_entities = json.dumps(images)
        m.type = 'images'
    elif message.is_video:
        m.type = 'video'
        video_url = getVideoUrl(m.href)
        m.extended_entities = json.dumps({
            'url': message.display_url,
            'video_url': video_url
        })
    else:
        m.type = 'image'
    id = pg.insertIfNotExist('message', m,
                             "id_str='%s' and m_type='instagram'" % m.id_str)
    if id is not None:
        print '%s new instagram message %s' % (m.name, m.id_str)
    # 肯定会有一条重复
    # else:
    #    print '%s 重复记录 %s' % (m.user_name, m.id_str)
    return id
Example #15
0
 def addTimeLine(self, oper, user_id, target_type, target_id, other_info=None):
     timeline = storage()
     timeline.oper = oper
     timeline.user_id = user_id
     timeline.target_type = target_type
     timeline.target_id = target_id
     if other_info:
         timeline.other_info = json.dumps(other_info, cls=public_bz.ExtEncoder)
     return self.pg.db.insert('timeline', **timeline)
Example #16
0
def getTenContent(name):
    ten_names = getList(name)[0][1][:SHOW_COUNT]
    lists = []
    for i in ten_names:
        c = public_bz.storage()
        c.name = i[0]
        c.time = i[1]
        c.content = getHtmlContent(c.name)
        lists.append(c)
    return lists
Example #17
0
def sendGithub(openid, text, user_name, id):
    '''
    create by bigzhu at 15/07/22 15:05:01 发送github的消息
    '''
    articles = []
    article = storage()
    article.title = user_name
    article.url = "http://follow.center/message?t=github&id=%s" % id
    article.description = text
    articles = [article]
    sendArticle(openid, articles)
Example #18
0
 def post(self):
     self.set_header("Content-Type", "application/json")
     data = json.loads(self.request.body)
     anki_info = storage()
     anki_info.user_name = data['user_name']
     anki_info.password = data['password']
     anki_info.user_id = self.current_user
     db_bz.insertOrUpdate(pg, 'anki', anki_info,
                          "user_id='%s'" % anki_info.user_id)
     anki.getMidAndCsrfTokenHolder(anki_info.user_id, reset_cookie=True)
     self.write(json.dumps(self.data))
Example #19
0
def sendGithub(openid, text, user_name, id):
    '''
    create by bigzhu at 15/07/22 15:05:01 发送github的消息
    '''
    articles = []
    article = storage()
    article.title = user_name
    article.url = "http://follow.center/message?t=github&id=%s" % id
    article.description = text
    articles = [article]
    sendArticle(openid, articles)
Example #20
0
def getUserEvent(user_name, etag):
    '''
    create by bigzhu at 15/07/15 17:54:08 取github
    modify by bigzhu at 15/07/22 16:20:42 时间同样要加入8小时,否则不正确
    '''
    headers = {'If-None-Match': etag}
    try:
        r = requests.get('https://api.github.com/users/%s/events' % user_name,
                         headers=headers)
    except requests.exceptions.ConnectionError:
        print public_bz.getExpInfoAll()
        return
    if r.status_code == 200:
        messages = r.json()
        if not messages:
            delGithubUser(user_name)
            # 没有这个github用户,取消
            return
        actor = messages[0]['actor']
        # actor不定是作者名字,有可能org才是
        if actor['login'].lower() == user_name.lower():
            the_user = actor
        else:
            org = messages[0]['org']
            if org['login'].lower() == user_name.lower():
                the_user = org
                # 如果是org,那么url不同
                the_user['url'] = "https://api.github.com/users/" + user_name
            else:
                raise "in this github can't find user_name=%s"

        user_id = saveUser(the_user['id'], the_user['url'])

        # 更新etag
        etag = r.headers['etag']
        updateEtag(user_name, etag)

        for i in r.json():
            i['actor'] = user_id
            message = storage(i)
            message.created_at = time_bz.unicodeToDateTIme(message.created_at)
            message.created_at += timedelta(hours=8)
            saveMessage(copy.deepcopy(message))
            #id = saveMessage(copy.deepcopy(message))
            # if id is not None:
            #    text = formatInfo(message)
            #    print text
            #    openids = public_db.getOpenidsByName('github', user_name)
            #    for data in openids:
            #        wechat_oper.sendGithub(data.openid, text, user_name, id)
    if r.status_code == 404:
        delGithubUser(user_name)
    else:
        print r.status_code
Example #21
0
def sendInstagram(openid, text, img_url, user_name, id):
    '''
    create by bigzhu at 15/08/01 00:35:08
    modify by bigzhu at 15/08/01 00:57:28 不用代理,没被屏蔽
    '''
    article = storage()
    article.title = user_name
    article.picurl = 'http://follow.center/sp/' + oper.encodeUrl(img_url)
    article.url = "http://follow.center/message?t=instagram&id=%s" % id
    article.description = text
    articles = [article]
    sendArticle(openid, articles)
Example #22
0
    def initialize(self):
        '''
        针对 oauth2 ,需要你重载的时候来设置为你自己的参数, 以下是 google twitter douban 的例子
        modify by bigzhu at 15/04/26 21:56:09 对应的oauth登录的参数,应该在对应的oauth里面来设置,这里不用再设置了
        '''

        UserInfoHandler.initialize(self)
        oauth2 = storage()
        oauth2.google = storage(enabled=False, url='/google')
        oauth2.twitter = storage(enabled=False, url='/twitter')
        oauth2.douban = storage(enabled=False, url='/douban')
        oauth2.github = storage(enabled=False, url='/github')
        self.oauth2 = oauth2

        # 用户操作相关的
        self.user_oper = user_bz.UserOper(self.pg)
        # 是否要验证
        self.validate = False

        #salt
        self.salt = "hold is watching you"
Example #23
0
def getUserEvent(user_name, etag):
    '''
    create by bigzhu at 15/07/15 17:54:08 取github
    modify by bigzhu at 15/07/22 16:20:42 时间同样要加入8小时,否则不正确
    '''
    headers = {'If-None-Match': etag}
    try:
        r = requests.get('https://api.github.com/users/%s/events' % user_name, headers=headers)
    except requests.exceptions.ConnectionError:
        print public_bz.getExpInfoAll()
        return
    if r.status_code == 200:
        messages = r.json()
        if not messages:
            delGithubUser(user_name)
            # 没有这个github用户,取消
            return
        actor = messages[0]['actor']
        # actor不定是作者名字,有可能org才是
        if actor['login'].lower() == user_name.lower():
            the_user = actor
        else:
            org = messages[0]['org']
            if org['login'].lower() == user_name.lower():
                the_user = org
                # 如果是org,那么url不同
                the_user['url'] = "https://api.github.com/users/" + user_name
            else:
                raise "in this github can't find user_name=%s"

        user_id = saveUser(the_user['id'], the_user['url'])

        # 更新etag
        etag = r.headers['etag']
        updateEtag(user_name, etag)

        for i in r.json():
            i['actor'] = user_id
            message = storage(i)
            message.created_at = time_bz.unicodeToDateTIme(message.created_at)
            message.created_at += timedelta(hours=8)
            saveMessage(copy.deepcopy(message))
            #id = saveMessage(copy.deepcopy(message))
            # if id is not None:
            #    text = formatInfo(message)
            #    print text
            #    openids = public_db.getOpenidsByName('github', user_name)
            #    for data in openids:
            #        wechat_oper.sendGithub(data.openid, text, user_name, id)
    if r.status_code == 404:
        delGithubUser(user_name)
    else:
        print r.status_code
Example #24
0
 def post(self):
     self.set_header("Content-Type", "application/json")
     data = json.loads(self.request.body)
     god_id = data['god_id']
     remark = data['remark']
     where = "user_id='%s' and god_id=%s" % (self.current_user, god_id)
     values = storage()
     values.user_id = self.current_user
     values.remark = remark
     values.god_id = god_id
     db_bz.insertOrUpdate(pg, 'remark', values, where)
     self.write(json.dumps({'error': '0'}))
Example #25
0
def sendInstagram(openid, text, img_url, user_name, id):
    '''
    create by bigzhu at 15/08/01 00:35:08
    modify by bigzhu at 15/08/01 00:57:28 不用代理,没被屏蔽
    '''
    article = storage()
    article.title = user_name
    article.picurl = 'http://follow.center/sp/' + oper.encodeUrl(img_url)
    article.url = "http://follow.center/message?t=instagram&id=%s" % id
    article.description = text
    articles = [article]
    sendArticle(openid, articles)
Example #26
0
def saveUser(god_name, tumblr_name, user):
    social_user = public_bz.storage()
    social_user.type = 'tumblr'
    # social_user.name = user['name']
    social_user.name = tumblr_name
    social_user.count = user.get('likes', -1)  # 有人会不分享likes数
    avatar_url = 'https://api.tumblr.com/v2/blog/%s.tumblr.com/avatar/512' % user[
        'name']
    social_user.avatar = tumblrRealAvatar(avatar_url)
    social_user.description = user['description']
    social_user.sync_key = user['updated']

    pg.update('god', where={'name': god_name}, tumblr=json.dumps(social_user))
    return social_user
Example #27
0
def saveUser(god_name, twitter_name, twitter_user):
    social_user = public_bz.storage()
    # 不要用返回的name, 大小写会发生变化
    # social_user.name = twitter_user.screen_name
    social_user.name = twitter_name
    social_user.type = 'twitter'
    social_user.count = twitter_user.followers_count
    social_user.avatar = twitter_user.profile_image_url_https.replace('_normal', '_400x400')
    social_user.description = twitter_user.description
    # 没有找到
    # social_user.sync_key = twitter_user.description

    pg.update('god', where={'name': god_name}, twitter=json.dumps(social_user))
    return social_user
Example #28
0
def saveUser(god_name, ins_name, user, sync_key):
    social_user = public_bz.storage()
    social_user.type = 'instagram'
    # social_user.name = user['username']
    social_user.name = ins_name
    social_user.count = user['followed_by']['count']
    social_user.avatar = user['profile_pic_url']
    social_user.description = user['biography']
    social_user.id = user['id']
    social_user.sync_key = sync_key
    pg.update('god',
              where={'name': god_name},
              instagram=json.dumps(social_user))
    return social_user
Example #29
0
def getContent(name, count=15, lenght=None):
    ten_names = getList(name)[0][1][:count]
    lists = []
    for i in ten_names:
        c = public_bz.storage()
        c.name = i[0]
        c.time = i[1]
        c.content = getHtmlContent(c.name)
        if lenght:
            print len(c.content)
            if len(c.content) < lenght:
                print 'continue ' + c.name
                continue
        lists.append(c)
    return lists
Example #30
0
def getContent(name, count=15, lenght=None):
    ten_names = getList(name)[0][1][:count]
    lists = []
    for i in ten_names:
        c = public_bz.storage()
        c.name = i[0]
        c.time = i[1]
        c.content = getHtmlContent(c.name)
        if lenght:
            print len(c.content)
            if len(c.content) < lenght:
                print 'continue ' + c.name
                continue
        lists.append(c)
    return lists
Example #31
0
def sendTwitter(openid, tweet, screen_name, id):
    '''
    发送twitter的消息
    '''

    articles = []
    if hasattr(tweet, 'extended_entities') and tweet.extended_entities['media']:
        for media in tweet.extended_entities['media']:
            article = storage()
            article.picurl = "http://follow.center/ProxyHandler/%s" % media['media_url_https']
            article.url = "http://follow.center/message?t=twitter&id=%s" % id
            articles.append(article)
        if len(articles) == 1:
            articles[0].title = screen_name
        else:
            articles[0].title = screen_name + ': ' + tweet.text
        articles[0].description = tweet.text
    else:
        article = storage()
        article.title = screen_name
        article.url = "http://follow.center/message?t=twitter&id=%s" % id
        article.description = tweet.text
        articles = [article]
    sendArticle(openid, articles)
Example #32
0
def saveMessage(message):
    '''
    create by bigzhu at 15/07/16 09:44:39 为了抽取数据方便,合并数据到 content 里
    '''
    message.id_str = message.pop('id')

    content = storage()
    content.type = message.type
    content.repo = message.pop('repo')
    content.payload = message.pop('payload')
    message.content = json.dumps(content)

    if message.get('org'):
        message.org = json.dumps(message.org)

    return db_bz.insertIfNotExist(pg, 'github_message', message, "id_str='%s'" % message.id_str)
Example #33
0
def saveUser(god_name, facebook_name, user, etag):
    social_user = public_bz.storage()
    social_user.type = 'facebook'
    # social_user.name = user['username']
    social_user.name = facebook_name
    # facebook 取不到 friend count or followed count
    social_user.count = -1
    social_user.avatar = user['picture']['data']['url']
    social_user.description = user.get('bio')  # bio 可能没有
    if etag is not None:
        social_user.sync_key = etag
    social_user.out_id = user['id']

    pg.update('god',
              where={'name': god_name},
              facebook=json.dumps(social_user))
    return social_user
Example #34
0
 def groupByCreatedDateDay(self, timelines):
     '''
     create by bigzhu at 15/02/03 13:44:24 按照天的精度,归并timeline
     modify by bigzhu at 15/02/06 13:29:58 修改为返回 list 而不是一个 dic, 以便于对时间按天的排序
     '''
     day_time_lines = []
     group_time_line = {}
     for timeline in timelines:
         day = timeline.created_date.strftime('%Y年%m月%d日')
         this_day_timeline = group_time_line.get(day)
         if this_day_timeline:
             #this_day_timeline.insert(0, timeline)
             this_day_timeline.append(timeline)
         else:
             group_time_line[day] = [timeline]
             day_time_lines.append(storage(day = day, timelines = group_time_line[day]))
     return day_time_lines
Example #35
0
    def put(self):
        self.set_header("Content-Type", "application/json")
        data = json.loads(self.request.body)
        last_time = int(data.get('last_time'))
        last_time = time_bz.timestampToDateTime(last_time, True)
        # last_message_id = data.get('message_id')
        user_id = self.current_user

        if user_id is None:
            pass
        else:
            last_oper.saveLast(last_time, user_id)
        data = storage()
        data.error = OK
        data.unread_message_count = oper.getUnreadCount(user_id)

        self.write(json.dumps(data, cls=json_bz.ExtEncoder))
Example #36
0
def getUser(user_name, always_check=False):
    '''
    modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
    '''
    users = list(
        pg.select('instagram_user',
                  where="lower(username)=lower('%s')" % user_name))
    if users and not always_check:
        return users[0]
    else:
        user = None
        try:
            for this_user in api.user_search(user_name, 10):
                if this_user.username.lower() == user_name.lower():
                    user = this_user
                    break
        except instagram.bind.InstagramClientError:
            print public_bz.getExpInfoAll()
            return
        # 如果没有这个用户
        if user is None:
            public_db.delNoName('instagram', user_name)
            return
        try:
            user = api.user(user.id)
        except (instagram.bind.InstagramAPIError,
                instagram.bind.InstagramClientError):
            # 通常是没有访问权限
            print public_bz.getExpInfoAll()
            public_db.delNoName('instagram', user_name)
            return

        db_user = storage()
        db_user.id_str = user.id
        db_user.username = user.username
        db_user.full_name = user.full_name
        db_user.profile_picture = user.profile_picture
        db_user.bio = user.bio
        db_user.website = user.website
        db_user.counts = json.dumps(user.counts)
        #pg.insert('instagram_user', **db_user)
        pg.insertOrUpdate(pg, 'instagram_user', db_user,
                          "id_str='%s'" % db_user.id_str)
        #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
        return getUser(user_name)
Example #37
0
def getUser(user_name, always_check=False):
    '''
    modify by bigzhu at 15/07/31 16:41:16 api找用户时是按专注度来排序的,名字绝对匹配的未必是第一位, 从10个里面找
    '''
    users = list(pg.select('instagram_user', where="lower(username)=lower('%s')" % user_name))
    if users and not always_check:
        return users[0]
    else:
        user = None
        try:
            for this_user in api.user_search(user_name, 10):
                if this_user.username.lower() == user_name.lower():
                    user = this_user
                    break
        except instagram.bind.InstagramClientError:
            print public_bz.getExpInfoAll()
            return
        # 如果没有这个用户
        if user is None:
            public_db.delNoName('instagram', user_name)
            return
        try:
            user = api.user(user.id)
        except (instagram.bind.InstagramAPIError, instagram.bind.InstagramClientError):
            # 通常是没有访问权限
            print public_bz.getExpInfoAll()
            public_db.delNoName('instagram', user_name)
            return

        db_user = storage()
        db_user.id_str = user.id
        db_user.username = user.username
        db_user.full_name = user.full_name
        db_user.profile_picture = user.profile_picture
        db_user.bio = user.bio
        db_user.website = user.website
        db_user.counts = json.dumps(user.counts)
        #pg.insert('instagram_user', **db_user)
        pg.insertOrUpdate(pg, 'instagram_user', db_user, "id_str='%s'" % db_user.id_str)
        #db_bz.insertIfNotExist(pg, 'instagram_user', db_user, "id=%s" % db_user.id)
        return getUser(user_name)
Example #38
0
def saveUser(god_name, github_name, user, sync_key=None):
    social_user = public_bz.storage()

    try:
        # social_user.name = user['login']
        social_user.name = github_name
    except Exception as e:
        print e
        print user
    social_user.type = 'github'
    if user.get('followers') is None:
        social_user.count = -1
    else:
        social_user.count = user['followers']
    social_user.avatar = user.get('avatar_url', '')
    social_user.description = user.get('bio')
    if sync_key is not None:
        social_user.sync_key = sync_key

    pg.update('god', where={'name': god_name}, github=json.dumps(social_user))
    return social_user
Example #39
0
    def get(self, parm=None):

        starttime = datetime.datetime.now()
        self.set_header("Content-Type", "application/json")
        after = None
        limit = None
        search_key = None
        god_name = None
        if parm:
            parm = json.loads(parm)
            after = parm.get('after')  # 晚于这个时间的
            limit = parm.get('limit')
            search_key = parm.get('search_key')
            god_name = parm.get('god_name')  # 只查这个god

        user_id = self.current_user
        if after:
            after = time_bz.timestampToDateTime(after, True)
        elif search_key is None and god_name is None:  # 这些条件查询不能卡上次看到那条的时间
            after = last_oper.getLastTime(user_id)

        messages = public_db.getNewMessages(user_id=user_id,
                                            after=after,
                                            limit=limit,
                                            god_name=god_name,
                                            search_key=search_key)
        data = storage()
        data.error = OK
        data.messages = messages
        data.unread_message_count = oper.getUnreadCount(user_id)
        if (len(messages) == 0):
            if (user_id):
                data.followed_god_count = god_oper.getFollowedGodCount(user_id)
            else:
                data.followed_god_count = 0

        endtime = datetime.datetime.now()

        print((endtime - starttime).seconds)
        self.write(json.dumps(data, cls=json_bz.ExtEncoder))
Example #40
0
def saveUser(id, url):
    '''
    create by bigzhu at 15/07/15 21:27:19 保存github信息
    create by bigzhu at 15/07/22 16:17:37 fix bug, not return id
    '''
    if list(pg.select('github_user', where='id=%s' % id)):
        return id
    else:
        r = requests.get(url)
        user = storage(r.json())
        del user.url
        del user.followers_url
        del user.following_url
        del user.gists_url
        del user.starred_url
        del user.subscriptions_url
        del user.organizations_url
        del user.repos_url
        del user.events_url
        del user.received_events_url
        del user.type
        return db_bz.insertIfNotExist(pg, 'github_user', user)
Example #41
0
def saveUser(id, url):
    '''
    create by bigzhu at 15/07/15 21:27:19 保存github信息
    create by bigzhu at 15/07/22 16:17:37 fix bug, not return id
    '''
    if list(pg.select('github_user', where='id=%s' % id)):
        return id
    else:
        r = requests.get(url)
        user = storage(r.json())
        print user
        if hasattr(user, 'url'):
            del user.url
        del user.followers_url
        del user.following_url
        del user.gists_url
        del user.starred_url
        del user.subscriptions_url
        del user.organizations_url
        del user.repos_url
        del user.events_url
        del user.received_events_url
        del user.type
        return db_bz.insertIfNotExist(pg, 'github_user', user)
Example #42
0
def getHtmlByName(name):
    the_html = storage()
    the_html.name = name
    the_html.time = getModifyTime(name)
    the_html.content = getHtmlContent(name)
    return the_html
Example #43
0
def saveMedias(user, medias):
    '''
    create by bigzhu at 15/09/04 20:58:54 保存meedias

         "attribution":null,
         "tags":[  ],
         "type":"image",
         "location":{  },
         "comments":{  },
         "filter":"Normal",
         "created_time":"1441362020",
         "link":"https:\/\/instagram.com\/p\/7NIHiLJJs3\/",
         "likes":{  },
         "images":{  },
         "users_in_photo":[  ],
         "caption":{  },
         "user_has_liked":false,
         "id":"1066544388859271991_262341",
         "user":{  }
    '''

    for media_d in medias['data']:
        media = storage(media_d)
        db_media = storage()

        #db_media.attribution = media.attribution
        #db_media.tags = json.dumps(media.tags, cls=public_bz.ExtEncoder)
        db_media.type = media.type
        #db_media.location = json.dumps(media.location, cls=public_bz.ExtEncoder)
        db_media.comments = json.dumps(media.comments, cls=public_bz.ExtEncoder)
        db_media.filter = media.filter
        #db_media.created_time = time_bz.timestampToDateTime(media.created_time) + timedelta(hours=8)
        db_media.created_time = time_bz.timestampToDateTime(media.created_time)
        db_media.link = media.link
        #db_media.likes = json.dumps(media.likes, cls=public_bz.ExtEncoder)
        db_media.low_resolution = json.dumps(media.images['low_resolution'])
        db_media.standard_resolution = json.dumps(media.images['standard_resolution'])
        db_media.thumbnail = json.dumps(media.images['thumbnail'])
        #db_media.users_in_photo = json.dumps(media.users_in_photo, cls=public_bz.ExtEncoder)
        if media.caption:
            caption = media.caption
            caption['user_id'] = caption['from']['id']
            del caption['from']
        else:
            caption = ''
        db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder)

        db_media.id_str = media.id
        db_media.user_id = user.id

        id = pg.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str)
        if id is None:
            raise Exception('重复记录 id=%s, name=%s' % (media.id, user.username))
        else:
            print 'new=', media.id, user.username
        if id is not None and len(medias) <= 2:  # 新增加消息,微信通知只通知2条以内
            openids = public_db.getOpenidsByName('instagram', user.username)
            for data in openids:
                if caption != '':
                    text = caption.get('text')
                else:
                    text = ''
                wechat_oper.sendInstagram(data.openid, text, media.images['low_resolution']['url'], user.username, id)

    if medias['pagination']:
        next_url = medias['pagination']['next_url']
        medias = callGetMeidaApi(next_url=next_url)
        saveMedias(user, medias)
Example #44
0
def getMedia(user_name=None, with_next_url=None, user=None):
    if user_name:
        user = getUser(user_name)
        if user is None:
            return
        # min_id 会查出大于等于这个id的
        try:
            medias, next_ = api.user_recent_media(user_id=user.id, min_id=user.last_id)
        except instagram.bind.InstagramClientError:
            print public_bz.getExpInfoAll()
            public_db.delNoName('instagram', user_name)
            return
        if medias:
            last_id = medias[0].id
            pg.update('instagram_user', where="lower(username)=lower('%s')" % user_name, last_id=last_id)
    else:
        medias, next_ = api.user_recent_media(with_next_url=with_next_url)

    for media in medias:
        db_media = storage()
        if media.caption:
            caption = media.caption.__dict__
            caption['user_id'] = caption['user'].id
            del caption['user']
        else:
            caption = ''
        db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder)
        db_media.comment_count = media.comment_count

        if media.comments:
            media.comments = [d.__dict__ for d in media.comments]
            for comment in media.comments:
                comment['user'] = comment['user'].__dict__
        db_media.comments = json.dumps(media.comments, cls=public_bz.ExtEncoder)
        db_media.created_time = media.created_time
        # 8小时的问题
        db_media.created_time += timedelta(hours=8)
        db_media.filter = media.filter
        db_media.low_resolution = json.dumps(media.images['low_resolution'].__dict__)
        db_media.standard_resolution = json.dumps(media.images['standard_resolution'].__dict__)
        db_media.thumbnail = json.dumps(media.images['thumbnail'].__dict__)
        db_media.id_str = media.id
        db_media.like_count = media.like_count
        # likes里有User对象,暂时不存了
        #db_media.likes = json.dumps(media.likes)
        db_media.link = media.link
        db_media.type = media.type
        db_media.user_id = user.id
        id = db_bz.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str)
        print 'new=', media.id, user.username
        if id is not None and len(medias) <= 2:  # 新增加消息,微信通知只通知2条以内
            openids = public_db.getOpenidsByName('instagram', user.username)
            for data in openids:
                if caption != '':
                    text = caption.get('text')
                else:
                    text = ''
                wechat_oper.sendInstagram(data.openid, text, media.images['standard_resolution'].url, user.username, id)
    # 递归查出
    if next_ != with_next_url:
        getMedia(with_next_url=next_, user=user)
Example #45
0
 def getEmptyUserInfo(self):
     user_info = public_bz.storage()
     for p in model_oper_bz.getModelAttributes(model_bz.user_info):
         user_info[p] = ""
     return user_info
Example #46
0
def saveMedias(user, medias):
    '''
    create by bigzhu at 15/09/04 20:58:54 保存meedias

         "attribution":null,
         "tags":[  ],
         "type":"image",
         "location":{  },
         "comments":{  },
         "filter":"Normal",
         "created_time":"1441362020",
         "link":"https:\/\/instagram.com\/p\/7NIHiLJJs3\/",
         "likes":{  },
         "images":{  },
         "users_in_photo":[  ],
         "caption":{  },
         "user_has_liked":false,
         "id":"1066544388859271991_262341",
         "user":{  }
    '''

    for media_d in medias['data']:
        media = storage(media_d)
        db_media = storage()

        #db_media.attribution = media.attribution
        #db_media.tags = json.dumps(media.tags, cls=public_bz.ExtEncoder)
        db_media.type = media.type
        #db_media.location = json.dumps(media.location, cls=public_bz.ExtEncoder)
        db_media.comments = json.dumps(media.comments,
                                       cls=public_bz.ExtEncoder)
        db_media.filter = media.filter
        #db_media.created_time = time_bz.timestampToDateTime(media.created_time) + timedelta(hours=8)
        db_media.created_time = time_bz.timestampToDateTime(media.created_time)
        db_media.link = media.link
        #db_media.likes = json.dumps(media.likes, cls=public_bz.ExtEncoder)
        db_media.low_resolution = json.dumps(media.images['low_resolution'])
        db_media.standard_resolution = json.dumps(
            media.images['standard_resolution'])
        db_media.thumbnail = json.dumps(media.images['thumbnail'])
        #db_media.users_in_photo = json.dumps(media.users_in_photo, cls=public_bz.ExtEncoder)
        if media.caption:
            caption = media.caption
            caption['user_id'] = caption['from']['id']
            del caption['from']
        else:
            caption = ''
        db_media.caption = json.dumps(caption, cls=public_bz.ExtEncoder)

        db_media.id_str = media.id
        db_media.user_id_str = user.id

        #id = pg.insertIfNotExist(pg, 'instagram_media', db_media, "id_str='%s'" % db_media.id_str)
        m = public_bz.storage()
        m.id_str = db_media.id_str
        m.m_type = 'instagram'
        m.m_user_id = db_media.user_id_str
        m.created_at = db_media.created_time
        m.extended_entities = db_media.standard_resolution
        m.content = db_media.comments
        if media.caption:
            m.text = media.caption['text']
        else:
            m.text = None
        m.href = db_media.link
        m.type = db_media.type

        id = pg.insertIfNotExist(pg, 'm', m, "id_str='%s'" % db_media.id_str)
        if id is None:
            # 似乎就是会有重复的
            error = '重复记录 id=%s, name=%s' % (media.id, user.username)
            print error
            #raise Exception(error)
        else:
            print 'new=', media.id, user.username
        # if id is not None and len(medias) <= 2:  # 新增加消息,微信通知只通知2条以内
        #    openids = public_db.getOpenidsByName('instagram', user.username)
        #    for data in openids:
        #        if caption != '':
        #            text = caption.get('text')
        #        else:
        #            text = ''
        #        wechat_oper.sendInstagram(data.openid, text, media.images['low_resolution']['url'], user.username, id)

    if medias['pagination']:
        next_url = medias['pagination']['next_url']
        medias = callGetMeidaApi(next_url=next_url)
        saveMedias(user, medias)
Example #47
0
def saveTwitter(tweet):
    '''
    create by bigzhu at 15/07/10 14:39:48
        保存twitter
    '''

    del tweet.user._json
    #del tweet.user._api
    tweet.user.entities = json.dumps(tweet.user.entities)

    del tweet.user.id
    pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user), "id_str='%s'" % tweet.user.id_str)

    tweet.t_user_id = tweet.user.id_str
    screen_name = tweet.user.screen_name
    del tweet.user

    if hasattr(tweet, 'author'):
        #del tweet.author.id
        pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author), "id_str='%s'" % tweet.author.id_str)
        tweet.t_author_id = tweet.author.id_str

        del tweet.author

    if hasattr(tweet, '_api'):
        del tweet._api
    if hasattr(tweet, '_json'):
        del tweet._json
    # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
    if hasattr(tweet, 'id'):
        del tweet.id

    if hasattr(tweet, 'entities'):
        tweet.entities = json.dumps(tweet.entities)
    if hasattr(tweet, 'geo'):
        tweet.geo = json.dumps(tweet.geo)
    if hasattr(tweet, 'coordinates'):
        tweet.coordinates = json.dumps(tweet.coordinates)
    if hasattr(tweet, 'extended_entities'):
        tweet.extended_entities = json.dumps(tweet.extended_entities)
    if hasattr(tweet, 'scopes'):
        tweet.scopes = json.dumps(tweet.scopes)

    if hasattr(tweet, 'retweeted_status'):
        saveTwitter(tweet.retweeted_status)
        tweet.retweeted_status = tweet.retweeted_status.id_str
    if hasattr(tweet, 'quoted_status'):
        # print tweet.quoted_status
        del tweet.quoted_status
    #    saveTwitter(tweet.quoted_status)
    #    tweet.quoted_status = tweet.quoted_status.id_str

    # place  是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
    if hasattr(tweet, 'place'):
        del tweet.place

    # for k, v in vars(tweet).items():
    #    print '%s=%s' % (k, v)

    #return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)

    m = public_bz.storage()
    m.id_str = tweet.id_str
    m.m_type = 'twitter'
    m.m_user_id = tweet.t_user_id
    m.created_at = tweet.created_at
    if hasattr(tweet, 'extended_entities'):
        m.extended_entities = tweet.extended_entities
    m.content = None
    m.text = tweet.text
    m.href = 'https://twitter.com/'+screen_name+'/status/'+tweet.id_str

    return pg.insertIfNotExist(pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)
Example #48
0
def saveTwitter(tweet):
    '''
    create by bigzhu at 15/07/10 14:39:48
        保存twitter
    '''

    del tweet.user._json
    #del tweet.user._api
    tweet.user.entities = json.dumps(tweet.user.entities)

    del tweet.user.id
    pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.user),
                      "id_str='%s'" % tweet.user.id_str)

    tweet.t_user_id = tweet.user.id_str
    screen_name = tweet.user.screen_name
    del tweet.user

    if hasattr(tweet, 'author'):
        #del tweet.author.id
        pg.insertOrUpdate(pg, 'twitter_user', vars(tweet.author),
                          "id_str='%s'" % tweet.author.id_str)
        tweet.t_author_id = tweet.author.id_str

        del tweet.author

    if hasattr(tweet, '_api'):
        del tweet._api
    if hasattr(tweet, '_json'):
        del tweet._json
    # twitter id 太大了 "id": 618948810941673472 导致 psycopg2.DataError: integer out of range
    if hasattr(tweet, 'id'):
        del tweet.id

    if hasattr(tweet, 'entities'):
        tweet.entities = json.dumps(tweet.entities)
    if hasattr(tweet, 'geo'):
        tweet.geo = json.dumps(tweet.geo)
    if hasattr(tweet, 'coordinates'):
        tweet.coordinates = json.dumps(tweet.coordinates)
    if hasattr(tweet, 'extended_entities'):
        tweet.extended_entities = json.dumps(tweet.extended_entities)
    if hasattr(tweet, 'scopes'):
        tweet.scopes = json.dumps(tweet.scopes)

    if hasattr(tweet, 'retweeted_status'):
        saveTwitter(tweet.retweeted_status)
        tweet.retweeted_status = tweet.retweeted_status.id_str
    if hasattr(tweet, 'quoted_status'):
        # print tweet.quoted_status
        del tweet.quoted_status
    #    saveTwitter(tweet.quoted_status)
    #    tweet.quoted_status = tweet.quoted_status.id_str

    # place  是一个对象(我不知道如何处理): Place(_api=<tweepy.api.API object at 0x1808050>
    if hasattr(tweet, 'place'):
        del tweet.place

    # for k, v in vars(tweet).items():
    #    print '%s=%s' % (k, v)

    #return pg.insertIfNotExist(pg, 'twitter_message', vars(tweet), "id_str='%s'" % tweet.id_str)

    m = public_bz.storage()
    m.id_str = tweet.id_str
    m.m_type = 'twitter'
    m.m_user_id = tweet.t_user_id
    m.created_at = tweet.created_at
    if hasattr(tweet, 'extended_entities'):
        m.extended_entities = tweet.extended_entities
    m.content = None
    m.text = tweet.text
    m.href = 'https://twitter.com/' + screen_name + '/status/' + tweet.id_str

    return pg.insertIfNotExist(
        pg, 'm', m, "id_str='%s' and m_type='twitter'" % tweet.id_str)