Beispiel #1
0
def move_status():
    STATUS_REDIS_KEY = "/status/text/%s"
    RAW_STATUS_REDIS_KEY = "/status/raw/%s"

    start = 3720000
    limit = 100000
    #r =db_conn.execute("select count(1) from status")
    #total = r.fetchone()[0]
    total = 4423725
    print '----total status:', total
    sys.stdout.flush()

    ef = open("error.log", "a")
    #cf = open("cmd.txt", "w")
    while (start <= int(total)):
        f = open("./midfile.txt", "w")
        print '-------start ', start
        sys.stdout.flush()
        cursor = db_conn.execute(
            "select id from status order by id limit %s,%s", (start, limit))
        rows = cursor.fetchall()
        for row in rows:
            text = mongo_conn.get(STATUS_REDIS_KEY % row[0])
            raw = mongo_conn.get(RAW_STATUS_REDIS_KEY % row[0])
            if text and raw:
                text = json_encode(text) if not isinstance(
                    text, basestring) else text
                raw = json_encode(raw) if not isinstance(raw,
                                                         basestring) else raw

                db_conn.execute(
                    '''replace into raw_status (status_id, text, raw) 
                    values(%s,%s,%s)''', (row[0], text, raw))
        db_conn.commit()
        start += limit
def move_status():
    STATUS_REDIS_KEY = "/status/text/%s"
    RAW_STATUS_REDIS_KEY = "/status/raw/%s"

    start = 3720000
    limit = 100000
    #r =db_conn.execute("select count(1) from status")
    #total = r.fetchone()[0]
    total = 4423725
    print '----total status:', total
    sys.stdout.flush()

    ef = open("error.log", "a")
    #cf = open("cmd.txt", "w")
    while (start <= int(total)):
        f = open("./midfile.txt", "w")
        print '-------start ', start
        sys.stdout.flush()
        cursor = db_conn.execute("select id from status order by id limit %s,%s", (start, limit))
        rows = cursor.fetchall()
        for row in rows:
            text = mongo_conn.get(STATUS_REDIS_KEY % row[0])
            raw = mongo_conn.get(RAW_STATUS_REDIS_KEY% row[0])
            if text and raw:
                text = json_encode(text) if not isinstance(text, basestring) else text
                raw = json_encode(raw) if not isinstance(raw, basestring) else raw

                db_conn.execute('''replace into raw_status (status_id, text, raw) 
                    values(%s,%s,%s)''', (row[0], text, raw))
        db_conn.commit()
        start += limit
Beispiel #3
0
 def raw(self):
     if self.category == config.CATE_THEPAST_NOTE:
         note = Note.get(self.origin_id)
         return note
     else:
         _raw = mongo_conn.get(Status.RAW_STATUS_REDIS_KEY % self.id)
         return json_decode(_raw) if _raw else ""
Beispiel #4
0
 def text(self):
     if self.category == config.CATE_THEPAST_NOTE:
         note = Note.get(self.origin_id)
         return note and note.content
     else:
         _text = mongo_conn.get(Status.STATUS_REDIS_KEY % self.id)
         return json_decode(_text) if _text else ""
Beispiel #5
0
def move_user_profile():
    RAW_USER_REDIS_KEY = "/user/raw/%s"

    cursor = db_conn.execute("select id from user order by id")
    rows = cursor.fetchall()
    cursor and cursor.close()
    for row in rows:
        print '--------user raw id:', row[0]
        sys.stdout.flush()
        r1 = mongo_conn.get(RAW_USER_REDIS_KEY % row[0])
        if r1:
            print "r1"
            #UserProfile.set(row[0], r1)
            Kv.set('/profile/%s' % row[0], r1)
        r2 = mongo_conn.get("/profile/%s" % row[0])
        if r2:
            #Kv.set('/profile/%s' %row[0], r2)
            UserProfile.set(row[0], r2)
def move_user_profile():
    RAW_USER_REDIS_KEY = "/user/raw/%s" 

    cursor = db_conn.execute("select id from user order by id")
    rows = cursor.fetchall()
    cursor and cursor.close()
    for row in rows:
        print '--------user raw id:', row[0]
        sys.stdout.flush()
        r1 = mongo_conn.get(RAW_USER_REDIS_KEY % row[0])
        if r1:
            print "r1"
            #UserProfile.set(row[0], r1)
            Kv.set('/profile/%s' %row[0], r1)
        r2 = mongo_conn.get("/profile/%s" % row[0])
        if r2:
            #Kv.set('/profile/%s' %row[0], r2)
            UserProfile.set(row[0], r2)
Beispiel #7
0
    def __init__(self, id, user_id, origin_id, create_time, site, category, title=""):
        self.id = str(id)
        self.user_id = str(user_id)
        self.origin_id = str(origin_id)
        self.create_time = create_time
        self.site = site
        self.category = category
        self.title = title
        self.text = mongo_conn.get(self.__class__.STATUS_REDIS_KEY % self.id)
        self.text = json_decode(self.text) if self.text else ""
        self.origin_user_id = UserAlias.get_by_user_and_type(self.user_id, self.site).alias
        if self.site == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]:
            self.create_time += datetime.timedelta(seconds=8 * 3600)

        self.bare_text = self._generate_bare_text()
Beispiel #8
0
 def raw(self):
     _raw = mongo_conn.get(Status.RAW_STATUS_REDIS_KEY % self.id)
     return json_decode(_raw) if _raw else ""
Beispiel #9
0
 def text(self):
     _text = mongo_conn.get(Status.STATUS_REDIS_KEY % self.id)
     return json_decode(_text) if _text else ""
Beispiel #10
0
 def raw(self):
     _raw = mongo_conn.get(User.RAW_USER_REDIS_KEY % self.id)
     return json_decode(_raw) if _raw else ""
Beispiel #11
0
 def get_profile(self):
     r = mongo_conn.get('/profile/%s' %self.id)
     return json_decode(r) if r else {}
#-*- coding:utf-8 -*-

import sys
sys.path.append('../')

import datetime
import past
from past.store import db_conn, mongo_conn
from past.utils.escape import json_decode

cursor = db_conn.execute("select id from status where category=200")
rows = cursor.fetchall()
cursor and cursor.close()
ids = [x[0] for x in rows]

for x in ids:
    try:
        raw = mongo_conn.get("/status/raw/%s" %x)
        if raw:
            print x
            data = json_decode(raw)
            t = data.get("created_at")
            created_at = datetime.datetime.strptime(t, "%a %b %d %H:%M:%S +0800 %Y")
            db_conn.execute("update status set create_time = %s where id=%s", (created_at, x))
            db_conn.commit()
    except:
        import traceback
        print traceback.format_exc()
        sys.stdout.flush()