def move_status():
STATUS_REDIS_KEY = "/status/text/%s"
RAW_STATUS_REDIS_KEY = "/status/raw/%s"
start = 3720000
limit = 100000
#r =db_conn.execute("select count(1) from status")
#total = r.fetchone()[0]
total = 4423725
print '----total status:', total
sys.stdout.flush()
ef = open("error.log", "a")
#cf = open("cmd.txt", "w")
while (start <= int(total)):
f = open("./midfile.txt", "w")
print '-------start ', start
sys.stdout.flush()
cursor = db_conn.execute(
"select id from status order by id limit %s,%s", (start, limit))
rows = cursor.fetchall()
for row in rows:
text = mongo_conn.get(STATUS_REDIS_KEY % row[0])
raw = mongo_conn.get(RAW_STATUS_REDIS_KEY % row[0])
if text and raw:
text = json_encode(text) if not isinstance(
text, basestring) else text
raw = json_encode(raw) if not isinstance(raw,
basestring) else raw
db_conn.execute(
'''replace into raw_status (status_id, text, raw)
values(%s,%s,%s)''', (row[0], text, raw))
db_conn.commit()
start += limit
def move_status():
STATUS_REDIS_KEY = "/status/text/%s"
RAW_STATUS_REDIS_KEY = "/status/raw/%s"
start = 3720000
limit = 100000
#r =db_conn.execute("select count(1) from status")
#total = r.fetchone()[0]
total = 4423725
print '----total status:', total
sys.stdout.flush()
ef = open("error.log", "a")
#cf = open("cmd.txt", "w")
while (start <= int(total)):
f = open("./midfile.txt", "w")
print '-------start ', start
sys.stdout.flush()
cursor = db_conn.execute("select id from status order by id limit %s,%s", (start, limit))
rows = cursor.fetchall()
for row in rows:
text = mongo_conn.get(STATUS_REDIS_KEY % row[0])
raw = mongo_conn.get(RAW_STATUS_REDIS_KEY% row[0])
if text and raw:
text = json_encode(text) if not isinstance(text, basestring) else text
raw = json_encode(raw) if not isinstance(raw, basestring) else raw
db_conn.execute('''replace into raw_status (status_id, text, raw)
values(%s,%s,%s)''', (row[0], text, raw))
db_conn.commit()
start += limit
def raw(self):
if self.category == config.CATE_THEPAST_NOTE:
note = Note.get(self.origin_id)
return note
else:
_raw = mongo_conn.get(Status.RAW_STATUS_REDIS_KEY % self.id)
return json_decode(_raw) if _raw else ""
def text(self):
if self.category == config.CATE_THEPAST_NOTE:
note = Note.get(self.origin_id)
return note and note.content
else:
_text = mongo_conn.get(Status.STATUS_REDIS_KEY % self.id)
return json_decode(_text) if _text else ""
def move_user_profile():
RAW_USER_REDIS_KEY = "/user/raw/%s"
cursor = db_conn.execute("select id from user order by id")
rows = cursor.fetchall()
cursor and cursor.close()
for row in rows:
print '--------user raw id:', row[0]
sys.stdout.flush()
r1 = mongo_conn.get(RAW_USER_REDIS_KEY % row[0])
if r1:
print "r1"
#UserProfile.set(row[0], r1)
Kv.set('/profile/%s' % row[0], r1)
r2 = mongo_conn.get("/profile/%s" % row[0])
if r2:
#Kv.set('/profile/%s' %row[0], r2)
UserProfile.set(row[0], r2)
def move_user_profile():
RAW_USER_REDIS_KEY = "/user/raw/%s"
cursor = db_conn.execute("select id from user order by id")
rows = cursor.fetchall()
cursor and cursor.close()
for row in rows:
print '--------user raw id:', row[0]
sys.stdout.flush()
r1 = mongo_conn.get(RAW_USER_REDIS_KEY % row[0])
if r1:
print "r1"
#UserProfile.set(row[0], r1)
Kv.set('/profile/%s' %row[0], r1)
r2 = mongo_conn.get("/profile/%s" % row[0])
if r2:
#Kv.set('/profile/%s' %row[0], r2)
UserProfile.set(row[0], r2)
def __init__(self, id, user_id, origin_id, create_time, site, category, title=""):
self.id = str(id)
self.user_id = str(user_id)
self.origin_id = str(origin_id)
self.create_time = create_time
self.site = site
self.category = category
self.title = title
self.text = mongo_conn.get(self.__class__.STATUS_REDIS_KEY % self.id)
self.text = json_decode(self.text) if self.text else ""
self.origin_user_id = UserAlias.get_by_user_and_type(self.user_id, self.site).alias
if self.site == config.OPENID_TYPE_DICT[config.OPENID_TWITTER]:
self.create_time += datetime.timedelta(seconds=8 * 3600)
self.bare_text = self._generate_bare_text()
def raw(self):
_raw = mongo_conn.get(Status.RAW_STATUS_REDIS_KEY % self.id)
return json_decode(_raw) if _raw else ""
def text(self):
_text = mongo_conn.get(Status.STATUS_REDIS_KEY % self.id)
return json_decode(_text) if _text else ""
def raw(self):
_raw = mongo_conn.get(User.RAW_USER_REDIS_KEY % self.id)
return json_decode(_raw) if _raw else ""
def get_profile(self):
r = mongo_conn.get('/profile/%s' %self.id)
return json_decode(r) if r else {}
#-*- coding:utf-8 -*-
import sys
sys.path.append('../')
import datetime
import past
from past.store import db_conn, mongo_conn
from past.utils.escape import json_decode
cursor = db_conn.execute("select id from status where category=200")
rows = cursor.fetchall()
cursor and cursor.close()
ids = [x[0] for x in rows]
for x in ids:
try:
raw = mongo_conn.get("/status/raw/%s" %x)
if raw:
print x
data = json_decode(raw)
t = data.get("created_at")
created_at = datetime.datetime.strptime(t, "%a %b %d %H:%M:%S +0800 %Y")
db_conn.execute("update status set create_time = %s where id=%s", (created_at, x))
db_conn.commit()
except:
import traceback
print traceback.format_exc()
sys.stdout.flush()