def move_url_from_mongo_redis(self):
#更新到的时间 1431048036
conn = db.connect('192.168.241.23', 'zhihu')
try:
start_time = db.find_one(conn, 'record')['time']
except:
start_time = 0
end_time = time.time()
authorids = db.find(conn, 'author',
{'tg': {
'$lt': end_time,
'$gt': start_time
}})
print 'get from mongo: ', authorids.count()
num = 0
for authorid in authorids:
try:
gender = authorid['gender']
fansnum = authorid['fansnum']
except:
redis.add_url(self.redis_conn,
authorid['id'],
key=self.rediskey)
num += 1
record = db.find_one(conn, 'record', {'id': '1'})
print "record: ", record
if not record:
db.insert(conn, 'record', {'time': end_time, 'id': '1'})
else:
db.update(conn, 'record', {'id': '1'}, {'time': end_time})
print 'add to redis: ', num
db.close(conn)
def move_url_from_mongo_redis(self):
#更新到的时间 1431048036
conn = db.connect('192.168.241.23', 'zhihu')
try:
start_time = db.find_one(conn, 'record')['time']
except:
start_time = 0
end_time = time.time()
authorids = db.find(conn, 'author',{'tg':{'$lt':end_time,'$gt':start_time}})
print 'get from mongo: ',authorids.count()
num = 0
for authorid in authorids:
try:
gender = authorid['gender']
fansnum = authorid['fansnum']
except:
redis.add_url(self.redis_conn, authorid['id'],key=self.rediskey)
num += 1
record = db.find_one(conn, 'record',{'id':'1'})
print "record: ",record
if not record:
db.insert(conn, 'record', {'time':end_time,'id':'1'})
else:
db.update(conn, 'record', {'id':'1'}, {'time':end_time})
print 'add to redis: ',num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.25', 'zhihu')
num = 0
num1 = 0
# authors = db.find(conn, 'author',{},{'id':'true','fansnum':'true','follownum':'true','hash_id':'true'})
authors = db.find(conn, 'author', {'id': 'liao-qi-can'}, {
'id': 'true',
'fansnum': 'true',
'follownum': 'true',
'hash_id': 'true'
})
for author in authors:
try:
fansnum = author['fansnum']
if int(fansnum) >= 1000:
data = author['id'] + '`$`' + author[
'hash_id'] + '`$`' + author['follownum']
num1 += 1
redis.add_url(self.redis_conn, data)
num += 1
except:
pass
print 'filter num: ', num1
print 'authors num: ', num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.23', 'zhihu')
authors = db.find(conn, 'kol_author',{},{'id':'true','fansnum':'true'})
num = 0
for author in authors:
redis.add_url(self.redis_conn, author['id'],key='zhihu_answer')
num += 1
print 'add to redis:',num
db.close(conn)
def get_author_from_mongo(self):
author_ids = []
conn = db.connect('192.168.241.25', 'zhihu')
authors = db.find(conn, 'author')
for author in authors:
# author_url = author['url']
author_id = author['id']
author_ids.append(author_id)
db.close(conn)
return author_ids
def get_author_from_mongo(self):
author_ids = []
conn = db.connect('192.168.241.25', 'zhihu')
authors = db.find(conn, 'author')
for author in authors:
# author_url = author['url']
author_id = author['id']
author_ids.append(author_id)
db.close(conn)
return author_ids
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.23', 'zhihu')
post_urls = db.find(conn, 'post', {}, {'url': 'true'})
num = 0
key = self.store_req_urls_redis_key
redis.delete_key(self.redis_conn, key)
for post_url in post_urls:
redis.add_url(self.redis_conn, post_url['url'], key=key)
num += 1
print 'add to redis: ', num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.23', 'zhihu')
post_urls = db.find(conn, 'post',{},{'url':'true'})
num = 0
key=self.store_req_urls_redis_key
redis.delete_key(self.redis_conn,key )
for post_url in post_urls:
redis.add_url(self.redis_conn, post_url['url'],key=key)
num += 1
print 'add to redis: ',num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.23', 'zhihu')
authors = db.find(conn, 'kol_author', {}, {
'id': 'true',
'fansnum': 'true'
})
num = 0
for author in authors:
redis.add_url(self.redis_conn, author['id'], key='zhihu_answer')
num += 1
print 'add to redis:', num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.25', 'zhihu')
authors = db.find(conn, 'author',{},{'id':'true','fansnum':'true'})
num = 0
for author in authors:
try:
fansnum = author['fansnum']
if int(fansnum)>=500:
redis.add_url(self.redis_conn, author['id'],key='zhihu_update_post')
num += 1
except:
pass
print 'add to redis: ',num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.25', 'zhihu')
num = 0
num1 = 0
# authors = db.find(conn, 'author',{},{'id':'true','fansnum':'true','follownum':'true','hash_id':'true'})
authors = db.find(conn, 'author',{'id':'liao-qi-can'},{'id':'true','fansnum':'true','follownum':'true','hash_id':'true'})
for author in authors:
try:
fansnum = author['fansnum']
if int(fansnum)>=1000:
data = author['id']+'`$`'+author['hash_id']+'`$`'+author['follownum']
num1 += 1
redis.add_url(self.redis_conn, data)
num += 1
except:
pass
print 'filter num: ' ,num1
print 'authors num: ',num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.25', 'zhihu')
authors = db.find(conn, 'author', {}, {
'id': 'true',
'fansnum': 'true'
})
num = 0
for author in authors:
try:
fansnum = author['fansnum']
if int(fansnum) >= 500:
redis.add_url(self.redis_conn,
author['id'],
key='zhihu_update_post')
num += 1
except:
pass
print 'add to redis: ', num
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.25', 'zhihu')
authorids = db.find(conn, 'author2', {}, {'id': 'true'})
for authorid in authorids:
redis.add_url(self.redis_conn, authorid['id'])
db.close(conn)
def move_url_from_mongo_redis(self):
conn = db.connect('192.168.241.25', 'zhihu')
authorids = db.find(conn, 'author2',{},{'id':'true'})
for authorid in authorids:
redis.add_url(self.redis_conn, authorid['id'])
db.close(conn)