def create_comgeneral():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
tablename = 'comgeneral'
mongo.create_unique_index(mongo_conn,
tablename,
uniques=[[('objectid', 1), ('url', 1)]])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime', 'terrace'])
def dispatcher():
hosts = open('./hosts.conf').read().strip().split('\n')
redis_conns = []
for host in hosts:
conn = redis.redis_connect(host)
redis_conns.append(conn)
mongo_conn = mongo.connect('192.168.241.12', 'stream')
servernum = len(hosts)
packer = msgpack.Packer()
tablename = 'general'
key = 'stream'
maps = {}
rawdatas = mongo.find(mongo_conn, tablename, {}, 30000)
for raw in rawdatas:
raw.pop('_id')
#print raw
url = raw['url']
num = disp_strategy(url, servernum, 'utf-8')
data = packer.pack(raw)
redis.add_set_value(redis_conns[num], key, data)
if num not in maps:
maps.update({num: 1})
else:
maps[num] += 1
print maps
def dispatcher():
hosts = open('./hosts.conf').read().strip().split('\n')
redis_conns = []
for host in hosts:
conn = redis.redis_connect(host)
redis_conns.append(conn)
mongo_conn = mongo.connect('192.168.241.12', 'stream')
servernum = len(hosts)
packer = msgpack.Packer()
tablename = 'general'
key = 'stream'
maps = {}
rawdatas = mongo.find(mongo_conn, tablename, {}, 30000)
for raw in rawdatas:
raw.pop('_id')
#print raw
url = raw['url']
num = disp_strategy(url, servernum, 'utf-8')
data = packer.pack(raw)
redis.add_set_value(redis_conns[num], key, data)
if num not in maps:
maps.update({num:1})
else:
maps[num] += 1
print maps
def create_general():
global mongo_conn
if not mongo_conn:
tablename = 'general'
mongo_conn = mongo.connect('192.168.241.12', 'stream')
mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])
return mongo_conn
def create_general():
global mongo_conn
if not mongo_conn:
tablename = 'general'
mongo_conn = mongo.connect('192.168.241.12', 'stream')
mongo.create_unique_index(mongo_conn,
tablename,
uniques=[[('objectid', 1), ('url', 1)]])
mongo.create_index(mongo_conn,
tablename,
indexs=['updatetime', 'terrace'])
return mongo_conn
def dispatcher():
redisconns = redis_conns()
servernum = len(redisconns)
mongo_conn = mongo.connect('192.168.241.12', 'stream')
packer = msgpack.Packer()
tablename = 'general'
key = 'stream'
try:
total = int(open('total.dat').read().strip())
except:
total = 0
while True:
rawdatas = mongo.find(mongo_conn, tablename, {}, 3600)
tag = False
for raw in rawdatas:
tag = True
raw.pop('_id')
url = raw['url']
num = disp_strategy(url, servernum, 'utf-8')
data = packer.pack(raw)
try:
redis.add_set_value(redisconns[num], key, data)
except:
fh = open('./error_server.dat', 'w')
fh.write(
'%s\t%s\n' %
(num,
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
serverlists = range(servernum)
while True:
if len(serverlists) == 0:
open('stop.dat', 'w').write('所有服务器均已无法使用')
raise Exception('所有服务器均已无法使用')
serverlists.remove(num)
num = random.choice(serverlists)
try:
redis.add_set_value(redisconns[num], key, data)
break
except:
fh.write('%s\t%s\n' %
(num, datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')))
fh.close()
total += 1
mongo.delete(mongo_conn, tablename, prerequisite={'url': url})
f = open('total.dat', 'w')
f.write('%s\n' % (total))
f.flush()
f.close()
if not tag:
print 'wait urls...'
time.sleep(120)
def dispatcher():
redisconns = redis_conns()
servernum = len(redisconns)
mongo_conn = mongo.connect('192.168.241.12', 'stream')
packer = msgpack.Packer()
tablename = 'general'
key = 'stream'
try:
total = int(open('total.dat').read().strip())
except:
total = 0
while True:
rawdatas = mongo.find(mongo_conn, tablename, {}, 3600)
tag = False
for raw in rawdatas:
tag = True
raw.pop('_id')
url = raw['url']
num = disp_strategy(url, servernum, 'utf-8')
data = packer.pack(raw)
try:
redis.add_set_value(redisconns[num], key, data)
except:
fh = open('./error_server.dat','w')
fh.write('%s\t%s\n'%(num,datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
serverlists = range(servernum)
while True:
if len(serverlists)==0:
open('stop.dat','w').write('所有服务器均已无法使用')
raise Exception('所有服务器均已无法使用')
serverlists.remove(num)
num = random.choice(serverlists)
try:
redis.add_set_value(redisconns[num], key, data)
break
except:
fh.write('%s\t%s\n'%(num,datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
fh.close()
total += 1
mongo.delete(mongo_conn,tablename,prerequisite={'url':url})
f = open('total.dat','w')
f.write('%s\n'%(total))
f.flush()
f.close()
if not tag:
print 'wait urls...'
time.sleep(120)
def create_comgeneral():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
tablename = 'comgeneral'
mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])
import mongo_util as mongo
global mongo_conn
#mongo_conn = ''
mongo_conn = mongo.connect('192.168.241.12', 'stream')
def create_general():
global mongo_conn
if not mongo_conn:
tablename = 'general'
mongo_conn = mongo.connect('192.168.241.12', 'stream')
mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])
return mongo_conn
def create_weixin():
global mongo_conn
tablename = 'weixin'
mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime'])
def create_assemble():
global mongo_conn
tablename = 'assemble'
mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])
def create_comgeneral():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
tablename = 'comgeneral'
mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])
#create_general()
import mongo_util as mongo
global mongo_conn
#mongo_conn = ''
mongo_conn = mongo.connect('192.168.241.12', 'stream')
def create_general():
global mongo_conn
if not mongo_conn:
tablename = 'general'
mongo_conn = mongo.connect('192.168.241.12', 'stream')
mongo.create_unique_index(mongo_conn,
tablename,
uniques=[[('objectid', 1), ('url', 1)]])
mongo.create_index(mongo_conn,
tablename,
indexs=['updatetime', 'terrace'])
return mongo_conn
def create_weixin():
global mongo_conn
tablename = 'weixin'
mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])
mongo.create_index(mongo_conn, tablename, indexs=['updatetime'])
def create_assemble():
global mongo_conn
tablename = 'assemble'
mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])
def get_mongo_conn():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
return mongo_conn
def get_mongo_conn():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
return mongo_conn
def delete_weixin():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
tablename = 'weixin'
mongo.delete(mongo_conn, tablename)
def delete_comgeneral():
mongo_conn = mongo.connect('192.168.241.12', 'stream')
tablename = 'comgeneral'
mongo.delete(mongo_conn, tablename)
