Python connect 예제들

예제 #1

파일: create_mongo_indexes.py 프로젝트: easyshell/stream

def create_comgeneral():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    tablename = 'comgeneral'
    mongo.create_unique_index(mongo_conn,
                              tablename,
                              uniques=[[('objectid', 1), ('url', 1)]])
    mongo.create_index(mongo_conn, tablename, indexs=['updatetime', 'terrace'])

예제 #2

파일: dispatcher_test.py 프로젝트: easyshell/stream

def dispatcher():
    hosts = open('./hosts.conf').read().strip().split('\n')
    redis_conns = []
    for host in hosts:
        conn = redis.redis_connect(host)
        redis_conns.append(conn)
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    servernum = len(hosts)
    packer = msgpack.Packer()
    tablename = 'general'
    key = 'stream'
    maps = {}
    rawdatas = mongo.find(mongo_conn, tablename, {}, 30000)
    for raw in rawdatas:
        raw.pop('_id')
        #print raw
        url = raw['url']
        num = disp_strategy(url, servernum, 'utf-8')
        data = packer.pack(raw)
        redis.add_set_value(redis_conns[num], key, data)
        if num not in maps:
            maps.update({num: 1})
        else:
            maps[num] += 1
    print maps

예제 #3

파일: dispatcher_test.py 프로젝트: demon108/stream

def dispatcher():
    hosts = open('./hosts.conf').read().strip().split('\n')
    redis_conns = []
    for host in hosts:
        conn = redis.redis_connect(host)
        redis_conns.append(conn)
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    servernum = len(hosts)
    packer = msgpack.Packer()
    tablename = 'general'
    key = 'stream'
    maps = {}
    rawdatas = mongo.find(mongo_conn, tablename, {}, 30000)
    for raw in rawdatas:
	raw.pop('_id')
        #print raw
        url = raw['url']
        num = disp_strategy(url, servernum, 'utf-8')
        data = packer.pack(raw)
        redis.add_set_value(redis_conns[num], key, data)
        if num not in maps:
            maps.update({num:1})
        else:
            maps[num] += 1
    print maps

예제 #4

파일: create_mongo_indexes.py 프로젝트: demon108/stream

def create_general():
    global mongo_conn
    if not mongo_conn:
        tablename = 'general'
        mongo_conn = mongo.connect('192.168.241.12', 'stream')
        mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
        mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])
    return mongo_conn

예제 #5

파일: create_mongo_indexes.py 프로젝트: easyshell/stream

def create_general():
    global mongo_conn
    if not mongo_conn:
        tablename = 'general'
        mongo_conn = mongo.connect('192.168.241.12', 'stream')
        mongo.create_unique_index(mongo_conn,
                                  tablename,
                                  uniques=[[('objectid', 1), ('url', 1)]])
        mongo.create_index(mongo_conn,
                           tablename,
                           indexs=['updatetime', 'terrace'])
    return mongo_conn

예제 #6

파일: dispatcher.py 프로젝트: easyshell/stream

def dispatcher():
    redisconns = redis_conns()
    servernum = len(redisconns)
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    packer = msgpack.Packer()
    tablename = 'general'
    key = 'stream'
    try:
        total = int(open('total.dat').read().strip())
    except:
        total = 0
    while True:
        rawdatas = mongo.find(mongo_conn, tablename, {}, 3600)
        tag = False
        for raw in rawdatas:
            tag = True
            raw.pop('_id')
            url = raw['url']
            num = disp_strategy(url, servernum, 'utf-8')
            data = packer.pack(raw)
            try:
                redis.add_set_value(redisconns[num], key, data)
            except:
                fh = open('./error_server.dat', 'w')
                fh.write(
                    '%s\t%s\n' %
                    (num,
                     datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
                serverlists = range(servernum)
                while True:
                    if len(serverlists) == 0:
                        open('stop.dat', 'w').write('所有服务器均已无法使用')
                        raise Exception('所有服务器均已无法使用')
                    serverlists.remove(num)
                    num = random.choice(serverlists)
                    try:
                        redis.add_set_value(redisconns[num], key, data)
                        break
                    except:
                        fh.write('%s\t%s\n' %
                                 (num, datetime.datetime.now().strftime(
                                     '%Y-%m-%d %H:%M:%S')))
                fh.close()
            total += 1
            mongo.delete(mongo_conn, tablename, prerequisite={'url': url})
        f = open('total.dat', 'w')
        f.write('%s\n' % (total))
        f.flush()
        f.close()
        if not tag:
            print 'wait urls...'
            time.sleep(120)

예제 #7

파일: dispatcher.py 프로젝트: demon108/stream

def dispatcher():
    redisconns = redis_conns()
    servernum = len(redisconns)
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    packer = msgpack.Packer()
    tablename = 'general'
    key = 'stream'
    try:
        total = int(open('total.dat').read().strip())
    except:
        total = 0
    while True:
        rawdatas = mongo.find(mongo_conn, tablename, {}, 3600)
        tag = False
        for raw in rawdatas:
            tag = True
            raw.pop('_id')
            url = raw['url']
            num = disp_strategy(url, servernum, 'utf-8')
            data = packer.pack(raw)
            try:
                redis.add_set_value(redisconns[num], key, data)
            except:
                fh = open('./error_server.dat','w')
                fh.write('%s\t%s\n'%(num,datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
                serverlists = range(servernum)
                while True:
                    if len(serverlists)==0:
                        open('stop.dat','w').write('所有服务器均已无法使用')
                        raise Exception('所有服务器均已无法使用')
                    serverlists.remove(num)
                    num = random.choice(serverlists)
                    try:
                        redis.add_set_value(redisconns[num], key, data)
                        break
                    except:
                        fh.write('%s\t%s\n'%(num,datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
                fh.close()
            total += 1
            mongo.delete(mongo_conn,tablename,prerequisite={'url':url})
        f = open('total.dat','w')
        f.write('%s\n'%(total))
        f.flush()
        f.close()
        if not tag:
            print 'wait urls...'
            time.sleep(120)

예제 #8

파일: create_mongo_indexes.py 프로젝트: demon108/stream

def create_comgeneral():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    tablename = 'comgeneral'
    mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
    mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])

예제 #9

파일: create_mongo_indexes.py 프로젝트: demon108/stream

import mongo_util as mongo
global mongo_conn
#mongo_conn = ''
mongo_conn = mongo.connect('192.168.241.12', 'stream')
def create_general():
    global mongo_conn
    if not mongo_conn:
        tablename = 'general'
        mongo_conn = mongo.connect('192.168.241.12', 'stream')
        mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
        mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])
    return mongo_conn

def create_weixin():
    global mongo_conn
    tablename = 'weixin'
    mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])
    mongo.create_index(mongo_conn, tablename, indexs=['updatetime'])

def create_assemble():
    global mongo_conn
    tablename = 'assemble'
    mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])

def create_comgeneral():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    tablename = 'comgeneral'
    mongo.create_unique_index(mongo_conn, tablename, uniques=[[('objectid',1),('url',1)]])
    mongo.create_index(mongo_conn, tablename, indexs=['updatetime','terrace'])

#create_general()

예제 #10

파일: create_mongo_indexes.py 프로젝트: easyshell/stream

import mongo_util as mongo
global mongo_conn
#mongo_conn = ''
mongo_conn = mongo.connect('192.168.241.12', 'stream')


def create_general():
    global mongo_conn
    if not mongo_conn:
        tablename = 'general'
        mongo_conn = mongo.connect('192.168.241.12', 'stream')
        mongo.create_unique_index(mongo_conn,
                                  tablename,
                                  uniques=[[('objectid', 1), ('url', 1)]])
        mongo.create_index(mongo_conn,
                           tablename,
                           indexs=['updatetime', 'terrace'])
    return mongo_conn


def create_weixin():
    global mongo_conn
    tablename = 'weixin'
    mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])
    mongo.create_index(mongo_conn, tablename, indexs=['updatetime'])


def create_assemble():
    global mongo_conn
    tablename = 'assemble'
    mongo.create_unique_index(mongo_conn, tablename, uniques=['url'])

예제 #11

파일: weixin_process.py 프로젝트: demon108/stream

def get_mongo_conn():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    return mongo_conn

예제 #12

def get_mongo_conn():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    return mongo_conn

예제 #13

def delete_weixin():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    tablename = 'weixin'
    mongo.delete(mongo_conn, tablename)

예제 #14

def delete_comgeneral():
    mongo_conn = mongo.connect('192.168.241.12', 'stream')
    tablename = 'comgeneral'
    mongo.delete(mongo_conn, tablename)