Python DB.scan_iter Exemples

Exemple #1

Fichier : index.py Projet : xlqian/addok

def create_edge_ngrams():
    start = time.time()
    pool = Pool()
    count = 0
    chunk = []
    for key in DB.scan_iter(match='w|*'):
        count += 1
        chunk.append(key)
        if count % 10000 == 0:
            pool.map(index_ngram_key, chunk)
            print("Done", count, time.time() - start)
            chunk = []
    if chunk:
        pool.map(index_ngram_key, chunk)
    pool.close()
    pool.join()
    print('Done', count, 'in', time.time() - start)

Exemple #2

Fichier : autocomplete.py Projet : eric-pommereau/addok

def create_edge_ngrams(*args):
    start = time.time()
    pool = Pool()
    count = 0
    chunk = []
    for key in DB.scan_iter(match='w|*'):
        count += 1
        chunk.append(key)
        if count % 10000 == 0:
            pool.map(index_ngram_key, chunk)
            print("Done", count, time.time() - start)
            chunk = []
    if chunk:
        pool.map(index_ngram_key, chunk)
    pool.close()
    pool.join()
    print('Done', count, 'in', time.time() - start)

Exemple #3

Fichier : autocomplete.py Projet : phreephree/addok

def create_edge_ngrams(*args):
    pattern = '{}*'.format(dbkeys.TOKEN_PREFIX)
    parallelize(index_ngram_keys,
                DB.scan_iter(match=pattern),
                chunk_size=10000,
                throttle=1000)

Exemple #4

Fichier : autocomplete.py Projet : magopian/addok

def create_edge_ngrams(*args):
    parallelize(index_ngram_key,
                DB.scan_iter(match='w|*'),
                prefix="Computing ngrams",
                throttle=1000)

Exemple #5

Fichier : autocomplete.py Projet : digideskio/addok

def create_edge_ngrams(*args):
    parallelize(index_ngram_key, DB.scan_iter(match='w|*'),
                prefix="Computing ngrams", throttle=1000)

Exemple #6

def create_edge_ngrams(*args):
    parallelize(index_ngram_keys,
                DB.scan_iter(match='w|*'),
                chunk_size=10000,
                throttle=1000)