Exemplo n.º 1
0
def get_batch(num_workers, **kwargs):
    try:
        enqueuer = GeneratorEnqueuer(generator(**kwargs),
                                     use_multiprocessing=False)
        enqueuer.start(max_queue_size=1, workers=1)
        # enqueuer = GeneratorEnqueuer(generator(**kwargs), use_multiprocessing=True)
        print(
            'Generator use 10 batches for buffering, this may take a while, you can tune this yourself.'
        )
        # enqueuer.start(max_queue_size=10, workers=num_workers)
        generator_output = None
        while True:
            print("开始取数据")
            while not enqueuer.queue.empty():
                if not enqueuer.queue.empty():
                    print("我来取数据了a " + str(enqueuer.queue.qsize()))
                    generator_output = enqueuer.queue.get()
                    print("我来取数据了b " + str(enqueuer.queue.qsize()))
                    break
                else:
                    time.sleep(0.1)
            yield generator_output
            generator_output = None
    finally:
        print('Generator use 10 batches完成了')
        if enqueuer is not None:
            enqueuer.stop()
Exemplo n.º 2
0
def get_batch(num_workers, **kwargs):
    try:
        enqueuer = GeneratorEnqueuer(generator(**kwargs), use_multiprocessing=True)
        print('Generator use 10 batches for buffering, this may take a while, you can tune this yourself.')
        enqueuer.start(max_queue_size=10, workers=num_workers)
        generator_output = None
        while True:
            while enqueuer.is_running():
                if not enqueuer.queue.empty():
                    generator_output = enqueuer.queue.get()
                    break
                else:
                    time.sleep(0.01)
            yield generator_output
            generator_output = None
    finally:
        if enqueuer is not None:
            enqueuer.stop()