async def master_server(event_loop, log_to_stdout):
task = asyncio.create_task(
master(SERVER_URL, [DummyBuilder(dummy_prechunk=False)], num_chunks=10)
)
yield task
task.cancel()
async def master_server():
task = asyncio.create_task(
master("tcp://127.0.0.1:8234", [DummyBuilder(dummy_prechunk=False)],
num_chunks=10))
yield
task.cancel()
async def test_no_prechunk(caplog):
asyncio.create_task(
master(
SERVER_URL,
[DummyBuilderWithNoPrechunk(dummy_prechunk=False)],
num_chunks=10,
))
await asyncio.sleep(1)
assert (
f"Can't distributed process DummyBuilderWithNoPrechunk. Skipping for now"
in caplog.text)
def run(builders, verbosity, reporting_store, num_workers, url, num_chunks,
no_bars):
# Set Logging
levels = [logging.WARNING, logging.INFO, logging.DEBUG]
level = levels[min(len(levels) - 1,
verbosity)] # capped to number of levels
root = logging.getLogger()
root.setLevel(level)
ch = TqdmLoggingHandler()
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch.setFormatter(formatter)
root.addHandler(ch)
builder_objects = []
for b in builders:
if str(b).endswith(".py") or str(b).endswith(".ipynb"):
builder_objects.append(load_builder_from_source(b))
else:
builder_objects.append(loadfn(b))
builder_objects = [
b if isinstance(b, list) else [b] for b in builder_objects
]
builder_objects = list(chain.from_iterable(builder_objects))
if reporting_store:
reporting_store = loadfn(reporting_store)
root.addHandler(ReportingHandler(reporting_store))
if url:
loop = asyncio.get_event_loop()
if num_chunks > 0:
# Master
loop.run_until_complete(master(url, builder_objects, num_chunks))
else:
# worker
loop.run_until_complete(worker(url, num_workers))
else:
if num_workers == 1:
for builder in builder_objects:
serial(builder, no_bars)
else:
loop = asyncio.get_event_loop()
for builder in builder_objects:
loop.run_until_complete(multi(builder, num_workers, no_bars))
def run(builders, verbosity, reporting_store, num_workers, url, num_chunks):
# Set Logging
levels = [logging.WARNING, logging.INFO, logging.DEBUG]
level = levels[min(len(levels) - 1, verbosity)] # capped to number of levels
root = logging.getLogger()
root.setLevel(level)
ch = TqdmLoggingHandler()
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
ch.setFormatter(formatter)
root.addHandler(ch)
builders = [loadfn(b) for b in builders]
builders = [b if isinstance(b, list) else [b] for b in builders]
builders = list(chain.from_iterable(builders))
if reporting_store:
reporting_store = loadfn(reporting_store)
root.addHandler(ReportingHandler(reporting_store))
if url:
if num_chunks > 0:
# Master
asyncio.run(master(url, builders, num_chunks))
else:
# worker
asyncio.run(worker(url, num_workers))
else:
if num_workers == 1:
for builder in builders:
serial(builder)
else:
for builder in builders:
asyncio.run(multi(builder, num_workers))