def batch(files, host, preprocessing, binarize, ocr, segmentation, stats,
postprocessing, output, grayscale, help_tasks):
"""
Add a new job to the pipeline.
"""
if host:
batch = NetworkSimpleBatch(host)
click.echo(u'Preparing filestore\t\t[', nl=False)
try:
batch.create_batch()
except:
click.secho(u'\u2717', fg='red', nl=False)
click.echo(']')
exit()
click.secho(u'\u2713', fg='green', nl=False)
click.echo(']')
for doc in files:
def callback(monitor):
spin(u'Uploading {}'.format(doc))
batch.add_document(doc, callback)
click.secho(u'\b\u2713', fg='green', nl=False)
click.echo('\033[?25h\n', nl=False)
else:
from nidaba import storage
click.echo(u'Preparing filestore\t\t[', nl=False)
try:
batch = SimpleBatch()
except:
click.secho(u'\u2717', fg='red', nl=False)
click.echo(']')
exit()
for doc in files:
shutil.copy2(doc,
storage.get_abs_path(batch.id, os.path.basename(doc)))
batch.add_document((batch.id, os.path.basename(doc)))
click.secho(u'\u2713', fg='green', nl=False)
click.echo(']')
click.echo(u'Building batch\t\t\t[', nl=False)
if not grayscale:
batch.add_task('img', 'rgb_to_gray')
if preprocessing:
for alg in preprocessing:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('img', alg[0], **kwargs)
if binarize:
for alg in binarize:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('binarize', alg[0], **kwargs)
if segmentation:
for alg in segmentation:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('segmentation', alg[0], **kwargs)
if ocr:
for alg in ocr:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('ocr', alg[0], **kwargs)
if stats:
for alg in stats:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('stats', alg[0], **kwargs)
if postprocessing:
for alg in postprocessing:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('postprocessing', alg[0], **kwargs)
if output:
for alg in output:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('output', alg[0], **kwargs)
batch.run()
click.secho(u'\u2713', fg='green', nl=False)
click.echo(']')
click.echo(batch.id)
def batch(files, host, preprocessing, binarize, ocr, segmentation, stats,
postprocessing, output, grayscale, help_tasks):
"""
Add a new job to the pipeline.
"""
if host:
batch = NetworkSimpleBatch(host)
click.echo(u'Preparing filestore\t\t[', nl=False)
try:
batch.create_batch()
except:
click.secho(u'\u2717', fg='red', nl=False)
click.echo(']')
exit()
click.secho(u'\u2713', fg='green', nl=False)
click.echo(']')
for doc in files:
def callback(monitor):
spin(u'Uploading {}'.format(doc))
batch.add_document(doc, callback)
click.secho(u'\b\u2713', fg='green', nl=False)
click.echo('\033[?25h\n', nl=False)
else:
from nidaba import storage
click.echo(u'Preparing filestore\t\t[', nl=False)
try:
batch = SimpleBatch()
except:
click.secho(u'\u2717', fg='red', nl=False)
click.echo(']')
exit()
for doc in files:
shutil.copy2(doc, storage.get_abs_path(batch.id, os.path.basename(doc)))
batch.add_document((batch.id, os.path.basename(doc)))
click.secho(u'\u2713', fg='green', nl=False)
click.echo(']')
click.echo(u'Building batch\t\t\t[', nl=False)
if not grayscale:
batch.add_task('img', 'rgb_to_gray')
if preprocessing:
for alg in preprocessing:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('img', alg[0], **kwargs)
if binarize:
for alg in binarize:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('binarize', alg[0], **kwargs)
if segmentation:
for alg in segmentation:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('segmentation', alg[0], **kwargs)
if ocr:
for alg in ocr:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('ocr', alg[0], **kwargs)
if stats:
for alg in stats:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('stats', alg[0], **kwargs)
if postprocessing:
for alg in postprocessing:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('postprocessing', alg[0], **kwargs)
if output:
for alg in output:
for kwargs in alg[1]:
kwargs = move_to_storage(batch, kwargs)
batch.add_task('output', alg[0], **kwargs)
batch.run()
click.secho(u'\u2713', fg='green', nl=False)
click.echo(']')
click.echo(batch.id)
