def add_document(self, doc): """Add a document to the batch. Adds a document tuple to the batch and checks if it exists. Args: doc (tuple): A standard document tuple. Raises: NidabaInputException: The document tuple does not refer to a file. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') if not self.storage.is_file(*doc): raise NidabaInputException('Input document is not a file.') with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) self.docs.append(doc) self.scratchpad['scratchpad']['docs'] = self.docs pipe.set(self.id, json.dumps(self.scratchpad)) pipe.execute() break except self.redis.WatchError: continue
def rm_document(self, doc): """Removes a document from the (unexecuted) batch. Removes a document tuple from the batch. Args: doc (tuple): A standard document tuple. Raises: NidabaInputException: The document tuple does not refer to a file. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) self.docs.remove(list(doc)) self.scratchpad['scratchpad']['docs'] = self.docs pipe.set(self.id, json.dumps(self.scratchpad)) pipe.execute() break except WatchError: continue except ValueError: raise NidabaInputException( 'Document not part of the batch')
def task_arg_validator(arg_values, **kwargs): """ Validates keyword arguments against the list of valid argument values contained in the task definition. Raises: NidabaInputException if validation failed. """ kwc = kwargs.copy() def _val_single_arg(arg, type): if type == 'float': if not isinstance(val, float) and not isinstance(val, int): raise NidabaInputException('{} is not a float'.format(val)) elif type == 'int': if not isinstance(val, int): raise NidabaInputException('{} is not an int'.format(val)) elif type == 'str': if not isinstance(val, basestring): raise NidabaInputException('{} is not a string'.format(val)) # XXX: Add file/files checker for local case elif type == 'file': pass elif type == 'files': pass else: raise NidabaInputException('Argument type {} unknown'.format(type)) for k, v in arg_values.iteritems(): try: val = kwc.pop(k) except: raise NidabaInputException('Missing argument: {}'.format(k)) if isinstance(v, tuple): if not isinstance(val, type(v[0])): raise NidabaInputException( '{} of different type than range fields'.format(val)) if val < v[0] or val > v[1]: raise NidabaInputException( '{} outside of allowed range {}-{}'.format(val, *v)) elif isinstance(v, list): if isinstance(val, Iterable) and not isinstance(val, basestring): va = set(val) else: va = set([val]) if not set(v).issuperset(va): raise NidabaInputException( '{} not in list of valid values'.format(val)) else: _val_single_arg(val, v) if kwc: raise NidabaInputException('Superfluous arguments present')
def get_state(self): """ Retrieves the current state of a batch. Returns: (unicode): A string containing one of the following states: NONE: The batch ID is not registered in the backend. FAILURE: Batch execution has failed. PENDING: The batch is currently running. SUCCESS: The batch has completed successfully. """ if not self.id: raise NidabaInputException('Object not attached to batch.') r = requests.get('{}/batch/{}'.format(self.host, self.id)) r.raise_for_status() batch = r.json() if 'scratchpad' in batch: return u'NONE' elif 'chains' in batch: self.lock = True batch = batch['chains'] st = u'SUCCESS' for subtask in batch.itervalues(): if subtask['state'] == 'PENDING' or subtask[ 'state'] == 'RUNNING': st = u'PENDING' if subtask['state'] == 'FAILURE': return u'FAILURE' return st else: return u'NONE'
def run(self): """Executes the current batch definition. Expands the current batch definition to a series of celery chains and executes them asynchronously. Additionally a batch record is written to the celery result backend. Returns: (unicode): Batch identifier. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') # reorder task definitions keys = [ 'img', 'binarize', 'segmentation', 'ocr', 'stats', 'postprocessing', 'output' ] tasks = OrderedDict((key, self.tasks[key]) for key in keys) self.add_step() for group, btasks in tasks.iteritems(): self.add_tick() for task in btasks: super(SimpleBatch, self).add_task('{}.{}'.format(group, task[0]), **task[1]) if self.order[group] == 'sequence': self.add_tick() self.lock = True return super(SimpleBatch, self).run()
def _val_single_arg(arg, type): if type == 'float': if not isinstance(val, float) and not isinstance(val, int): raise NidabaInputException('{} is not a float'.format(val)) elif type == 'int': if not isinstance(val, int): raise NidabaInputException('{} is not an int'.format(val)) elif type == 'str': if not isinstance(val, basestring): raise NidabaInputException('{} is not a string'.format(val)) # XXX: Add file/files checker for local case elif type == 'file': pass elif type == 'files': pass else: raise NidabaInputException('Argument type {} unknown'.format(type))
def add_task(self, group, method, **kwargs): """Add a task. Adds a ``task``, a single executable task gathering one or more input documents and returning a single output document, to the current tick. Multiple jobs are run in parallel. Args: group (unicode): A task group identifier method (unicode): A task identifier **kwargs: Arguments to the task Raises: NidabaInputException: Trying to modify executed task. NidabaNoSuchAlgorithmException: Invalid method given. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') # validate that the task exists if group not in self.tasks: raise NidabaNoSuchAlgorithmException( 'Unknown task group {}'.format(group)) if u'nidaba.{}.{}'.format(group, method) not in self.celery.app.tasks: raise NidabaNoSuchAlgorithmException('Unknown task {} {}'.format( group, method)) task = self.celery.app.tasks[u'nidaba.{}.{}'.format(group, method)] # validate arguments first against getcallargs try: getcallargs(task.run, ('', ''), **kwargs) except TypeError as e: raise NidabaInputException(str(e)) # validate against arg_values field of the task task_arg_validator(task.get_valid_args(), **kwargs) with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) self.tasks[group].append((method, kwargs)) self.scratchpad['scratchpad']['simple_tasks'] = self.tasks pipe.set(self.id, json.dumps(self.scratchpad)) pipe.execute() break except WatchError: continue
def get_documents(self): """ Returns the list of input document for this task. """ if not self.id: raise NidabaInputException('Object not attached to batch.') r = requests.get('{}/batch/{}/pages'.format(self.host, self.id)) r.raise_for_status() return r.json()
def run(self): """ Executes the current batch definition. Expands the current batch definition to a series of celery chains and executes them asynchronously. Additionally a batch record is written to the celery result backend. Returns: (unicode): Batch identifier. """ if not self.id: raise NidabaInputException('Object not attached to batch.') if self.lock: raise NidabaInputException('Executed batch may not be reexecuted') r = requests.post('{}/batch/{}'.format(self.host, self.id)) r.raise_for_status() return self.id
def next(self): row = self.reader.next() if len(row) != 5: raise NidabaInputException('Incorrect number of columns') coords = [int(s) for s in row[:-1]] coordinates = [ coords[0], coords[1], coords[0] + coords[2], coords[1] + coords[3] ] return coordinates + [row[-1]]
def get_tasks(self): """ Returns the task tree either from the scratchpad or from the pipeline when already in execution. """ if not self.id: raise NidabaInputException('Object not attached to batch.') r = requests.get('{}/batch/{}/tasks'.format(self.host, self.id)) r.raise_for_status() return r.json()
def add_document(self, path, callback, auxiliary=False): """ Add a document to the batch. Uploads a document to the API server and adds it to the batch. ..note:: Note that this function accepts a standard file system path and NOT a storage tuple as a client using the web API is not expected to keep a separate, local storage medium. Args: path (unicode): Path to the document callback (function): A function that is called with a ``requests_toolbelt.multipart.encoder.MultipartEncoderMonitor`` instance. auxiliary (bool): Switch to disable setting the file as an input document. May be used to upload ground truths, metadata, and other ancillary files.. Raises: NidabaInputException: The document does not refer to a file or the batch is locked because the run() method has been called. """ if not self.id: raise NidabaInputException('Object not attached to batch.') if self.lock: raise NidabaInputException('Executed batch may not be modified') if auxiliary: params = {'auxiliary': True} else: params = {} m = encoder.MultipartEncoderMonitor.from_fields( fields={'scans': (os.path.basename(path), open(path, 'rb'))}, callback=callback) r = requests.post('{}/batch/{}/pages'.format(self.host, self.id), data=m, headers={'Content-Type': m.content_type}, params=params) r.raise_for_status() return r.json()[0]['url']
def add_task(self, group, method, *args, **kwargs): """ Add a particular task configuration to a task group. Args: group (unicode): Group the task belongs to method (unicode): Name of the task kwargs: Arguments to the task """ if not self.id: raise NidabaInputException('Object not attached to batch.') if self.lock: raise NidabaInputException('Executed batch may not be modified') # validate that the task exists if group not in self.allowed_tasks or method not in self.allowed_tasks[ group]: raise NidabaInputException('Unknown task {}'.format(method)) r = requests.post('{}/batch/{}/tasks/{}/{}'.format( self.host, self.id, group, method), json=kwargs) r.raise_for_status()
def __init__(self, id=None): # stuff depending on a valid configuration from nidaba import storage from nidaba import config self.storage = storage # slowly importing stuff from nidaba import tasks from nidaba import plugins from nidaba import celery self.task_reg = tasks self.celery = celery self.id = id if self.id is None: self.id = uuid.uuid4().get_hex() self.storage.prepare_filestore(self.id) if not self.storage.is_valid_job(self.id): raise NidabaInputException('Storage not prepared for task') self.docs = [] self.scratchpad = {} self.redis = config.Redis self.tasks = OrderedDict([('img', []), ('binarize', []), ('segmentation', []), ('ocr', []), ('stats', []), ('postprocessing', []), ('output', []), ('archive', [])]) # defines if tasks in a group are run in parallel or in sequence and their merge mode self.order = { 'img': ('sequence', False), 'binarize': ('parallel', False), 'segmentation': ('parallel', False), 'ocr': ('parallel', False), 'stats': ('parallel', False), 'postprocessing': ('sequence', 'doc'), 'output': ('sequence', False), 'archive': ('parallel', True) } self.lock = False with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) if 'scratchpad' not in self.scratchpad: self.lock = True pipe.execute() break except WatchError: continue
def rm_task(self, group, method, **kwargs): """Removes a task from the (unexecuted) batch. Removes a task from the batch. Args: group (unicode): A task group identifier method (unicode): A task identifier **kwargs: Arguments to the task Raises: NidabaInputException: Trying to modify executed task. NidabaNoSuchAlgorithmException: Invalid method given. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') # validate that the task exists if group not in self.tasks: raise NidabaNoSuchAlgorithmException( 'Unknown task group {}'.format(group)) if u'nidaba.{}.{}'.format(group, method) not in self.celery.app.tasks: raise NidabaNoSuchAlgorithmException('Unknown task {} {}'.format( group, method)) task = self.celery.app.tasks[u'nidaba.{}.{}'.format(group, method)] with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) self.tasks[group].remove([method, kwargs]) self.scratchpad['scratchpad']['simple_tasks'] = self.tasks pipe.set(self.id, json.dumps(self.scratchpad)) pipe.execute() break except WatchError: continue except ValueError: raise NidabaInputException('Task not part of the batch')
def get_extended_state(self): """ Returns the extended batch state. Raises: NidabaInputException if the batch hasn't been executed yet. """ if not self.id: raise NidabaInputException('Object not attached to batch.') r = requests.get('{}/batch/{}'.format(self.host, self.id)) r.raise_for_status() if 'chains' in r.json(): self.lock = True return r.json()['chains']
def create_batch(self): """ Creates a batch on the server. Also synchronizes the list of available tasks and their parameters. """ if self.id is not None: raise NidabaInputException( 'SimpleBatch object already initialized') r = requests.post('{}/batch'.format(self.host)) r.raise_for_status() self.id = r.json()['id'] self.lock = False self.get_available_tasks() return self.id
def add_task(self, group, method, **kwargs): """ Add a particular task configuration to a task group. Args: group (unicode): Group the task belongs to method (unicode): Name of the task kwargs: Arguments to the task """ if self.lock: raise NidabaInputException('Executed batch may not be modified') # validate that the task exists if group not in self.tasks: raise NidabaNoSuchAlgorithmException('Unknown task group') if u'nidaba.{}.{}'.format(group, method) not in self.celery.app.tasks: raise NidabaNoSuchAlgorithmException('Unknown task') task = self.celery.app.tasks[u'nidaba.{}.{}'.format(group, method)] # validate arguments first against getcallargs try: getcallargs(task.run, ('', ''), **kwargs) except TypeError as e: raise NidabaInputException(str(e)) # validate against arg_values field of the task task_arg_validator(task.get_valid_args(), **kwargs) with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) self.tasks[group].append((method, kwargs)) self.scratchpad['scratchpad']['tasks'] = self.tasks pipe.set(self.id, json.dumps(self.scratchpad)) pipe.execute() break except WatchError: continue
def rm_document(self, path): """ Removes a document from the batch. ..note:: Note that this function accepts a standard file system path and NOT a storage tuple as a client using the web API is not expected to keep a separate, local storage medium. Args: path (unicode): Path to the document Raises: NidabaInputException: The document does not refer to a file or the batch is locked because the run() method has been called. """ if not self.id: raise NidabaInputException('Object not attached to batch.') if self.lock: raise NidabaInputException('Executed batch may not be modified') r = requests.delete('{}/batch/{}/pages'.format(self.host, self.id), json={'scans': [path]}) r.raise_for_status()
def is_running(self): """ Returns True if the batch's run() method has been successfully called, otherwise False. """ if not self.id: raise NidabaInputException('Object not attached to batch.') r = requests.get('{}/batch/{}'.format(self.host, self.id)) r.raise_for_status() self.lock = True if r.json(): self.lock = True return True else: self.lock = False return False
def add_document(self, doc): """Add a document to the batch. Adds a document tuple to the batch and checks if it exists. Args: doc (tuple): A standard document tuple. Raises: NidabaInputException: The document tuple does not refer to a file or the batch is locked because the run() method has been called. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') super(SimpleBatch, self).add_document(doc)
def __init__(self, id=None): # stuff depending on a valid configuration from nidaba import storage self.storage = storage # slowly importing stuff (tasks and plugins also needed so the task # registry is complete) from nidaba import tasks from nidaba import plugins from nidaba import celery self.celery = celery self.tasks = OrderedDict([('img', []), ('binarize', []), ('segmentation', []), ('ocr', []), ('stats', []), ('postprocessing', []), ('output', [])]) # defines if tasks in a group are run in parallel or in sequence self.order = { 'img': 'sequence', 'binarize': 'parallel', 'segmentation': 'parallel', 'ocr': 'parallel', 'stats': 'parallel', 'postprocessing': 'sequence', 'output': 'sequence' } if id is None: id = unicode(uuid.uuid4()) self.storage.prepare_filestore(id) if not self.storage.is_valid_job(id): raise NidabaInputException('Storage not prepared for task') super(SimpleBatch, self).__init__(id) self.lock = False with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) if 'scratchpad' not in self.scratchpad: self.lock = True pipe.execute() break except WatchError: continue
def get_results(self): """ Retrieves the storage tuples of a successful batch. Returns: """ if not self.id: raise NidabaInputException('Object not attached to batch.') r = requests.get('{}/batch/{}'.format(self.host, self.id)) r.raise_for_status() if 'chains' in r.json(): self.lock = True batch = r.json()['chains'] outfiles = [] for subtask in batch.itervalues(): if len(subtask['children']) == 0 and not subtask[ 'housekeeping'] and subtask['result'] is not None: outfiles.append( (subtask['result'], subtask['root_document'])) return outfiles else: return None
def run(self): """Executes the current batch definition. Expands the current batch definition to a series of celery chains and executes them asynchronously. Additionally a batch record is written to the celery result backend. Returns: (unicode): Batch identifier. """ if self.lock: raise NidabaInputException('Executed batch may not be modified') # reorder task definitions keys = [ 'img', 'binarize', 'segmentation', 'ocr', 'stats', 'postprocessing', 'output', 'archive' ] tasks = OrderedDict((key, self.tasks[key]) for key in keys) first = [] prev = None result_data = {} self.lock = True # build chain root_docs = self.docs prev = [] for group, step in tasks.iteritems(): # skip groups without tasks if not step: continue sequential = True if self.order[group][0] == 'sequence' else False mmode = self.order[group][1] def _repeat(lst, n): return list( itertools.chain.from_iterable( itertools.repeat(x, n) for x in lst)) if sequential: step = [step] # multiply number of tasks in this step by number of tasks in # previous step if not merging if not mmode: step = _repeat(step, len(root_docs)) root_docs = root_docs * (len(step) / len(root_docs)) # by number of root docs if doc merging elif mmode == 'doc': step = _repeat(step, len(self.docs)) root_docs = self.docs else: root_docs = [root_docs] * len(step) if not sequential: step = [[x] for x in step] nprev = [] r = [] for rd_idx, (rdoc, c) in enumerate(zip(root_docs, step)): if sequential: r.append([]) for idx, (fun, kwargs) in enumerate(c): # if idx > 0 (sequential == true) parent is previous task in sequence if idx > 0: parents = [task_id] # if merge mode is 'doc' base parents are tasks n * (len(prev)/len(docs)) to n+1 ... elif mmode == 'doc': parents = prev[rd_idx::len(root_docs)] # if merging everything all tasks in previous step are parents elif mmode: parents = prev # if not merging a single task in previous step is the parent elif mmode is False: parents = [prev[rd_idx % len(prev)]] if prev else prev task_id = uuid.uuid4().get_hex() # last task in a sequence is entered into new prev array if idx + 1 == len(c): nprev.append(task_id) result_data[task_id] = { 'children': [], 'parents': parents, 'root_documents': [rdoc], 'state': 'PENDING', 'result': None, 'task': (group, fun, kwargs) } for parent in parents: result_data[parent]['children'].append(task_id) task = self.celery.app.tasks[u'nidaba.{}.{}'.format( group, fun)] if sequential: r[-1].append( task.s(batch_id=self.id, task_id=task_id, **kwargs)) else: r.append( task.s(batch_id=self.id, task_id=task_id, **kwargs)) prev = nprev t = self.celery.app.tasks[u'nidaba.util.barrier'].s( merging=mmode, sequential=sequential, replace=r, root_docs=self.docs) first.append(t) with self.redis.pipeline() as pipe: while (1): try: pipe.watch(self.id) self._restore_and_create_scratchpad(pipe) # also deletes the scratchpad pipe.set(self.id, json.dumps(result_data)) pipe.execute() break except self.redis.WatchError: continue chain(first).apply_async(args=[self.docs]) return self.id