def __init__(self, **kwargs): self.num_lines, self.num_chars = 0, 0 # The flag sequential=True is necessary to # allow for the shared variables above. Otherwise, # due to parallelism, special procedures would be # needed to safely increment shared vars. Builder.__init__(self, **kwargs)
def __init__(self, *args, **kwargs): self.checker_name = type(self).__name__ _log.info(self.checker_name) Builder.__init__(self, *args, **kwargs)
self._mismatch_dict._getvalue() if not self._seq else self._mismatch_dict) def _increase_counter(self, nrow, ncol, mismatch_dict): # https://docs.python.org/2/library/multiprocessing.html#multiprocessing.managers.SyncManager.list if self._lock is not None: self._lock.acquire() mc = self._mismatch_counter for k in categories[self.checker_name]: mc[categories[self.checker_name].index(k)] += len(mismatch_dict[k]) self._mismatch_counter = mc for k, v in mismatch_dict.iteritems(): self._mismatch_dict[k] += v currow = self._counter[nrow] currow[ncol] += 1 self._counter[nrow] = currow self._counter_total.value += 1 if py is not None and not \ self._counter_total.value % (10*self._ncols*self._nrows): self._push_to_plotly() if (not self._counter_total.value % 2500): _log.info('processed %d items', self._counter_total.value) if self._lock is not None: self._lock.release() def finalize(self, errors): if py is not None: self._push_to_plotly() _log.info("%d items processed.", self._counter_total.value) return True Builder.register(SNLGroupBaseChecker)
self._mismatch_dict._getvalue() if not self._seq else self._mismatch_dict ) def _increase_counter(self, nrow, ncol, mismatch_dict): # https://docs.python.org/2/library/multiprocessing.html#multiprocessing.managers.SyncManager.list if self._lock is not None: self._lock.acquire() mc = self._mismatch_counter for k in categories[self.checker_name]: mc[categories[self.checker_name].index(k)] += len(mismatch_dict[k]) self._mismatch_counter = mc for k,v in mismatch_dict.iteritems(): self._mismatch_dict[k] += v currow = self._counter[nrow] currow[ncol] += 1 self._counter[nrow] = currow self._counter_total.value += 1 if py is not None and not \ self._counter_total.value % (10*self._ncols*self._nrows): self._push_to_plotly() if (not self._counter_total.value%2500): _log.info('processed %d items', self._counter_total.value) if self._lock is not None: self._lock.release() def finalize(self, errors): if py is not None: self._push_to_plotly() _log.info("%d items processed.", self._counter_total.value) return True Builder.register(SNLGroupBaseChecker)