def __init__(self, **kwargs):
self.num_lines, self.num_chars = 0, 0
# The flag sequential=True is necessary to
# allow for the shared variables above. Otherwise,
# due to parallelism, special procedures would be
# needed to safely increment shared vars.
Builder.__init__(self, **kwargs)
def __init__(self, **kwargs):
self.num_lines, self.num_chars = 0, 0
# The flag sequential=True is necessary to
# allow for the shared variables above. Otherwise,
# due to parallelism, special procedures would be
# needed to safely increment shared vars.
Builder.__init__(self, **kwargs)
def __init__(self, *args, **kwargs):
self.checker_name = type(self).__name__
_log.info(self.checker_name)
Builder.__init__(self, *args, **kwargs)
self._mismatch_dict._getvalue()
if not self._seq else self._mismatch_dict)
def _increase_counter(self, nrow, ncol, mismatch_dict):
# https://docs.python.org/2/library/multiprocessing.html#multiprocessing.managers.SyncManager.list
if self._lock is not None: self._lock.acquire()
mc = self._mismatch_counter
for k in categories[self.checker_name]:
mc[categories[self.checker_name].index(k)] += len(mismatch_dict[k])
self._mismatch_counter = mc
for k, v in mismatch_dict.iteritems():
self._mismatch_dict[k] += v
currow = self._counter[nrow]
currow[ncol] += 1
self._counter[nrow] = currow
self._counter_total.value += 1
if py is not None and not \
self._counter_total.value % (10*self._ncols*self._nrows):
self._push_to_plotly()
if (not self._counter_total.value % 2500):
_log.info('processed %d items', self._counter_total.value)
if self._lock is not None: self._lock.release()
def finalize(self, errors):
if py is not None: self._push_to_plotly()
_log.info("%d items processed.", self._counter_total.value)
return True
Builder.register(SNLGroupBaseChecker)
def __init__(self, *args, **kwargs):
self.checker_name = type(self).__name__
_log.info(self.checker_name)
Builder.__init__(self, *args, **kwargs)
self._mismatch_dict._getvalue() if not self._seq
else self._mismatch_dict
)
def _increase_counter(self, nrow, ncol, mismatch_dict):
# https://docs.python.org/2/library/multiprocessing.html#multiprocessing.managers.SyncManager.list
if self._lock is not None: self._lock.acquire()
mc = self._mismatch_counter
for k in categories[self.checker_name]:
mc[categories[self.checker_name].index(k)] += len(mismatch_dict[k])
self._mismatch_counter = mc
for k,v in mismatch_dict.iteritems():
self._mismatch_dict[k] += v
currow = self._counter[nrow]
currow[ncol] += 1
self._counter[nrow] = currow
self._counter_total.value += 1
if py is not None and not \
self._counter_total.value % (10*self._ncols*self._nrows):
self._push_to_plotly()
if (not self._counter_total.value%2500):
_log.info('processed %d items', self._counter_total.value)
if self._lock is not None: self._lock.release()
def finalize(self, errors):
if py is not None: self._push_to_plotly()
_log.info("%d items processed.", self._counter_total.value)
return True
Builder.register(SNLGroupBaseChecker)