class LoggerSection(object):
classProvides(ISectionBlueprint)
implements(ISection)
def __init__(self, transmogrifier, name, options, previous):
self.transmogrifier = transmogrifier
keys = options.get('keys') or ''
self.pathkey = options.get('path-key', '_path').strip()
self.keys = Matcher(*keys.splitlines())
self.previous = previous
self.logger = name
self.storage = IAnnotations(transmogrifier).setdefault(
VALIDATIONKEY, [])
def __iter__(self):
start_time = time()
count = 0
problematic = 0
for item in self.previous:
# source sections add store path of current generated item in annotation
# it gives posibility to monitor what items go through all pipeline
# sections between source section and this section and what don't
if self.pathkey in item and item[self.pathkey] in self.storage:
self.storage.remove(item[self.pathkey])
count += 1
# print item data stored on keys given as option
items = []
for key in item.keys():
if self.keys(key)[0] is not None:
items.append("%s=%s" % (key, item[key]))
if items:
msg = ", ".join(items)
logging.getLogger(self.logger).info(msg)
yield item
working_time = int(round(time() - start_time))
# log items that maybe have some problems
if self.storage:
problematic = len(self.storage)
logging.getLogger(self.logger).warning('\nNext objects didn\'t go through full pipeline:\n%s' % \
'\n'.join(['\t'+i for i in self.storage]))
# delete validation data from annotations
anno = IAnnotations(self.transmogrifier)
if VALIDATIONKEY in anno:
del anno[VALIDATIONKEY]
seconds = working_time % 60
minutes = working_time / 60 % 60
hours = working_time / 3600
stats = "\nPipeline processing time: %02d:%02d:%02d\n" % (
hours, minutes, seconds)
stats += "\t%4d items were generated in source sections\n" % (
count + problematic)
stats += "\t%4d went through full pipeline\n" % count
stats += "\t%4d were discarded in some section" % problematic
logging.getLogger(self.logger).info(stats)
class LoggerSection(object):
classProvides(ISectionBlueprint)
implements(ISection)
def __init__(self, transmogrifier, name, options, previous):
self.transmogrifier = transmogrifier
keys = options.get('keys') or ''
self.pathkey = options.get('path-key', '_path').strip()
self.keys = Matcher(*keys.splitlines())
self.previous = previous
self.logger = name
self.storage = IAnnotations(transmogrifier).setdefault(VALIDATIONKEY, [])
def __iter__(self):
start_time = time()
count = 0
problematic = 0
for item in self.previous:
# source sections add store path of current generated item in annotation
# it gives posibility to monitor what items go through all pipeline
# sections between source section and this section and what don't
if self.pathkey in item and item[self.pathkey] in self.storage:
self.storage.remove(item[self.pathkey])
count += 1
# print item data stored on keys given as option
items = []
for key in item.keys():
if self.keys(key)[0] is not None:
items.append("%s=%s" % (key, item[key]))
if items:
msg = ", ".join(items)
logging.getLogger(self.logger).info(msg)
yield item
working_time = int(round(time() - start_time))
# log items that maybe have some problems
if self.storage:
problematic = len(self.storage)
logging.getLogger(self.logger).warning('\nNext objects didn\'t go through full pipeline:\n%s' % \
'\n'.join(['\t'+i for i in self.storage]))
# delete validation data from annotations
anno = IAnnotations(self.transmogrifier)
if VALIDATIONKEY in anno:
del anno[VALIDATIONKEY]
seconds = working_time % 60
minutes = working_time / 60 % 60
hours = working_time / 3600
stats = "\nPipeline processing time: %02d:%02d:%02d\n" % (hours, minutes, seconds)
stats += "\t%4d items were generated in source sections\n" % (count + problematic)
stats += "\t%4d went through full pipeline\n" % count
stats += "\t%4d were discarded in some section" % problematic
logging.getLogger(self.logger).info(stats)
class LoggerSection(object):
def __init__(self, transmogrifier, name, options, previous):
self.transmogrifier = transmogrifier
keys = options.get("keys") or ""
self.pathkey = options.get("path-key", "_path").strip()
self.keys = Matcher(*keys.splitlines())
self.previous = previous
self.logger = name
self.storage = IAnnotations(api.portal.get().REQUEST).setdefault(
VALIDATIONKEY, [])
self.errored = IAnnotations(api.portal.get().REQUEST).setdefault(
ERROREDKEY, [])
self.count = IAnnotations(api.portal.get().REQUEST).setdefault(
COUNTKEY, {})
def __iter__(self):
start_time = time()
count = 0
problematic = 0
problematic2 = 0
for item in self.previous:
# source sections add store path of current generated item in annotation
# it gives posibility to monitor what items go through all pipeline
# sections between source section and this section and what don't
if self.pathkey in item and item[self.pathkey] in self.storage:
self.storage.remove(item[self.pathkey])
count += 1
# print item data stored on keys given as option
items = []
for key in item.keys():
if self.keys(key)[0] is not None:
items.append("%s=%s" % (key, item[key]))
if items:
msg = ", ".join(items)
logging.getLogger(self.logger).info(msg)
self.count["remaining"] = self.count["remaining"] - 1
logging.getLogger(self.logger).info(
"Remaining {} of {}.".format(self.count["remaining"],
self.count["total"]))
yield item
working_time = int(round(time() - start_time))
# log items that maybe have some problems
if self.storage:
problematic = len(self.storage)
logging.getLogger(self.logger).warning(
"\nNext objects didn't go through full pipeline:\n%s" %
"\n".join(["\t" + i for i in self.storage]))
if self.errored:
problematic2 = len(self.errored)
logger = logging.getLogger(self.logger)
logger.warning("\nNext objects errored somewhere in the pipeline:")
for error in self.errored:
logger.warning("- {path}: {reason}".format(path=error["path"],
reason=error.get(
"reason", "")))
# self.log_errors(self.errored)
# delete validation data from annotations
annotations = IAnnotations(api.portal.get().REQUEST)
if VALIDATIONKEY in annotations:
del annotations[VALIDATIONKEY]
if ERROREDKEY in annotations:
del annotations[ERROREDKEY]
seconds = working_time % 60
minutes = working_time / 60 % 60
hours = working_time / 3600
stats = "\nPipeline processing time: %02d:%02d:%02d\n" % (
hours,
minutes,
seconds,
)
stats += "\t%4d items were generated in source sections\n" % (
count + problematic)
stats += "\t%4d went through full pipeline\n" % count
stats += "\t%4d were discarded in some section\n" % problematic
stats += "\t%4d were errored in some section" % problematic2
logging.getLogger(self.logger).info(stats)
