def enable_logging(self, result_log_path, runnable_log_path):
"""Causes the extraction runner to keep logs of what it does
If a log file from the same day exists, it is used. If not, new logs files are created for the current day
If this method is not called, by default, info will be printed to stdout and stderr
Args:
result_log_path: String file path that indicates where to store the results log.
This log stores when processing starts and ends for each run. It also notes if any errors occur during the run.
For example, if this is '/path/to/result' the logs will be stored in files named '/path/to/log.[year]-[month]-[day].log'
runnable_log_path: String file path that indicates where to store the runnables log.
This log contains any log messages that runnables log.
Path and day rotation is handled the same as with the results log
"""
result_log_path = utils.expand_path(result_log_path)
runnable_log_path = utils.expand_path(runnable_log_path)
if not os.path.exists(os.path.dirname(result_log_path)): os.makedirs(os.path.dirname(result_log_path))
if not os.path.exists(os.path.dirname(runnable_log_path)): os.makedirs(os.path.dirname(runnable_log_path))
result_log_handler = extraction.ParallelTimedRotatingFileHandler(result_log_path, when='D', delay=True)
runnable_log_handler = extraction.ParallelTimedRotatingFileHandler(runnable_log_path, when='D', delay=True)
formatter = logging.Formatter('%(asctime)s: %(message)s')
result_log_handler.setFormatter(formatter)
runnable_log_handler.setFormatter(formatter)
self.result_logger.handlers = []
self.runnable_logger.handlers = []
self.result_logger.addHandler(result_log_handler)
self.runnable_logger.addHandler(runnable_log_handler)
def enable_logging(self, result_log_path, runnable_log_path):
"""Causes the extraction runner to keep logs of what it does
If a log file from the same day exists, it is used. If not, new logs files are created for the current day
If this method is not called, by default, info will be printed to stdout and stderr
Args:
result_log_path: String file path that indicates where to store the results log.
This log stores when processing starts and ends for each run. It also notes if any errors occur during the run.
For example, if this is '/path/to/result' the logs will be stored in files named '/path/to/log.[year]-[month]-[day].log'
runnable_log_path: String file path that indicates where to store the runnables log.
This log contains any log messages that runnables log.
Path and day rotation is handled the same as with the results log
"""
result_log_path = utils.expand_path(result_log_path)
runnable_log_path = utils.expand_path(runnable_log_path)
if not os.path.exists(os.path.dirname(result_log_path)): os.makedirs(os.path.dirname(result_log_path))
if not os.path.exists(os.path.dirname(runnable_log_path)): os.makedirs(os.path.dirname(runnable_log_path))
result_log_handler = extraction.log.ParallelTimedRotatingFileHandler(result_log_path, when='D', delay=True)
runnable_log_handler = extraction.log.ParallelTimedRotatingFileHandler(runnable_log_path, when='D', delay=True)
formatter = logging.Formatter('%(asctime)s: %(message)s')
result_log_handler.setFormatter(formatter)
runnable_log_handler.setFormatter(formatter)
self.result_logger.handlers = []
self.runnable_logger.handlers = []
self.result_logger.addHandler(result_log_handler)
self.runnable_logger.addHandler(runnable_log_handler)
def run_from_file(self, file_path, output_dir=None, **kwargs):
"""Runs the extractor on the file at file_path
Reads the file at file_path from disk into a string. Then runs the extractors
on this data string.
Args:
file_path: Reads this file and passes its data to the extractors and filters
output_dir: An optional string that specifies the directory to write the results to
If this isn't provided, results will be written to the same directory as the file
**kwargs: Optional keyword arguments
write_dep_errors: A Boolean. If True, extractors that fail because dependencies fail
will still write a short xml file with this error to disk. (Good for clarity)
If False, extractors with failing dependencies won't write anything to disk
file_prefix: A string to prepend to all filenames that get written to disk
"""
file_path = utils.expand_path(file_path)
if not output_dir:
output_dir = os.path.dirname(file_path)
return self.run(open(file_path, 'rb').read(),
output_dir,
run_name=file_path,
**kwargs)
def run_from_file(self, file_path, output_dir=None, **kwargs):
"""Runs the extractor on the file at file_path
Reads the file at file_path from disk into a string. Then runs the extractors
on this data string.
Args:
file_path: Reads this file and passes its data to the extractors and filters
output_dir: An optional string that specifies the directory to write the results to
If this isn't provided, results will be written to the same directory as the file
**kwargs: Optional keyword arguments
write_dep_errors: A Boolean. If True, extractors that fail because dependencies fail
will still write a short xml file with this error to disk. (Good for clarity)
If False, extractors with failing dependencies won't write anything to disk
file_prefix: A string to prepend to all filenames that get written to disk
"""
file_path = utils.expand_path(file_path)
if not output_dir:
output_dir = os.path.dirname(file_path)
return self.run(open(file_path, 'rb').read(), output_dir, run_name=file_path, **kwargs)
