def __init__(self, verbose=False, quiet=True):
ETL.__init__(self, verbose=verbose)
self.read_configfiles()
self.config["plugins"] = []
def __init__(self, verbose=False):
ETL.__init__(self, verbose=verbose)
self.verbose = verbose
self.read_configfiles()
# Watched events
#
# We need IN_MOVE_SELF to track moved folder paths
# pyinotify-internally. If omitted, the os instructions
# mv /docs/src /docs/dest; touch /docs/dest/doc.pdf
# will produce a IN_MOVED_TO pathname=/docs/dest/ followed by
# IN_CLOSE_WRITE pathname=/docs/src/doc.pdf
# where we would like a IN_CLOSE_WRITE pathname=/docs/dest/doc.pdf
self.mask = (pyinotify.IN_DELETE
| pyinotify.IN_CLOSE_WRITE
| pyinotify.IN_MOVED_TO
| pyinotify.IN_MOVED_FROM
| pyinotify.IN_MOVE_SELF)
self.watchmanager = pyinotify.WatchManager() # Watch Manager
self.handler = EventHandler()
self.notifier = pyinotify.Notifier(self.watchmanager, self.handler)
def __init__(self, verbose=False, quiet=True): ETL.__init__(self, verbose=verbose) self.read_configfiles() self.config["plugins"] = []
def __init__(self, verbose=False, quiet=True):
ETL.__init__(self, verbose=verbose)
self.quiet = quiet
self.set_configdefaults()
self.read_configfiles()
def __init__(self, verbose=False, quiet=True): ETL.__init__(self, verbose=verbose) self.quiet = quiet self.set_configdefaults() self.read_configfiles()
def __init__(self, verbose = False ): ETL.__init__(self, verbose=verbose) self.verbose=verbose self.read_configfiles() self.mask = pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_MOVED_FROM # watched events self.watchmanager = pyinotify.WatchManager() # Watch Manager self.handler = EventHandler() self.notifier = pyinotify.Notifier(self.watchmanager, self.handler)
def __init__(self, verbose=False):
ETL.__init__(self, verbose=verbose)
self.verbose = verbose
self.read_configfiles()
self.mask = pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO | pyinotify.IN_MOVED_FROM # watched events
self.watchmanager = pyinotify.WatchManager() # Watch Manager
self.handler = EventHandler()
self.notifier = pyinotify.Notifier(self.watchmanager, self.handler)
def __init__(self, plugins=[], verbose=False ):
ETL.__init__(self, plugins=plugins, verbose=verbose)
self.read_configfile ('/etc/etl/config')
self.read_configfile ('/etc/opensemanticsearch/etl')
self.read_configfile ('/etc/opensemanticsearch/enhancer-rdf')
self.fields = self.getfieldnames_from_plugins()
# init exporter (todo: exporter as extended PySolr)
self.export_solr = export_solr.export_solr()
# init PySolr
solr_uri = self.config['solr']
if not solr_uri.endswith('/'):
solr_uri += '/'
solr_uri += self.config['index']
self.solr = pysolr.Solr( solr_uri )
self.threads_max = None
# if not set explicit, autodetection of count of CPUs for amount of threads
if not self.threads_max:
import multiprocessing
self.threads_max = multiprocessing.cpu_count()
if self.verbose:
print ( "Setting threads to count of CPUs: " + str(self.threads_max) )
self.rows_per_step = 100
if self.rows_per_step < self.threads_max * 2:
self.rows_per_step = self.threads_max * 2
self.work_in_progress = []
self.delete_from_work_in_progress_lock = threading.Lock()
self.delete_from_work_in_progress_after_commit = []
self.work_in_progress_lock = threading.Lock()
self.e_job_done = threading.Event()
def __init__(self, verbose=False, quiet=True):
ETL.__init__(self, verbose=verbose)
self.quiet = quiet
self.set_configdefaults()
self.read_configfiles()
# read on what DB or search server software our index is
export = self.config['export']
# call delete function of the configured exporter
module = importlib.import_module(export)
objectreference = getattr(module, export)
self.connector = objectreference()
def __init__(self, verbose=False, quiet=True):
ETL.__init__(self, verbose=verbose)
self.quiet = quiet
self.set_configdefaults()
self.read_configfiles()
# read on what DB or search server software our index is
export = self.config['export']
# call delete function of the configured exporter
module = importlib.import_module(export)
objectreference = getattr(module, export)
self.connector = objectreference()
def __init__(self, verbose=False, quiet=True):
ETL.__init__(self, verbose=verbose)
self.quiet = quiet
self.set_configdefaults()
self.read_configfiles()
# if not set explicit, autodetection of count of CPUs for amount of threads
if not self.threads_max:
import multiprocessing
self.threads_max = multiprocessing.cpu_count()
if self.verbose:
print("Setting threads to count of CPUs: " +
str(self.threads_max))
self.e_job_done = threading.Event()
