def main():
"""
main entry point for script
"""
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
config = Config(opts['config'])
dwconfig = config.getsection("datawarehouse")
dbif = DbHelper(dwconfig, 'modw_supremm.batchscripts')
for resourcename, settings in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(settings['resource_id'])):
logging.debug("Processing %s (id=%s)", resourcename, settings['resource_id'])
if "script_dir" in settings:
total = processfor(settings['resource_id'], settings['script_dir'], dbif, opts['deltadays'])
logging.info("Processed %s files for %s", total, resourcename)
else:
logging.debug("Skip resource %s no script dir defined", resourcename)
dbif.postinsert()
def runindexing():
""" main script entry point """
opts = getoptions()
keep_csv = opts["keep_csv"]
setuplogger(opts['log'], opts['debugfile'], filelevel=logging.INFO)
config = Config(opts['config'])
logging.info("archive indexer starting")
pool = None
if opts['num_threads'] > 1:
logging.debug("Using %s processes", opts['num_threads'])
pool = Pool(opts['num_threads'])
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename,
str(resource['resource_id'])):
if not resource.get('pcp_log_dir'):
continue
acache = PcpArchiveProcessor(resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'],
opts['all'])
if pool is not None:
index_resource_multiprocessing(config, resource, acache, afind,
pool, keep_csv)
else:
fast_index_allowed = bool(resource.get("fast_index", False))
with LoadFileIndexUpdater(config, resource, keep_csv) as index:
for archivefile, fast_index, hostname in afind.find(
resource['pcp_log_dir']):
start_time = time.time()
data = acache.processarchive(
archivefile, fast_index and fast_index_allowed,
hostname)
parse_end = time.time()
if data is not None:
index.insert(*data)
db_end = time.time()
logging.debug(
"processed archive %s (fileio %s, dbacins %s)",
archivefile, parse_end - start_time,
db_end - parse_end)
logging.info("archive indexer complete")
if pool is not None:
pool.close()
pool.join()
def runindexing():
""" main script entry point """
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log'])
logging.captureWarnings(True)
config = Config(opts['config'])
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(resource['resource_id'])):
acache = PcpArchiveProcessor(config, resource)
afind = PcpArchiveFinder(opts['mindate'])
for archivefile in afind.find(resource['pcp_log_dir']):
acache.processarchive(archivefile)
acache.close()
def main():
"""
main entry point for script
"""
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
config = Config(opts['config'])
dwconfig = config.getsection("datawarehouse")
for resourcename, settings in config.resourceconfigs():
if opts['resource'] in (None, resourcename,
str(settings['resource_id'])):
logging.debug("Processing %s (id=%s)", resourcename,
settings['resource_id'])
respath, timestamp_mode = parse_resource_config(settings)
if respath:
dbif = DbHelper(dwconfig, "modw_supremm", timestamp_mode)
total = processfor(settings['resource_id'], respath, dbif,
opts['deltadays'])
dbif.postinsert()
logging.info("Processed %s files for %s", total, resourcename)
else:
logging.debug("Skip resource %s no script dir defined",
resourcename)
def runindexing():
""" main script entry point """
opts = getoptions()
setuplogger(opts['log'], opts['debugfile'], logging.DEBUG)
config = Config(opts['config'])
logging.info("archive indexer starting")
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(resource['resource_id'])):
acache = PcpArchiveProcessor(config, resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'])
for archivefile in afind.find(resource['pcp_log_dir']):
acache.processarchive(archivefile)
acache.close()
logging.info("archive indexer complete")
def runindexing():
""" main script entry point """
opts = getoptions()
setuplogger(opts['log'], opts['debugfile'], logging.DEBUG)
config = Config(opts['config'])
logging.info("archive indexer starting")
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(resource['resource_id'])):
acache = PcpArchiveProcessor(config, resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'])
for archivefile in afind.find(resource['pcp_log_dir']):
acache.processarchive(archivefile)
acache.close()
logging.info("archive indexer complete")
