def main():
"""
main entry point for script
"""
comm = MPI.COMM_WORLD
opts = getoptions(True)
opts['threads'] = comm.Get_size()
logout = "mpiOutput-{}.log".format(comm.Get_rank())
# For MPI jobs, do something sane with logging.
setuplogger(logging.ERROR, logout, opts['log'])
config = Config()
if comm.Get_size() < 2:
logging.error("Must run MPI job with at least 2 processes")
sys.exit(1)
myhost = MPI.Get_processor_name()
logging.info("Nodename: %s", myhost)
processjobs(config, opts, comm.Get_rank(), comm)
logging.info("Rank: %s FINISHED", comm.Get_rank())
def runindexing():
""" main script entry point """
opts = getoptions()
keep_csv = opts["keep_csv"]
setuplogger(opts['log'], opts['debugfile'], filelevel=logging.INFO)
config = Config(opts['config'])
logging.info("archive indexer starting")
pool = None
if opts['num_threads'] > 1:
logging.debug("Using %s processes", opts['num_threads'])
pool = Pool(opts['num_threads'])
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename,
str(resource['resource_id'])):
if not resource.get('pcp_log_dir'):
continue
acache = PcpArchiveProcessor(resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'],
opts['all'])
if pool is not None:
index_resource_multiprocessing(config, resource, acache, afind,
pool, keep_csv)
else:
fast_index_allowed = bool(resource.get("fast_index", False))
with LoadFileIndexUpdater(config, resource, keep_csv) as index:
for archivefile, fast_index, hostname in afind.find(
resource['pcp_log_dir']):
start_time = time.time()
data = acache.processarchive(
archivefile, fast_index and fast_index_allowed,
hostname)
parse_end = time.time()
if data is not None:
index.insert(*data)
db_end = time.time()
logging.debug(
"processed archive %s (fileio %s, dbacins %s)",
archivefile, parse_end - start_time,
db_end - parse_end)
logging.info("archive indexer complete")
if pool is not None:
pool.close()
pool.join()
def main():
"""
main entry point for script
"""
opts = getoptions(False)
setuplogger(opts['log'])
config = Config()
threads = opts['threads']
process_pool = mp.Pool(threads) if threads > 1 else None
processjobs(config, opts, process_pool)
if process_pool is not None:
# wait for all processes to finish
process_pool.close()
process_pool.join()
def runindexing():
""" main script entry point """
opts = getoptions()
setuplogger(opts['log'], opts['debugfile'], logging.DEBUG)
config = Config(opts['config'])
logging.info("archive indexer starting")
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(resource['resource_id'])):
acache = PcpArchiveProcessor(config, resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'])
for archivefile in afind.find(resource['pcp_log_dir']):
acache.processarchive(archivefile)
acache.close()
logging.info("archive indexer complete")
def runindexing():
""" main script entry point """
opts = getoptions()
setuplogger(opts['log'], opts['debugfile'], logging.DEBUG)
config = Config(opts['config'])
logging.info("archive indexer starting")
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(resource['resource_id'])):
acache = PcpArchiveProcessor(config, resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'])
for archivefile in afind.find(resource['pcp_log_dir']):
acache.processarchive(archivefile)
acache.close()
logging.info("archive indexer complete")
def main():
"""
main entry point for script
"""
opts = getoptions(False)
setuplogger(opts['log'])
config = Config()
threads = opts['threads']
if threads <= 1:
processjobs(config, opts, None)
return
else:
proclist = []
for procid in xrange(threads):
p = Process(target=processjobs, args=(config, opts, procid))
p.start()
proclist.append(p)
for proc in proclist:
p.join()
