def runingest():
if len(sys.argv) > 1:
config = Config(sys.argv[1])
else:
config = Config()
if len(sys.argv) > 2:
end_time = sys.argv[2]
else:
end_time = 9223372036854775807L
#ingest(config, end_time)
ingestall(config)
def main():
"""
main entry point for script
"""
comm = MPI.COMM_WORLD
opts = getoptions(True)
opts['threads'] = comm.Get_size()
logout = "mpiOutput-{}.log".format(comm.Get_rank())
# For MPI jobs, do something sane with logging.
setuplogger(logging.ERROR, logout, opts['log'])
config = Config()
if comm.Get_size() < 2:
logging.error("Must run MPI job with at least 2 processes")
sys.exit(1)
myhost = MPI.Get_processor_name()
logging.info("Nodename: %s", myhost)
processjobs(config, opts, comm.Get_rank(), comm)
logging.info("Rank: %s FINISHED", comm.Get_rank())
def test():
""" simple test function """
config = Config()
xdm = XDMoDAcct(13, config, None, None)
for job in xdm.get(1444151688, None):
print job
def main():
"""
main entry point for script
"""
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
config = Config()
threads = opts['threads']
if threads <= 1:
processjobs(config, opts, None)
return
else:
proclist = []
for procid in xrange(threads):
p = Process(target=processjobs, args=(config, opts, procid))
p.start()
proclist.append(p)
for proc in proclist:
p.join()
def default_settings(confpath):
""" populate the default settings for the configuration.
will use the values specifed in the configuration file if it exists.
"""
defaults = {}
defaults['usexdmodconfig'] = 'y'
defaults['xdmodpath'] = '/etc/xdmod'
defaults['archiveoutdir'] = '/dev/shm/supremm'
defaults['mysqlhostname'] = 'localhost'
defaults['mysqlport'] = 3306
defaults['mysqlusername'] = '******'
defaults['mycnffilename'] = '~/.supremm.my.cnf'
defaults['mongouri'] = 'mongodb://localhost:27017/supremm'
defaults['mongodb'] = 'supremm'
try:
existingconf = Config(confpath)
rawconfig = existingconf._config.copy()
if 'xdmodroot' in rawconfig and 'datawarehouse' in rawconfig and 'include' in rawconfig['datawarehouse']:
defaults['usexdmodconfig'] = 'y'
defaults['xdmodpath'] = rawconfig['xdmodroot']
else:
dwconfig = existingconf.getsection('datawarehouse')
defaults['usexdmodconfig'] = 'n'
defaults['mysqlhostname'] = dwconfig['host']
defaults['mysqlport'] = dwconfig.get('port', defaults['mysqlport'])
defaults['mycnffilename'] = dwconfig['defaultsfile']
try:
mycnf = ConfigParser.RawConfigParser()
mycnf.read(os.path.expanduser(dwconfig['defaultsfile']))
if mycnf.has_section('client'):
defaults['mysqlusername'] = mycnf.get('client', 'user')
except ConfigParser.Error:
pass
outputconfig = existingconf.getsection('outputdatabase')
defaults['mongouri'] = outputconfig['uri']
defaults['mongodb'] = outputconfig['dbname']
summarycnf = existingconf.getsection('summary')
defaults['archiveoutdir'] = summarycnf['archive_out_dir']
defaults['resources'] = rawconfig['resources']
except Exception as e:
# ignore missing or broken existing config files.
pass
return defaults
def runindexing():
""" main script entry point """
opts = getoptions()
keep_csv = opts["keep_csv"]
setuplogger(opts['log'], opts['debugfile'], filelevel=logging.INFO)
config = Config(opts['config'])
logging.info("archive indexer starting")
pool = None
if opts['num_threads'] > 1:
logging.debug("Using %s processes", opts['num_threads'])
pool = Pool(opts['num_threads'])
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename,
str(resource['resource_id'])):
if not resource.get('pcp_log_dir'):
continue
acache = PcpArchiveProcessor(resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'],
opts['all'])
if pool is not None:
index_resource_multiprocessing(config, resource, acache, afind,
pool, keep_csv)
else:
fast_index_allowed = bool(resource.get("fast_index", False))
with LoadFileIndexUpdater(config, resource, keep_csv) as index:
for archivefile, fast_index, hostname in afind.find(
resource['pcp_log_dir']):
start_time = time.time()
data = acache.processarchive(
archivefile, fast_index and fast_index_allowed,
hostname)
parse_end = time.time()
if data is not None:
index.insert(*data)
db_end = time.time()
logging.debug(
"processed archive %s (fileio %s, dbacins %s)",
archivefile, parse_end - start_time,
db_end - parse_end)
logging.info("archive indexer complete")
if pool is not None:
pool.close()
pool.join()
def updateMysqlTables(display, opts):
""" Interactive mysql script execution """
config = Config()
dbsettings = config.getsection("datawarehouse")
checkForPreviousInstall(display, dbsettings)
migration = pkg_resources.resource_filename(
__name__, "migrations/1.0-1.1/modw_supremm.sql")
host = dbsettings['host']
port = dbsettings['port'] if 'port' in dbsettings else 3306
display.newpage("MySQL Database setup")
myrootuser = display.prompt_string("DB Admin Username", "root")
myrootpass = display.prompt_password("DB Admin Password")
pflag = "-p{0}".format(myrootpass) if myrootpass != "" else ""
shellcmd = "mysql -u {0} {1} -h {2} -P {3} < {4}".format(
myrootuser, pflag, host, port, migration)
try:
if opts.debug:
display.print_text(shellcmd)
retval = subprocess.call(shellcmd, shell=True)
if retval != 0:
display.print_warning("""
An error occurred migrating the tables. Please create the tables manually
following the documentation in the install guide.
""")
else:
display.print_text("Sucessfully migrated tables")
except OSError as e:
display.print_warning("""
An error:
\"{0}\"
occurred running the mysql command. Please create the tables manually
following the documentation in the install guide.
""".format(e.strerror))
display.hitanykey("Press ENTER to continue.")
def main():
""" print out config data according to cmdline args """
opts = getoptions()
conf = Config()
try:
section = conf.getsection(opts['section'])
if opts['item'] != None:
print section[opts['item']]
else:
print json.dumps(section, indent=4)
except KeyError:
sys.stderr.write(
"Error section \"%s\" not defined in configuration file.\n" %
(opts['section']))
sys.exit(1)
def generatetempconfig(confdata, mycnf):
""" Generate a configuration object based on the config settings """
confpath = tempfile.mkdtemp()
tmpconfdata = copy.deepcopy(confdata)
if mycnf != None:
mycnfpath = os.path.join(confpath, "my.cnf")
tmpconfdata['datawarehouse']['defaultsfile'] = mycnfpath
with open(mycnfpath, "w") as tmpmycnf:
tmpmycnf.write(mycnf)
with open(os.path.join(confpath, "config.json"), "w") as tmpconfig:
json.dump(tmpconfdata, tmpconfig, indent=4)
config = Config(confpath)
return config
def main():
"""
main entry point for script
"""
opts = getoptions(False)
setuplogger(opts['log'])
config = Config()
threads = opts['threads']
process_pool = mp.Pool(threads) if threads > 1 else None
processjobs(config, opts, process_pool)
if process_pool is not None:
# wait for all processes to finish
process_pool.close()
process_pool.join()
def promptconfig(display):
""" prompt user for configuration path """
config = None
while config == None:
confpath = display.prompt_string("Enter path to configuration files", Config.autodetectconfpath())
try:
config = Config(confpath)
config.getsection('datawarehouse')
config.getsection('outputdatabase')
except Exception as e:
errmsg = """
Error unable to read valid configuration file in directory
{0}
Please enter a valid config path (or ctrl-c to exit this program)
"""
display.newpage()
display.print_text(errmsg.format(confpath))
config = None
return config
def main():
"""
main entry point for script
"""
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
config = Config(opts['config'])
dwconfig = config.getsection("datawarehouse")
for resourcename, settings in config.resourceconfigs():
if opts['resource'] in (None, resourcename,
str(settings['resource_id'])):
logging.debug("Processing %s (id=%s)", resourcename,
settings['resource_id'])
respath, timestamp_mode = parse_resource_config(settings)
if respath:
dbif = DbHelper(dwconfig, "modw_supremm", timestamp_mode)
total = processfor(settings['resource_id'], respath, dbif,
opts['deltadays'])
dbif.postinsert()
logging.info("Processed %s files for %s", total, resourcename)
else:
logging.debug("Skip resource %s no script dir defined",
resourcename)
def runindexing():
""" main script entry point """
opts = getoptions()
setuplogger(opts['log'], opts['debugfile'], logging.DEBUG)
config = Config(opts['config'])
logging.info("archive indexer starting")
for resourcename, resource in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(resource['resource_id'])):
acache = PcpArchiveProcessor(config, resource)
afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'])
for archivefile in afind.find(resource['pcp_log_dir']):
acache.processarchive(archivefile)
acache.close()
logging.info("archive indexer complete")
def main():
"""
main entry point for script
"""
opts, args = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
preprocs = loadpreprocessors()
plugins = loadplugins()
if len(opts['plugin_whitelist']) > 0:
preprocs, plugins = filter_plugins(
{"plugin_whitelist": opts['plugin_whitelist']}, preprocs, plugins)
elif len(opts['plugin_blacklist']) > 0:
preprocs, plugins = filter_plugins(
{"plugin_blacklist": opts['plugin_blacklist']}, preprocs, plugins)
logging.debug("Loaded %s preprocessors", len(preprocs))
logging.debug("Loaded %s plugins", len(plugins))
archivelist = args
job = MockJob(archivelist)
config = Config(confpath=opts['config'])
preprocessors = [x(job) for x in preprocs]
analytics = [x(job) for x in plugins]
s = Summarize(preprocessors, analytics, job, config)
s.process()
result = s.get()
print json.dumps(result, indent=4)
def main():
"""
main entry point for script
"""
opts = getoptions(False)
setuplogger(opts['log'])
config = Config()
threads = opts['threads']
if threads <= 1:
processjobs(config, opts, None)
return
else:
proclist = []
for procid in xrange(threads):
p = Process(target=processjobs, args=(config, opts, procid))
p.start()
proclist.append(p)
for proc in proclist:
p.join()
