def runingest(): if len(sys.argv) > 1: config = Config(sys.argv[1]) else: config = Config() if len(sys.argv) > 2: end_time = sys.argv[2] else: end_time = 9223372036854775807L #ingest(config, end_time) ingestall(config)
def main(): """ main entry point for script """ comm = MPI.COMM_WORLD opts = getoptions(True) opts['threads'] = comm.Get_size() logout = "mpiOutput-{}.log".format(comm.Get_rank()) # For MPI jobs, do something sane with logging. setuplogger(logging.ERROR, logout, opts['log']) config = Config() if comm.Get_size() < 2: logging.error("Must run MPI job with at least 2 processes") sys.exit(1) myhost = MPI.Get_processor_name() logging.info("Nodename: %s", myhost) processjobs(config, opts, comm.Get_rank(), comm) logging.info("Rank: %s FINISHED", comm.Get_rank())
def test(): """ simple test function """ config = Config() xdm = XDMoDAcct(13, config, None, None) for job in xdm.get(1444151688, None): print job
def main(): """ main entry point for script """ opts = getoptions() logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log']) if sys.version.startswith("2.7"): logging.captureWarnings(True) config = Config() threads = opts['threads'] if threads <= 1: processjobs(config, opts, None) return else: proclist = [] for procid in xrange(threads): p = Process(target=processjobs, args=(config, opts, procid)) p.start() proclist.append(p) for proc in proclist: p.join()
def default_settings(confpath): """ populate the default settings for the configuration. will use the values specifed in the configuration file if it exists. """ defaults = {} defaults['usexdmodconfig'] = 'y' defaults['xdmodpath'] = '/etc/xdmod' defaults['archiveoutdir'] = '/dev/shm/supremm' defaults['mysqlhostname'] = 'localhost' defaults['mysqlport'] = 3306 defaults['mysqlusername'] = '******' defaults['mycnffilename'] = '~/.supremm.my.cnf' defaults['mongouri'] = 'mongodb://localhost:27017/supremm' defaults['mongodb'] = 'supremm' try: existingconf = Config(confpath) rawconfig = existingconf._config.copy() if 'xdmodroot' in rawconfig and 'datawarehouse' in rawconfig and 'include' in rawconfig['datawarehouse']: defaults['usexdmodconfig'] = 'y' defaults['xdmodpath'] = rawconfig['xdmodroot'] else: dwconfig = existingconf.getsection('datawarehouse') defaults['usexdmodconfig'] = 'n' defaults['mysqlhostname'] = dwconfig['host'] defaults['mysqlport'] = dwconfig.get('port', defaults['mysqlport']) defaults['mycnffilename'] = dwconfig['defaultsfile'] try: mycnf = ConfigParser.RawConfigParser() mycnf.read(os.path.expanduser(dwconfig['defaultsfile'])) if mycnf.has_section('client'): defaults['mysqlusername'] = mycnf.get('client', 'user') except ConfigParser.Error: pass outputconfig = existingconf.getsection('outputdatabase') defaults['mongouri'] = outputconfig['uri'] defaults['mongodb'] = outputconfig['dbname'] summarycnf = existingconf.getsection('summary') defaults['archiveoutdir'] = summarycnf['archive_out_dir'] defaults['resources'] = rawconfig['resources'] except Exception as e: # ignore missing or broken existing config files. pass return defaults
def runindexing(): """ main script entry point """ opts = getoptions() keep_csv = opts["keep_csv"] setuplogger(opts['log'], opts['debugfile'], filelevel=logging.INFO) config = Config(opts['config']) logging.info("archive indexer starting") pool = None if opts['num_threads'] > 1: logging.debug("Using %s processes", opts['num_threads']) pool = Pool(opts['num_threads']) for resourcename, resource in config.resourceconfigs(): if opts['resource'] in (None, resourcename, str(resource['resource_id'])): if not resource.get('pcp_log_dir'): continue acache = PcpArchiveProcessor(resource) afind = PcpArchiveFinder(opts['mindate'], opts['maxdate'], opts['all']) if pool is not None: index_resource_multiprocessing(config, resource, acache, afind, pool, keep_csv) else: fast_index_allowed = bool(resource.get("fast_index", False)) with LoadFileIndexUpdater(config, resource, keep_csv) as index: for archivefile, fast_index, hostname in afind.find( resource['pcp_log_dir']): start_time = time.time() data = acache.processarchive( archivefile, fast_index and fast_index_allowed, hostname) parse_end = time.time() if data is not None: index.insert(*data) db_end = time.time() logging.debug( "processed archive %s (fileio %s, dbacins %s)", archivefile, parse_end - start_time, db_end - parse_end) logging.info("archive indexer complete") if pool is not None: pool.close() pool.join()
def updateMysqlTables(display, opts): """ Interactive mysql script execution """ config = Config() dbsettings = config.getsection("datawarehouse") checkForPreviousInstall(display, dbsettings) migration = pkg_resources.resource_filename( __name__, "migrations/1.0-1.1/modw_supremm.sql") host = dbsettings['host'] port = dbsettings['port'] if 'port' in dbsettings else 3306 display.newpage("MySQL Database setup") myrootuser = display.prompt_string("DB Admin Username", "root") myrootpass = display.prompt_password("DB Admin Password") pflag = "-p{0}".format(myrootpass) if myrootpass != "" else "" shellcmd = "mysql -u {0} {1} -h {2} -P {3} < {4}".format( myrootuser, pflag, host, port, migration) try: if opts.debug: display.print_text(shellcmd) retval = subprocess.call(shellcmd, shell=True) if retval != 0: display.print_warning(""" An error occurred migrating the tables. Please create the tables manually following the documentation in the install guide. """) else: display.print_text("Sucessfully migrated tables") except OSError as e: display.print_warning(""" An error: \"{0}\" occurred running the mysql command. Please create the tables manually following the documentation in the install guide. """.format(e.strerror)) display.hitanykey("Press ENTER to continue.")
def main(): """ print out config data according to cmdline args """ opts = getoptions() conf = Config() try: section = conf.getsection(opts['section']) if opts['item'] != None: print section[opts['item']] else: print json.dumps(section, indent=4) except KeyError: sys.stderr.write( "Error section \"%s\" not defined in configuration file.\n" % (opts['section'])) sys.exit(1)
def generatetempconfig(confdata, mycnf): """ Generate a configuration object based on the config settings """ confpath = tempfile.mkdtemp() tmpconfdata = copy.deepcopy(confdata) if mycnf != None: mycnfpath = os.path.join(confpath, "my.cnf") tmpconfdata['datawarehouse']['defaultsfile'] = mycnfpath with open(mycnfpath, "w") as tmpmycnf: tmpmycnf.write(mycnf) with open(os.path.join(confpath, "config.json"), "w") as tmpconfig: json.dump(tmpconfdata, tmpconfig, indent=4) config = Config(confpath) return config
def main(): """ main entry point for script """ opts = getoptions(False) setuplogger(opts['log']) config = Config() threads = opts['threads'] process_pool = mp.Pool(threads) if threads > 1 else None processjobs(config, opts, process_pool) if process_pool is not None: # wait for all processes to finish process_pool.close() process_pool.join()
def promptconfig(display): """ prompt user for configuration path """ config = None while config == None: confpath = display.prompt_string("Enter path to configuration files", Config.autodetectconfpath()) try: config = Config(confpath) config.getsection('datawarehouse') config.getsection('outputdatabase') except Exception as e: errmsg = """ Error unable to read valid configuration file in directory {0} Please enter a valid config path (or ctrl-c to exit this program) """ display.newpage() display.print_text(errmsg.format(confpath)) config = None return config
def main(): """ main entry point for script """ opts = getoptions() logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log']) if sys.version.startswith("2.7"): logging.captureWarnings(True) config = Config(opts['config']) dwconfig = config.getsection("datawarehouse") for resourcename, settings in config.resourceconfigs(): if opts['resource'] in (None, resourcename, str(settings['resource_id'])): logging.debug("Processing %s (id=%s)", resourcename, settings['resource_id']) respath, timestamp_mode = parse_resource_config(settings) if respath: dbif = DbHelper(dwconfig, "modw_supremm", timestamp_mode) total = processfor(settings['resource_id'], respath, dbif, opts['deltadays']) dbif.postinsert() logging.info("Processed %s files for %s", total, resourcename) else: logging.debug("Skip resource %s no script dir defined", resourcename)
def runindexing(): """ main script entry point """ opts = getoptions() setuplogger(opts['log'], opts['debugfile'], logging.DEBUG) config = Config(opts['config']) logging.info("archive indexer starting") for resourcename, resource in config.resourceconfigs(): if opts['resource'] in (None, resourcename, str(resource['resource_id'])): acache = PcpArchiveProcessor(config, resource) afind = PcpArchiveFinder(opts['mindate'], opts['maxdate']) for archivefile in afind.find(resource['pcp_log_dir']): acache.processarchive(archivefile) acache.close() logging.info("archive indexer complete")
def main(): """ main entry point for script """ opts, args = getoptions() logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log']) if sys.version.startswith("2.7"): logging.captureWarnings(True) preprocs = loadpreprocessors() plugins = loadplugins() if len(opts['plugin_whitelist']) > 0: preprocs, plugins = filter_plugins( {"plugin_whitelist": opts['plugin_whitelist']}, preprocs, plugins) elif len(opts['plugin_blacklist']) > 0: preprocs, plugins = filter_plugins( {"plugin_blacklist": opts['plugin_blacklist']}, preprocs, plugins) logging.debug("Loaded %s preprocessors", len(preprocs)) logging.debug("Loaded %s plugins", len(plugins)) archivelist = args job = MockJob(archivelist) config = Config(confpath=opts['config']) preprocessors = [x(job) for x in preprocs] analytics = [x(job) for x in plugins] s = Summarize(preprocessors, analytics, job, config) s.process() result = s.get() print json.dumps(result, indent=4)
def main(): """ main entry point for script """ opts = getoptions(False) setuplogger(opts['log']) config = Config() threads = opts['threads'] if threads <= 1: processjobs(config, opts, None) return else: proclist = [] for procid in xrange(threads): p = Process(target=processjobs, args=(config, opts, procid)) p.start() proclist.append(p) for proc in proclist: p.join()