def default_settings(confpath):
""" populate the default settings for the configuration.
will use the values specifed in the configuration file if it exists.
"""
defaults = {}
defaults['usexdmodconfig'] = 'y'
defaults['xdmodpath'] = '/etc/xdmod'
defaults['archiveoutdir'] = '/dev/shm/supremm'
defaults['mysqlhostname'] = 'localhost'
defaults['mysqlport'] = 3306
defaults['mysqlusername'] = '******'
defaults['mycnffilename'] = '~/.supremm.my.cnf'
defaults['mongouri'] = 'mongodb://localhost:27017/supremm'
defaults['mongodb'] = 'supremm'
try:
existingconf = Config(confpath)
rawconfig = existingconf._config.copy()
if 'xdmodroot' in rawconfig and 'datawarehouse' in rawconfig and 'include' in rawconfig['datawarehouse']:
defaults['usexdmodconfig'] = 'y'
defaults['xdmodpath'] = rawconfig['xdmodroot']
else:
dwconfig = existingconf.getsection('datawarehouse')
defaults['usexdmodconfig'] = 'n'
defaults['mysqlhostname'] = dwconfig['host']
defaults['mysqlport'] = dwconfig.get('port', defaults['mysqlport'])
defaults['mycnffilename'] = dwconfig['defaultsfile']
try:
mycnf = ConfigParser.RawConfigParser()
mycnf.read(os.path.expanduser(dwconfig['defaultsfile']))
if mycnf.has_section('client'):
defaults['mysqlusername'] = mycnf.get('client', 'user')
except ConfigParser.Error:
pass
outputconfig = existingconf.getsection('outputdatabase')
defaults['mongouri'] = outputconfig['uri']
defaults['mongodb'] = outputconfig['dbname']
summarycnf = existingconf.getsection('summary')
defaults['archiveoutdir'] = summarycnf['archive_out_dir']
defaults['resources'] = rawconfig['resources']
except Exception as e:
# ignore missing or broken existing config files.
pass
return defaults
def main():
"""
main entry point for script
"""
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
config = Config(opts['config'])
dwconfig = config.getsection("datawarehouse")
dbif = DbHelper(dwconfig, 'modw_supremm.batchscripts')
for resourcename, settings in config.resourceconfigs():
if opts['resource'] in (None, resourcename, str(settings['resource_id'])):
logging.debug("Processing %s (id=%s)", resourcename, settings['resource_id'])
if "script_dir" in settings:
total = processfor(settings['resource_id'], settings['script_dir'], dbif, opts['deltadays'])
logging.info("Processed %s files for %s", total, resourcename)
else:
logging.debug("Skip resource %s no script dir defined", resourcename)
dbif.postinsert()
def promptconfig(display):
""" prompt user for configuration path """
config = None
while config == None:
confpath = display.prompt_string("Enter path to configuration files", Config.autodetectconfpath())
try:
config = Config(confpath)
config.getsection('datawarehouse')
config.getsection('outputdatabase')
except Exception as e:
errmsg = """
Error unable to read valid configuration file in directory
{0}
Please enter a valid config path (or ctrl-c to exit this program)
"""
display.newpage()
display.print_text(errmsg.format(confpath))
config = None
return config
def main():
""" print out config data according to cmdline args """
opts = getoptions()
conf = Config()
try:
section = conf.getsection(opts['section'])
if opts['item'] != None:
print section[opts['item']]
else:
print json.dumps(section, indent=4)
except KeyError:
sys.stderr.write("Error section \"%s\" not defined in configuration file.\n" % (opts['section']))
sys.exit(1)
def updateMysqlTables(display, opts):
""" Interactive mysql script execution """
config = Config()
dbsettings = config.getsection("datawarehouse")
checkForPreviousInstall(display, dbsettings)
migration = pkg_resources.resource_filename(
__name__, "migrations/1.0-1.1/modw_supremm.sql")
host = dbsettings['host']
port = dbsettings['port'] if 'port' in dbsettings else 3306
display.newpage("MySQL Database setup")
myrootuser = display.prompt_string("DB Admin Username", "root")
myrootpass = display.prompt_password("DB Admin Password")
pflag = "-p{0}".format(myrootpass) if myrootpass != "" else ""
shellcmd = "mysql -u {0} {1} -h {2} -P {3} < {4}".format(
myrootuser, pflag, host, port, migration)
try:
if opts.debug:
display.print_text(shellcmd)
retval = subprocess.call(shellcmd, shell=True)
if retval != 0:
display.print_warning("""
An error occurred migrating the tables. Please create the tables manually
following the documentation in the install guide.
""")
else:
display.print_text("Sucessfully migrated tables")
except OSError as e:
display.print_warning("""
An error:
\"{0}\"
occurred running the mysql command. Please create the tables manually
following the documentation in the install guide.
""".format(e.strerror))
display.hitanykey("Press ENTER to continue.")
def main():
""" print out config data according to cmdline args """
opts = getoptions()
conf = Config()
try:
section = conf.getsection(opts['section'])
if opts['item'] != None:
print section[opts['item']]
else:
print json.dumps(section, indent=4)
except KeyError:
sys.stderr.write(
"Error section \"%s\" not defined in configuration file.\n" %
(opts['section']))
sys.exit(1)
def main():
"""
main entry point for script
"""
opts = getoptions()
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S',
level=opts['log'])
if sys.version.startswith("2.7"):
logging.captureWarnings(True)
config = Config(opts['config'])
dwconfig = config.getsection("datawarehouse")
for resourcename, settings in config.resourceconfigs():
if opts['resource'] in (None, resourcename,
str(settings['resource_id'])):
logging.debug("Processing %s (id=%s)", resourcename,
settings['resource_id'])
respath, timestamp_mode = parse_resource_config(settings)
if respath:
dbif = DbHelper(dwconfig, "modw_supremm", timestamp_mode)
total = processfor(settings['resource_id'], respath, dbif,
opts['deltadays'])
dbif.postinsert()
logging.info("Processed %s files for %s", total, resourcename)
else:
logging.debug("Skip resource %s no script dir defined",
resourcename)
