def test_value_is_callable(): # pylint: disable=W0612
logger = Logger("test", handler=handler, template="{foo}: {message}")
func = lambda: "yay!"
logger.info("from lambda", foo=func)
output = get_output()
assert output == "yay!: from lambda\n"
def method_level_gte_logger_level(): # pylint: disable=W0612
logger = Logger("test", level=LogLevel.INFO, handler=handler)
logger.info("this should print")
output = get_output()
assert re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6} INFO test: this should print\n",
output,
)
def test_default_provided(): # pylint: disable=W0612
logger = Logger("test",
handler=handler,
template="{foo}: {message}",
foo="bar")
logger.info("ok")
output = get_output()
assert output == "bar: ok\n"
def test_exception_capturing():
logger = Logger("test",
handler=handler,
template="{level} {name}: {message}")
try:
1 / 0
except ZeroDivisionError:
logger.exception("caught")
output = get_output()
assert re.match(
r"ERROR test: caught\nTraceback \(most recent call last\):\n File .+\n 1 / 0\nZeroDivisionError: division by zero",
output,
)
def setup_logging(self):
log_writer = None
log_writer_class = self.ini.get("DEFAULT", "logger_class", None)
if log_writer_class:
if log_writer_class == "SyslogWriter":
facility = self.ini.get("DEFAULT", "syslog_facility", None)
priority = self.ini.get("DEFAULT", "syslog_priority", None)
log_writer = SyslogWriter(facility=facility, priority=priority)
elif log_writer_class == "ConsoleWriter":
output = self.ini.get("DEFAULT", "console_output", "stdout")
log_writer = ConsoleWriter(output=output)
else:
log_writer = FileWriter(self.home_dir)
else:
#log_writer = FileWriter(self.home_dir)
log_writer = FileWriter('/var/log/glideinwms-pilot')
self.log = Logger(log_writer)
self.log.log_info("Pilot Launcher started...")
def setup_logging(self):
log_writer = None
log_writer_class = self.ini.get("DEFAULTS", "logger_class", None)
if log_writer_class:
if log_writer_class == "SyslogWriter":
facility = self.ini.get("DEFAULTS", "syslog_facility", None)
priority = self.ini.get("DEFAULTS", "syslog_priority", None)
log_writer = SyslogWriter(facility=facility, priority=priority)
elif log_writer_class == "ConsoleWriter":
output = self.ini.get("DEFAULTS", "console_output", "stdout")
log_writer = ConsoleWriter(output=output)
else:
log_writer = FileWriter(self.home_dir)
else:
log_writer = FileWriter(self.home_dir)
self.log = Logger(log_writer)
self.log.log_info("Pilot Launcher started...")
if line.strip() != "":
c.execute("INSERT INTO names VALUES(?);", [line.strip()])
count += 1
log.debug("Added {} names.".format(count))
rows = c.execute("SELECT * FROM names;")
found_count = 0
for row in rows:
found_count += 1
log.debug("Found {} names.".format(found_count))
db.commit()
c.close()
log.debug("All changes committed.")
if count != found_count:
log.warn(
"Looks like some names have not gotten added. Perhaps duplicates? This should be investigated.")
log.success("Database created and set up successfully!")
return True
if __name__ == '__main__':
log = Logger('db_setup', log_folder=Path(LOG_FOLDER).resolve(),
log_level='info', output_level='debug')
success = setup_db(log)
if success:
log.debug("Success!")
else:
log.error("Could not setup database!")
class Config(object):
"""
Base class for the handling configuration
Derived class needs to implement contextualization logic
"""
def __init__(self, config_ini="/etc/glideinwms/glidein-pilot.ini"):
if not os.path.exists(config_ini):
raise ConfigError("%s does not exist" % config_ini)
self.ini = ini_handler.Ini(config_ini)
self.contextualization_type = self.ini.get("DEFAULT",
"contextualize_protocol")
if self.contextualization_type is not None:
self.contextualization_type = self.contextualization_type.upper()
self.default_max_lifetime = self.ini.get("DEFAULT",
"default_max_lifetime",
"172800") # 48 hours
self.max_lifetime = self.default_max_lifetime
self.disable_shutdown = self.ini.getBoolean("DEFAULT",
"disable_shutdown", False)
self.max_script_runtime = self.ini.get("DEFAULT", "max_script_runtime",
"60")
self.pre_script_dir = self.ini.get(
"DIRECTORIES", "pre_script_dir",
"/usr/libexec/glideinwms_pilot/PRE")
self.post_script_dir = self.ini.get(
"DIRECTORIES", "post_script_dir",
"/usr/libexec/glideinwms_pilot/POST")
self.glidein_user = "******"
# home directory is created by the rpm
self.home_dir = os.path.expanduser('~%s' % self.glidein_user)
self.scratch_dir = "/home/scratchgwms"
# glidein_startup.sh specific attributes
self.factory_url = ""
self.pilot_args = ""
self.proxy_file = ""
self.pilot_args = ""
def setup(self):
self.setup_logging()
self.setup_pilot_files()
self.setup_contextualization()
def setup_pilot_files(self):
self.ini_file = "%s/glidein_userdata" % self.home_dir
self.userdata_file = "%s/userdata_file" % self.home_dir
self.log.log_info("Default ini file: %s" % self.ini_file)
self.log.log_info("Default userdata file: %s" % self.userdata_file)
def validate_contextualization(self):
self.log.log_info("Contextualization Type identified as: %s" %
self.contextualization_type)
if not is_context_valid(self.contextualization_type):
raise ConfigError("context_type %s not in the supported list %s" %
(self.contextualization_type, valid_contexts()))
def setup_contextualization(self):
raise NotImplementedError(
'Implementation for contextualize_protocol %s not available' %
self.contextualization_type)
def setup_logging(self):
log_writer = None
log_writer_class = self.ini.get("DEFAULT", "logger_class", None)
if log_writer_class:
if log_writer_class == "SyslogWriter":
facility = self.ini.get("DEFAULT", "syslog_facility", None)
priority = self.ini.get("DEFAULT", "syslog_priority", None)
log_writer = SyslogWriter(facility=facility, priority=priority)
elif log_writer_class == "ConsoleWriter":
output = self.ini.get("DEFAULT", "console_output", "stdout")
log_writer = ConsoleWriter(output=output)
else:
log_writer = FileWriter(self.home_dir)
else:
#log_writer = FileWriter(self.home_dir)
log_writer = FileWriter('/var/log/glideinwms-pilot')
self.log = Logger(log_writer)
self.log.log_info("Pilot Launcher started...")
def export_custom_env(self):
"""
@returns: string containing the shell (sh, bash, etc) directives to
export the environment variables
"""
environment = ""
try:
env = self.get_custom_env()
for option in env:
environment += "export %s=%s; " % (option, env[option])
except:
# pylint: disable=W0702
pass
return environment
def get_custom_env(self):
"""
Returns a dictionary of the parent process environment plus the custom
environment variables defined in the pilot config file.
NOTE: All custom environment variable names will be upper cased. The
values for the custom environment variables will not be modified.
@returns: dictionary containing the desired process environment
"""
environment = {}
# inherit the parent process environment
for var in os.environ.keys():
environment[var] = os.environ[var]
try:
# add in the custom environment
for option in self.ini.cp.options("CUSTOM_ENVIRONMENT"):
environment[str(option).upper()] = self.ini.get(
"CUSTOM_ENVIRONMENT", option)
except:
# pylint: disable=W0702
pass
# Add in the pilot proxy
environment["X509_USER_PROXY"] = self.proxy_file
#environment["HOME"] = self.home_dir
#environment["LOGNAME"] = self.glidein_user
environment["SCRATCH"] = self.scratch_dir
return environment
def test_value_not_provided(): # pylint: disable=W0612
logger = Logger("test", handler=handler, template="{foo}: {message}")
logger.info("not provided")
output = get_output()
assert output == "\b: not provided\n"
def test_value_provided_in_log_call(): # pylint: disable=W0612
logger = Logger("test", handler=handler, template="{foo}: {message}")
logger.info("ok", foo="baz")
output = get_output()
assert output == "baz: ok\n"
def method_level_lt_logger_level(): # pylint: disable=W0612
logger = Logger("test", level=LogLevel.INFO, handler=handler)
logger.debug("this shouldn't print")
output = get_output()
assert output == ""
class Config(object):
valid_context_types = [CONTEXT_TYPE_EC2, CONTEXT_TYPE_NIMBUS, CONTEXT_TYPE_OPENNEBULA]
def __init__(self, config_ini="/etc/glideinwms/glidein-pilot.ini"):
if not os.path.exists(config_ini):
raise ConfigError("%s does not exist" % config_ini)
self.ini = ini_handler.Ini(config_ini)
self.default_max_lifetime = self.ini.get("DEFAULT", "default_max_lifetime", "172800") # 48 hours
self.max_lifetime = self.default_max_lifetime # can be overridden
self.disable_shutdown = self.ini.getBoolean("DEFAULT", "disable_shutdown", False)
self.max_script_runtime = self.ini.get("DEFAULT", "max_script_runtime", "60")
self.pre_script_dir = self.ini.get("DIRECTORIES", "pre_script_dir", "/usr/libexec/glideinwms_pilot/PRE")
self.post_script_dir = self.ini.get("DIRECTORIES", "post_script_dir", "/usr/libexec/glideinwms_pilot/POST")
# home directory is created by the rpm
self.home_dir = "/home/glidein_pilot"
self.glidein_user = "******"
self.scratch_dir = "/home/scratchgwms"
# glidein_startup.sh specific attributes
self.factory_url = ""
self.pilot_args = ""
self.proxy_file = ""
self.pilot_args = ""
def setup(self):
self.setup_logging()
self.setup_pilot_files()
self.setup_contextualization()
def setup_pilot_files(self):
self.ini_file = "%s/glidein_userdata" % self.home_dir
self.userdata_file = "%s/userdata_file" % self.home_dir
self.log.log_info("Default ini file: %s" % self.ini_file)
self.log.log_info("Default userdata file: %s" % self.userdata_file)
def setup_contextualization(self):
self.contextualization_type = self.ini.get("DEFAULT", "contextualize_protocol")
self.log.log_info("Contextualization Type identified as: %s" % self.contextualization_type)
if self.contextualization_type in Config.valid_context_types:
if self.contextualization_type == CONTEXT_TYPE_EC2:
self.ec2_url = self.ini.get("DEFAULT", "ec2_url")
elif self.contextualization_type == CONTEXT_TYPE_NIMBUS:
self.nimbus_url_file = self.ini.get("DEFAULT", "nimbus_url_file")
elif self.contextualization_type == CONTEXT_TYPE_OPENNEBULA:
self.one_user_data_file = self.ini.get("DEFAULT", "one_user_data_file")
else:
raise ConfigError("configured context type not valid")
def setup_logging(self):
log_writer = None
log_writer_class = self.ini.get("DEFAULT", "logger_class", None)
if log_writer_class:
if log_writer_class == "SyslogWriter":
facility = self.ini.get("DEFAULT", "syslog_facility", None)
priority = self.ini.get("DEFAULT", "syslog_priority", None)
log_writer = SyslogWriter(facility=facility, priority=priority)
elif log_writer_class == "ConsoleWriter":
output = self.ini.get("DEFAULT", "console_output", "stdout")
log_writer = ConsoleWriter(output=output)
else:
log_writer = FileWriter(self.home_dir)
else:
#log_writer = FileWriter(self.home_dir)
log_writer = FileWriter('/var/log/glideinwms-pilot')
self.log = Logger(log_writer)
self.log.log_info("Pilot Launcher started...")
def export_custom_env(self):
"""
@returns: string containing the shell (sh, bash, etc) directives to
export the environment variables
"""
environment = ""
try:
env = self.get_custom_env()
for option in env:
environment += "export %s=%s; " % (option, env[option])
except:
# pylint: disable=W0702
pass
return environment
def get_custom_env(self):
"""
Returns a dictionary of the parent process environment plus the custom
environment variables defined in the pilot config file.
NOTE: All custom environment variable names will be upper cased. The
values for the custom environment variables will not be modified.
@returns: dictionary containing the desired process environment
"""
environment = {}
# inherit the parent process environment
for var in os.environ.keys():
environment[var] = os.environ[var]
try:
# add in the custom environment
for option in self.ini.cp.options("CUSTOM_ENVIRONMENT"):
environment[str(option).upper()] = self.ini.get("CUSTOM_ENVIRONMENT", option)
except:
# pylint: disable=W0702
pass
# Add in the pilot proxy
environment["X509_USER_PROXY"] = self.proxy_file
#environment["HOME"] = self.home_dir
#environment["LOGNAME"] = self.glidein_user
environment["SCRATCH"] = self.scratch_dir
return environment
from pathlib import Path
import sys, os
import traceback
import psycopg2
import discord
from discord.ext import commands
from discord.ext.commands import has_role
import discord.ext.commands.errors as discord_errors
import typing
from simple_logging import Logger
from db_setup import setup_db
from config import *
log = Logger('ug_bot', log_folder=Path(LOG_FOLDER).resolve(),
log_level='debug', output_level='debug')
log.debug("Loaded all modules.")
log.info("Starting the bot...")
if not setup_db(log):
log.error("Unable to setup the database.")
sys.exit(1)
DB_URL = os.environ['DATABASE_URL']
db = psycopg2.connect(DB_URL)
log.debug("Connected to database.")
TOKEN = os.environ['DISCORD_TOKEN']
log.info("Token:", TOKEN)
class Config(object):
valid_context_types = ["EC2", "NIMBUS", "OPENNEBULA"]
def __init__(self, config_ini="/etc/glideinwms/glidein-pilot.ini"):
if not os.path.exists(config_ini):
raise ConfigError("%s does not exist" % config_ini)
self.cp = ConfigParser.ConfigParser()
self.cp.read(config_ini)
self.default_max_lifetime = self.cp_get("DEFAULTS", "default_max_lifetime", "172800") # 48 hours
self.max_lifetime = self.default_max_lifetime # can be overridden
self.disable_shutdown = self.cp_getBoolean("DEFAULTS", "disable_shutdown", False)
# glidein_startup.sh specific attributes
self.factory_url = ""
self.pilot_args = ""
self.proxy_file = ""
self.pilot_args = ""
def setup(self):
# must call setup_user to get the home directory for logging
self.setup_user()
self.setup_logging()
self.setup_pilot_files()
self.setup_contextualization()
def setup_pilot_files(self):
self.ini_file = "%s/glidein_userdata" % self.home_dir
self.userdata_file = "%s/userdata_file" % self.home_dir
def setup_contextualization(self):
self.contextualization_type = self.cp_get("DEFAULTS", "contextualize_protocol")
if self.contextualization_type in Config.valid_context_types:
if self.contextualization_type == "EC2":
self.ec2_url = self.cp_get("DEFAULTS", "ec2_url")
elif self.contextualization_type == "NIMBUS":
self.nimbus_url_file = self.cp_get("DEFAULTS", "nimbus_url_file")
elif self.contextualization_type == "OPENNEBULA":
self.one_user_data_file = self.cp_get("DEFAULTS", "one_user_data_file")
else:
raise ConfigError("configured context type not valid")
def setup_user(self):
self.glidein_user = self.cp_get("GLIDEIN_USER", "user_name")
self.user_ids = self.cp_get("GLIDEIN_USER", "user_ids")
# The home directory may or may not exist yet, if not, it will be created
self.home_dir = self.cp_get("GLIDEIN_USER", "user_home")
self.validate_user(self.glidein_user, self.user_ids)
self.make_directories()
def setup_logging(self):
log_writer = None
log_writer_class = self.cp_get("DEFAULTS", "logger_class", None)
if log_writer_class:
if log_writer_class == "SyslogWriter":
facility = self.cp_get("DEFAULTS", "syslog_facility", None)
priority = self.cp_get("DEFAULTS", "syslog_priority", None)
log_writer = SyslogWriter(facility=facility, priority=priority)
elif log_writer_class == "ConsoleWriter":
output = self.cp_get("DEFAULTS", "console_output", "stdout")
log_writer = ConsoleWriter(output=output)
else:
log_writer = FileWriter(self.home_dir)
else:
log_writer = FileWriter(self.home_dir)
self.log = Logger(log_writer)
self.log.log_info("Pilot Launcher started...")
def export_grid_env(self):
environment = ""
try:
for option in self.cp.options("GRID_ENV"):
environment += "export %s=%s; " % (str(option).upper(), self.cp.get("GRID_ENV", option))
# I don't believe it is necessary to run the pilot as specific UID/GID anymore
#environment += "export GLIDEIN_Condor_IDS=%s;" % self.user_ids
environment += "export X509_USER_PROXY=%s;" % self.proxy_file
except:
# pylint: disable=W0702
pass
return environment
def get_grid_env(self):
environment = {}
try:
# inherit the parent process environment
for var in os.environ.keys():
environment[var] = os.environ[var]
# add in the "grid" specific environment
for option in self.cp.options("GRID_ENV"):
environment[str(option).upper()] = self.cp.get("GRID_ENV", option)
# I don't believe it is necessary to run the pilot as specific UID/GID anymore
#environment["GLIDEIN_Condor_IDS"]= self.user_ids
environment["X509_USER_PROXY"] = self.proxy_file
except:
# pylint: disable=W0702
pass
return environment
def make_directories(self):
# Make GLIDEIN_HOME
mkdir_p(self.home_dir)
chown(self.user_ids, self.home_dir)
try:
for option in self.cp.options("DIRECTORIES"):
directory = self.cp.get("DIRECTORIES", option)
mkdir_p(directory)
chown(self.user_ids, directory)
except:
# pylint: disable=W0702
pass
def validate_user(self, username, user_ids):
try:
pwd_tuple = pwd.getpwnam(username)
pw_uid = pwd_tuple[2]
pw_gid = pwd_tuple[3]
ids = user_ids.split(".")
if not int(ids[0]) == int(pw_uid):
msg = "User specified in configuration present on system, but "\
"the uids don't match. (system uid: %s, configured uid: "\
"%s" % (str(pw_uid), str(ids[0]))
raise ConfigError(msg)
if not int(ids[1]) == int(pw_gid):
msg = "User specified in configuration present on system, but "\
"the gids don't match. (system gid: %s, configured gid: "\
"%s" % (str(pw_gid), str(ids[1]))
raise ConfigError(msg)
except:
raise ConfigError("User specified in configuration not on system")
def cp_get(self, section, option, default=""):
"""
Helper function for ConfigParser objects which allows setting the default.
ConfigParser objects throw an exception if one tries to access an option
which does not exist; this catches the exception and returns the default
value instead.
@param section: Section of config parser to read
@param option: Option in section to retrieve
@param default: Default value if the section/option is not present.
@returns: Value stored in CP for section/option, or default if it is not
present.
"""
if not isinstance(self.cp, ConfigParser.ConfigParser):
raise RuntimeError('cp_get called without a proper cp as first arg')
try:
return self.cp.get(section, option)
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
return default
def cp_getBoolean(self, section, option, default=True):
"""
Helper function for ConfigParser objects which allows setting the default.
If the cp object has a section/option of the proper name, and if that value
has a 'y' or 't', we assume it's supposed to be true. Otherwise, if it
contains a 'n' or 'f', we assume it's supposed to be true.
If neither applies - or the option doesn't exist, return the default
@param section: Section of config parser to read
@param option: Option in section to retrieve
@param default: Default value if the section/option is not present.
@returns: Value stored in CP for section/option, or default if it is not
present.
"""
val = str(self.cp_get(section, option, default)).lower()
if val.find('t') >= 0 or val.find('y') >= 0 or val.find('1') >= 0:
return True
if val.find('f') >= 0 or val.find('n') >= 0 or val.find('0') >= 0:
return False
return default