def bootstrap_scheduler(self):
bootstrapped = False
try:
self.save("plugin_modules_library", self._plugin_modules_library)
Scheduler.clear_locks(self)
self.scheduler = Scheduler()
for plugin_info, fn, function_name in self.periodic_tasks:
meta = fn.will_fn_metadata
self.add_periodic_task(
plugin_info["full_module_name"],
plugin_info["name"],
function_name,
meta["sched_args"],
meta["sched_kwargs"],
meta["function_name"],
)
for plugin_info, fn, function_name in self.random_tasks:
meta = fn.will_fn_metadata
self.add_random_tasks(
plugin_info["full_module_name"],
plugin_info["name"],
function_name,
meta["start_hour"],
meta["end_hour"],
meta["day_of_week"],
meta["num_times_per_day"]
)
bootstrapped = True
except Exception as e:
self.startup_error("Error bootstrapping scheduler", e)
if bootstrapped:
show_valid("Scheduler started.")
self.scheduler.start_loop(self)
def bootstrap_generation(self):
self.generation_backends = []
self.generation_threads = []
for b in settings.GENERATION_BACKENDS:
module = import_module(b)
for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
try:
if (
hasattr(cls, "is_will_generationbackend")
and cls.is_will_generationbackend
and class_name != "GenerationBackend"
):
c = cls()
thread = self.create_process(
target=c.start,
args=(b,),
kwargs={"bot": self},
)
thread.start()
self.generation_threads.append(thread)
show_valid("Generation: %s Backend started." % cls.__name__)
except Exception as e:
self.startup_error("Error bootstrapping %s io" % b, e)
self.generation_backends.append(b)
pass
def verify_plugin_settings(self):
puts("Verifying settings requested by plugins...")
missing_settings = False
missing_setting_error_messages = []
with indent(2):
for name, meta in self.required_settings_from_plugins.items():
if not hasattr(settings, name):
error_message = (
"%(setting_name)s is missing. It's required by the"
"%(plugin_name)s plugin's '%(function_name)s' method."
) % meta
puts(colored.red("✗ %(setting_name)s" % meta))
missing_setting_error_messages.append(error_message)
missing_settings = True
else:
show_valid("%(setting_name)s" % meta)
if missing_settings:
puts("")
warn(
"Will is missing settings required by some plugins. "
"He's starting up anyway, but you will run into errors"
" if you try to use those plugins!"
)
self.add_startup_error("\n".join(missing_setting_error_messages))
else:
puts("")
def bootstrap_analysis(self):
self.analysis_backends = []
self.analysis_threads = []
for b in settings.ANALYZE_BACKENDS:
module = import_module(b)
for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
try:
if (
hasattr(cls, "is_will_analysisbackend") and
cls.is_will_analysisbackend and
class_name != "AnalysisBackend"
):
c = cls()
thread = Process(
target=c.start,
args=(b,),
kwargs={"bot": self},
)
thread.start()
self.analysis_threads.append(thread)
show_valid("Analysis: %s Backend started." % cls.__name__)
except Exception as e:
self.startup_error("Error bootstrapping %s io" % b, e)
self.analysis_backends.append(b)
pass
def bootstrap_bottle(self):
bootstrapped = False
try:
for cls, function_name in self.bottle_routes:
instantiated_cls = cls(bot=self)
instantiated_fn = getattr(instantiated_cls, function_name)
bottle_route_args = {}
for k, v in instantiated_fn.will_fn_metadata.items():
if "bottle_" in k and k != "bottle_route":
bottle_route_args[k[len("bottle_"):]] = v
bottle.route(instantiated_fn.will_fn_metadata["bottle_route"], **bottle_route_args)(instantiated_fn)
bootstrapped = True
except Exception as e:
self.startup_error("Error bootstrapping bottle", e)
if bootstrapped:
show_valid("Web server started at %s." % (settings.PUBLIC_URL,))
bottle.run(host='0.0.0.0', port=settings.HTTPSERVER_PORT, server='wsgiref', quiet=True)
def bootstrap_pubsub_mixin(self):
puts("Bootstrapping pubsub...")
try:
self.bootstrap_pubsub()
# Make sure settings are there.
self.pubsub.verify_settings()
with indent(2):
show_valid("Bootstrapped!")
puts("")
except ImportError:
module_name = traceback.format_exc().split(" ")[-1]
error("Unable to bootstrap pubsub - attempting to load %s" % module_name)
puts(traceback.format_exc())
sys.exit(1)
except Exception:
error("Unable to bootstrap pubsub!")
puts(traceback.format_exc())
sys.exit(1)
def verify_individual_setting(self, test_setting, quiet=False):
if not test_setting.get("only_if", True):
return True
if hasattr(settings, test_setting["name"][5:]):
with indent(2):
show_valid(test_setting["name"])
return True
else:
error("%(name)s... missing!" % test_setting)
with indent(2):
puts("""To obtain a %(name)s: \n%(obtain_at)s
To set your %(name)s:
1. On your local machine, add this to your virtual environment's bin/postactivate file:
export %(name)s=YOUR_ACTUAL_%(name)s
2. If you've deployed will on heroku, run
heroku config:set %(name)s=YOUR_ACTUAL_%(name)s
""" % test_setting)
return False
def bootstrap_io(self):
# puts("Bootstrapping IO...")
self.has_stdin_io_backend = False
self.io_backends = []
self.io_threads = []
self.stdin_io_backends = []
for b in self.valid_io_backends:
module = import_module(b)
for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
try:
if (
hasattr(cls, "is_will_iobackend")
and cls.is_will_iobackend
and class_name != "IOBackend"
and class_name != "StdInOutIOBackend"
):
c = cls()
if hasattr(c, "stdin_process") and c.stdin_process:
thread = self.create_process(
target=c._start,
args=(b,),
)
thread.start()
self.has_stdin_io_backend = True
self.io_threads.append(thread)
else:
thread = self.create_process(
target=c._start,
args=(
b,
)
)
thread.start()
self.io_threads.append(thread)
show_valid("IO: %s Backend started." % cls.friendly_name)
except Exception as e:
self.startup_error("Error bootstrapping %s io" % b, e)
self.io_backends.append(b)
def verify_execution(self):
puts("Verifying Execution backend...")
missing_settings = False
missing_setting_error_messages = []
one_valid_backend = False
if not hasattr(settings, "EXECUTION_BACKENDS"):
settings.EXECUTION_BACKENDS = ["will.backends.execution.all", ]
with indent(2):
for b in settings.EXECUTION_BACKENDS:
try:
path_name = None
for mod in b.split('.'):
if path_name is not None:
path_name = [path_name]
file_name, path_name, description = imp.find_module(mod, path_name)
one_valid_backend = True
show_valid("%s" % b)
except ImportError:
error_message = (
"Execution backend %s is missing. Please either remove it \nfrom config.py "
"or WILL_EXECUTION_BACKENDS, or provide it somehow (pip install, etc)."
) % b
puts(colored.red("✗ %s" % b))
puts()
puts(error_message)
puts()
puts(traceback.format_exc())
missing_setting_error_messages.append(error_message)
missing_settings = True
if missing_settings and not one_valid_backend:
puts("")
error(
"Unable to find a valid IO backend - will has no way to talk "
"or listen!\n Quitting now, please look at the above errors!\n"
)
sys.exit(1)
puts()
def bootstrap_execution(self):
missing_setting_error_messages = []
self.execution_backends = []
self.running_execution_threads = []
execution_backends = getattr(settings, "EXECUTION_BACKENDS", ["will.backends.execution.all", ])
for b in execution_backends:
module = import_module(b)
for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
try:
if (
hasattr(cls, "is_will_execution_backend")
and cls.is_will_execution_backend
and class_name != "ExecutionBackend"
):
c = cls(bot=self)
self.execution_backends.append(c)
show_valid("Execution: %s Backend started." % cls.__name__)
except ImportError:
error_message = (
"Execution backend %s is missing. Please either remove it \nfrom config.py "
"or WILL_EXECUTION_BACKENDS, or provide it somehow (pip install, etc)."
) % settings.EXECUTION_BACKENDS
puts(colored.red("✗ %s" % settings.EXECUTION_BACKENDS))
puts()
puts(error_message)
puts()
puts(traceback.format_exc())
missing_setting_error_messages.append(error_message)
if len(self.execution_backends) == 0:
puts("")
error(
"Unable to find a valid execution backend - will has no way to make decisions!"
"\n Quitting now, please look at the above error!\n"
)
sys.exit(1)
def import_settings(quiet=True):
"""This method takes care of importing settings from the environment, and config.py file.
Order of operations:
1. Imports all WILL_ settings from the environment, and strips off the WILL_
2. Imports settings from config.py
3. Sets defaults for any missing, required settings.
This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping.
"""
settings = {}
# Import from environment, handle environment-specific parsing.
for k, v in os.environ.items():
if k[:5] == "WILL_":
k = k[5:]
settings[k] = v
if "HIPCHAT_ROOMS" in settings and type(
settings["HIPCHAT_ROOMS"]) is type("tes"):
settings["HIPCHAT_ROOMS"] = settings["HIPCHAT_ROOMS"].split(";")
if "ROOMS" in settings:
settings["ROOMS"] = settings["ROOMS"].split(";")
if "PLUGINS" in settings:
settings["PLUGINS"] = settings["PLUGINS"].split(";")
if 'PLUGIN_BLACKLIST' in settings:
settings["PLUGIN_BLACKLIST"] = (settings["PLUGIN_BLACKLIST"].split(";")
if settings["PLUGIN_BLACKLIST"] else
[])
# If HIPCHAT_SERVER is set, we need to change the USERNAME slightly
# for XMPP to work.
if "HIPCHAT_SERVER" in settings:
settings["USERNAME"] = "******".\
format(user=settings["USERNAME"].split("@")[0],
host=settings["HIPCHAT_SERVER"])
else:
settings["HIPCHAT_SERVER"] = "api.hipchat.com"
# Import from config
if not quiet:
puts("Importing config.py... ")
with indent(2):
try:
had_warning = False
try:
import config
except ImportError:
# Missing config.py. Check for config.py.dist
if os.path.isfile("config.py.dist"):
confirm = input(
"Hi, looks like you're just starting up!\nI didn't find a config.py, but I do see config.py.dist here. Want me to use that? (y/n) "
).lower()
if confirm in ["y", "yes"]:
print("Great! One moment.\n\n")
os.rename("config.py.dist", "config.py")
import config
else:
print(
"Ok. I can't start without one though. Quitting now!"
)
sys.exit(1)
else:
error(
"I'm missing my config.py file. Usually one comes with the installation - maybe it got lost?"
)
sys.exit(1)
for k, v in config.__dict__.items():
# Ignore private variables
if "__" not in k:
if k in os.environ and v != os.environ[k] and not quiet:
warn(
"%s is set in the environment as '%s', but overridden in"
" config.py as '%s'." % (k, os.environ[k], v))
had_warning = True
if k not in settings:
settings[k] = v
if not had_warning and not quiet:
show_valid("Valid.")
except:
# TODO: Check to see if there's a config.py.dist
if not quiet:
warn("no config.py found. This might be ok, but more likely, "
"you haven't copied config.py.dist over to config.py")
if not quiet:
puts("Verifying settings... ")
with indent(2):
# Deprecation and backwards-compatibility for Will 1.x-> 2.x
DEPRECATED_BUT_MAPPED_SETTINGS = {
"USERNAME": "******",
"PASSWORD": "******",
"V1_TOKEN": "HIPCHAT_V1_TOKEN",
"V2_TOKEN": "HIPCHAT_V2_TOKEN",
"TOKEN": "HIPCHAT_V1_TOKEN",
"ROOMS": "HIPCHAT_ROOMS",
"NAME": "HIPCHAT_NAME",
"HANDLE": "HIPCHAT_HANDLE",
"DEFAULT_ROOM": "HIPCHAT_DEFAULT_ROOM",
"SLACK_DEFAULT_ROOM": "SLACK_DEFAULT_CHANNEL",
}
deprecation_warn_shown = False
for k, v in DEPRECATED_BUT_MAPPED_SETTINGS.items():
if not v in settings and k in settings:
if not deprecation_warn_shown and not quiet:
error(
"Deprecated settings. The following settings will stop working in Will 2.2:"
)
deprecation_warn_shown = True
if not quiet:
warn("Please update %s to %s. " % (k, v))
settings[v] = settings[k]
del settings[k]
# Migrate from 1.x
if "CHAT_BACKENDS" in settings and "IO_BACKENDS" not in settings:
IO_BACKENDS = []
for c in settings["CHAT_BACKENDS"]:
IO_BACKENDS.append("will.backends.io_adapters.%s" % c)
settings["IO_BACKENDS"] = IO_BACKENDS
if not quiet:
warn("Deprecated settings. Please update your config.py from:"
"\n CHAT_BACKENDS = %s\n to\n IO_BACKENDS = %s" %
(settings["CHAT_BACKENDS"], IO_BACKENDS))
if "CHAT_BACKENDS" not in settings and "IO_BACKENDS" not in settings:
if not quiet:
warn(
"""Deprecated settings. No backend found, so we're defaulting to hipchat and shell only.
Please add this to your config.py:
IO_BACKENDS = "
"will.backends.io_adapters.hipchat",
"will.backends.io_adapters.shell",
# "will.backends.io_adapters.slack",
# "will.backends.io_adapters.rocketchat",
]
""")
settings["IO_BACKENDS"] = [
"will.backends.io_adapters.hipchat",
"will.backends.io_adapters.shell",
]
if "ANALYZE_BACKENDS" not in settings:
if not quiet:
note(
"No ANALYZE_BACKENDS specified. Defaulting to history only."
)
settings["ANALYZE_BACKENDS"] = [
"will.backends.analysis.nothing",
"will.backends.analysis.history",
]
if "GENERATION_BACKENDS" not in settings:
if not quiet:
note(
"No GENERATION_BACKENDS specified. Defaulting to fuzzy_all_matches and strict_regex."
)
settings["GENERATION_BACKENDS"] = [
"will.backends.generation.fuzzy_all_matches",
"will.backends.generation.strict_regex",
]
if "EXECUTION_BACKENDS" not in settings:
if not quiet:
note(
"No EXECUTION_BACKENDS specified. Defaulting to best_score."
)
settings["EXECUTION_BACKENDS"] = [
"will.backends.execution.best_score",
]
# Set for hipchat
for b in settings["IO_BACKENDS"]:
if "hipchat" in b:
if "ALLOW_INSECURE_HIPCHAT_SERVER" in settings \
and (
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] is True
or settings["ALLOW_INSECURE_HIPCHAT_SERVER"].lower() == "true"
):
warn(
"You are choosing to run will with SSL disabled. "
"This is INSECURE and should NEVER be deployed outside a development environment."
)
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = True
settings["REQUESTS_OPTIONS"] = {
"verify": False,
}
else:
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = False
if "HIPCHAT_ROOMS" not in settings:
if not quiet:
warn(
"no HIPCHAT_ROOMS list found in the environment or config. "
"This is ok - Will will just join all available HIPCHAT_rooms."
)
settings["HIPCHAT_ROOMS"] = None
if ("HIPCHAT_DEFAULT_ROOM" not in settings
and "HIPCHAT_ROOMS" in settings
and settings["HIPCHAT_ROOMS"]
and len(settings["HIPCHAT_ROOMS"]) > 0):
if not quiet:
warn(
"no HIPCHAT_DEFAULT_ROOM found in the environment or config. "
"Defaulting to '%s', the first one." %
settings["HIPCHAT_ROOMS"][0])
settings["HIPCHAT_DEFAULT_ROOM"] = settings[
"HIPCHAT_ROOMS"][0]
if "HIPCHAT_HANDLE" in settings and "HIPCHAT_HANDLE_NOTED" not in settings:
if not quiet:
note(
"""HIPCHAT_HANDLE is no longer required (or used), as Will knows how to get\n
his current handle from the HipChat servers."""
)
settings["HIPCHAT_HANDLE_NOTED"] = True
if "HIPCHAT_NAME" in settings and "HIPCHAT_NAME_NOTED" not in settings:
if not quiet:
note(
"""HIPCHAT_NAME is no longer required (or used), as Will knows how to get\n
his current name from the HipChat servers.""")
settings["HIPCHAT_NAME_NOTED"] = True
# Rocket.chat
for b in settings["IO_BACKENDS"]:
if "rocketchat" in b:
if "ROCKETCHAT_USERNAME" in settings and "ROCKETCHAT_EMAIL" not in settings:
settings["ROCKETCHAT_EMAIL"] = settings[
"ROCKETCHAT_USERNAME"]
if "ROCKETCHAT_URL" in settings:
if settings["ROCKETCHAT_URL"].endswith("/"):
settings["ROCKETCHAT_URL"] = settings[
"ROCKETCHAT_URL"][:-1]
if ("DEFAULT_BACKEND" not in settings and "IO_BACKENDS" in settings
and settings["IO_BACKENDS"]
and len(settings["IO_BACKENDS"]) > 0):
if not quiet:
note(
"no DEFAULT_BACKEND found in the environment or config.\n "
" Defaulting to '%s', the first one." %
settings["IO_BACKENDS"][0])
settings["DEFAULT_BACKEND"] = settings["IO_BACKENDS"][0]
for b in settings["IO_BACKENDS"]:
if "slack" in b and "SLACK_DEFAULT_CHANNEL" not in settings and not quiet:
warn(
"No SLACK_DEFAULT_CHANNEL set - any messages sent without an explicit channel will go "
"to a non-deterministic channel that will has access to "
"- this is almost certainly not what you want.")
if "HTTPSERVER_PORT" not in settings:
# For heroku
if "PORT" in os.environ:
settings["HTTPSERVER_PORT"] = os.environ["PORT"]
else:
if not quiet:
warn(
"no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'."
)
settings["HTTPSERVER_PORT"] = "80"
if "STORAGE_BACKEND" not in settings:
if not quiet:
warn("No STORAGE_BACKEND specified. Defaulting to redis.")
settings["STORAGE_BACKEND"] = "redis"
if "PUBSUB_BACKEND" not in settings:
if not quiet:
warn("No PUBSUB_BACKEND specified. Defaulting to redis.")
settings["PUBSUB_BACKEND"] = "redis"
if settings["STORAGE_BACKEND"] == "redis" or settings[
"PUBSUB_BACKEND"] == "redis":
if "REDIS_URL" not in settings:
# For heroku
if "REDIS_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDIS_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using Heroku Redis or another standard REDIS_URL. If so, all good."
)
if "REDISCLOUD_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good."
)
elif "REDISTOGO_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISTOGO_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good."
)
elif "OPENREDIS_URL" in os.environ:
settings["REDIS_URL"] = os.environ["OPENREDIS_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good."
)
else:
settings["REDIS_URL"] = "redis://localhost:6379/7"
if not quiet:
note(
"WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7."
)
if not settings["REDIS_URL"].startswith("redis://"):
settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"]
if "REDIS_MAX_CONNECTIONS" not in settings or not settings[
"REDIS_MAX_CONNECTIONS"]:
settings["REDIS_MAX_CONNECTIONS"] = 4
if not quiet:
note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.")
if settings["STORAGE_BACKEND"] == "file":
if "FILE_DIR" not in settings:
settings["FILE_DIR"] = "~/.will/"
if not quiet:
note("FILE_DIR not set. Defaulting to ~/.will/")
if settings["STORAGE_BACKEND"] == "couchbase":
if "COUCHBASE_URL" not in settings:
settings["COUCHBASE_URL"] = "couchbase:///will"
if not quiet:
note(
"COUCHBASE_URL not set. Defaulting to couchbase:///will"
)
if "PUBLIC_URL" not in settings:
default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"]
settings["PUBLIC_URL"] = default_public
if not quiet:
note(
"no PUBLIC_URL found in the environment or config.\n Defaulting to '%s'."
% default_public)
if not "REQUESTS_OPTIONS" in settings:
settings["REQUESTS_OPTIONS"] = {}
if "TEMPLATE_DIRS" not in settings:
if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ:
# All good
pass
else:
settings["TEMPLATE_DIRS"] = []
if "WILL_HANDLE" not in settings:
if "HANDLE" in settings:
settings["WILL_HANDLE"] = settings["HANDLE"]
elif "SLACK_HANDLE" in settings:
settings["WILL_HANDLE"] = settings["SLACK_HANDLE"]
elif "HIPCHAT_HANDLE" in settings:
settings["WILL_HANDLE"] = settings["HIPCHAT_HANDLE"]
elif "ROCKETCHAT_HANDLE" in settings:
settings["WILL_HANDLE"] = settings["ROCKETCHAT_HANDLE"]
else:
settings["WILL_HANDLE"] = "will"
if "ADMINS" not in settings:
settings["ADMINS"] = "*"
else:
if "WILL_ADMINS" in os.environ:
settings["ADMINS"] = [
a.strip().lower()
for a in settings.get('ADMINS', '').split(';')
if a.strip()
]
if "ADMINS" in settings and settings["ADMINS"] != "*":
warn(
"ADMINS is now deprecated, and will be removed at the end of 2017. Please use ACL instead. See below for details"
)
note("Change your config.py to:\n ACL = {\n 'admins': %s\n }"
% settings["ADMINS"])
if "DISABLE_ACL" not in settings:
settings["DISABLE_ACL"] = False
if "PROXY_URL" in settings:
parsed_proxy_url = parse.urlparse(settings["PROXY_URL"])
settings["USE_PROXY"] = True
settings["PROXY_HOSTNAME"] = parsed_proxy_url.hostname
settings["PROXY_USERNAME"] = parsed_proxy_url.username
settings["PROXY_PASSWORD"] = parsed_proxy_url.password
settings["PROXY_PORT"] = parsed_proxy_url.port
else:
settings["USE_PROXY"] = False
if "EVENT_LOOP_INTERVAL" not in settings:
settings["EVENT_LOOP_INTERVAL"] = 0.025
if "LOGLEVEL" not in settings:
settings["LOGLEVEL"] = "ERROR"
if "ENABLE_INTERNAL_ENCRYPTION" not in settings:
settings["ENABLE_INTERNAL_ENCRYPTION"] = True
if "SECRET_KEY" not in settings:
if not quiet:
if "ENABLE_INTERNAL_ENCRYPTION" in settings and settings[
"ENABLE_INTERNAL_ENCRYPTION"]:
key = auto_key()
if key:
warn(
"""No SECRET_KEY specified and ENABLE_INTERNAL_ENCRYPTION is on.\n
Temporarily auto-generating a key specific to this computer:\n {}\n
Please set WILL_SECRET_KEY in the environment as soon as possible to ensure \n
Will is able to access information from previous runs."""
.format(key))
else:
error(
"""ENABLE_INTERNAL_ENCRYPTION is turned on, but a SECRET_KEY has not been given.\n
We tried to automatically generate temporary SECRET_KEY, but this appears to be a \n"
shared or virtualized environment.\n Please set a unique secret key in the
environment as WILL_SECRET_KEY to run will.""")
print(
" Unable to start will without a SECRET_KEY while encryption is turned on. Shutting down."
)
sys.exit(1)
settings["SECRET_KEY"] = key
os.environ["WILL_SECRET_KEY"] = settings["SECRET_KEY"]
os.environ["WILL_EPHEMERAL_SECRET_KEY"] = "True"
if "FUZZY_MINIMUM_MATCH_CONFIDENCE" not in settings:
settings["FUZZY_MINIMUM_MATCH_CONFIDENCE"] = 91
if "FUZZY_REGEX_ALLOWABLE_ERRORS" not in settings:
settings["FUZZY_REGEX_ALLOWABLE_ERRORS"] = 3
# Set them in the module namespace
for k in sorted(settings, key=lambda x: x[0]):
if not quiet:
show_valid(k)
globals()[k] = settings[k]
def bootstrap_plugins(self):
puts("Bootstrapping plugins...")
OTHER_HELP_HEADING = "Other"
plugin_modules = {}
plugin_modules_library = {}
# NOTE: You can't access self.storage here, or it will deadlock when the threads try to access redis.
with indent(2):
parent_help_text = None
for plugin_name, plugin_root in self.plugins_dirs.items():
for root, dirs, files in os.walk(plugin_root, topdown=False):
for f in files:
if f[-3:] == ".py" and f != "__init__.py":
try:
module_path = os.path.join(root, f)
path_components = module_path.split(os.sep)
module_name = path_components[-1][:-3]
full_module_name = ".".join(path_components)
# Check blacklist.
blacklisted = False
for b in settings.PLUGIN_BLACKLIST:
if b in full_module_name:
blacklisted = True
break
parent_mod = path_components[-2].split("/")[-1]
parent_help_text = parent_mod.title()
# Don't even *try* to load a blacklisted module.
if not blacklisted:
try:
plugin_modules[full_module_name] = imp.load_source(module_name, module_path)
parent_root = os.path.join(root, "__init__.py")
parent = imp.load_source(parent_mod, parent_root)
parent_help_text = getattr(parent, "MODULE_DESCRIPTION", parent_help_text)
except:
# If it's blacklisted, don't worry if this blows up.
if blacklisted:
pass
else:
raise
plugin_modules_library[full_module_name] = {
"full_module_name": full_module_name,
"file_path": module_path,
"name": module_name,
"parent_name": plugin_name,
"parent_module_name": parent_mod,
"parent_help_text": parent_help_text,
"blacklisted": blacklisted,
}
except Exception as e:
self.startup_error("Error loading %s" % (module_path,), e)
self.plugins = []
for name, module in plugin_modules.items():
try:
for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
try:
if hasattr(cls, "is_will_plugin") and cls.is_will_plugin and class_name != "WillPlugin":
self.plugins.append({
"name": class_name,
"class": cls,
"module": module,
"full_module_name": name,
"parent_name": plugin_modules_library[name]["parent_name"],
"parent_path": plugin_modules_library[name]["file_path"],
"parent_module_name": plugin_modules_library[name]["parent_module_name"],
"parent_help_text": plugin_modules_library[name]["parent_help_text"],
"blacklisted": plugin_modules_library[name]["blacklisted"],
})
except Exception as e:
self.startup_error("Error bootstrapping %s" % (class_name,), e)
except Exception as e:
self.startup_error("Error bootstrapping %s" % (name,), e)
self._plugin_modules_library = plugin_modules_library
# Sift and Sort.
self.message_listeners = {}
self.periodic_tasks = []
self.random_tasks = []
self.bottle_routes = []
self.all_listener_regexes = []
self.help_modules = {}
self.help_modules[OTHER_HELP_HEADING] = []
self.some_listeners_include_me = False
self.plugins.sort(key=operator.itemgetter("parent_module_name"))
self.required_settings_from_plugins = {}
last_parent_name = None
for plugin_info in self.plugins:
try:
if last_parent_name != plugin_info["parent_help_text"]:
friendly_name = "%(parent_help_text)s " % plugin_info
module_name = " %(parent_name)s" % plugin_info
# Justify
friendly_name = friendly_name.ljust(50, '-')
module_name = module_name.rjust(40, '-')
puts("")
puts("%s%s" % (friendly_name, module_name))
last_parent_name = plugin_info["parent_help_text"]
with indent(2):
plugin_name = plugin_info["name"]
plugin_warnings = []
# Just a little nicety
if plugin_name[-6:] == "Plugin":
plugin_name = plugin_name[:-6]
if plugin_info["blacklisted"]:
puts("✗ %s (blacklisted)" % plugin_name)
else:
plugin_instances = {}
for function_name, fn in inspect.getmembers(
plugin_info["class"],
predicate=lambda x: inspect.ismethod(x) or inspect.isfunction(x)
):
try:
# Check for required_settings
with indent(2):
if hasattr(fn, "will_fn_metadata"):
meta = fn.will_fn_metadata
if "warnings" in meta:
plugin_warnings.append(meta["warnings"])
if "required_settings" in meta:
for s in meta["required_settings"]:
self.required_settings_from_plugins[s] = {
"plugin_name": plugin_name,
"function_name": function_name,
"setting_name": s,
}
if (
"listens_to_messages" in meta
and meta["listens_to_messages"]
and "listener_regex" in meta
):
# puts("- %s" % function_name)
regex = meta["listener_regex"]
if not meta["case_sensitive"]:
regex = "(?i)%s" % regex
help_regex = meta["listener_regex"]
if meta["listens_only_to_direct_mentions"]:
help_regex = "@%s %s" % (settings.WILL_HANDLE, help_regex)
self.all_listener_regexes.append(help_regex)
if meta["__doc__"]:
pht = plugin_info.get("parent_help_text", None)
if pht:
if pht in self.help_modules:
self.help_modules[pht].append(u"%s" % meta["__doc__"])
else:
self.help_modules[pht] = [u"%s" % meta["__doc__"]]
else:
self.help_modules[OTHER_HELP_HEADING].append(u"%s" % meta["__doc__"])
if meta["multiline"]:
compiled_regex = re.compile(regex, re.MULTILINE | re.DOTALL)
else:
compiled_regex = re.compile(regex)
if plugin_info["class"] in plugin_instances:
instance = plugin_instances[plugin_info["class"]]
else:
instance = plugin_info["class"](bot=self)
plugin_instances[plugin_info["class"]] = instance
full_method_name = "%s.%s" % (plugin_info["name"], function_name)
cleaned_info = copy.copy(plugin_info)
del cleaned_info["module"]
del cleaned_info["class"]
self.message_listeners[full_method_name] = {
"full_method_name": full_method_name,
"function_name": function_name,
"class_name": plugin_info["name"],
"regex_pattern": meta["listener_regex"],
"regex": compiled_regex,
"fn": getattr(instance, function_name),
"args": meta["listener_args"],
"include_me": meta["listener_includes_me"],
"case_sensitive": meta["case_sensitive"],
"multiline": meta["multiline"],
"direct_mentions_only": meta["listens_only_to_direct_mentions"],
"admin_only": meta["listens_only_to_admin"],
"acl": meta["listeners_acl"],
"plugin_info": cleaned_info,
}
if meta["listener_includes_me"]:
self.some_listeners_include_me = True
elif "periodic_task" in meta and meta["periodic_task"]:
# puts("- %s" % function_name)
self.periodic_tasks.append((plugin_info, fn, function_name))
elif "random_task" in meta and meta["random_task"]:
# puts("- %s" % function_name)
self.random_tasks.append((plugin_info, fn, function_name))
elif "bottle_route" in meta:
# puts("- %s" % function_name)
self.bottle_routes.append((plugin_info["class"], function_name))
except Exception:
error(plugin_name)
self.startup_error(
"Error bootstrapping %s.%s" % (
plugin_info["class"],
function_name,
)
)
if len(plugin_warnings) > 0:
show_invalid(plugin_name)
for w in plugin_warnings:
warn(w)
else:
show_valid(plugin_name)
except Exception as e:
self.startup_error("Error bootstrapping %s" % (plugin_info["class"],), e)
self.save("all_listener_regexes", self.all_listener_regexes)
puts("")
def verify_io(self):
puts("Verifying IO backends...")
missing_settings = False
missing_setting_error_messages = []
one_valid_backend = False
self.valid_io_backends = []
if not hasattr(settings, "IO_BACKENDS"):
settings.IO_BACKENDS = ["will.backends.io_adapters.shell", ]
# Try to import them all, catch errors and output trouble if we hit it.
for b in settings.IO_BACKENDS:
with indent(2):
try:
path_name = None
for mod in b.split('.'):
if path_name is not None:
path_name = [path_name]
file_name, path_name, description = imp.find_module(mod, path_name)
# show_valid("%s" % b)
module = import_module(b)
for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
if (
hasattr(cls, "is_will_iobackend")
and cls.is_will_iobackend
and class_name != "IOBackend"
and class_name != "StdInOutIOBackend"
):
c = cls()
show_valid(c.friendly_name)
c.verify_settings()
one_valid_backend = True
self.valid_io_backends.append(b)
except EnvironmentError:
puts(colored.red(" ✗ %s is missing settings, and will be disabled." % b))
puts()
missing_settings = True
except Exception:
error_message = (
"IO backend %s is missing. Please either remove it \nfrom config.py "
"or WILL_IO_BACKENDS, or provide it somehow (pip install, etc)."
) % b
puts(colored.red("✗ %s" % b))
puts()
puts(error_message)
puts()
puts(traceback.format_exc())
missing_setting_error_messages.append(error_message)
missing_settings = True
if missing_settings and not one_valid_backend:
puts("")
error(
"Unable to find a valid IO backend - will has no way to talk "
"or listen!\n Quitting now, please look at the above errors!\n"
)
self.handle_sys_exit()
return False
puts()
return True
