def verify_plugin_settings(self):
puts("Verifying settings requested by plugins...")
missing_settings = False
missing_setting_error_messages = []
with indent(2):
for name, meta in self.required_settings_from_plugins.items():
if not hasattr(settings, name):
error_message = (
"%(setting_name)s is missing. It's required by the"
"%(plugin_name)s plugin's '%(function_name)s' method."
) % meta
puts(colored.red("✗ %(setting_name)s" % meta))
missing_setting_error_messages.append(error_message)
missing_settings = True
else:
show_valid("%(setting_name)s" % meta)
if missing_settings:
puts("")
warn(
"Will is missing settings required by some plugins. "
"He's starting up anyway, but you will run into errors"
" if you try to use those plugins!"
)
self.add_startup_error("\n".join(missing_setting_error_messages))
else:
puts("")
def load_config(self):
puts("Loading configuration...")
with indent(2):
settings.import_settings(quiet=False)
puts("")
puts("Verifying rooms...")
# If we're missing ROOMS, join all of them.
with indent(2):
if settings.ROOMS == None:
# Yup. Thanks, BSDs.
q = Queue()
p = Process(target=self.update_available_rooms,
args=(),
kwargs={
"q": q,
})
p.start()
rooms_list = q.get()
show_valid("Joining all %s known rooms." % len(rooms_list))
os.environ["WILL_ROOMS"] = ";".join(rooms_list)
p.join()
settings.import_settings()
else:
show_valid("Joining the %s room%s specified." % (len(
settings.ROOMS), "s" if len(settings.ROOMS) > 1 else ""))
puts("")
def load_config(self):
puts("Loading configuration...")
with indent(2):
settings.import_settings(quiet=False)
puts("")
puts("Verifying rooms...")
# If we're missing ROOMS, join all of them.
with indent(2):
if settings.ROOMS is None:
# Yup. Thanks, BSDs.
q = Queue()
p = Process(target=self.update_available_rooms, args=(), kwargs={"q": q, })
p.start()
rooms_list = q.get()
show_valid("Joining all %s known rooms." % len(rooms_list))
os.environ["WILL_ROOMS"] = ";".join(rooms_list)
p.join()
settings.import_settings()
else:
show_valid(
"Joining the %s room%s specified." % (
len(settings.ROOMS),
"s" if len(settings.ROOMS) > 1 else ""
)
)
puts("")
def verify_plugin_settings(self):
puts("Verifying settings requested by plugins...")
missing_settings = False
missing_setting_error_messages = []
with indent(2):
for name, meta in self.required_settings_from_plugins.items():
if not hasattr(settings, name):
error_message = (
"%(setting_name)s is missing. It's required by the"
"%(plugin_name)s plugin's '%(function_name)s' method."
) % meta
puts(colored.red("✗ %(setting_name)s" % meta))
missing_setting_error_messages.append(error_message)
missing_settings = True
else:
show_valid("%(setting_name)s" % meta)
if missing_settings:
puts("")
warn("Will is missing settings required by some plugins. "
"He's starting up anyway, but you will run into errors"
" if you try to use those plugins!")
self.add_startup_error(
"\n".join(missing_setting_error_messages))
else:
puts("")
def bootstrap_storage_mixin(self):
puts("Bootstrapping storage...")
try:
self.bootstrap_storage()
with indent(2):
show_valid("Connection to %s successful." % settings.REDIS_URL)
puts("")
except:
error("Unable to connect to %s" % settings.REDIS_URL)
sys.exit(1)
def bootstrap_storage_mixin(self):
puts("Bootstrapping storage...")
try:
self.bootstrap_storage()
with indent(2):
show_valid("Connection to %s successful." % settings.REDIS_URL)
puts("")
except:
error("Unable to connect to %s" % settings.REDIS_URL)
sys.exit(1)
def verify_individual_setting(self, test_setting, quiet=False):
if hasattr(settings, test_setting["name"][5:]):
with indent(2):
show_valid(test_setting["name"])
return True
else:
error("%(name)s... missing!" % test_setting)
with indent(2):
puts("""To obtain a %(name)s: \n%(obtain_at)s
To set your %(name)s:
1. On your local machine, add this to your virtual environment's bin/postactivate file:
export %(name)s=YOUR_ACTUAL_%(name)s
2. If you've deployed will on heroku, run
heroku config:set %(name)s=YOUR_ACTUAL_%(name)s
""" % test_setting)
return False
def verify_individual_setting(self, test_setting, quiet=False):
if hasattr(settings, test_setting["name"][5:]):
with indent(2):
show_valid(test_setting["name"])
return True
else:
error("%(name)s... missing!" % test_setting)
with indent(2):
puts("""To obtain a %(name)s: \n%(obtain_at)s
To set your %(name)s:
1. On your local machine, add this to your virtual environment's bin/postactivate file:
export %(name)s=YOUR_ACTUAL_%(name)s
2. If you've deployed will on heroku, run
heroku config:set %(name)s=YOUR_ACTUAL_%(name)s
""" % test_setting)
return False
def verify_environment(self):
missing_settings = False
required_settings = [
{
"name": "WILL_USERNAME",
"obtain_at": """1. Go to hipchat, and create a new user for will.
2. Log into will, and go to Account settings>XMPP/Jabber Info.
3. On that page, the 'Jabber ID' is the value you want to use.""",
},
{
"name": "WILL_PASSWORD",
"obtain_at": (
"1. Go to hipchat, and create a new user for will. "
"Note that password - this is the value you want. "
"It's used for signing in via XMPP."
),
},
{
"name": "WILL_V2_TOKEN",
"obtain_at": """1. Log into hipchat using will's user.
2. Go to https://your-org.hipchat.com/account/api
3. Create a token.
4. Copy the value - this is the WILL_V2_TOKEN.""",
},
{
"name": "WILL_REDIS_URL",
"obtain_at": """1. Set up an accessible redis host locally or in production
2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""",
}
]
puts("")
puts("Verifying environment...")
for r in required_settings:
if not self.verify_individual_setting(r):
missing_settings = True
if missing_settings:
error(
"Will was unable to start because some required environment "
"variables are missing. Please fix them and try again!"
)
sys.exit(1)
else:
puts("")
puts("Verifying credentials...")
# Parse [email protected] into id, where 222222 is the id. Yup.
user_id = settings.USERNAME[0:settings.USERNAME.find("@")][settings.USERNAME.find("_") + 1:]
# Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS.
# http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests
q = Queue()
p = Process(target=self.get_hipchat_user, args=(user_id,), kwargs={"q": q, })
p.start()
user_data = q.get()
p.join()
if "error" in user_data:
error("We ran into trouble: '%(message)s'" % user_data["error"])
sys.exit(1)
with indent(2):
show_valid("%s authenticated" % user_data["name"])
os.environ["WILL_NAME"] = user_data["name"]
show_valid("@%s verified as handle" % user_data["mention_name"])
os.environ["WILL_HANDLE"] = user_data["mention_name"]
puts("")
def bootstrap_bottle(self):
bootstrapped = False
try:
for cls, function_name in self.bottle_routes:
instantiated_cls = cls()
instantiated_fn = getattr(instantiated_cls, function_name)
bottle_route_args = {}
for k, v in instantiated_fn.will_fn_metadata.items():
if "bottle_" in k and k != "bottle_route":
bottle_route_args[k[len("bottle_"):]] = v
bottle.route(instantiated_fn.will_fn_metadata["bottle_route"], **bottle_route_args)(instantiated_fn)
bootstrapped = True
except Exception, e:
self.startup_error("Error bootstrapping bottle", e)
if bootstrapped:
show_valid("Web server started.")
bottle.run(host='0.0.0.0', port=settings.HTTPSERVER_PORT, server='cherrypy', quiet=True)
def bootstrap_xmpp(self):
bootstrapped = False
try:
self.start_xmpp_client()
sorted_help = {}
for k, v in self.help_modules.items():
sorted_help[k] = sorted(v)
self.save("help_modules", sorted_help)
self.save("all_listener_regexes", self.all_listener_regexes)
self.connect()
bootstrapped = True
except Exception, e:
bootstrapped = False
try:
for cls, function_name in self.bottle_routes:
instantiated_cls = cls()
instantiated_fn = getattr(instantiated_cls, function_name)
bottle_route_args = {}
for k, v in instantiated_fn.will_fn_metadata.items():
if "bottle_" in k and k != "bottle_route":
bottle_route_args[k[len("bottle_"):]] = v
bottle.route(instantiated_fn.will_fn_metadata["bottle_route"],
**bottle_route_args)(instantiated_fn)
bootstrapped = True
except Exception, e:
self.startup_error("Error bootstrapping bottle", e)
if bootstrapped:
show_valid("Web server started.")
bottle.run(host='0.0.0.0',
port=settings.HTTPSERVER_PORT,
server='cherrypy',
quiet=True)
def bootstrap_xmpp(self):
bootstrapped = False
try:
self.start_xmpp_client()
sorted_help = {}
for k, v in self.help_modules.items():
sorted_help[k] = sorted(v)
self.save("help_modules", sorted_help)
self.save("all_listener_regexes", self.all_listener_regexes)
def import_settings(quiet=True):
"""This method takes care of importing settings from the environment, and config.py file.
Order of operations:
1. Imports all WILL_ settings from the environment, and strips off the WILL_
2. Imports settings from config.py
3. Sets defaults for any missing, required settings.
This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping.
"""
settings = {}
# Import from environment, handle environment-specific parsing.
for k, v in os.environ.items():
if k[:5] == "WILL_":
k = k[5:]
settings[k] = v
if "ROOMS" in settings:
settings["ROOMS"] = settings["ROOMS"].split(";")
# Import from config
if not quiet:
puts("Importing config.py... ")
with indent(2):
try:
had_warning = False
import config
for k, v in config.__dict__.items():
# Ignore private variables
if "__" not in k:
if k in os.environ and v != os.environ[k] and not quiet:
warn("%s is set in the environment as '%s', but overridden in config.py as '%s'." % (k, os.environ[k], v))
had_warning = True
settings[k] = v
if not had_warning and not quiet:
show_valid("Valid.")
except:
# TODO: Check to see if there's a config.py.dist
if not quiet:
warn("no config.py found. This might be ok, but more likely, you haven't copied config.py.dist over to config.py")
if not quiet:
puts("Verifying settings... ")
with indent(2):
# Set defaults
if "ROOMS" not in settings:
if not quiet:
warn("no ROOMS list found in the environment or config. This is ok - Will will just join all available rooms.")
settings["ROOMS"] = None
if not "DEFAULT_ROOM" in settings and "ROOMS" in settings and settings["ROOMS"] and len(settings["ROOMS"]) > 0:
if not quiet:
warn("no DEFAULT_ROOM found in the environment or config. Defaulting to '%s', the first one." % settings["ROOMS"][0])
settings["DEFAULT_ROOM"] = settings["ROOMS"][0]
if not "HTTPSERVER_PORT" in settings:
# For heroku
if "PORT" in os.environ:
settings["HTTPSERVER_PORT"] = os.environ["PORT"]
else:
if not quiet:
warn("no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'.")
settings["HTTPSERVER_PORT"] = "80"
if not "REDIS_URL" in settings:
# For heroku
if "REDISCLOUD_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"]
if not quiet:
note("WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good.")
elif "REDISTOGO_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISTOGO_URL"]
if not quiet:
note("WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good.")
elif "OPENREDIS_URL" in os.environ:
settings["REDIS_URL"] = os.environ["OPENREDIS_URL"]
if not quiet:
note("WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good.")
else:
settings["REDIS_URL"] = "redis://localhost:6379/7"
if not quiet:
note("WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7.")
if not "PUBLIC_URL" in settings:
default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"]
settings["PUBLIC_URL"] = default_public
if not quiet:
warn("no PUBLIC_URL found in the environment or config. Defaulting to '%s'." % default_public)
if not "V1_TOKEN" in settings:
if not quiet:
warn("no V1_TOKEN found in the environment or config. This is generally ok, but if you have more than 30 rooms, you may recieve rate-limit errors without one.")
if not "TEMPLATE_DIRS" in settings:
if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ:
# All good
pass
else:
settings["TEMPLATE_DIRS"] = []
if not "ADMINS" in settings:
settings["ADMINS"] = "*"
else:
if "WILL_ADMINS" in os.environ:
settings["ADMINS"] = [a.strip().lower() for a in settings.get('ADMINS', '').split(';') if a.strip()]
# Set them in the module namespace
for k in sorted(settings, key=lambda x: x[0]):
if not quiet:
show_valid(k)
globals()[k] = settings[k]
class WillBot(EmailMixin, WillXMPPClientMixin, StorageMixin, ScheduleMixin,
ErrorMixin, RoomMixin, HipChatMixin, PluginModulesLibraryMixin):
def __init__(self, **kwargs):
if "template_dirs" in kwargs:
warn("template_dirs is now depreciated")
if "plugin_dirs" in kwargs:
warn("plugin_dirs is now depreciated")
log_level = getattr(settings, 'LOGLEVEL', logging.ERROR)
logging.basicConfig(level=log_level,
format='%(levelname)-8s %(message)s')
# Find all the PLUGINS modules
plugins = settings.PLUGINS
self.plugins_dirs = {}
# Set template dirs.
full_path_template_dirs = []
for t in settings.TEMPLATE_DIRS:
full_path_template_dirs.append(os.path.abspath(t))
# Add will's templates_root
if TEMPLATES_ROOT not in full_path_template_dirs:
full_path_template_dirs += [
TEMPLATES_ROOT,
]
# Add this project's templates_root
if PROJECT_TEMPLATE_ROOT not in full_path_template_dirs:
full_path_template_dirs += [
PROJECT_TEMPLATE_ROOT,
]
# Convert those to dirs
for plugin in plugins:
path_name = None
for mod in plugin.split('.'):
if path_name is not None:
path_name = [path_name]
file_name, path_name, description = imp.find_module(
mod, path_name)
# Add, uniquely.
self.plugins_dirs[os.path.abspath(path_name)] = plugin
if os.path.exists(
os.path.join(os.path.abspath(path_name), "templates")):
full_path_template_dirs.append(
os.path.join(os.path.abspath(path_name), "templates"))
# Key by module name
self.plugins_dirs = dict(
zip(self.plugins_dirs.values(), self.plugins_dirs.keys()))
# Storing here because storage hasn't been bootstrapped yet.
os.environ["WILL_TEMPLATE_DIRS_PICKLED"] =\
";;".join(full_path_template_dirs)
def bootstrap(self):
print_head()
self.verify_environment()
self.load_config()
self.bootstrap_storage_mixin()
self.bootstrap_plugins()
self.verify_plugin_settings()
puts("Bootstrapping complete.")
puts("\nStarting core processes:")
# Scheduler
scheduler_thread = Process(target=self.bootstrap_scheduler)
# scheduler_thread.daemon = True
# Bottle
bottle_thread = Process(target=self.bootstrap_bottle)
# bottle_thread.daemon = True
# XMPP Listener
xmpp_thread = Process(target=self.bootstrap_xmpp)
# xmpp_thread.daemon = True
with indent(2):
try:
# Start up threads.
xmpp_thread.start()
scheduler_thread.start()
bottle_thread.start()
errors = self.get_startup_errors()
if len(errors) > 0:
default_room = self.get_room_from_name_or_id(
settings.DEFAULT_ROOM)["room_id"]
error_message = "FYI, I ran into some problems while starting up:"
for err in errors:
error_message += "\n%s\n" % err
self.send_room_message(default_room,
error_message,
color="yellow")
puts(colored.red(error_message))
while True:
time.sleep(100)
except (KeyboardInterrupt, SystemExit):
scheduler_thread.terminate()
bottle_thread.terminate()
xmpp_thread.terminate()
print '\n\nReceived keyboard interrupt, quitting threads.',
while (scheduler_thread.is_alive() or bottle_thread.is_alive()
or xmpp_thread.is_alive()):
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(0.5)
def verify_individual_setting(self, test_setting, quiet=False):
if hasattr(settings, test_setting["name"][5:]):
with indent(2):
show_valid(test_setting["name"])
return True
else:
error("%(name)s... missing!" % test_setting)
with indent(2):
puts("""To obtain a %(name)s: \n%(obtain_at)s
To set your %(name)s:
1. On your local machine, add this to your virtual environment's bin/postactivate file:
export %(name)s=YOUR_ACTUAL_%(name)s
2. If you've deployed will on heroku, run
heroku config:set %(name)s=YOUR_ACTUAL_%(name)s
""" % test_setting)
return False
def verify_environment(self):
missing_settings = False
required_settings = [{
"name":
"WILL_USERNAME",
"obtain_at":
"""1. Go to hipchat, and create a new user for will.
2. Log into will, and go to Account settings>XMPP/Jabber Info.
3. On that page, the 'Jabber ID' is the value you want to use.""",
}, {
"name":
"WILL_PASSWORD",
"obtain_at":
"1. Go to hipchat, and create a new user for will. Note that password - this is the value you want. It's used for signing in via XMPP.",
}, {
"name":
"WILL_V2_TOKEN",
"obtain_at":
"""1. Log into hipchat using will's user.
2. Go to https://your-org.hipchat.com/account/api
3. Create a token.
4. Copy the value - this is the WILL_V2_TOKEN.""",
}, {
"name":
"WILL_REDIS_URL",
"obtain_at":
"""1. Set up an accessible redis host locally or in production
2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""",
}]
puts("")
puts("Verifying environment...")
for r in required_settings:
if not self.verify_individual_setting(r):
missing_settings = True
if missing_settings:
error(
"Will was unable to start because some required environment variables are missing. Please fix them and try again!"
)
sys.exit(1)
else:
puts("")
puts("Verifying credentials...")
# Parse [email protected] into id, where 222222 is the id. Yup.
user_id = settings.USERNAME[0:settings.USERNAME.
find("@")][settings.USERNAME.find("_") +
1:]
# Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS.
# http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests
q = Queue()
p = Process(target=self.get_hipchat_user,
args=(user_id, ),
kwargs={
"q": q,
})
p.start()
user_data = q.get()
p.join()
if "error" in user_data:
error("We ran into trouble: '%(message)s'" % user_data["error"])
sys.exit(1)
with indent(2):
show_valid("%s authenticated" % user_data["name"])
os.environ["WILL_NAME"] = user_data["name"]
show_valid("@%s verified as handle" % user_data["mention_name"])
os.environ["WILL_HANDLE"] = user_data["mention_name"]
puts("")
def load_config(self):
puts("Loading configuration...")
with indent(2):
settings.import_settings(quiet=False)
puts("")
puts("Verifying rooms...")
# If we're missing ROOMS, join all of them.
with indent(2):
if settings.ROOMS == None:
# Yup. Thanks, BSDs.
q = Queue()
p = Process(target=self.update_available_rooms,
args=(),
kwargs={
"q": q,
})
p.start()
rooms_list = q.get()
show_valid("Joining all %s known rooms." % len(rooms_list))
os.environ["WILL_ROOMS"] = ";".join(rooms_list)
p.join()
settings.import_settings()
else:
show_valid("Joining the %s room%s specified." % (len(
settings.ROOMS), "s" if len(settings.ROOMS) > 1 else ""))
puts("")
def verify_plugin_settings(self):
puts("Verifying settings requested by plugins...")
missing_settings = False
missing_setting_error_messages = []
with indent(2):
for name, meta in self.required_settings_from_plugins.items():
if not hasattr(settings, name):
error_message = "%(setting_name)s is missing. It's required by the %(plugin_name)s plugin's '%(function_name)s' method." % meta
puts(colored.red("✗ %(setting_name)s" % meta))
missing_setting_error_messages.append(error_message)
missing_settings = True
else:
show_valid("%(setting_name)s" % meta)
if missing_settings:
puts("")
warn(
"Will is missing settings required by some plugins. He's starting up anyway, but you will run into errors if you try to use those plugins!"
)
self.add_startup_error(
"\n".join(missing_setting_error_messages))
else:
puts("")
def bootstrap_storage_mixin(self):
puts("Bootstrapping storage...")
try:
self.bootstrap_storage()
with indent(2):
show_valid("Connection to %s successful." % settings.REDIS_URL)
puts("")
except:
error("Unable to connect to %s" % settings.REDIS_URL)
sys.exit(1)
def bootstrap_scheduler(self):
bootstrapped = False
try:
self.save("plugin_modules_library", self._plugin_modules_library)
Scheduler.clear_locks(self)
self.scheduler = Scheduler()
for plugin_info, fn, function_name in self.periodic_tasks:
meta = fn.will_fn_metadata
self.add_periodic_task(
plugin_info["full_module_name"],
plugin_info["name"],
function_name,
meta["sched_args"],
meta["sched_kwargs"],
meta["function_name"],
)
for plugin_info, fn, function_name in self.random_tasks:
meta = fn.will_fn_metadata
self.add_random_tasks(plugin_info["full_module_name"],
plugin_info["name"], function_name,
meta["start_hour"], meta["end_hour"],
meta["day_of_week"],
meta["num_times_per_day"])
bootstrapped = True
except Exception, e:
self.startup_error("Error bootstrapping scheduler", e)
if bootstrapped:
show_valid("Scheduler started.")
self.scheduler.start_loop(self)
def verify_environment(self):
missing_settings = False
required_settings = [{
"name":
"WILL_USERNAME",
"obtain_at":
"""1. Go to hipchat, and create a new user for will.
2. Log into will, and go to Account settings>XMPP/Jabber Info.
3. On that page, the 'Jabber ID' is the value you want to use.""",
}, {
"name":
"WILL_PASSWORD",
"obtain_at":
"1. Go to hipchat, and create a new user for will. Note that password - this is the value you want. It's used for signing in via XMPP.",
}, {
"name":
"WILL_V2_TOKEN",
"obtain_at":
"""1. Log into hipchat using will's user.
2. Go to https://your-org.hipchat.com/account/api
3. Create a token.
4. Copy the value - this is the WILL_V2_TOKEN.""",
}, {
"name":
"WILL_REDIS_URL",
"obtain_at":
"""1. Set up an accessible redis host locally or in production
2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""",
}]
puts("")
puts("Verifying environment...")
for r in required_settings:
if not self.verify_individual_setting(r):
missing_settings = True
if missing_settings:
error(
"Will was unable to start because some required environment variables are missing. Please fix them and try again!"
)
sys.exit(1)
else:
puts("")
puts("Verifying credentials...")
# Parse [email protected] into id, where 222222 is the id. Yup.
user_id = settings.USERNAME[0:settings.USERNAME.
find("@")][settings.USERNAME.find("_") +
1:]
# Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS.
# http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests
q = Queue()
p = Process(target=self.get_hipchat_user,
args=(user_id, ),
kwargs={
"q": q,
})
p.start()
user_data = q.get()
p.join()
if "error" in user_data:
error("We ran into trouble: '%(message)s'" % user_data["error"])
sys.exit(1)
with indent(2):
show_valid("%s authenticated" % user_data["name"])
os.environ["WILL_NAME"] = user_data["name"]
show_valid("@%s verified as handle" % user_data["mention_name"])
os.environ["WILL_HANDLE"] = user_data["mention_name"]
puts("")
def train(epoch=10, batch_size=10, dataset_path=None, one_hot=False):
if one_hot:
loss_func = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=LR)
else:
loss_func = nn.BCELoss()
optimizer = optim.RMSprop(net.parameters(), lr=LR, alpha=0.9)
if dataset_path is not None and DEVICE != "kaggle":
if sys.platform.startswith('win'):
TRAIN_PATH = dataset_path + '\\train'
VALID_PATH = dataset_path + '\\test'
elif sys.platform.startswith('linux'):
TRAIN_PATH = dataset_path + '/train'
VALID_PATH = dataset_path + '/test'
elif DEVICE == "kaggle":
TRAIN_PATH = '../input/dogs-vs-cats/train/train'
VALID_PATH = '../input/dogs-vs-cats/test/test'
DATASET_PATH = '../input/dogs-vs-cats'
# print(TRAIN_PATH)
else:
raise ValueError("Dataset can not be None")
cat_dog_dataset = dataloader.CatVsDogDataset(TRAIN_PATH, mode="train", one_hot=one_hot)
train_loader = Data(cat_dog_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
# train_loader = Data(cat_dog_dataset, batch_size=batch_size, shuffle=True)
cat_dog_dataset_test = dataloader.CatVsDogDataset(TRAIN_PATH, mode="test", one_hot=one_hot)
test_loader = Data(cat_dog_dataset_test, batch_size=batch_size, shuffle=True, num_workers=0)
# test_loader = Data(cat_dog_dataset_test, batch_size=batch_size, shuffle=True)
cat_dog_dataset_valid = dataloader.CatVsDogValid(VALID_PATH)
valid_loader = Data(cat_dog_dataset_valid, batch_size=batch_size, shuffle=True, num_workers=0)
start_time = time.time()
print("Net: VGG%s, Total epoch: %d, Batch_size: %d, LR: %f, Device: %s"%(NET, epoch, batch_size, LR, DEVICE))
time.sleep(0.1)
for epoch in range(epoch):
print("\nEpoch: %d"%(epoch + 1))
time.sleep(0.1)
train_loss_sum, train_acc_sum, n = 0.0, 0.0, 0
for batch, (x, y) in enumerate(tqdm(train_loader)):
y_hat = net(x)
# if batch_size > 1, use sum() to calculate per batch loss
if one_hot:
loss = loss_func(y_hat, y).sum()
else:
loss = loss_func(y_hat, y)
# print("\t\tBatch #{0}/{1}".format(batch+1, len(train_loader)) + "Loss = %.6f"%float(loss))
if optimizer is not None:
optimizer.zero_grad()
elif params is not None and params[0].grad is not None:
for param in params:
param.grad.data.zero_()
loss.backward()
if optimizer is None:
optimizer = optim.SGD(net.parameters(), lr=globals(LR))
optimizer.step()
else:
optimizer.step()
# convert tensor data type to float data type
# train_loss_sum += loss.item()
# train_acc_sum += (y_hat == y).sum().item()
if one_hot:
train_loss_sum += loss_func(y_hat, y).sum().item()
train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()
else:
train_loss_sum += loss.item()
train_acc_sum += (torch.round(y_hat) == y).float().mean().item()
# print(train_loss_sum)
# print(train_acc_sum)
# train_loss_sum += float(loss_func(y_hat, y))
print('Epoch: {epoch}, Loss:{loss}, Accuracy:{accuracy}, Average_loss:{average_loss}, Average_accuracy:{average_accuracy}%'.\
format(epoch=epoch+1, loss=float('%.6f' % train_loss_sum), accuracy=float('%.6f' % train_acc_sum), \
average_loss=float('%.6f' %(train_loss_sum/(batch+1))), \
average_accuracy=float('%.6f' % (train_acc_sum/(batch+1)*100))))
if (epoch+1) % RECORD_EPOCH == 0:
test_acc = evaluate_accuracy(test_loader, net)
print('Epoch: {epoch}, Valid accuracy: {valid:.6f}%'.format(epoch=epoch+1, valid=test_acc*100))
end_time = time.time()
h, m, s = second2clock(end_time - start_time)
print("Total trainning time: " + "%d hours %02d mins %.2f seconds" % (h, m, s))
start_time = time.time()
test_acc = evaluate_accuracy(test_loader, net)
end_time = time.time()
h, m, s = second2clock(end_time - start_time)
print("Test accuracy: {:.6f}".format(test_acc*100) + "%, Eval time: " + "%d hours %02d mins %.2f seconds" % (h, m, s))
test_img, test_label = iter(test_loader).__next__()
show_result(net, test_img[0:SHOW_PIC_NUM], test_label[0:SHOW_PIC_NUM], rgb=RGB)
if SHOW_VALID_PIC:
valid_img = iter(valid_loader).__next__()
show_valid(net, valid_img[0:SHOW_PIC_NUM], rgb=RGB)
if CSV:
valid_loader = Data(cat_dog_dataset_valid, batch_size=1, shuffle=False, num_workers=0)
creat_csv(net, valid_loader)
def import_settings(quiet=True):
"""This method takes care of importing settings from the environment, and config.py file.
Order of operations:
1. Imports all WILL_ settings from the environment, and strips off the WILL_
2. Imports settings from config.py
3. Sets defaults for any missing, required settings.
This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping.
"""
settings = {}
# Import from environment, handle environment-specific parsing.
for k, v in os.environ.items():
if k[:5] == "WILL_":
k = k[5:]
settings[k] = v
if "ROOMS" in settings:
settings["ROOMS"] = settings["ROOMS"].split(";")
# If HIPCHAT_SERVER is set, we need to change the USERNAME slightly
# for XMPP to work.
if "HIPCHAT_SERVER" in settings:
settings["USERNAME"] = "******". \
format(user=settings["USERNAME"].split("@")[0],
host=settings["HIPCHAT_SERVER"])
else:
settings["HIPCHAT_SERVER"] = "api.hipchat.com"
# Import from config
if not quiet:
puts("Importing config.py... ")
with indent(2):
try:
had_warning = False
import config
for k, v in config.__dict__.items():
# Ignore private variables
if "__" not in k:
if k in os.environ and v != os.environ[k] and not quiet:
warn("%s is set in the environment as '%s', but overridden in"
" config.py as '%s'." % (k, os.environ[k], v))
had_warning = True
settings[k] = v
if not had_warning and not quiet:
show_valid("Valid.")
except:
# TODO: Check to see if there's a config.py.dist
if not quiet:
warn("no config.py found. This might be ok, but more likely, "
"you haven't copied config.py.dist over to config.py")
if not quiet:
puts("Verifying settings... ")
with indent(2):
# Set defaults
if "ROOMS" not in settings:
if not quiet:
warn("no ROOMS list found in the environment or config. "
"This is ok - Will will just join all available rooms.")
settings["ROOMS"] = None
if "DEFAULT_ROOM" not in settings and "ROOMS" in settings and settings["ROOMS"] and len(settings["ROOMS"]) > 0:
if not quiet:
warn("no DEFAULT_ROOM found in the environment or config. "
"Defaulting to '%s', the first one." % settings["ROOMS"][0])
settings["DEFAULT_ROOM"] = settings["ROOMS"][0]
if "HTTPSERVER_PORT" not in settings:
# For heroku
if "PORT" in os.environ:
settings["HTTPSERVER_PORT"] = os.environ["PORT"]
else:
if not quiet:
warn("no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'.")
settings["HTTPSERVER_PORT"] = "8000"
if "STORAGE_BACKEND" not in settings:
settings["STORAGE_BACKEND"] = "redis"
if settings["STORAGE_BACKEND"] == "redis":
if "REDIS_URL" not in settings:
# For heroku
if "REDISCLOUD_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"]
if not quiet:
note("WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good.")
elif "REDISTOGO_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISTOGO_URL"]
if not quiet:
note("WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good.")
elif "OPENREDIS_URL" in os.environ:
settings["REDIS_URL"] = os.environ["OPENREDIS_URL"]
if not quiet:
note("WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good.")
else:
settings["REDIS_URL"] = "redis://localhost:6379/7"
if not quiet:
note("WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7.")
if not settings["REDIS_URL"].startswith("redis://"):
settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"]
if "REDIS_MAX_CONNECTIONS" not in settings:
settings["REDIS_MAX_CONNECTIONS"] = 4
if not quiet:
note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.")
if settings["STORAGE_BACKEND"] == "file":
if "FILE_DIR" not in settings:
settings["FILE_DIR"] = "~/.will/"
if not quiet:
note("FILE_DIR not set. Defaulting to ~/.will/")
if settings["STORAGE_BACKEND"] == "couchbase":
if "COUCHBASE_URL" not in settings:
settings["COUCHBASE_URL"] = "couchbase:///will"
if not quiet:
note("COUCHBASE_URL not set. Defaulting to couchbase:///will")
if "PUBLIC_URL" not in settings:
default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"]
settings["PUBLIC_URL"] = default_public
if not quiet:
warn("no PUBLIC_URL found in the environment or config. Defaulting to '%s'." % default_public)
if "V1_TOKEN" not in settings:
if not quiet:
warn(
"no V1_TOKEN found in the environment or config."
"This is generally ok, but if you have more than 30 rooms, "
"you may recieve rate-limit errors without one."
)
if "TEMPLATE_DIRS" not in settings:
if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ:
# All good
pass
else:
settings["TEMPLATE_DIRS"] = []
if "ALLOW_INSECURE_HIPCHAT_SERVER" in settings and \
(settings["ALLOW_INSECURE_HIPCHAT_SERVER"] is True or
settings["ALLOW_INSECURE_HIPCHAT_SERVER"].lower() == "true"):
warn("You are choosing to run will with SSL disabled. "
"This is INSECURE and should NEVER be deployed outside a development environment.")
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = True
settings["REQUESTS_OPTIONS"] = {
"verify": False,
}
else:
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = False
settings["REQUESTS_OPTIONS"] = {}
if "ADMINS" not in settings:
settings["ADMINS"] = "*"
else:
if "WILL_ADMINS" in os.environ:
settings["ADMINS"] = [a.strip().lower() for a in settings.get('ADMINS', '').split(';') if a.strip()]
if "PROXY_URL" in settings:
parsed_proxy_url = urlparse(settings["PROXY_URL"])
settings["USE_PROXY"] = True
settings["PROXY_HOSTNAME"] = parsed_proxy_url.hostname
settings["PROXY_USERNAME"] = parsed_proxy_url.username
settings["PROXY_PASSWORD"] = parsed_proxy_url.password
settings["PROXY_PORT"] = parsed_proxy_url.port
else:
settings["USE_PROXY"] = False
# Set them in the module namespace
for k in sorted(settings, key=lambda x: x[0]):
if not quiet:
show_valid(k)
globals()[k] = settings[k]
def import_settings(quiet=True):
"""This method takes care of importing settings from the environment, and config.py file.
Order of operations:
1. Imports all WILL_ settings from the environment, and strips off the WILL_
2. Imports settings from config.py
3. Sets defaults for any missing, required settings.
This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping.
"""
settings = {}
# Import from environment, handle environment-specific parsing.
for k, v in os.environ.items():
if k[:5] == "WILL_":
k = k[5:]
settings[k] = v
if "ROOMS" in settings:
settings["ROOMS"] = settings["ROOMS"].split(";")
# If HIPCHAT_SERVER is set, we need to change the USERNAME slightly
# for XMPP to work.
if "HIPCHAT_SERVER" in settings:
settings["USERNAME"] = "******".\
format(user=settings["USERNAME"].split("@")[0],
host=settings["HIPCHAT_SERVER"])
else:
settings["HIPCHAT_SERVER"] = "api.hipchat.com"
# Import from config
if not quiet:
puts("Importing config.py... ")
with indent(2):
try:
had_warning = False
import config
for k, v in config.__dict__.items():
# Ignore private variables
if "__" not in k:
if k in os.environ and v != os.environ[k] and not quiet:
warn(
"%s is set in the environment as '%s', but overridden in"
" config.py as '%s'." % (k, os.environ[k], v))
had_warning = True
settings[k] = v
if not had_warning and not quiet:
show_valid("Valid.")
except:
# TODO: Check to see if there's a config.py.dist
if not quiet:
warn("no config.py found. This might be ok, but more likely, "
"you haven't copied config.py.dist over to config.py")
if not quiet:
puts("Verifying settings... ")
with indent(2):
# Set defaults
if "ROOMS" not in settings:
if not quiet:
warn("no ROOMS list found in the environment or config. "
"This is ok - Will will just join all available rooms.")
settings["ROOMS"] = None
if "DEFAULT_ROOM" not in settings and "ROOMS" in settings and settings[
"ROOMS"] and len(settings["ROOMS"]) > 0:
if not quiet:
warn("no DEFAULT_ROOM found in the environment or config. "
"Defaulting to '%s', the first one." %
settings["ROOMS"][0])
settings["DEFAULT_ROOM"] = settings["ROOMS"][0]
if "HTTPSERVER_PORT" not in settings:
# For heroku
if "PORT" in os.environ:
settings["HTTPSERVER_PORT"] = os.environ["PORT"]
else:
if not quiet:
warn(
"no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'."
)
settings["HTTPSERVER_PORT"] = "80"
if "STORAGE_BACKEND" not in settings:
settings["STORAGE_BACKEND"] = "redis"
if settings["STORAGE_BACKEND"] == "redis":
if "REDIS_URL" not in settings:
# For heroku
if "REDISCLOUD_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good."
)
elif "REDISTOGO_URL" in os.environ:
settings["REDIS_URL"] = os.environ["REDISTOGO_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good."
)
elif "OPENREDIS_URL" in os.environ:
settings["REDIS_URL"] = os.environ["OPENREDIS_URL"]
if not quiet:
note(
"WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good."
)
else:
settings["REDIS_URL"] = "redis://localhost:6379/7"
if not quiet:
note(
"WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7."
)
if not settings["REDIS_URL"].startswith("redis://"):
settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"]
if "REDIS_MAX_CONNECTIONS" not in settings:
settings["REDIS_MAX_CONNECTIONS"] = 4
if not quiet:
note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.")
if settings["STORAGE_BACKEND"] == "file":
if "FILE_DIR" not in settings:
settings["FILE_DIR"] = "~/.will/"
if not quiet:
note("FILE_DIR not set. Defaulting to ~/.will/")
if settings["STORAGE_BACKEND"] == "couchbase":
if "COUCHBASE_URL" not in settings:
settings["COUCHBASE_URL"] = "couchbase:///will"
if not quiet:
note(
"COUCHBASE_URL not set. Defaulting to couchbase:///will"
)
if "PUBLIC_URL" not in settings:
default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"]
settings["PUBLIC_URL"] = default_public
if not quiet:
warn(
"no PUBLIC_URL found in the environment or config. Defaulting to '%s'."
% default_public)
if "V1_TOKEN" not in settings:
if not quiet:
warn(
"no V1_TOKEN found in the environment or config."
"This is generally ok, but if you have more than 30 rooms, "
"you may recieve rate-limit errors without one.")
if "TEMPLATE_DIRS" not in settings:
if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ:
# All good
pass
else:
settings["TEMPLATE_DIRS"] = []
if "ALLOW_INSECURE_HIPCHAT_SERVER" in settings and\
(settings["ALLOW_INSECURE_HIPCHAT_SERVER"] is True or
settings["ALLOW_INSECURE_HIPCHAT_SERVER"].lower() == "true"):
warn(
"You are choosing to run will with SSL disabled. "
"This is INSECURE and should NEVER be deployed outside a development environment."
)
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = True
settings["REQUESTS_OPTIONS"] = {
"verify": False,
}
else:
settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = False
settings["REQUESTS_OPTIONS"] = {}
if "ADMINS" not in settings:
settings["ADMINS"] = "*"
else:
if "WILL_ADMINS" in os.environ:
settings["ADMINS"] = [
a.strip().lower()
for a in settings.get('ADMINS', '').split(';')
if a.strip()
]
if "PROXY_URL" in settings:
parsed_proxy_url = urlparse(settings["PROXY_URL"])
settings["USE_PROXY"] = True
settings["PROXY_HOSTNAME"] = parsed_proxy_url.hostname
settings["PROXY_USERNAME"] = parsed_proxy_url.username
settings["PROXY_PASSWORD"] = parsed_proxy_url.password
settings["PROXY_PORT"] = parsed_proxy_url.port
else:
settings["USE_PROXY"] = False
# Set them in the module namespace
for k in sorted(settings, key=lambda x: x[0]):
if not quiet:
show_valid(k)
globals()[k] = settings[k]
