def verify_plugin_settings(self): puts("Verifying settings requested by plugins...") missing_settings = False missing_setting_error_messages = [] with indent(2): for name, meta in self.required_settings_from_plugins.items(): if not hasattr(settings, name): error_message = ( "%(setting_name)s is missing. It's required by the" "%(plugin_name)s plugin's '%(function_name)s' method." ) % meta puts(colored.red("✗ %(setting_name)s" % meta)) missing_setting_error_messages.append(error_message) missing_settings = True else: show_valid("%(setting_name)s" % meta) if missing_settings: puts("") warn( "Will is missing settings required by some plugins. " "He's starting up anyway, but you will run into errors" " if you try to use those plugins!" ) self.add_startup_error("\n".join(missing_setting_error_messages)) else: puts("")
def load_config(self): puts("Loading configuration...") with indent(2): settings.import_settings(quiet=False) puts("") puts("Verifying rooms...") # If we're missing ROOMS, join all of them. with indent(2): if settings.ROOMS == None: # Yup. Thanks, BSDs. q = Queue() p = Process(target=self.update_available_rooms, args=(), kwargs={ "q": q, }) p.start() rooms_list = q.get() show_valid("Joining all %s known rooms." % len(rooms_list)) os.environ["WILL_ROOMS"] = ";".join(rooms_list) p.join() settings.import_settings() else: show_valid("Joining the %s room%s specified." % (len( settings.ROOMS), "s" if len(settings.ROOMS) > 1 else "")) puts("")
def load_config(self): puts("Loading configuration...") with indent(2): settings.import_settings(quiet=False) puts("") puts("Verifying rooms...") # If we're missing ROOMS, join all of them. with indent(2): if settings.ROOMS is None: # Yup. Thanks, BSDs. q = Queue() p = Process(target=self.update_available_rooms, args=(), kwargs={"q": q, }) p.start() rooms_list = q.get() show_valid("Joining all %s known rooms." % len(rooms_list)) os.environ["WILL_ROOMS"] = ";".join(rooms_list) p.join() settings.import_settings() else: show_valid( "Joining the %s room%s specified." % ( len(settings.ROOMS), "s" if len(settings.ROOMS) > 1 else "" ) ) puts("")
def verify_plugin_settings(self): puts("Verifying settings requested by plugins...") missing_settings = False missing_setting_error_messages = [] with indent(2): for name, meta in self.required_settings_from_plugins.items(): if not hasattr(settings, name): error_message = ( "%(setting_name)s is missing. It's required by the" "%(plugin_name)s plugin's '%(function_name)s' method." ) % meta puts(colored.red("✗ %(setting_name)s" % meta)) missing_setting_error_messages.append(error_message) missing_settings = True else: show_valid("%(setting_name)s" % meta) if missing_settings: puts("") warn("Will is missing settings required by some plugins. " "He's starting up anyway, but you will run into errors" " if you try to use those plugins!") self.add_startup_error( "\n".join(missing_setting_error_messages)) else: puts("")
def bootstrap_storage_mixin(self): puts("Bootstrapping storage...") try: self.bootstrap_storage() with indent(2): show_valid("Connection to %s successful." % settings.REDIS_URL) puts("") except: error("Unable to connect to %s" % settings.REDIS_URL) sys.exit(1)
def verify_individual_setting(self, test_setting, quiet=False): if hasattr(settings, test_setting["name"][5:]): with indent(2): show_valid(test_setting["name"]) return True else: error("%(name)s... missing!" % test_setting) with indent(2): puts("""To obtain a %(name)s: \n%(obtain_at)s To set your %(name)s: 1. On your local machine, add this to your virtual environment's bin/postactivate file: export %(name)s=YOUR_ACTUAL_%(name)s 2. If you've deployed will on heroku, run heroku config:set %(name)s=YOUR_ACTUAL_%(name)s """ % test_setting) return False
def verify_environment(self): missing_settings = False required_settings = [ { "name": "WILL_USERNAME", "obtain_at": """1. Go to hipchat, and create a new user for will. 2. Log into will, and go to Account settings>XMPP/Jabber Info. 3. On that page, the 'Jabber ID' is the value you want to use.""", }, { "name": "WILL_PASSWORD", "obtain_at": ( "1. Go to hipchat, and create a new user for will. " "Note that password - this is the value you want. " "It's used for signing in via XMPP." ), }, { "name": "WILL_V2_TOKEN", "obtain_at": """1. Log into hipchat using will's user. 2. Go to https://your-org.hipchat.com/account/api 3. Create a token. 4. Copy the value - this is the WILL_V2_TOKEN.""", }, { "name": "WILL_REDIS_URL", "obtain_at": """1. Set up an accessible redis host locally or in production 2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""", } ] puts("") puts("Verifying environment...") for r in required_settings: if not self.verify_individual_setting(r): missing_settings = True if missing_settings: error( "Will was unable to start because some required environment " "variables are missing. Please fix them and try again!" ) sys.exit(1) else: puts("") puts("Verifying credentials...") # Parse [email protected] into id, where 222222 is the id. Yup. user_id = settings.USERNAME[0:settings.USERNAME.find("@")][settings.USERNAME.find("_") + 1:] # Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS. # http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests q = Queue() p = Process(target=self.get_hipchat_user, args=(user_id,), kwargs={"q": q, }) p.start() user_data = q.get() p.join() if "error" in user_data: error("We ran into trouble: '%(message)s'" % user_data["error"]) sys.exit(1) with indent(2): show_valid("%s authenticated" % user_data["name"]) os.environ["WILL_NAME"] = user_data["name"] show_valid("@%s verified as handle" % user_data["mention_name"]) os.environ["WILL_HANDLE"] = user_data["mention_name"] puts("")
def bootstrap_bottle(self): bootstrapped = False try: for cls, function_name in self.bottle_routes: instantiated_cls = cls() instantiated_fn = getattr(instantiated_cls, function_name) bottle_route_args = {} for k, v in instantiated_fn.will_fn_metadata.items(): if "bottle_" in k and k != "bottle_route": bottle_route_args[k[len("bottle_"):]] = v bottle.route(instantiated_fn.will_fn_metadata["bottle_route"], **bottle_route_args)(instantiated_fn) bootstrapped = True except Exception, e: self.startup_error("Error bootstrapping bottle", e) if bootstrapped: show_valid("Web server started.") bottle.run(host='0.0.0.0', port=settings.HTTPSERVER_PORT, server='cherrypy', quiet=True) def bootstrap_xmpp(self): bootstrapped = False try: self.start_xmpp_client() sorted_help = {} for k, v in self.help_modules.items(): sorted_help[k] = sorted(v) self.save("help_modules", sorted_help) self.save("all_listener_regexes", self.all_listener_regexes) self.connect() bootstrapped = True except Exception, e:
bootstrapped = False try: for cls, function_name in self.bottle_routes: instantiated_cls = cls() instantiated_fn = getattr(instantiated_cls, function_name) bottle_route_args = {} for k, v in instantiated_fn.will_fn_metadata.items(): if "bottle_" in k and k != "bottle_route": bottle_route_args[k[len("bottle_"):]] = v bottle.route(instantiated_fn.will_fn_metadata["bottle_route"], **bottle_route_args)(instantiated_fn) bootstrapped = True except Exception, e: self.startup_error("Error bootstrapping bottle", e) if bootstrapped: show_valid("Web server started.") bottle.run(host='0.0.0.0', port=settings.HTTPSERVER_PORT, server='cherrypy', quiet=True) def bootstrap_xmpp(self): bootstrapped = False try: self.start_xmpp_client() sorted_help = {} for k, v in self.help_modules.items(): sorted_help[k] = sorted(v) self.save("help_modules", sorted_help) self.save("all_listener_regexes", self.all_listener_regexes)
def import_settings(quiet=True): """This method takes care of importing settings from the environment, and config.py file. Order of operations: 1. Imports all WILL_ settings from the environment, and strips off the WILL_ 2. Imports settings from config.py 3. Sets defaults for any missing, required settings. This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping. """ settings = {} # Import from environment, handle environment-specific parsing. for k, v in os.environ.items(): if k[:5] == "WILL_": k = k[5:] settings[k] = v if "ROOMS" in settings: settings["ROOMS"] = settings["ROOMS"].split(";") # Import from config if not quiet: puts("Importing config.py... ") with indent(2): try: had_warning = False import config for k, v in config.__dict__.items(): # Ignore private variables if "__" not in k: if k in os.environ and v != os.environ[k] and not quiet: warn("%s is set in the environment as '%s', but overridden in config.py as '%s'." % (k, os.environ[k], v)) had_warning = True settings[k] = v if not had_warning and not quiet: show_valid("Valid.") except: # TODO: Check to see if there's a config.py.dist if not quiet: warn("no config.py found. This might be ok, but more likely, you haven't copied config.py.dist over to config.py") if not quiet: puts("Verifying settings... ") with indent(2): # Set defaults if "ROOMS" not in settings: if not quiet: warn("no ROOMS list found in the environment or config. This is ok - Will will just join all available rooms.") settings["ROOMS"] = None if not "DEFAULT_ROOM" in settings and "ROOMS" in settings and settings["ROOMS"] and len(settings["ROOMS"]) > 0: if not quiet: warn("no DEFAULT_ROOM found in the environment or config. Defaulting to '%s', the first one." % settings["ROOMS"][0]) settings["DEFAULT_ROOM"] = settings["ROOMS"][0] if not "HTTPSERVER_PORT" in settings: # For heroku if "PORT" in os.environ: settings["HTTPSERVER_PORT"] = os.environ["PORT"] else: if not quiet: warn("no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'.") settings["HTTPSERVER_PORT"] = "80" if not "REDIS_URL" in settings: # For heroku if "REDISCLOUD_URL" in os.environ: settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"] if not quiet: note("WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good.") elif "REDISTOGO_URL" in os.environ: settings["REDIS_URL"] = os.environ["REDISTOGO_URL"] if not quiet: note("WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good.") elif "OPENREDIS_URL" in os.environ: settings["REDIS_URL"] = os.environ["OPENREDIS_URL"] if not quiet: note("WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good.") else: settings["REDIS_URL"] = "redis://localhost:6379/7" if not quiet: note("WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7.") if not "PUBLIC_URL" in settings: default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"] settings["PUBLIC_URL"] = default_public if not quiet: warn("no PUBLIC_URL found in the environment or config. Defaulting to '%s'." % default_public) if not "V1_TOKEN" in settings: if not quiet: warn("no V1_TOKEN found in the environment or config. This is generally ok, but if you have more than 30 rooms, you may recieve rate-limit errors without one.") if not "TEMPLATE_DIRS" in settings: if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ: # All good pass else: settings["TEMPLATE_DIRS"] = [] if not "ADMINS" in settings: settings["ADMINS"] = "*" else: if "WILL_ADMINS" in os.environ: settings["ADMINS"] = [a.strip().lower() for a in settings.get('ADMINS', '').split(';') if a.strip()] # Set them in the module namespace for k in sorted(settings, key=lambda x: x[0]): if not quiet: show_valid(k) globals()[k] = settings[k]
class WillBot(EmailMixin, WillXMPPClientMixin, StorageMixin, ScheduleMixin, ErrorMixin, RoomMixin, HipChatMixin, PluginModulesLibraryMixin): def __init__(self, **kwargs): if "template_dirs" in kwargs: warn("template_dirs is now depreciated") if "plugin_dirs" in kwargs: warn("plugin_dirs is now depreciated") log_level = getattr(settings, 'LOGLEVEL', logging.ERROR) logging.basicConfig(level=log_level, format='%(levelname)-8s %(message)s') # Find all the PLUGINS modules plugins = settings.PLUGINS self.plugins_dirs = {} # Set template dirs. full_path_template_dirs = [] for t in settings.TEMPLATE_DIRS: full_path_template_dirs.append(os.path.abspath(t)) # Add will's templates_root if TEMPLATES_ROOT not in full_path_template_dirs: full_path_template_dirs += [ TEMPLATES_ROOT, ] # Add this project's templates_root if PROJECT_TEMPLATE_ROOT not in full_path_template_dirs: full_path_template_dirs += [ PROJECT_TEMPLATE_ROOT, ] # Convert those to dirs for plugin in plugins: path_name = None for mod in plugin.split('.'): if path_name is not None: path_name = [path_name] file_name, path_name, description = imp.find_module( mod, path_name) # Add, uniquely. self.plugins_dirs[os.path.abspath(path_name)] = plugin if os.path.exists( os.path.join(os.path.abspath(path_name), "templates")): full_path_template_dirs.append( os.path.join(os.path.abspath(path_name), "templates")) # Key by module name self.plugins_dirs = dict( zip(self.plugins_dirs.values(), self.plugins_dirs.keys())) # Storing here because storage hasn't been bootstrapped yet. os.environ["WILL_TEMPLATE_DIRS_PICKLED"] =\ ";;".join(full_path_template_dirs) def bootstrap(self): print_head() self.verify_environment() self.load_config() self.bootstrap_storage_mixin() self.bootstrap_plugins() self.verify_plugin_settings() puts("Bootstrapping complete.") puts("\nStarting core processes:") # Scheduler scheduler_thread = Process(target=self.bootstrap_scheduler) # scheduler_thread.daemon = True # Bottle bottle_thread = Process(target=self.bootstrap_bottle) # bottle_thread.daemon = True # XMPP Listener xmpp_thread = Process(target=self.bootstrap_xmpp) # xmpp_thread.daemon = True with indent(2): try: # Start up threads. xmpp_thread.start() scheduler_thread.start() bottle_thread.start() errors = self.get_startup_errors() if len(errors) > 0: default_room = self.get_room_from_name_or_id( settings.DEFAULT_ROOM)["room_id"] error_message = "FYI, I ran into some problems while starting up:" for err in errors: error_message += "\n%s\n" % err self.send_room_message(default_room, error_message, color="yellow") puts(colored.red(error_message)) while True: time.sleep(100) except (KeyboardInterrupt, SystemExit): scheduler_thread.terminate() bottle_thread.terminate() xmpp_thread.terminate() print '\n\nReceived keyboard interrupt, quitting threads.', while (scheduler_thread.is_alive() or bottle_thread.is_alive() or xmpp_thread.is_alive()): sys.stdout.write(".") sys.stdout.flush() time.sleep(0.5) def verify_individual_setting(self, test_setting, quiet=False): if hasattr(settings, test_setting["name"][5:]): with indent(2): show_valid(test_setting["name"]) return True else: error("%(name)s... missing!" % test_setting) with indent(2): puts("""To obtain a %(name)s: \n%(obtain_at)s To set your %(name)s: 1. On your local machine, add this to your virtual environment's bin/postactivate file: export %(name)s=YOUR_ACTUAL_%(name)s 2. If you've deployed will on heroku, run heroku config:set %(name)s=YOUR_ACTUAL_%(name)s """ % test_setting) return False def verify_environment(self): missing_settings = False required_settings = [{ "name": "WILL_USERNAME", "obtain_at": """1. Go to hipchat, and create a new user for will. 2. Log into will, and go to Account settings>XMPP/Jabber Info. 3. On that page, the 'Jabber ID' is the value you want to use.""", }, { "name": "WILL_PASSWORD", "obtain_at": "1. Go to hipchat, and create a new user for will. Note that password - this is the value you want. It's used for signing in via XMPP.", }, { "name": "WILL_V2_TOKEN", "obtain_at": """1. Log into hipchat using will's user. 2. Go to https://your-org.hipchat.com/account/api 3. Create a token. 4. Copy the value - this is the WILL_V2_TOKEN.""", }, { "name": "WILL_REDIS_URL", "obtain_at": """1. Set up an accessible redis host locally or in production 2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""", }] puts("") puts("Verifying environment...") for r in required_settings: if not self.verify_individual_setting(r): missing_settings = True if missing_settings: error( "Will was unable to start because some required environment variables are missing. Please fix them and try again!" ) sys.exit(1) else: puts("") puts("Verifying credentials...") # Parse [email protected] into id, where 222222 is the id. Yup. user_id = settings.USERNAME[0:settings.USERNAME. find("@")][settings.USERNAME.find("_") + 1:] # Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS. # http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests q = Queue() p = Process(target=self.get_hipchat_user, args=(user_id, ), kwargs={ "q": q, }) p.start() user_data = q.get() p.join() if "error" in user_data: error("We ran into trouble: '%(message)s'" % user_data["error"]) sys.exit(1) with indent(2): show_valid("%s authenticated" % user_data["name"]) os.environ["WILL_NAME"] = user_data["name"] show_valid("@%s verified as handle" % user_data["mention_name"]) os.environ["WILL_HANDLE"] = user_data["mention_name"] puts("") def load_config(self): puts("Loading configuration...") with indent(2): settings.import_settings(quiet=False) puts("") puts("Verifying rooms...") # If we're missing ROOMS, join all of them. with indent(2): if settings.ROOMS == None: # Yup. Thanks, BSDs. q = Queue() p = Process(target=self.update_available_rooms, args=(), kwargs={ "q": q, }) p.start() rooms_list = q.get() show_valid("Joining all %s known rooms." % len(rooms_list)) os.environ["WILL_ROOMS"] = ";".join(rooms_list) p.join() settings.import_settings() else: show_valid("Joining the %s room%s specified." % (len( settings.ROOMS), "s" if len(settings.ROOMS) > 1 else "")) puts("") def verify_plugin_settings(self): puts("Verifying settings requested by plugins...") missing_settings = False missing_setting_error_messages = [] with indent(2): for name, meta in self.required_settings_from_plugins.items(): if not hasattr(settings, name): error_message = "%(setting_name)s is missing. It's required by the %(plugin_name)s plugin's '%(function_name)s' method." % meta puts(colored.red("✗ %(setting_name)s" % meta)) missing_setting_error_messages.append(error_message) missing_settings = True else: show_valid("%(setting_name)s" % meta) if missing_settings: puts("") warn( "Will is missing settings required by some plugins. He's starting up anyway, but you will run into errors if you try to use those plugins!" ) self.add_startup_error( "\n".join(missing_setting_error_messages)) else: puts("") def bootstrap_storage_mixin(self): puts("Bootstrapping storage...") try: self.bootstrap_storage() with indent(2): show_valid("Connection to %s successful." % settings.REDIS_URL) puts("") except: error("Unable to connect to %s" % settings.REDIS_URL) sys.exit(1) def bootstrap_scheduler(self): bootstrapped = False try: self.save("plugin_modules_library", self._plugin_modules_library) Scheduler.clear_locks(self) self.scheduler = Scheduler() for plugin_info, fn, function_name in self.periodic_tasks: meta = fn.will_fn_metadata self.add_periodic_task( plugin_info["full_module_name"], plugin_info["name"], function_name, meta["sched_args"], meta["sched_kwargs"], meta["function_name"], ) for plugin_info, fn, function_name in self.random_tasks: meta = fn.will_fn_metadata self.add_random_tasks(plugin_info["full_module_name"], plugin_info["name"], function_name, meta["start_hour"], meta["end_hour"], meta["day_of_week"], meta["num_times_per_day"]) bootstrapped = True except Exception, e: self.startup_error("Error bootstrapping scheduler", e) if bootstrapped: show_valid("Scheduler started.") self.scheduler.start_loop(self)
def verify_environment(self): missing_settings = False required_settings = [{ "name": "WILL_USERNAME", "obtain_at": """1. Go to hipchat, and create a new user for will. 2. Log into will, and go to Account settings>XMPP/Jabber Info. 3. On that page, the 'Jabber ID' is the value you want to use.""", }, { "name": "WILL_PASSWORD", "obtain_at": "1. Go to hipchat, and create a new user for will. Note that password - this is the value you want. It's used for signing in via XMPP.", }, { "name": "WILL_V2_TOKEN", "obtain_at": """1. Log into hipchat using will's user. 2. Go to https://your-org.hipchat.com/account/api 3. Create a token. 4. Copy the value - this is the WILL_V2_TOKEN.""", }, { "name": "WILL_REDIS_URL", "obtain_at": """1. Set up an accessible redis host locally or in production 2. Set WILL_REDIS_URL to its full value, i.e. redis://localhost:6379/7""", }] puts("") puts("Verifying environment...") for r in required_settings: if not self.verify_individual_setting(r): missing_settings = True if missing_settings: error( "Will was unable to start because some required environment variables are missing. Please fix them and try again!" ) sys.exit(1) else: puts("") puts("Verifying credentials...") # Parse [email protected] into id, where 222222 is the id. Yup. user_id = settings.USERNAME[0:settings.USERNAME. find("@")][settings.USERNAME.find("_") + 1:] # Splitting into a thread. Necessary because *BSDs (including OSX) don't have threadsafe DNS. # http://stackoverflow.com/questions/1212716/python-interpreter-blocks-multithreaded-dns-requests q = Queue() p = Process(target=self.get_hipchat_user, args=(user_id, ), kwargs={ "q": q, }) p.start() user_data = q.get() p.join() if "error" in user_data: error("We ran into trouble: '%(message)s'" % user_data["error"]) sys.exit(1) with indent(2): show_valid("%s authenticated" % user_data["name"]) os.environ["WILL_NAME"] = user_data["name"] show_valid("@%s verified as handle" % user_data["mention_name"]) os.environ["WILL_HANDLE"] = user_data["mention_name"] puts("")
def train(epoch=10, batch_size=10, dataset_path=None, one_hot=False): if one_hot: loss_func = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=LR) else: loss_func = nn.BCELoss() optimizer = optim.RMSprop(net.parameters(), lr=LR, alpha=0.9) if dataset_path is not None and DEVICE != "kaggle": if sys.platform.startswith('win'): TRAIN_PATH = dataset_path + '\\train' VALID_PATH = dataset_path + '\\test' elif sys.platform.startswith('linux'): TRAIN_PATH = dataset_path + '/train' VALID_PATH = dataset_path + '/test' elif DEVICE == "kaggle": TRAIN_PATH = '../input/dogs-vs-cats/train/train' VALID_PATH = '../input/dogs-vs-cats/test/test' DATASET_PATH = '../input/dogs-vs-cats' # print(TRAIN_PATH) else: raise ValueError("Dataset can not be None") cat_dog_dataset = dataloader.CatVsDogDataset(TRAIN_PATH, mode="train", one_hot=one_hot) train_loader = Data(cat_dog_dataset, batch_size=batch_size, shuffle=True, num_workers=0) # train_loader = Data(cat_dog_dataset, batch_size=batch_size, shuffle=True) cat_dog_dataset_test = dataloader.CatVsDogDataset(TRAIN_PATH, mode="test", one_hot=one_hot) test_loader = Data(cat_dog_dataset_test, batch_size=batch_size, shuffle=True, num_workers=0) # test_loader = Data(cat_dog_dataset_test, batch_size=batch_size, shuffle=True) cat_dog_dataset_valid = dataloader.CatVsDogValid(VALID_PATH) valid_loader = Data(cat_dog_dataset_valid, batch_size=batch_size, shuffle=True, num_workers=0) start_time = time.time() print("Net: VGG%s, Total epoch: %d, Batch_size: %d, LR: %f, Device: %s"%(NET, epoch, batch_size, LR, DEVICE)) time.sleep(0.1) for epoch in range(epoch): print("\nEpoch: %d"%(epoch + 1)) time.sleep(0.1) train_loss_sum, train_acc_sum, n = 0.0, 0.0, 0 for batch, (x, y) in enumerate(tqdm(train_loader)): y_hat = net(x) # if batch_size > 1, use sum() to calculate per batch loss if one_hot: loss = loss_func(y_hat, y).sum() else: loss = loss_func(y_hat, y) # print("\t\tBatch #{0}/{1}".format(batch+1, len(train_loader)) + "Loss = %.6f"%float(loss)) if optimizer is not None: optimizer.zero_grad() elif params is not None and params[0].grad is not None: for param in params: param.grad.data.zero_() loss.backward() if optimizer is None: optimizer = optim.SGD(net.parameters(), lr=globals(LR)) optimizer.step() else: optimizer.step() # convert tensor data type to float data type # train_loss_sum += loss.item() # train_acc_sum += (y_hat == y).sum().item() if one_hot: train_loss_sum += loss_func(y_hat, y).sum().item() train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item() else: train_loss_sum += loss.item() train_acc_sum += (torch.round(y_hat) == y).float().mean().item() # print(train_loss_sum) # print(train_acc_sum) # train_loss_sum += float(loss_func(y_hat, y)) print('Epoch: {epoch}, Loss:{loss}, Accuracy:{accuracy}, Average_loss:{average_loss}, Average_accuracy:{average_accuracy}%'.\ format(epoch=epoch+1, loss=float('%.6f' % train_loss_sum), accuracy=float('%.6f' % train_acc_sum), \ average_loss=float('%.6f' %(train_loss_sum/(batch+1))), \ average_accuracy=float('%.6f' % (train_acc_sum/(batch+1)*100)))) if (epoch+1) % RECORD_EPOCH == 0: test_acc = evaluate_accuracy(test_loader, net) print('Epoch: {epoch}, Valid accuracy: {valid:.6f}%'.format(epoch=epoch+1, valid=test_acc*100)) end_time = time.time() h, m, s = second2clock(end_time - start_time) print("Total trainning time: " + "%d hours %02d mins %.2f seconds" % (h, m, s)) start_time = time.time() test_acc = evaluate_accuracy(test_loader, net) end_time = time.time() h, m, s = second2clock(end_time - start_time) print("Test accuracy: {:.6f}".format(test_acc*100) + "%, Eval time: " + "%d hours %02d mins %.2f seconds" % (h, m, s)) test_img, test_label = iter(test_loader).__next__() show_result(net, test_img[0:SHOW_PIC_NUM], test_label[0:SHOW_PIC_NUM], rgb=RGB) if SHOW_VALID_PIC: valid_img = iter(valid_loader).__next__() show_valid(net, valid_img[0:SHOW_PIC_NUM], rgb=RGB) if CSV: valid_loader = Data(cat_dog_dataset_valid, batch_size=1, shuffle=False, num_workers=0) creat_csv(net, valid_loader)
def import_settings(quiet=True): """This method takes care of importing settings from the environment, and config.py file. Order of operations: 1. Imports all WILL_ settings from the environment, and strips off the WILL_ 2. Imports settings from config.py 3. Sets defaults for any missing, required settings. This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping. """ settings = {} # Import from environment, handle environment-specific parsing. for k, v in os.environ.items(): if k[:5] == "WILL_": k = k[5:] settings[k] = v if "ROOMS" in settings: settings["ROOMS"] = settings["ROOMS"].split(";") # If HIPCHAT_SERVER is set, we need to change the USERNAME slightly # for XMPP to work. if "HIPCHAT_SERVER" in settings: settings["USERNAME"] = "******". \ format(user=settings["USERNAME"].split("@")[0], host=settings["HIPCHAT_SERVER"]) else: settings["HIPCHAT_SERVER"] = "api.hipchat.com" # Import from config if not quiet: puts("Importing config.py... ") with indent(2): try: had_warning = False import config for k, v in config.__dict__.items(): # Ignore private variables if "__" not in k: if k in os.environ and v != os.environ[k] and not quiet: warn("%s is set in the environment as '%s', but overridden in" " config.py as '%s'." % (k, os.environ[k], v)) had_warning = True settings[k] = v if not had_warning and not quiet: show_valid("Valid.") except: # TODO: Check to see if there's a config.py.dist if not quiet: warn("no config.py found. This might be ok, but more likely, " "you haven't copied config.py.dist over to config.py") if not quiet: puts("Verifying settings... ") with indent(2): # Set defaults if "ROOMS" not in settings: if not quiet: warn("no ROOMS list found in the environment or config. " "This is ok - Will will just join all available rooms.") settings["ROOMS"] = None if "DEFAULT_ROOM" not in settings and "ROOMS" in settings and settings["ROOMS"] and len(settings["ROOMS"]) > 0: if not quiet: warn("no DEFAULT_ROOM found in the environment or config. " "Defaulting to '%s', the first one." % settings["ROOMS"][0]) settings["DEFAULT_ROOM"] = settings["ROOMS"][0] if "HTTPSERVER_PORT" not in settings: # For heroku if "PORT" in os.environ: settings["HTTPSERVER_PORT"] = os.environ["PORT"] else: if not quiet: warn("no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'.") settings["HTTPSERVER_PORT"] = "8000" if "STORAGE_BACKEND" not in settings: settings["STORAGE_BACKEND"] = "redis" if settings["STORAGE_BACKEND"] == "redis": if "REDIS_URL" not in settings: # For heroku if "REDISCLOUD_URL" in os.environ: settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"] if not quiet: note("WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good.") elif "REDISTOGO_URL" in os.environ: settings["REDIS_URL"] = os.environ["REDISTOGO_URL"] if not quiet: note("WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good.") elif "OPENREDIS_URL" in os.environ: settings["REDIS_URL"] = os.environ["OPENREDIS_URL"] if not quiet: note("WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good.") else: settings["REDIS_URL"] = "redis://localhost:6379/7" if not quiet: note("WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7.") if not settings["REDIS_URL"].startswith("redis://"): settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"] if "REDIS_MAX_CONNECTIONS" not in settings: settings["REDIS_MAX_CONNECTIONS"] = 4 if not quiet: note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.") if settings["STORAGE_BACKEND"] == "file": if "FILE_DIR" not in settings: settings["FILE_DIR"] = "~/.will/" if not quiet: note("FILE_DIR not set. Defaulting to ~/.will/") if settings["STORAGE_BACKEND"] == "couchbase": if "COUCHBASE_URL" not in settings: settings["COUCHBASE_URL"] = "couchbase:///will" if not quiet: note("COUCHBASE_URL not set. Defaulting to couchbase:///will") if "PUBLIC_URL" not in settings: default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"] settings["PUBLIC_URL"] = default_public if not quiet: warn("no PUBLIC_URL found in the environment or config. Defaulting to '%s'." % default_public) if "V1_TOKEN" not in settings: if not quiet: warn( "no V1_TOKEN found in the environment or config." "This is generally ok, but if you have more than 30 rooms, " "you may recieve rate-limit errors without one." ) if "TEMPLATE_DIRS" not in settings: if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ: # All good pass else: settings["TEMPLATE_DIRS"] = [] if "ALLOW_INSECURE_HIPCHAT_SERVER" in settings and \ (settings["ALLOW_INSECURE_HIPCHAT_SERVER"] is True or settings["ALLOW_INSECURE_HIPCHAT_SERVER"].lower() == "true"): warn("You are choosing to run will with SSL disabled. " "This is INSECURE and should NEVER be deployed outside a development environment.") settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = True settings["REQUESTS_OPTIONS"] = { "verify": False, } else: settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = False settings["REQUESTS_OPTIONS"] = {} if "ADMINS" not in settings: settings["ADMINS"] = "*" else: if "WILL_ADMINS" in os.environ: settings["ADMINS"] = [a.strip().lower() for a in settings.get('ADMINS', '').split(';') if a.strip()] if "PROXY_URL" in settings: parsed_proxy_url = urlparse(settings["PROXY_URL"]) settings["USE_PROXY"] = True settings["PROXY_HOSTNAME"] = parsed_proxy_url.hostname settings["PROXY_USERNAME"] = parsed_proxy_url.username settings["PROXY_PASSWORD"] = parsed_proxy_url.password settings["PROXY_PORT"] = parsed_proxy_url.port else: settings["USE_PROXY"] = False # Set them in the module namespace for k in sorted(settings, key=lambda x: x[0]): if not quiet: show_valid(k) globals()[k] = settings[k]
def import_settings(quiet=True): """This method takes care of importing settings from the environment, and config.py file. Order of operations: 1. Imports all WILL_ settings from the environment, and strips off the WILL_ 2. Imports settings from config.py 3. Sets defaults for any missing, required settings. This method takes a quiet kwarg, that when False, prints helpful output. Called that way during bootstrapping. """ settings = {} # Import from environment, handle environment-specific parsing. for k, v in os.environ.items(): if k[:5] == "WILL_": k = k[5:] settings[k] = v if "ROOMS" in settings: settings["ROOMS"] = settings["ROOMS"].split(";") # If HIPCHAT_SERVER is set, we need to change the USERNAME slightly # for XMPP to work. if "HIPCHAT_SERVER" in settings: settings["USERNAME"] = "******".\ format(user=settings["USERNAME"].split("@")[0], host=settings["HIPCHAT_SERVER"]) else: settings["HIPCHAT_SERVER"] = "api.hipchat.com" # Import from config if not quiet: puts("Importing config.py... ") with indent(2): try: had_warning = False import config for k, v in config.__dict__.items(): # Ignore private variables if "__" not in k: if k in os.environ and v != os.environ[k] and not quiet: warn( "%s is set in the environment as '%s', but overridden in" " config.py as '%s'." % (k, os.environ[k], v)) had_warning = True settings[k] = v if not had_warning and not quiet: show_valid("Valid.") except: # TODO: Check to see if there's a config.py.dist if not quiet: warn("no config.py found. This might be ok, but more likely, " "you haven't copied config.py.dist over to config.py") if not quiet: puts("Verifying settings... ") with indent(2): # Set defaults if "ROOMS" not in settings: if not quiet: warn("no ROOMS list found in the environment or config. " "This is ok - Will will just join all available rooms.") settings["ROOMS"] = None if "DEFAULT_ROOM" not in settings and "ROOMS" in settings and settings[ "ROOMS"] and len(settings["ROOMS"]) > 0: if not quiet: warn("no DEFAULT_ROOM found in the environment or config. " "Defaulting to '%s', the first one." % settings["ROOMS"][0]) settings["DEFAULT_ROOM"] = settings["ROOMS"][0] if "HTTPSERVER_PORT" not in settings: # For heroku if "PORT" in os.environ: settings["HTTPSERVER_PORT"] = os.environ["PORT"] else: if not quiet: warn( "no HTTPSERVER_PORT found in the environment or config. Defaulting to ':80'." ) settings["HTTPSERVER_PORT"] = "80" if "STORAGE_BACKEND" not in settings: settings["STORAGE_BACKEND"] = "redis" if settings["STORAGE_BACKEND"] == "redis": if "REDIS_URL" not in settings: # For heroku if "REDISCLOUD_URL" in os.environ: settings["REDIS_URL"] = os.environ["REDISCLOUD_URL"] if not quiet: note( "WILL_REDIS_URL not set, but it appears you're using RedisCloud. If so, all good." ) elif "REDISTOGO_URL" in os.environ: settings["REDIS_URL"] = os.environ["REDISTOGO_URL"] if not quiet: note( "WILL_REDIS_URL not set, but it appears you're using RedisToGo. If so, all good." ) elif "OPENREDIS_URL" in os.environ: settings["REDIS_URL"] = os.environ["OPENREDIS_URL"] if not quiet: note( "WILL_REDIS_URL not set, but it appears you're using OpenRedis. If so, all good." ) else: settings["REDIS_URL"] = "redis://localhost:6379/7" if not quiet: note( "WILL_REDIS_URL not set. Defaulting to redis://localhost:6379/7." ) if not settings["REDIS_URL"].startswith("redis://"): settings["REDIS_URL"] = "redis://%s" % settings["REDIS_URL"] if "REDIS_MAX_CONNECTIONS" not in settings: settings["REDIS_MAX_CONNECTIONS"] = 4 if not quiet: note("REDIS_MAX_CONNECTIONS not set. Defaulting to 4.") if settings["STORAGE_BACKEND"] == "file": if "FILE_DIR" not in settings: settings["FILE_DIR"] = "~/.will/" if not quiet: note("FILE_DIR not set. Defaulting to ~/.will/") if settings["STORAGE_BACKEND"] == "couchbase": if "COUCHBASE_URL" not in settings: settings["COUCHBASE_URL"] = "couchbase:///will" if not quiet: note( "COUCHBASE_URL not set. Defaulting to couchbase:///will" ) if "PUBLIC_URL" not in settings: default_public = "http://localhost:%s" % settings["HTTPSERVER_PORT"] settings["PUBLIC_URL"] = default_public if not quiet: warn( "no PUBLIC_URL found in the environment or config. Defaulting to '%s'." % default_public) if "V1_TOKEN" not in settings: if not quiet: warn( "no V1_TOKEN found in the environment or config." "This is generally ok, but if you have more than 30 rooms, " "you may recieve rate-limit errors without one.") if "TEMPLATE_DIRS" not in settings: if "WILL_TEMPLATE_DIRS_PICKLED" in os.environ: # All good pass else: settings["TEMPLATE_DIRS"] = [] if "ALLOW_INSECURE_HIPCHAT_SERVER" in settings and\ (settings["ALLOW_INSECURE_HIPCHAT_SERVER"] is True or settings["ALLOW_INSECURE_HIPCHAT_SERVER"].lower() == "true"): warn( "You are choosing to run will with SSL disabled. " "This is INSECURE and should NEVER be deployed outside a development environment." ) settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = True settings["REQUESTS_OPTIONS"] = { "verify": False, } else: settings["ALLOW_INSECURE_HIPCHAT_SERVER"] = False settings["REQUESTS_OPTIONS"] = {} if "ADMINS" not in settings: settings["ADMINS"] = "*" else: if "WILL_ADMINS" in os.environ: settings["ADMINS"] = [ a.strip().lower() for a in settings.get('ADMINS', '').split(';') if a.strip() ] if "PROXY_URL" in settings: parsed_proxy_url = urlparse(settings["PROXY_URL"]) settings["USE_PROXY"] = True settings["PROXY_HOSTNAME"] = parsed_proxy_url.hostname settings["PROXY_USERNAME"] = parsed_proxy_url.username settings["PROXY_PASSWORD"] = parsed_proxy_url.password settings["PROXY_PORT"] = parsed_proxy_url.port else: settings["USE_PROXY"] = False # Set them in the module namespace for k in sorted(settings, key=lambda x: x[0]): if not quiet: show_valid(k) globals()[k] = settings[k]