def _load_lists(self): # Load content types for details self._tika_valid_content_types = set() if self.conf["tika"]["enabled"]: self.log("Reloading content types list for Tika details") for k, v in self.conf["tika"]["valid_content_types"].iteritems(): keywords = load_config(v) if not isinstance(keywords, list): raise ImproperlyConfigured("Keywords content types \ details list '{}' not valid".format(k)) keywords = {i.lower() for i in keywords} self._tika_valid_content_types |= keywords self.log("Content types Tika '{}' loaded".format(k)) # Load content types for blacklist self.log("Reloading content types list blacklist") self._cont_type_bl = set() for k, v in self.conf["content_types_blacklist"].iteritems(): keywords = load_config(v) if not isinstance(keywords, list): raise ImproperlyConfigured("Keywords content types blacklist \ list '{}' not valid".format(k)) keywords = {i.lower() for i in keywords} self._cont_type_bl |= keywords self.log("Content types blacklist '{}' loaded".format(k))
def test_read_string_from_config_key_empty_key(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") cfg["test"] = "" with self.assertRaises(utils.EmptyKeyValueException): utils.read_string_from_config(logger=logger, cfg=cfg, key="test")
def test_update_table(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") db = database.Database(cfg=cfg) df = pd.DataFrame(columns=["A"]) db._update_table(name="test", data=df)
def test_init_invalid_config(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") cfg.pop("resultsTable") with self.assertRaises(KeyError): database.Database(cfg=cfg)
def test_read_string_from_config_key_not_found(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") with self.assertRaises(KeyError): utils.read_string_from_config(logger=logger, cfg=cfg, key="not_found")
def test_read_table_not_found(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") db = database.Database(cfg=cfg) name = "not_available" with self.assertRaises(ValueError): db._read_table(name=name)
def _conf_loader(self): if not self.conf_file: raise ImproperlyConfigured( "Bolts configuration path NOT set for '{}'".format( self.component_name)) self.log("Reloading configuration for bolt") self._bolts_conf = load_config(self.conf_file) self._conf = self.bolts_conf[self.component_name]
def test_update_table_wrong_type(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") db = database.Database(cfg=cfg) d = 0.5 with self.assertRaises(TypeError): db._update_table(name="test", data=d)
def _load_lists(self): # Load subjects keywords self.log("Reloading phishing subjects keywords") self._s_keys = set() for k, v in self.conf["lists"]["subjects"].iteritems(): keywords = load_config(v) if not isinstance(keywords, list): raise ImproperlyConfigured( "Keywords subjects list '{}' not valid".format(k)) self._s_keys |= set(keywords) # Load targets keywords self.log("Reloading phishing targets keywords") self._t_keys = {} for k, v in self.conf["lists"]["targets"].iteritems(): keywords = load_config(v) if not isinstance(keywords, dict): raise ImproperlyConfigured( "Keywords targets dict '{}' not valid".format(k)) self._t_keys.update(keywords)
def _load_whitelist(self): self.log("Reloading whitelists domains for bolt") self._whitelist = set() for k, v in self.conf['whitelists'].iteritems(): expiry = v.get('expiry') now = datetime.utcnow() if (not expiry or datetime.strptime(expiry, "%Y-%m-%dT%H:%M:%S.%fZ") >= now): domains = load_config(v['path']) if not isinstance(domains, list): raise ImproperlyConfigured( "Whitelist {} not loaded".format(k)) domains = [i.lower() for i in domains] self._whitelist |= set(domains) self.log("Whitelist '{}' loaded".format(k))
def main(args): config = {k: v for k, v in args._get_kwargs()} config.update(load_config(path_to_config(config["experiment"]))) torch.manual_seed(config["seed"]) logging.info("creating europarl-v7 datasets.") dataset_train, dataset_valid = datasets(config) logging.info("creating model") model = BidirectionalTranslator(config).cuda() step = 0 if len(config["regularization"]["type"]) == 0: dis_opt_kwargs = None else: dis_opt_kwargs = config["rmsp"] if args.iter is not None: logging.info("loading model and optimizer parameters") ckpt = load_checkpoint(model, config["adam"], dis_opt_kwargs, args.iter, config["experiment"]) model, gen_optimizer, dis_optimizer, step = ckpt else: logging.info("creating optimizer(s)") gen_optimizer, dis_optimizer = build_optimizers( model, config["adam"], dis_opt_kwargs) logging.info("starting the {} routine from step {}.".format( config["mode"], step)) if config["dist"]: raise NotImplementedError else: if config["mode"] == "train": train(model, gen_optimizer, dis_optimizer, dataset_train, dataset_valid, step, config) else: evaluate(model, dataset_valid, config)
def main(args): config = load_config(args.config_path) device = torch.device('cuda:1') if torch.cuda.is_available() \ else torch.device('cpu') model = load_model(config, model_path=args.model_path) context = load_context(args.context_path) print('Context') print(context) print('=' * 50) while True: print('質問してください!') question = input() if question == 'q': break evaluate(question=question, context=context, model=model, config=config, device=device)
def test_load_config(self): logger = logging.getLogger(__name__) utils.load_config(logger, "config/data.json")
from datetime import date from telegram import ParseMode from modules.utils import load_config import modules.graphs as graphs import modules.c19api as c19api cfg = load_config() settings = cfg["bot"] def chatid(update, context): context.bot.send_message( chat_id=update.message.chat_id, text=f"Chatid: {update.message.chat_id}", parse_mode=ParseMode.HTML, ) def help(update, context): menuitems = "<b>" + settings["commands"]["title"] + "</b>\n" for command in list(settings["commands"].items())[1:]: menuitems += command[1] + "\n" context.bot.send_message( chat_id=update.message.chat_id, text=menuitems, parse_mode=ParseMode.HTML ) def stats(update, context): today = date.today().strftime("%d.%m.%Y")
def test_init_valid_config(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") database.Database(cfg=cfg)
def test_load_config_not_found(self): logger = logging.getLogger(__name__) with self.assertRaises(FileNotFoundError): utils.load_config(logger, "config/not_available.json")
def test_load_config_no_json(self): logger = logging.getLogger(__name__) with self.assertRaises(json.decoder.JSONDecodeError): utils.load_config(logger, "data/test.csv")
def test_read_string_from_config(self): logger = logging.getLogger(__name__) cfg = utils.load_config(logger, "config/data.json") utils.read_string_from_config(logger=logger, cfg=cfg, key="trainFile")
import json from modules.utils import load_config from modules.data.dataset_file_creating import transform_coco_to_dataframe if __name__ == '__main__': config = load_config( config_path='/home/vadbeg/Projects/Kaggle/herbarium-2020/config.ini') train_metadata_path = config.get('Data', 'train_metadata_path') with open(file=train_metadata_path, mode='r', encoding='UTF-8', errors='ignore') as file: train_metadata = json.load(file) dataframe = transform_coco_to_dataframe(coco_metadata=train_metadata, is_valid_dataset=False) dataframe.to_csv(config.get('Data', 'train_dataframe_path'))
import numpy as np import pandas as pd from modules.model.training import train_model from modules.model.network import HerbariumNet from modules.model.custom_layers import ArcMarginProduct, AddMarginProduct from modules.model.losses import LabelSmoothingLoss, SoftTripleLoss from modules.data.dataloader import create_dataloader, split_dataset from modules.data.augs import train_augmentations, valid_augmentations from modules.utils import load_config #, set_seed if __name__ == '__main__': # set_seed(seed_number=27) config_path = '/home/vadbeg/Projects/Kaggle/herbarium-2020/config.ini' config = load_config(config_path=config_path) train_images_path = config.get('Data', 'train_images_folder') train_dataframe_path = config.get('Data', 'train_dataframe_path') weights_dir = config.get('Model', 'weights_dir') reports_dir = config.get('Model', 'reports_dir') device = config.get('Model', 'device') model_type = config.get('Model', 'model_type') image_size = tuple(json.loads(config.get('Model', 'image_size'))) valid_size = config.getfloat('Model', 'valid_size') batch_size = config.getint('Model', 'batch_size') learning_rate = config.getfloat('Model', 'learning_rate') num_of_output_nodes = config.getint('Model', 'num_of_output_nodes') label_smoothing = config.getfloat('Model', 'label_smoothing')
def main(): # Set maximal threads to eight to avoid computation problems os.environ['NUMEXPR_MAX_THREADS'] = "8" # Configure logger logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - ' + '%(levelname)s - %(message)s', filename='applog.log', filemode='w') # Get logger with name of module logger = logging.getLogger(__name__) logger.info("Starting program.") # Read config data cfg = utils.load_config(logger, "config/data.json") # Set constant paths to where data is stored # Information is retrieved from config file logger.info("Reading config file...") TRAIN_FILE = utils.read_string_from_config(logger=logger, cfg=cfg, key="trainFile") IDEAL_FILE = utils.read_string_from_config(logger=logger, cfg=cfg, key="idealFile") TEST_FILE = utils.read_string_from_config(logger=logger, cfg=cfg, key="testFile") TRAIN_PATH = os.path.join(os.path.dirname(__file__), TRAIN_FILE) IDEAL_PATH = os.path.join(os.path.dirname(__file__), IDEAL_FILE) TEST_PATH = os.path.join(os.path.dirname(__file__), TEST_FILE) logger.info("Reading config file...Done") # Create database instance database = db.Database(cfg=cfg) # Save training data and ideal functions in db training_data = utils.read_data(logger=logger, path=TRAIN_PATH) ideal_data = utils.read_data(logger=logger, path=IDEAL_PATH) database.update_training_table(data=training_data) database.update_ideal_functions_table(data=ideal_data) # Initiate training process trainingData = data.TrainingData(training_data=training_data, ideal_data=ideal_data) training_result = trainingData.find_ideal_functions() # Initiate mapping process testData = data.TestData(ideal_data=ideal_data, result_data=training_result, test_filepath=TEST_PATH) test_result = testData.map_to_functions() # Save mapping results in db database.update_result_table(data=test_result) # Create plotter instance with results plotter = plot.Plotter(ideal_data=ideal_data, training_data=training_data, training_result=training_result, deviations=trainingData.deviations(), test_result=test_result) # Plot results plotter.plot_results() # Show results plotter.show_results() logger.info("Program has finished.")