def _load_lists(self):
# Load content types for details
self._tika_valid_content_types = set()
if self.conf["tika"]["enabled"]:
self.log("Reloading content types list for Tika details")
for k, v in self.conf["tika"]["valid_content_types"].iteritems():
keywords = load_config(v)
if not isinstance(keywords, list):
raise ImproperlyConfigured("Keywords content types \
details list '{}' not valid".format(k))
keywords = {i.lower() for i in keywords}
self._tika_valid_content_types |= keywords
self.log("Content types Tika '{}' loaded".format(k))
# Load content types for blacklist
self.log("Reloading content types list blacklist")
self._cont_type_bl = set()
for k, v in self.conf["content_types_blacklist"].iteritems():
keywords = load_config(v)
if not isinstance(keywords, list):
raise ImproperlyConfigured("Keywords content types blacklist \
list '{}' not valid".format(k))
keywords = {i.lower() for i in keywords}
self._cont_type_bl |= keywords
self.log("Content types blacklist '{}' loaded".format(k))
def test_read_string_from_config_key_empty_key(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
cfg["test"] = ""
with self.assertRaises(utils.EmptyKeyValueException):
utils.read_string_from_config(logger=logger, cfg=cfg, key="test")
def test_update_table(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
db = database.Database(cfg=cfg)
df = pd.DataFrame(columns=["A"])
db._update_table(name="test", data=df)
def test_init_invalid_config(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
cfg.pop("resultsTable")
with self.assertRaises(KeyError):
database.Database(cfg=cfg)
def test_read_string_from_config_key_not_found(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
with self.assertRaises(KeyError):
utils.read_string_from_config(logger=logger,
cfg=cfg,
key="not_found")
def test_read_table_not_found(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
db = database.Database(cfg=cfg)
name = "not_available"
with self.assertRaises(ValueError):
db._read_table(name=name)
def _conf_loader(self):
if not self.conf_file:
raise ImproperlyConfigured(
"Bolts configuration path NOT set for '{}'".format(
self.component_name))
self.log("Reloading configuration for bolt")
self._bolts_conf = load_config(self.conf_file)
self._conf = self.bolts_conf[self.component_name]
def test_update_table_wrong_type(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
db = database.Database(cfg=cfg)
d = 0.5
with self.assertRaises(TypeError):
db._update_table(name="test", data=d)
def _load_lists(self):
# Load subjects keywords
self.log("Reloading phishing subjects keywords")
self._s_keys = set()
for k, v in self.conf["lists"]["subjects"].iteritems():
keywords = load_config(v)
if not isinstance(keywords, list):
raise ImproperlyConfigured(
"Keywords subjects list '{}' not valid".format(k))
self._s_keys |= set(keywords)
# Load targets keywords
self.log("Reloading phishing targets keywords")
self._t_keys = {}
for k, v in self.conf["lists"]["targets"].iteritems():
keywords = load_config(v)
if not isinstance(keywords, dict):
raise ImproperlyConfigured(
"Keywords targets dict '{}' not valid".format(k))
self._t_keys.update(keywords)
def _load_whitelist(self):
self.log("Reloading whitelists domains for bolt")
self._whitelist = set()
for k, v in self.conf['whitelists'].iteritems():
expiry = v.get('expiry')
now = datetime.utcnow()
if (not expiry or
datetime.strptime(expiry, "%Y-%m-%dT%H:%M:%S.%fZ") >= now):
domains = load_config(v['path'])
if not isinstance(domains, list):
raise ImproperlyConfigured(
"Whitelist {} not loaded".format(k))
domains = [i.lower() for i in domains]
self._whitelist |= set(domains)
self.log("Whitelist '{}' loaded".format(k))
def main(args):
config = {k: v for k, v in args._get_kwargs()}
config.update(load_config(path_to_config(config["experiment"])))
torch.manual_seed(config["seed"])
logging.info("creating europarl-v7 datasets.")
dataset_train, dataset_valid = datasets(config)
logging.info("creating model")
model = BidirectionalTranslator(config).cuda()
step = 0
if len(config["regularization"]["type"]) == 0:
dis_opt_kwargs = None
else:
dis_opt_kwargs = config["rmsp"]
if args.iter is not None:
logging.info("loading model and optimizer parameters")
ckpt = load_checkpoint(model, config["adam"], dis_opt_kwargs,
args.iter, config["experiment"])
model, gen_optimizer, dis_optimizer, step = ckpt
else:
logging.info("creating optimizer(s)")
gen_optimizer, dis_optimizer = build_optimizers(
model, config["adam"], dis_opt_kwargs)
logging.info("starting the {} routine from step {}.".format(
config["mode"], step))
if config["dist"]:
raise NotImplementedError
else:
if config["mode"] == "train":
train(model, gen_optimizer, dis_optimizer, dataset_train,
dataset_valid, step, config)
else:
evaluate(model, dataset_valid, config)
def main(args):
config = load_config(args.config_path)
device = torch.device('cuda:1') if torch.cuda.is_available() \
else torch.device('cpu')
model = load_model(config, model_path=args.model_path)
context = load_context(args.context_path)
print('Context')
print(context)
print('=' * 50)
while True:
print('質問してください!')
question = input()
if question == 'q':
break
evaluate(question=question,
context=context,
model=model,
config=config,
device=device)
def test_load_config(self):
logger = logging.getLogger(__name__)
utils.load_config(logger, "config/data.json")
from datetime import date
from telegram import ParseMode
from modules.utils import load_config
import modules.graphs as graphs
import modules.c19api as c19api
cfg = load_config()
settings = cfg["bot"]
def chatid(update, context):
context.bot.send_message(
chat_id=update.message.chat_id,
text=f"Chatid: {update.message.chat_id}",
parse_mode=ParseMode.HTML,
)
def help(update, context):
menuitems = "<b>" + settings["commands"]["title"] + "</b>\n"
for command in list(settings["commands"].items())[1:]:
menuitems += command[1] + "\n"
context.bot.send_message(
chat_id=update.message.chat_id, text=menuitems, parse_mode=ParseMode.HTML
)
def stats(update, context):
today = date.today().strftime("%d.%m.%Y")
def test_init_valid_config(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
database.Database(cfg=cfg)
def test_load_config_not_found(self):
logger = logging.getLogger(__name__)
with self.assertRaises(FileNotFoundError):
utils.load_config(logger, "config/not_available.json")
def test_load_config_no_json(self):
logger = logging.getLogger(__name__)
with self.assertRaises(json.decoder.JSONDecodeError):
utils.load_config(logger, "data/test.csv")
def test_read_string_from_config(self):
logger = logging.getLogger(__name__)
cfg = utils.load_config(logger, "config/data.json")
utils.read_string_from_config(logger=logger, cfg=cfg, key="trainFile")
import json
from modules.utils import load_config
from modules.data.dataset_file_creating import transform_coco_to_dataframe
if __name__ == '__main__':
config = load_config(
config_path='/home/vadbeg/Projects/Kaggle/herbarium-2020/config.ini')
train_metadata_path = config.get('Data', 'train_metadata_path')
with open(file=train_metadata_path,
mode='r',
encoding='UTF-8',
errors='ignore') as file:
train_metadata = json.load(file)
dataframe = transform_coco_to_dataframe(coco_metadata=train_metadata,
is_valid_dataset=False)
dataframe.to_csv(config.get('Data', 'train_dataframe_path'))
import numpy as np
import pandas as pd
from modules.model.training import train_model
from modules.model.network import HerbariumNet
from modules.model.custom_layers import ArcMarginProduct, AddMarginProduct
from modules.model.losses import LabelSmoothingLoss, SoftTripleLoss
from modules.data.dataloader import create_dataloader, split_dataset
from modules.data.augs import train_augmentations, valid_augmentations
from modules.utils import load_config #, set_seed
if __name__ == '__main__':
# set_seed(seed_number=27)
config_path = '/home/vadbeg/Projects/Kaggle/herbarium-2020/config.ini'
config = load_config(config_path=config_path)
train_images_path = config.get('Data', 'train_images_folder')
train_dataframe_path = config.get('Data', 'train_dataframe_path')
weights_dir = config.get('Model', 'weights_dir')
reports_dir = config.get('Model', 'reports_dir')
device = config.get('Model', 'device')
model_type = config.get('Model', 'model_type')
image_size = tuple(json.loads(config.get('Model', 'image_size')))
valid_size = config.getfloat('Model', 'valid_size')
batch_size = config.getint('Model', 'batch_size')
learning_rate = config.getfloat('Model', 'learning_rate')
num_of_output_nodes = config.getint('Model', 'num_of_output_nodes')
label_smoothing = config.getfloat('Model', 'label_smoothing')
def main():
# Set maximal threads to eight to avoid computation problems
os.environ['NUMEXPR_MAX_THREADS'] = "8"
# Configure logger
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s - %(name)s - ' +
'%(levelname)s - %(message)s',
filename='applog.log',
filemode='w')
# Get logger with name of module
logger = logging.getLogger(__name__)
logger.info("Starting program.")
# Read config data
cfg = utils.load_config(logger, "config/data.json")
# Set constant paths to where data is stored
# Information is retrieved from config file
logger.info("Reading config file...")
TRAIN_FILE = utils.read_string_from_config(logger=logger,
cfg=cfg,
key="trainFile")
IDEAL_FILE = utils.read_string_from_config(logger=logger,
cfg=cfg,
key="idealFile")
TEST_FILE = utils.read_string_from_config(logger=logger,
cfg=cfg,
key="testFile")
TRAIN_PATH = os.path.join(os.path.dirname(__file__), TRAIN_FILE)
IDEAL_PATH = os.path.join(os.path.dirname(__file__), IDEAL_FILE)
TEST_PATH = os.path.join(os.path.dirname(__file__), TEST_FILE)
logger.info("Reading config file...Done")
# Create database instance
database = db.Database(cfg=cfg)
# Save training data and ideal functions in db
training_data = utils.read_data(logger=logger, path=TRAIN_PATH)
ideal_data = utils.read_data(logger=logger, path=IDEAL_PATH)
database.update_training_table(data=training_data)
database.update_ideal_functions_table(data=ideal_data)
# Initiate training process
trainingData = data.TrainingData(training_data=training_data,
ideal_data=ideal_data)
training_result = trainingData.find_ideal_functions()
# Initiate mapping process
testData = data.TestData(ideal_data=ideal_data,
result_data=training_result,
test_filepath=TEST_PATH)
test_result = testData.map_to_functions()
# Save mapping results in db
database.update_result_table(data=test_result)
# Create plotter instance with results
plotter = plot.Plotter(ideal_data=ideal_data,
training_data=training_data,
training_result=training_result,
deviations=trainingData.deviations(),
test_result=test_result)
# Plot results
plotter.plot_results()
# Show results
plotter.show_results()
logger.info("Program has finished.")
