class RegressionSetupExecuter(object):
def __init__(self,
regressor,
train_args_obtainer,
item_to_feature_list,
saver,
verbose=True):
self._regressor = regressor
self._train_args_obtainer = train_args_obtainer
self._saver = saver
self._item_to_feature_list = item_to_feature_list
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def _train(self):
self._logger.log("Train was started")
train_args = self._train_args_obtainer.obtain()
self._regressor.fit(*train_args)
self._logger.log("Train was finished")
def execute(self):
self._train()
data = {
'regressor': self._regressor,
'item_to_feature_list': self._item_to_feature_list
}
self._saver.save(data)
def __init__(self, trained_regressor, value_list_to_item, config, verbose=True):
self._trained_regressor = trained_regressor
self._value_list_to_item = value_list_to_item
self._config = config
self._verbose = verbose
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
class DeapCreator(object):
def __init__(self, trained_regressor, value_list_to_item, config, verbose=True):
self._trained_regressor = trained_regressor
self._value_list_to_item = value_list_to_item
self._config = config
self._verbose = verbose
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def _create(self):
self._logger.log("creation started with config {}".format(str_preview(self._config)))
def eval_func(genome_list):
value_list = ValueList(genome_list)
item = self._value_list_to_item(value_list)
pred = self._trained_regressor.predict_proba(item)
return pred,
creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", list, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
toolbox.register("attr_float", random.random)
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_float, self._config['genome_size'])
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("evaluate", eval_func)
toolbox.register("mate", tools.cxBlend, alpha=0.1)
toolbox.register("mutate", tools.mutGaussian, mu=0.5, sigma=0.5, indpb=0.05)
toolbox.register("select", tools.selBest)
pop = toolbox.population(n=self._config['population_size'])
hof = tools.HallOfFame(1)
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=self._config['num_of_generations'],
stats=stats, halloffame=hof, verbose=self._verbose)
pop = [self._value_list_to_item(ValueList(ind)) for ind in pop]
self._logger.log("creation finished")
return pop
def create(self):
return self._create()[0]
def create_many(self, n):
many = self._create()
while len(many) < n:
many += self._create()
return many[0:n]
def __init__(self,
regressor,
train_args_obtainer,
item_to_feature_list,
saver,
verbose=True):
self._regressor = regressor
self._train_args_obtainer = train_args_obtainer
self._saver = saver
self._item_to_feature_list = item_to_feature_list
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self,
folder_path,
file_path,
file_extension="",
save_method=shelve_save,
verbose=True):
self._folder_path = folder_path
self._file_path = file_path
self._save_method = save_method
self._file_extension = file_extension
self._full_path = os.path.join(self._folder_path,
self._file_path + self._file_extension)
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
class FileLoader(object):
def __init__(self, save_path, load_method=shelve_load, verbose=True):
self._save_path = save_path
self._load_method = load_method
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def load(self):
self._logger.log("Load from {} was started.".format(str_preview(self._save_path)))
if not os.path.isfile(self._save_path):
raise Exception("cannot load from path {}, file does not exist".format(self._save_path))
item = self._load_method(self._save_path)
self._logger.log("Load {} from {} was finished".format(str_preview(str(item)), str_preview(self._save_path)))
return item
class FileSaver(object):
def __init__(self, folder_path, file_path, file_extension="", save_method=shelve_save,
verbose=True):
self._folder_path = folder_path
self._file_path = file_path
self._save_method = save_method
self._file_extension = file_extension
self._full_path = os.path.join(self._folder_path, self._file_path + self._file_extension)
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def save(self, py_object):
self._logger.log("Save to {} was started.".format(str_preview(self._full_path)))
create_folder_if_needed(self._folder_path)
self._save_method(self._full_path, py_object)
self._logger.log("Save {} to {} was finished.".format(str_preview(py_object), str_preview(self._full_path)))
def __init__(self, regressor, train_args_obtainer, item_to_feature_list, saver, verbose=True):
self._regressor = regressor
self._train_args_obtainer = train_args_obtainer
self._saver = saver
self._item_to_feature_list = item_to_feature_list
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, folder_path, file_path, file_extension="", save_method=shelve_save,
verbose=True):
self._folder_path = folder_path
self._file_path = file_path
self._save_method = save_method
self._file_extension = file_extension
self._full_path = os.path.join(self._folder_path, self._file_path + self._file_extension)
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
class FileLoader(object):
def __init__(self, save_path, load_method=shelve_load, verbose=True):
self._save_path = save_path
self._load_method = load_method
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def load(self):
self._logger.log("Load from {} was started.".format(
str_preview(self._save_path)))
if not os.path.isfile(self._save_path):
raise Exception(
"cannot load from path {}, file does not exist".format(
self._save_path))
item = self._load_method(self._save_path)
self._logger.log("Load {} from {} was finished".format(
str_preview(str(item)), str_preview(self._save_path)))
return item
class RegressionSetupExecuter(object):
def __init__(self, regressor, train_args_obtainer, item_to_feature_list, saver, verbose=True):
self._regressor = regressor
self._train_args_obtainer = train_args_obtainer
self._saver = saver
self._item_to_feature_list = item_to_feature_list
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def _train(self):
self._logger.log("Train was started")
train_args = self._train_args_obtainer.obtain()
self._regressor.fit(*train_args)
self._logger.log("Train was finished")
def execute(self):
self._train()
data = {'regressor': self._regressor, 'item_to_feature_list': self._item_to_feature_list}
self._saver.save(data)
class FileSaver(object):
def __init__(self,
folder_path,
file_path,
file_extension="",
save_method=shelve_save,
verbose=True):
self._folder_path = folder_path
self._file_path = file_path
self._save_method = save_method
self._file_extension = file_extension
self._full_path = os.path.join(self._folder_path,
self._file_path + self._file_extension)
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def save(self, py_object):
self._logger.log("Save to {} was started.".format(
str_preview(self._full_path)))
create_folder_if_needed(self._folder_path)
self._save_method(self._full_path, py_object)
self._logger.log("Save {} to {} was finished.".format(
str_preview(py_object), str_preview(self._full_path)))
class TryBatchExecuter(object):
def __init__(self, executers, verbose=True):
self._executers = executers
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def execute(self):
index = len(self._executers) - 1
caught = True
self._logger.log("Starting execution on index {} of {}: {}".format(index + 1, len(self._executers),
self._executers[index]))
while caught:
caught = False
for i in range(index, len(self._executers)):
current_executer = self._executers[i]
try:
current_executer.execute()
except Exception as e:
self._logger.log(
"Stopping execution on index {} of {}: {}, because of error {}".format(i + 1,
len(
self._executers),
current_executer,
traceback.format_exc()))
if i > index or index == 0:
return
else:
self._logger.log(
"Stopping execution on index {} of {}: {}, because of error {}".format(i + 1,
len(
self._executers),
current_executer,
traceback.format_exc()))
self._logger.log("Starting execution on index {} of {}: {}".format(i, len(self._executers),
self._executers[i - 1]))
caught = True
index -= 1
break
class TryBatchExecuter(object):
def __init__(self, executers, verbose=True):
self._executers = executers
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def execute(self):
index = len(self._executers) - 1
caught = True
self._logger.log("Starting execution on index {} of {}: {}".format(
index + 1, len(self._executers), self._executers[index]))
while caught:
caught = False
for i in range(index, len(self._executers)):
current_executer = self._executers[i]
try:
current_executer.execute()
except Exception as e:
self._logger.log(
"Stopping execution on index {} of {}: {}, because of error {}"
.format(i + 1, len(self._executers), current_executer,
traceback.format_exc()))
if i > index or index == 0:
return
else:
self._logger.log(
"Stopping execution on index {} of {}: {}, because of error {}"
.format(i + 1, len(self._executers),
current_executer, traceback.format_exc()))
self._logger.log(
"Starting execution on index {} of {}: {}".format(
i, len(self._executers),
self._executers[i - 1]))
caught = True
index -= 1
break
class FolderCrawlerObtainer(object):
def __init__(self, path_to_item, folder, verbose=True):
self._path_to_item = path_to_item
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
self._folder = folder
def _crawl(self, folder):
self._logger.log("crawling started on {}".format(folder))
items = []
for root, _, file_names in os.walk(folder):
for file_name in file_names:
full_path = root + "/" + file_name
try:
items.append(self._path_to_item(full_path))
except Exception as e:
self._logger.log("Could not create item from path {} : {}".format(full_path, e.message))
self._logger.log("{} items were crawled.".format(len(items)))
return items
def obtain(self):
return self._crawl(self._folder)
def __init__(self, value_list_to_item, verbose=True):
self._value_list_to_item = value_list_to_item
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, regressor, item_to_feature_list, verbose=True):
self._regressor = regressor
self._item_to_feature_list = item_to_feature_list
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
from intelligent_creation.utilization.general_utilities.loggers.logger import Logger
global_logger = Logger(verbose=True)
def apply_carefully(items, func, verbose=True):
products = []
for item in items:
try:
products.append(func(item))
except Exception as e:
global_logger.log("Could not apply {} on {} with error {}".format(
func, item, e))
return products
DEFAULT_PREVIEW_LENGTH = 1000
def str_preview(str, max_len=DEFAULT_PREVIEW_LENGTH):
if len(str) > max_len:
char_len = (max_len - 5) / 2
return "{} ... {}".format(str[0:char_len],
str[len(str) - char_len:len(str)])
else:
return str
def __init__(self, creator, product_saver, num_of_products, verbose=True):
self._creator = creator
self._product_saver = product_saver
self._num_of_products = num_of_products
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, executers, verbose=True):
self._executers = executers
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, save_path, load_method=shelve_load, verbose=True):
self._save_path = save_path
self._load_method = load_method
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, save_path, load_method=shelve_load, verbose=True):
self._save_path = save_path
self._load_method = load_method
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, path_to_item, folder, verbose=True):
self._path_to_item = path_to_item
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
self._folder = folder
def __init__(self, obtainer, edit_method=unit_method, verbose=True):
self._obtainer = obtainer
self._edit_method = edit_method
self._verbose = verbose
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)
def __init__(self, executers, verbose=True):
self._executers = executers
self._logger = Logger(who=self.__class__.__name__, verbose=verbose)