def __setstate__(self, temp_d):
"""
Auxilixiary function used by pickle. Ovverriden to avoid problems with logger lock
"""
if '_logger' in temp_d:
temp_d['_logger'] = custom_logger.getLogger(temp_d['_logger'])
self.__dict__.update(temp_d)
def __init__(self, campaign_configuration):
"""
campaign_configuration: dict of dict:
The set of options specified by the user though command line and campaign configuration files
"""
assert campaign_configuration
self._campaign_configuration = campaign_configuration
self._logger = custom_logger.getLogger(__name__)
def __init__(self, seed):
"""
Parameters
----------
seed: float
The seed to be used for the internal random generator
"""
self._random_generator = random.Random(seed)
self._logger = custom_logger.getLogger(__name__)
def _get_selection_validation_generator(campaign_configuration, seed,
wrapped_generator, subclass_name,
is_validation):
"""
Static factory method to instantiate subclasses
Parameters
----------
campaign_configuration: dict of dict
The set of options specified by the user though command line and campaign configuration files
seed: integer
The seed to be used to initialize the random generator
wrapped_generator: ExpConfsGenerator
The generator to be wrapped
subclass_name: str
The name of the class to be generated
is_validation: bool
True if the instance to be created is for validating, false if it is for hp selection
Returns
-------
The generated instance
"""
logger = custom_logger.getLogger(__name__)
if subclass_name == "All":
return AllExpConfsGenerator(campaign_configuration, seed,
wrapped_generator, is_validation)
if subclass_name == "Extrapolation":
if is_validation:
# Split is performed as preprocessing step
return ExtrapolationExpConfsGenerator(campaign_configuration,
seed, wrapped_generator,
is_validation)
else:
logger.error(
"Extrapolation cannot be used to perform hp_selection")
sys.exit(1)
elif subclass_name == "HoldOut":
return HoldOutExpConfsGenerator(campaign_configuration, seed,
wrapped_generator, is_validation)
elif subclass_name == "KFold":
return KFoldExpConfsGenerator(
campaign_configuration, seed,
campaign_configuration['General']['folds'], wrapped_generator,
is_validation)
else:
logger.error("Unknown hp_selection/validation method %s",
subclass_name)
sys.exit(1)
def __init__(self, campaign_configuration, regressor, x_columns, scalers):
"""
Parameters
regressor
The wrapped regressor
"""
assert regressor
self._campaign_configuration = campaign_configuration
self._regressor = regressor
self._x_columns = x_columns
self._scalers = scalers
self._logger = custom_logger.getLogger(__name__)
def __init__(self, campaign_configuration, seed):
"""
Parameters
----------
campaign_configuration: dict of dict
The set of options specified by the user though command line and campaign configuration files
seed: integer
The seed to be used to initialize the random generator
"""
self._experiment_configurations = []
self._campaign_configuration = campaign_configuration
self._random_generator = random.Random(seed)
self._logger = custom_logger.getLogger(__name__)
def __init__(self, campaign_configuration, seed):
"""
Parameters
-
campaign_configuration: #TODO: add type
The set of options specified by the user though command line and campaign configuration files
seed: integer
The seed to be used in random based activities
Returns
-------
ExpConfsGenerator
The top level ExpConfsGenerator to be used to generate all the experiment configurations
"""
self._campaign_configuration = campaign_configuration
self._random_generator = random.Random(seed)
self._logger = custom_logger.getLogger(__name__)
def __init__(self, campaign_configuration, exp_confs: List[ec.ExperimentConfiguration]):
"""
Parameters
----------
campaign_configuration: dict of dict:
The set of options specified by the user though command line and campaign configuration files
exp_confs: List[ec.ExperimentConfiguration]
The list of the run experiment configurations
"""
self._campaign_configuration = campaign_configuration
self._exp_confs = exp_confs
self.raw_results: Dict[str, Dict] = {}
self._logger = custom_logger.getLogger(__name__)
# Logger writes to stdout and file
file_handler = logging.FileHandler(os.path.join(self._campaign_configuration['General']['output'], 'results'), 'a+')
self._logger.addHandler(file_handler)
def __init__(self, campaign_configuration, hyperparameters,
regression_inputs, prefix):
"""
campaign_configuration: dict of dict:
The set of options specified by the user though command line and campaign configuration files
hyperparameters: dictionary
The set of hyperparameters of this experiment configuration
regression_inputs: RegressionInputs
The input of the regression problem to be solved
"""
# Initialized attributes
self._campaign_configuration = campaign_configuration
self._hyperparameters = hyperparameters
self._regression_inputs = regression_inputs
self._signature = self._compute_signature(prefix)
self._logger = custom_logger.getLogger(self.get_signature_string())
self.mapes = {}
self._regressor = None
# Create experiment directory
self._experiment_directory = self._campaign_configuration['General'][
'output']
for token in self._signature:
self._experiment_directory = os.path.join(
self._experiment_directory, token)
# Import here to avoid problems with circular dependencies
# pylint: disable=import-outside-toplevel
import model_building.sfs_experiment_configuration
if isinstance(
self, model_building.sfs_experiment_configuration.
SFSExperimentConfiguration
) or 'FeatureSelection' not in self._campaign_configuration or 'method' not in self._campaign_configuration[
'FeatureSelection'] or self._campaign_configuration[
'FeatureSelection']['method'] != "SFS":
assert not os.path.exists(
self._experiment_directory), self._experiment_directory
os.makedirs(self._experiment_directory)
from IPython import embed
from custom_logger import getLogger
from driver import travian_driver, pages, humanlike_pauses, action_queue
from driver.selenium_server_utils import launch_selenium_standalone_server_if_necessary
from time import sleep
import json
import random
logger = getLogger('main')
with open('secrets.json', 'r') as secrets_file_handle:
secrets = json.load(secrets_file_handle)
def execute():
launch_selenium_standalone_server_if_necessary()
driver = travian_driver.TravianDriver(secrets['base_url'])
reports_page = pages.ReportsPage(driver)
messages_page = pages.MessagesPage(driver)
overview_page = pages.OverviewPage(driver)
driver.authenticate_if_necessary(secrets['username'], secrets['password'])
post_login_check = action_queue.ActionQueue(
action_queue=[
messages_page.go,
reports_page.go,
overview_page.go,
],
insert_pause_action=humanlike_pauses.stop_and_look_delay)
post_login_check.run(ordered=False)
from __future__ import division
from custom_logger import getLogger
from driver import pages, humanlike_pauses, action_queue
from os.path import isfile
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import numpy as np
import random
import re
import selenium
from IPython import embed
logger = getLogger('travian_driver')
REMOTE_SERVER_PORT = 4555
REMOTE_SERVER_URL = "http://localhost:{}/wd/hub".format(REMOTE_SERVER_PORT)
SESSION_ID_FILE_PATH = 'session_id.txt'
DEFAULT_WINDOW_SIZE = np.array([1280, 720], 'int')
class NoSessionIdFile(Exception):
pass
class InsufficientTroopsError(Exception):
pass
JAVA_PATH = '/usr/bin/java'
SHELL_PATH = '/bin/bash'
LAUNCH_SERVER_STANDALONE_SCRIPT_PATH = abspath('bin/launch_selenium_server_standalone.sh')
RUNNING_SERVER_TEST_SCRIPT_PATH = abspath('bin/running_selenium_server_test.sh')
LAUNCH_STANDALONE_SERVER_ARGS = [
SHELL_PATH,
LAUNCH_SERVER_STANDALONE_SCRIPT_PATH
]
RUNNING_STANDALONE_SERVER_TEST_ARGS = [
SHELL_PATH,
RUNNING_SERVER_TEST_SCRIPT_PATH
]
logger = getLogger('standalone_server_utils')
def launch_selenium_standalone_server():
logger.info('Launching a new selenium standalone server as a daemon')
p = Popen(LAUNCH_STANDALONE_SERVER_ARGS)
logger.info('Waiting for server to initialize')
sleep(15)
logger.info(f"New selenium standalone server successfully started, pid: {p.pid}")
def is_selenium_standalone_server_running():
p = Popen(RUNNING_STANDALONE_SERVER_TEST_ARGS)
p.communicate()
if p.returncode == 0:
logger.info('Detected running selenium standalone server')
return True
else:
import random
import numpy as np
from custom_logger import getLogger
logger = getLogger('action_queue')
def intersperse(lst, item):
result = [item] * (len(lst) * 2 - 1)
result[0::2] = lst
return result
class ActionQueueException(Exception):
pass
class ActionQueue(object):
def __init__(self, action_queue=None, insert_pause_action=None):
if action_queue is None:
self._action_queue = []
else:
self._action_queue = action_queue
if insert_pause_action is None or callable(insert_pause_action):
self._insert_pause_action = insert_pause_action
else:
raise ActionQueueException(
f'Cannot use a non-callable pause type: {insert_pause_action}')
def append(self, action):
def __init__(self, configuration_file, debug=False, seed=0, output="output", j=1, generate_plots=False, self_check=True, details=False):
"""
Constructor of the class
- Copy the parameters to member variables
- Initialize the logger
- Build the data preparation flow adding or not data preparation steps on the basis of the content of the loaded configuration file
Parameters
----------
configuration_file: str
The configuration file describing the experimental campaign to be performed
debug: bool
True if debug messages should be printed
seed: integer
The seed to be used to initialize the random generator engine
output: str
The directory where all the outputs will be written; it is created by this library and cannot exist before using this module
j: integer
The number of processes to be used in the grid search
generate_plots: bool
True if plots have to be used
self_check: bool
True if the generated regressor should be tested
details: bool
True if the results of the single experiments should be added
"""
self._data_preprocessing_list = []
self.random_generator = random.Random(seed)
self.debug = debug
self._self_check = self_check
if self.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
self._logger = custom_logger.getLogger(__name__)
# Check if the configuration file exists
if not os.path.exists(configuration_file):
self._logger.error("%s does not exist", configuration_file)
sys.exit(-1)
self.conf = cp.ConfigParser()
self.conf.optionxform = str
self.conf.read(configuration_file)
self.conf['General']['configuration_file'] = configuration_file
self.conf['General']['output'] = output
self.conf['General']['seed'] = str(seed)
self.conf['General']['j'] = str(j)
self.conf['General']['debug'] = str(debug)
self.conf['General']['generate_plots'] = str(generate_plots)
self.conf['General']['details'] = str(details)
self._campaign_configuration = {}
self.load_campaign_configuration(configuration_file)
# Check if output path already exist
if os.path.exists(output):
self._logger.error("%s already exists", output)
sys.exit(1)
os.mkdir(self._campaign_configuration['General']['output'])
shutil.copyfile(configuration_file, os.path.join(output, 'configuration_file.ini'))
self.conf.write(open(os.path.join(output, "enriched_configuration_file.ini"), 'w'))
# Check that validation method has been specified
if 'validation' not in self._campaign_configuration['General']:
self._logger.error("Validation not specified")
sys.exit(1)
# Check that if HoldOut is selected, hold_out_ratio is specified
if self._campaign_configuration['General']['validation'] == "HoldOut" or self._campaign_configuration['General']['hp_selection'] == "HoldOut":
if "hold_out_ratio" not in self._campaign_configuration['General']:
self._logger.error("hold_out_ratio not set")
sys.exit(1)
# Check that if Extrapolation is selected, extrapolation_columns is specified
if self._campaign_configuration['General']['validation'] == "Extrapolation":
if "extrapolation_columns" not in self._campaign_configuration['General']:
self._logger.error("extrapolation_columns not set")
sys.exit(1)
# Check that if XGBoost is used for feature selection tolerance is specified
if 'FeatureSelection' in self._campaign_configuration and self._campaign_configuration['FeatureSelection']['method'] == "XGBoost":
if "XGBoost_tolerance" not in self._campaign_configuration['FeatureSelection']:
self._logger.error("XGBoost tolerance not set")
sys.exit(1)
# Check that if ernest is used, normalization, product, column_selection, and inversion are disabled
if 'ernest' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['ernest']:
if 'use_columns' in self._campaign_configuration['DataPreparation'] or "skip_columns" in self._campaign_configuration['DataPreparation']:
logging.error("use_columns and skip_columns cannot be used with ernest")
sys.exit(1)
if 'inverse' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['inverse']:
logging.error("inverse cannot be used with ernest")
sys.exit(1)
if 'product_max_degree' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['product_max_degree']:
logging.error("product cannot be used with ernest")
sys.exit(1)
if 'normalization' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['normalization']:
logging.error("normalization cannot be used with ernest")
sys.exit(1)
# Adding read on input to data preprocessing step
self._data_preprocessing_list.append(data_preparation.data_loading.DataLoading(self._campaign_configuration))
# Adding column renaming if required
if 'rename_columns' in self._campaign_configuration['DataPreparation']:
self._data_preprocessing_list.append(data_preparation.rename_columns.RenameColumns(self._campaign_configuration))
# Adding column selection if required
if 'use_columns' in self._campaign_configuration['DataPreparation'] or "skip_columns" in self._campaign_configuration['DataPreparation']:
self._data_preprocessing_list.append(data_preparation.column_selection.ColumnSelection(self._campaign_configuration))
# Transform categorical features in onehot encoding
self._data_preprocessing_list.append(data_preparation.onehot_encoding.OnehotEncoding(self._campaign_configuration))
# Split according to extrapolation values if required
if self._campaign_configuration['General']['validation'] == "Extrapolation":
self._data_preprocessing_list.append(data_preparation.extrapolation.Extrapolation(self._campaign_configuration))
# Adding inverted features if required
if 'inverse' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['inverse']:
self._data_preprocessing_list.append(data_preparation.inversion.Inversion(self._campaign_configuration))
# Adding product features if required
if 'product_max_degree' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['product_max_degree']:
self._data_preprocessing_list.append(data_preparation.product.Product(self._campaign_configuration))
# Create ernest features if required
if 'ernest' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['ernest']:
self._data_preprocessing_list.append(data_preparation.ernest.Ernest(self._campaign_configuration))
# Adding data check
self._data_preprocessing_list.append(data_preparation.data_check.DataCheck(self._campaign_configuration))
self._model_building = model_building.model_building.ModelBuilding(self.random_generator.random())