def __setstate__(self, temp_d): """ Auxilixiary function used by pickle. Ovverriden to avoid problems with logger lock """ if '_logger' in temp_d: temp_d['_logger'] = custom_logger.getLogger(temp_d['_logger']) self.__dict__.update(temp_d)
def __init__(self, campaign_configuration): """ campaign_configuration: dict of dict: The set of options specified by the user though command line and campaign configuration files """ assert campaign_configuration self._campaign_configuration = campaign_configuration self._logger = custom_logger.getLogger(__name__)
def __init__(self, seed): """ Parameters ---------- seed: float The seed to be used for the internal random generator """ self._random_generator = random.Random(seed) self._logger = custom_logger.getLogger(__name__)
def _get_selection_validation_generator(campaign_configuration, seed, wrapped_generator, subclass_name, is_validation): """ Static factory method to instantiate subclasses Parameters ---------- campaign_configuration: dict of dict The set of options specified by the user though command line and campaign configuration files seed: integer The seed to be used to initialize the random generator wrapped_generator: ExpConfsGenerator The generator to be wrapped subclass_name: str The name of the class to be generated is_validation: bool True if the instance to be created is for validating, false if it is for hp selection Returns ------- The generated instance """ logger = custom_logger.getLogger(__name__) if subclass_name == "All": return AllExpConfsGenerator(campaign_configuration, seed, wrapped_generator, is_validation) if subclass_name == "Extrapolation": if is_validation: # Split is performed as preprocessing step return ExtrapolationExpConfsGenerator(campaign_configuration, seed, wrapped_generator, is_validation) else: logger.error( "Extrapolation cannot be used to perform hp_selection") sys.exit(1) elif subclass_name == "HoldOut": return HoldOutExpConfsGenerator(campaign_configuration, seed, wrapped_generator, is_validation) elif subclass_name == "KFold": return KFoldExpConfsGenerator( campaign_configuration, seed, campaign_configuration['General']['folds'], wrapped_generator, is_validation) else: logger.error("Unknown hp_selection/validation method %s", subclass_name) sys.exit(1)
def __init__(self, campaign_configuration, regressor, x_columns, scalers): """ Parameters regressor The wrapped regressor """ assert regressor self._campaign_configuration = campaign_configuration self._regressor = regressor self._x_columns = x_columns self._scalers = scalers self._logger = custom_logger.getLogger(__name__)
def __init__(self, campaign_configuration, seed): """ Parameters ---------- campaign_configuration: dict of dict The set of options specified by the user though command line and campaign configuration files seed: integer The seed to be used to initialize the random generator """ self._experiment_configurations = [] self._campaign_configuration = campaign_configuration self._random_generator = random.Random(seed) self._logger = custom_logger.getLogger(__name__)
def __init__(self, campaign_configuration, seed): """ Parameters - campaign_configuration: #TODO: add type The set of options specified by the user though command line and campaign configuration files seed: integer The seed to be used in random based activities Returns ------- ExpConfsGenerator The top level ExpConfsGenerator to be used to generate all the experiment configurations """ self._campaign_configuration = campaign_configuration self._random_generator = random.Random(seed) self._logger = custom_logger.getLogger(__name__)
def __init__(self, campaign_configuration, exp_confs: List[ec.ExperimentConfiguration]): """ Parameters ---------- campaign_configuration: dict of dict: The set of options specified by the user though command line and campaign configuration files exp_confs: List[ec.ExperimentConfiguration] The list of the run experiment configurations """ self._campaign_configuration = campaign_configuration self._exp_confs = exp_confs self.raw_results: Dict[str, Dict] = {} self._logger = custom_logger.getLogger(__name__) # Logger writes to stdout and file file_handler = logging.FileHandler(os.path.join(self._campaign_configuration['General']['output'], 'results'), 'a+') self._logger.addHandler(file_handler)
def __init__(self, campaign_configuration, hyperparameters, regression_inputs, prefix): """ campaign_configuration: dict of dict: The set of options specified by the user though command line and campaign configuration files hyperparameters: dictionary The set of hyperparameters of this experiment configuration regression_inputs: RegressionInputs The input of the regression problem to be solved """ # Initialized attributes self._campaign_configuration = campaign_configuration self._hyperparameters = hyperparameters self._regression_inputs = regression_inputs self._signature = self._compute_signature(prefix) self._logger = custom_logger.getLogger(self.get_signature_string()) self.mapes = {} self._regressor = None # Create experiment directory self._experiment_directory = self._campaign_configuration['General'][ 'output'] for token in self._signature: self._experiment_directory = os.path.join( self._experiment_directory, token) # Import here to avoid problems with circular dependencies # pylint: disable=import-outside-toplevel import model_building.sfs_experiment_configuration if isinstance( self, model_building.sfs_experiment_configuration. SFSExperimentConfiguration ) or 'FeatureSelection' not in self._campaign_configuration or 'method' not in self._campaign_configuration[ 'FeatureSelection'] or self._campaign_configuration[ 'FeatureSelection']['method'] != "SFS": assert not os.path.exists( self._experiment_directory), self._experiment_directory os.makedirs(self._experiment_directory)
from IPython import embed from custom_logger import getLogger from driver import travian_driver, pages, humanlike_pauses, action_queue from driver.selenium_server_utils import launch_selenium_standalone_server_if_necessary from time import sleep import json import random logger = getLogger('main') with open('secrets.json', 'r') as secrets_file_handle: secrets = json.load(secrets_file_handle) def execute(): launch_selenium_standalone_server_if_necessary() driver = travian_driver.TravianDriver(secrets['base_url']) reports_page = pages.ReportsPage(driver) messages_page = pages.MessagesPage(driver) overview_page = pages.OverviewPage(driver) driver.authenticate_if_necessary(secrets['username'], secrets['password']) post_login_check = action_queue.ActionQueue( action_queue=[ messages_page.go, reports_page.go, overview_page.go, ], insert_pause_action=humanlike_pauses.stop_and_look_delay) post_login_check.run(ordered=False)
from __future__ import division from custom_logger import getLogger from driver import pages, humanlike_pauses, action_queue from os.path import isfile from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait import numpy as np import random import re import selenium from IPython import embed logger = getLogger('travian_driver') REMOTE_SERVER_PORT = 4555 REMOTE_SERVER_URL = "http://localhost:{}/wd/hub".format(REMOTE_SERVER_PORT) SESSION_ID_FILE_PATH = 'session_id.txt' DEFAULT_WINDOW_SIZE = np.array([1280, 720], 'int') class NoSessionIdFile(Exception): pass class InsufficientTroopsError(Exception): pass
JAVA_PATH = '/usr/bin/java' SHELL_PATH = '/bin/bash' LAUNCH_SERVER_STANDALONE_SCRIPT_PATH = abspath('bin/launch_selenium_server_standalone.sh') RUNNING_SERVER_TEST_SCRIPT_PATH = abspath('bin/running_selenium_server_test.sh') LAUNCH_STANDALONE_SERVER_ARGS = [ SHELL_PATH, LAUNCH_SERVER_STANDALONE_SCRIPT_PATH ] RUNNING_STANDALONE_SERVER_TEST_ARGS = [ SHELL_PATH, RUNNING_SERVER_TEST_SCRIPT_PATH ] logger = getLogger('standalone_server_utils') def launch_selenium_standalone_server(): logger.info('Launching a new selenium standalone server as a daemon') p = Popen(LAUNCH_STANDALONE_SERVER_ARGS) logger.info('Waiting for server to initialize') sleep(15) logger.info(f"New selenium standalone server successfully started, pid: {p.pid}") def is_selenium_standalone_server_running(): p = Popen(RUNNING_STANDALONE_SERVER_TEST_ARGS) p.communicate() if p.returncode == 0: logger.info('Detected running selenium standalone server') return True else:
import random import numpy as np from custom_logger import getLogger logger = getLogger('action_queue') def intersperse(lst, item): result = [item] * (len(lst) * 2 - 1) result[0::2] = lst return result class ActionQueueException(Exception): pass class ActionQueue(object): def __init__(self, action_queue=None, insert_pause_action=None): if action_queue is None: self._action_queue = [] else: self._action_queue = action_queue if insert_pause_action is None or callable(insert_pause_action): self._insert_pause_action = insert_pause_action else: raise ActionQueueException( f'Cannot use a non-callable pause type: {insert_pause_action}') def append(self, action):
def __init__(self, configuration_file, debug=False, seed=0, output="output", j=1, generate_plots=False, self_check=True, details=False): """ Constructor of the class - Copy the parameters to member variables - Initialize the logger - Build the data preparation flow adding or not data preparation steps on the basis of the content of the loaded configuration file Parameters ---------- configuration_file: str The configuration file describing the experimental campaign to be performed debug: bool True if debug messages should be printed seed: integer The seed to be used to initialize the random generator engine output: str The directory where all the outputs will be written; it is created by this library and cannot exist before using this module j: integer The number of processes to be used in the grid search generate_plots: bool True if plots have to be used self_check: bool True if the generated regressor should be tested details: bool True if the results of the single experiments should be added """ self._data_preprocessing_list = [] self.random_generator = random.Random(seed) self.debug = debug self._self_check = self_check if self.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) self._logger = custom_logger.getLogger(__name__) # Check if the configuration file exists if not os.path.exists(configuration_file): self._logger.error("%s does not exist", configuration_file) sys.exit(-1) self.conf = cp.ConfigParser() self.conf.optionxform = str self.conf.read(configuration_file) self.conf['General']['configuration_file'] = configuration_file self.conf['General']['output'] = output self.conf['General']['seed'] = str(seed) self.conf['General']['j'] = str(j) self.conf['General']['debug'] = str(debug) self.conf['General']['generate_plots'] = str(generate_plots) self.conf['General']['details'] = str(details) self._campaign_configuration = {} self.load_campaign_configuration(configuration_file) # Check if output path already exist if os.path.exists(output): self._logger.error("%s already exists", output) sys.exit(1) os.mkdir(self._campaign_configuration['General']['output']) shutil.copyfile(configuration_file, os.path.join(output, 'configuration_file.ini')) self.conf.write(open(os.path.join(output, "enriched_configuration_file.ini"), 'w')) # Check that validation method has been specified if 'validation' not in self._campaign_configuration['General']: self._logger.error("Validation not specified") sys.exit(1) # Check that if HoldOut is selected, hold_out_ratio is specified if self._campaign_configuration['General']['validation'] == "HoldOut" or self._campaign_configuration['General']['hp_selection'] == "HoldOut": if "hold_out_ratio" not in self._campaign_configuration['General']: self._logger.error("hold_out_ratio not set") sys.exit(1) # Check that if Extrapolation is selected, extrapolation_columns is specified if self._campaign_configuration['General']['validation'] == "Extrapolation": if "extrapolation_columns" not in self._campaign_configuration['General']: self._logger.error("extrapolation_columns not set") sys.exit(1) # Check that if XGBoost is used for feature selection tolerance is specified if 'FeatureSelection' in self._campaign_configuration and self._campaign_configuration['FeatureSelection']['method'] == "XGBoost": if "XGBoost_tolerance" not in self._campaign_configuration['FeatureSelection']: self._logger.error("XGBoost tolerance not set") sys.exit(1) # Check that if ernest is used, normalization, product, column_selection, and inversion are disabled if 'ernest' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['ernest']: if 'use_columns' in self._campaign_configuration['DataPreparation'] or "skip_columns" in self._campaign_configuration['DataPreparation']: logging.error("use_columns and skip_columns cannot be used with ernest") sys.exit(1) if 'inverse' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['inverse']: logging.error("inverse cannot be used with ernest") sys.exit(1) if 'product_max_degree' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['product_max_degree']: logging.error("product cannot be used with ernest") sys.exit(1) if 'normalization' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['normalization']: logging.error("normalization cannot be used with ernest") sys.exit(1) # Adding read on input to data preprocessing step self._data_preprocessing_list.append(data_preparation.data_loading.DataLoading(self._campaign_configuration)) # Adding column renaming if required if 'rename_columns' in self._campaign_configuration['DataPreparation']: self._data_preprocessing_list.append(data_preparation.rename_columns.RenameColumns(self._campaign_configuration)) # Adding column selection if required if 'use_columns' in self._campaign_configuration['DataPreparation'] or "skip_columns" in self._campaign_configuration['DataPreparation']: self._data_preprocessing_list.append(data_preparation.column_selection.ColumnSelection(self._campaign_configuration)) # Transform categorical features in onehot encoding self._data_preprocessing_list.append(data_preparation.onehot_encoding.OnehotEncoding(self._campaign_configuration)) # Split according to extrapolation values if required if self._campaign_configuration['General']['validation'] == "Extrapolation": self._data_preprocessing_list.append(data_preparation.extrapolation.Extrapolation(self._campaign_configuration)) # Adding inverted features if required if 'inverse' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['inverse']: self._data_preprocessing_list.append(data_preparation.inversion.Inversion(self._campaign_configuration)) # Adding product features if required if 'product_max_degree' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['product_max_degree']: self._data_preprocessing_list.append(data_preparation.product.Product(self._campaign_configuration)) # Create ernest features if required if 'ernest' in self._campaign_configuration['DataPreparation'] and self._campaign_configuration['DataPreparation']['ernest']: self._data_preprocessing_list.append(data_preparation.ernest.Ernest(self._campaign_configuration)) # Adding data check self._data_preprocessing_list.append(data_preparation.data_check.DataCheck(self._campaign_configuration)) self._model_building = model_building.model_building.ModelBuilding(self.random_generator.random())