def generate_pattern(cls, interval_percentage: float, step_percentage: float, config: dict): start_rps = ErrorChecker.key_check_and_load('start_rps', config, default=0) line_coefficients = ErrorChecker.key_check_and_load('unit', config, defaul=dict()) vals = list() request_rate = start_rps for interval, coef in line_coefficients.items(): monotoneous_fragment_percentage = interval * interval_percentage monotoneous_fragment_end_rps = request_rate + coef * interval // interval_percentage monotoneous_config = { 'rps': { 'start': request_rate, 'end': monotoneous_fragment_end_rps } } vals.expand( MonotoneousTrendLoadGenerator.generate_pattern( monotoneous_fragment_percentage, step_percentage, monotoneous_config)) request_rate = monotoneous_fragment_end_rps return vals
def __init__(self, service_name : str, service_scaling_info_raw : dict, scaled_aspect_name : str): self.scaled_aspect_name = scaled_aspect_name self._booting_duration = collections.defaultdict(lambda: {'mean': 0, 'std': 0, 'unit': 'ms'}) self._termination_duration = collections.defaultdict(lambda: {'mean': 0, 'std': 0, 'unit': 'ms'}) self._booting_duration[self.__class__.DEFAULT_PROVIDER_NAME] self._termination_duration[self.__class__.DEFAULT_PROVIDER_NAME] booting_duration_raw = ErrorChecker.key_check_and_load('booting_duration', service_scaling_info_raw, 'service name', service_name) if 'value' in booting_duration_raw: self._booting_duration[self.__class__.DEFAULT_PROVIDER_NAME]['mean'] = ErrorChecker.key_check_and_load('value', booting_duration_raw, 'service name', service_name) self._booting_duration[self.__class__.DEFAULT_PROVIDER_NAME]['unit'] = ErrorChecker.key_check_and_load('unit', booting_duration_raw, 'service name', service_name) elif 'mean' in booting_duration_raw: self._update_distribution_parameters(self._booting_duration, self.__class__.DEFAULT_PROVIDER_NAME, provider_specific_config) else: for provider_name, provider_specific_config in booting_duration_raw.items(): self._update_distribution_parameters(self._booting_duration, provider_name, provider_specific_config) termination_duration_raw = ErrorChecker.key_check_and_load('termination_duration', service_scaling_info_raw, 'service name', service_name) if 'value' in termination_duration_raw: self._termination_duration[self.__class__.DEFAULT_PROVIDER_NAME]['mean'] = ErrorChecker.key_check_and_load('value', booting_duration_raw, 'service name', service_name) self._termination_duration[self.__class__.DEFAULT_PROVIDER_NAME]['unit'] = ErrorChecker.key_check_and_load('unit', booting_duration_raw, 'service name', service_name) elif 'mean' in booting_duration_raw: self._update_distribution_parameters(self._termination_duration, self.__class__.DEFAULT_PROVIDER_NAME, provider_specific_config) else: for provider_name, provider_specific_config in termination_duration_raw.items(): self._update_distribution_parameters(self._termination_duration, provider_name, provider_specific_config)
def __init__(self, config: dict): self.service_name = ErrorChecker.key_check_and_load( 'service_name', config) self.region_name = ErrorChecker.key_check_and_load('region', config) self.metric_name = ErrorChecker.key_check_and_load( 'metric_name', config) self.fhorizon_in_steps = ErrorChecker.key_check_and_load( 'horizon_in_steps', config, default=60) forecast_frequency_raw = ErrorChecker.key_check_and_load( 'forecast_frequency', config, default={ 'value': 1, 'unit': 's' }) forecast_frequency_val = ErrorChecker.key_check_and_load( 'value', forecast_frequency_raw) forecast_frequency_unit = ErrorChecker.key_check_and_load( 'unit', forecast_frequency_raw) self.forecast_frequency = pd.Timedelta(forecast_frequency_val, unit=forecast_frequency_unit) self._model_fitted = None self.fitted = False dir_with_pretrained_models = ErrorChecker.key_check_and_load( 'dir_with_pretrained_models', config, default=None) if not dir_with_pretrained_models is None: self.load_from_location(dir_with_pretrained_models) self.fitted = True self.do_not_adjust_model = ErrorChecker.key_check_and_load( 'do_not_adjust_model', config, default=False) self.dir_to_store_models = ErrorChecker.key_check_and_load( 'dir_to_store_models', config, default=None)
def parse(cls, pattern: dict, generation_bucket: pd.Timedelta): params = ErrorChecker.key_check_and_load('params', pattern) requests_count_per_unit_of_time = ErrorChecker.key_check_and_load( 'value', params) interval_of_time_raw = ErrorChecker.key_check_and_load( 'interval_of_time', params) unit_of_time = ErrorChecker.key_check_and_load('unit_of_time', params) interval_of_time = pd.Timedelta(interval_of_time_raw, unit=unit_of_time) if generation_bucket > interval_of_time: raise ValueError( 'The simulation step should be smaller or equal to the interval of time, for which the requests are generated' ) buckets_count = interval_of_time // generation_bucket requests_count_per_bucket = requests_count_per_unit_of_time // buckets_count load_distribution_in_steps_buckets = [requests_count_per_bucket ] * buckets_count # Distributing the leftovers (if any) leftover_requests_count = requests_count_per_unit_of_time % buckets_count for i in range(leftover_requests_count): load_distribution_in_steps_buckets[i] += 1 return (interval_of_time, load_distribution_in_steps_buckets)
def __init__(self, config: dict): super().__init__(config) forecasting_model_params = ErrorChecker.key_check_and_load( 'config', config) self.d = ErrorChecker.key_check_and_load('d', forecasting_model_params, 0)
def __init__(self, config_folder: str, regime_config: dict, simulator: 'Simulator', repetitions_count_per_simulation: int, keep_evaluated_configs: bool = False): super().__init__(simulator, repetitions_count_per_simulation, keep_evaluated_configs) self.unchanged_configs_folder = os.path.join( config_folder, ErrorChecker.key_check_and_load('unchanged_configs_folder', regime_config)) if not os.path.exists(self.unchanged_configs_folder): raise ValueError( f'Folder {self.unchanged_configs_folder} for the unchanged configs of the experiments does not exist' ) self.alternatives_folder = os.path.join( config_folder, ErrorChecker.key_check_and_load('alternatives_folder', regime_config)) if not os.path.exists(self.alternatives_folder): raise ValueError( f'Folder {self.alternatives_folder} for the evaluated alternative configs of the experiments does not exist' ) self.config_folder = config_folder self.simulations_results = collections.defaultdict(list)
def __init__(self, config): adjustment_heuristic_conf = ErrorChecker.key_check_and_load( 'adjustment_heuristic_conf', config, default=dict()) adjustment_heuristic_name = ErrorChecker.key_check_and_load( 'name', adjustment_heuristic_conf, default='none') self.post_processing_adjuster = AdjustmentHeuristic.get( adjustment_heuristic_name)(adjustment_heuristic_conf)
def __init__(self, config: dict): self.min_oscillations_period = ErrorChecker.key_check_and_load( 'min_oscillations_period', config, self.__class__.__name__) self.max_oscillations_period = ErrorChecker.key_check_and_load( 'max_oscillations_period', config, self.__class__.__name__) self.component_to_use = ErrorChecker.key_check_and_load( 'component', config, self.__class__.__name__, default='trend')
def __init__(self, config: dict): self.smoothing_level = ErrorChecker.key_check_and_load( 'smoothing_level', config, self.__class__.__name__) self.smoothing_trend = ErrorChecker.key_check_and_load( 'smoothing_trend', config, self.__class__.__name__) self.damping_trend = ErrorChecker.key_check_and_load( 'damping_trend', config, self.__class__.__name__)
def _update_distribution_parameters(self, parameters_storage : dict, provider_name : str, provider_specific_config : dict): parameters_storage[provider_name]['mean'] = ErrorChecker.key_check_and_load('mean', provider_specific_config, default = None) if parameters_storage[provider_name]['mean'] is None: parameters_storage[provider_name]['mean'] = ErrorChecker.key_check_and_load('value', provider_specific_config) parameters_storage[provider_name]['unit'] = ErrorChecker.key_check_and_load('unit', provider_specific_config) if 'std' in provider_specific_config: parameters_storage[provider_name]['std'] = ErrorChecker.key_check_and_load('std', provider_specific_config)
def __init__(self, region_name: str, count_of_services_affected: int, failure_type_conf: dict): super().__init__(region_name, count_of_services_affected) self.node_type = ErrorChecker.key_check_and_load( 'node_type', failure_type_conf) self.provider = ErrorChecker.key_check_and_load( 'provider', failure_type_conf)
def __init__(self, config: dict): super().__init__(config) forecasting_model_params = ErrorChecker.key_check_and_load( 'config', config) lags = ErrorChecker.key_check_and_load('lags', forecasting_model_params, default=[0]) self.lags = [lags] if isinstance(lags, numbers.Number) else lags
def __init__(self, config: dict): resolution_raw = ErrorChecker.key_check_and_load( 'resolution', config, self.__class__.__name__) resolution_value = ErrorChecker.key_check_and_load( 'value', resolution_raw, self.__class__.__name__) resolution_unit = ErrorChecker.key_check_and_load( 'unit', resolution_raw, self.__class__.__name__) self.resolution = pd.Timedelta(resolution_value, unit=resolution_unit)
def __init__(self, config : dict): super().__init__(config) forecasting_model_params = ErrorChecker.key_check_and_load('config', config) ar_lags = ErrorChecker.key_check_and_load('p', forecasting_model_params, default = [0]) self.ar_lags = [ar_lags] if isinstance(ar_lags, numbers.Number) else ar_lags ma_lags = ErrorChecker.key_check_and_load('q', forecasting_model_params, default = [0]) self.ma_lags = [ma_lags] if isinstance(ma_lags, numbers.Number) else ma_lags self.d = 0
def __init__(self, node_type: str, node_scaling_info_raw: dict): self.node_type = node_type self._booting_duration = self._update_distribution_parameters( ErrorChecker.key_check_and_load('booting_duration', node_scaling_info_raw, 'node type', node_type)) self._termination_duration = self._update_distribution_parameters( ErrorChecker.key_check_and_load('termination_duration', node_scaling_info_raw, 'node type', node_type))
def __init__(self, config: dict): self.order = ErrorChecker.key_check_and_load('order', config, self.__class__.__name__, default=1) self.critical_frequency = ErrorChecker.key_check_and_load( 'critical_frequency', config, self.__class__.__name__, default=15) self.type = ErrorChecker.key_check_and_load('type', config, self.__class__.__name__, default='lowpass')
def __init__(self, config: dict): super().__init__(config) forecasting_model_params = ErrorChecker.key_check_and_load( 'config', config) averaging_interval_raw = ErrorChecker.key_check_and_load( 'averaging_interval', forecasting_model_params) value = ErrorChecker.key_check_and_load('value', averaging_interval_raw) unit = ErrorChecker.key_check_and_load('unit', averaging_interval_raw) self._averaging_interval = pd.Timedelta(value, unit=unit) self._averaged_value = None
def __init__(self, simulation_step : pd.Timedelta, simulation_start : pd.Timestamp, filename : str, reqs_processing_infos : dict): self.region_models = {} if filename is None: raise ValueError('Configuration file not provided for the WorkloadModel.') else: with open(filename) as f: try: config = json.load(f) load_kind = ErrorChecker.key_check_and_load('load_kind', config) batch_size = ErrorChecker.key_check_and_load('batch_size', config, default = 1) generation_bucket_raw = ErrorChecker.key_check_and_load('generation_bucket', config, default = None) generation_bucket = ErrorChecker.parse_duration(generation_bucket_raw) if not generation_bucket_raw is None else simulation_step regions_configs = ErrorChecker.key_check_and_load('regions_configs', config) for region_config in regions_configs: region_name = ErrorChecker.key_check_and_load('region_name', region_config) pattern = ErrorChecker.key_check_and_load('pattern', region_config, 'region_name', region_name) load_configs = ErrorChecker.key_check_and_load('load_configs', region_config, 'region_name', region_name) self.region_models[region_name] = RegionalLoadModel.get(load_kind)(region_name, pattern, load_configs, generation_bucket, simulation_start, simulation_step, reqs_processing_infos, batch_size) except json.JSONDecodeError: raise ValueError(f'An invalid JSON when parsing for {self.__class__.__name__}')
def _add_failure(self, simulation_start : pd.Timestamp, simulation_step : pd.Timedelta, steps_count : int, failure_class : type, node_failure_conf : dict): """ Selects a particular timestamp during the whole simulation to add the failure based on a uniform distribution. """ failure_ts = simulation_start + simulation_step * np.random.randint(1, steps_count + 1) count_of_entities_affected = ErrorChecker.key_check_and_load('count_of_entities_affected', node_failure_conf) region_name = ErrorChecker.key_check_and_load('region_name', node_failure_conf) failure_type_conf = ErrorChecker.key_check_and_load('failure_type_conf', node_failure_conf) self.failures[failure_ts].append(failure_class(region_name, count_of_entities_affected, failure_type_conf))
def parse(load_configs: dict): reqs_types_ratios = {} for conf in load_configs: req_type = ErrorChecker.key_check_and_load('request_type', conf) load_config = ErrorChecker.key_check_and_load('load_config', conf) req_ratio = ErrorChecker.key_check_and_load('ratio', load_config) reqs_types_ratios[req_type] = req_ratio if sum(reqs_types_ratios.values()) != 1.0: raise ValueError('Request counts ratios do not add up to 1') return reqs_types_ratios
def generate_load_pattern_based_on_recipe(cls, config_file_path: str): if not os.path.isfile(config_file_path): raise ValueError( f'Path to the configuration file does not exist: {config_file_path}' ) with open(config_file_path, 'r') as f: config = json.load(f) step_duration_raw = ErrorChecker.key_check_and_load( 'step_duration', config) step_duration_value = ErrorChecker.key_check_and_load( 'value', step_duration_raw) step_duration_unit = ErrorChecker.key_check_and_load( 'unit', step_duration_raw) step_duration = pd.Timedelta(step_duration_value, unit=step_duration_unit) pattern_pieces = ErrorChecker.key_check_and_load('pieces', config, default=list()) pieces_intervals = list() for pattern_piece in pattern_pieces: interval_raw = ErrorChecker.key_check_and_load( 'interval', pattern_piece) interval_value = ErrorChecker.key_check_and_load( 'value', interval_raw) interval_unit = ErrorChecker.key_check_and_load( 'unit', interval_raw) pieces_intervals.append( pd.Timedelta(interval_value, unit=interval_unit)) joint_pattern_duration = sum(pieces_intervals, pd.Timedelta(0)) step_percentage = step_duration / joint_pattern_duration generated_vals = list() for pattern_piece, interval in zip(pattern_pieces, pieces_intervals): joint_pattern_percentage_for_piece = interval / joint_pattern_duration generator_class = cls._Registry.get( ErrorChecker.key_check_and_load('pattern', pattern_piece), None) if not generator_class is None: generated_vals.extend( generator_class.generate_pattern( joint_pattern_percentage_for_piece, step_percentage, ErrorChecker.key_check_and_load('config', pattern_piece, default=None))) return ',\n{'.join(json.dumps(generated_vals).split(', {'))
def _update_distribution_parameters(self, config: dict): distribution_parameters = {'mean': 0, 'std': 0, 'unit': 'ms'} distribution_parameters['mean'] = ErrorChecker.key_check_and_load( 'mean', config, default=None) if distribution_parameters['mean'] is None: distribution_parameters['mean'] = ErrorChecker.key_check_and_load( 'value', config) distribution_parameters['unit'] = ErrorChecker.key_check_and_load( 'unit', config) if 'std' in config: distribution_parameters['std'] = ErrorChecker.key_check_and_load( 'std', config) return distribution_parameters
def __init__(self, config): super().__init__(config) if self._model is None: model_params = ErrorChecker.key_check_and_load('model_params', config, default=dict()) model_config = { 'loss': ErrorChecker.key_check_and_load('loss', model_params, default='squared_loss'), 'penalty': ErrorChecker.key_check_and_load('penalty', model_params, default='l2'), 'alpha': ErrorChecker.key_check_and_load('alpha', model_params, default=0.0001), 'l1_ratio': ErrorChecker.key_check_and_load('l1_ratio', model_params, default=0.15), 'tol': ErrorChecker.key_check_and_load('tol', model_params, default=0.001), 'validation_fraction': ErrorChecker.key_check_and_load('validation_fraction', model_params, default=0.1), 'n_iter_no_change': ErrorChecker.key_check_and_load('n_iter_no_change', model_params, default=5), 'loss': ErrorChecker.key_check_and_load('loss', model_params, default='epsilon_insensitive'), 'epsilon': ErrorChecker.key_check_and_load('epsilon', model_params, default=0.1) } # TODO: consider leftovers? non-oblig self._model = SGDRegressor(**model_config)
def __init__(self, region_name: str, pattern: dict, load_configs: dict, generation_bucket: pd.Timedelta, simulation_start: pd.Timestamp, simulation_step: pd.Timedelta, reqs_processing_infos: dict, batch_size: int): super().__init__(region_name, generation_bucket, simulation_step, reqs_processing_infos, batch_size) self.monthly_vals = SeasonalLoadPatternParser.get( ErrorChecker.key_check_and_load('type', pattern, 'region_name', self.region_name)).parse(pattern) self.reqs_types_ratios = RatiosParser.parse(load_configs) self.reqs_generators = DistributionsParser.parse(load_configs) self.current_means_split_across_seconds = dict() self.current_req_split_across_simulation_steps = dict() for req_type in self.reqs_types_ratios: self.current_req_split_across_simulation_steps[req_type] = { ms_bucket_id: 0 for ms_bucket_id in range( pd.Timedelta(1000, unit='ms') // self.generation_bucket) } self.current_month = -1 self.current_time_unit = -1 self.cur_second_in_time_unit = -1
def to_metric(cls, config : dict): val = ErrorChecker.key_check_and_load('value', config) time_value, time_unit = 1, None resolution = ErrorChecker.key_check_and_load('resolution', config, default = None) if resolution is None: time_unit = ErrorChecker.key_check_and_load('unit', config) else: time_value = ErrorChecker.key_check_and_load('value', resolution) time_unit = ErrorChecker.key_check_and_load('unit', resolution) if time_value == 0: raise ValueError('Resolution should not be zero') return cls(val, time_interval = pd.Timedelta(time_value, unit = time_unit))
def __init__(self, config: dict): super().__init__(config) alpha = ErrorChecker.key_check_and_load('alpha', config, default=0.6) if alpha <= 0 or (alpha > 1 and alpha < 4): raise ValueError('Alpha should be in the range (0, 1] or [4, inf)') c = ErrorChecker.key_check_and_load('c', config, default=15) if c <= 0: raise ValueError('c has to be greater than 0') est = ErrorChecker.key_check_and_load('est', config, default='mic_approx') self.estimator = MINE(alpha=alpha, c=c, est=est)
def parse(load_configs: dict): reqs_generators = {} for conf in load_configs: req_type = ErrorChecker.key_check_and_load('request_type', conf) load_config = ErrorChecker.key_check_and_load('load_config', conf) sliced_distribution = ErrorChecker.key_check_and_load( 'sliced_distribution', load_config) req_distribution_type = ErrorChecker.key_check_and_load( 'type', sliced_distribution) req_distribution_params = ErrorChecker.key_check_and_load( 'params', sliced_distribution) reqs_generators[req_type] = SlicedRequestsNumDistribution.get( req_distribution_type)(req_distribution_params) return reqs_generators
def __init__(self, config): super().__init__(config) if self._model is None: model_params = ErrorChecker.key_check_and_load('model_params', config, default = dict()) learning_params = ErrorChecker.key_check_and_load('learning', model_params, default = {'loss' : 'mean_squared_error', 'optimizer' : 'adam'}) default_layers_params = ErrorChecker.key_check_and_load('default_layers_params', model_params, default = dict()) self._model = tf.keras.models.Sequential() model_layers = ErrorChecker.key_check_and_load('layers', config, default = list()) if len(model_layers) == 0: raise ValueError('No layers specified for the model') for layer_conf in model_layers: layer_type = ErrorChecker.key_check_and_load('type', layer_conf) layer_template = self.__class__._LAYERS.get(layer_type, None) # TODO: class? if layer_template is None: raise ValueError(f'Undefined layer {layer_type}') mandatory_layer_params = dict() for mandatory_param_name in layer_template['mandatory_params_names']: mandatory_param_value = ErrorChecker.key_check_and_load(mandatory_param_name, layer_conf) mandatory_layer_params[mandatory_param_name] = mandatory_param_value optional_params = ErrorChecker.key_check_and_load('params', layer_conf, default = default_layers_params.get(layer_type, layer_template['default_params'])) layer_params = {**mandatory_layer_params, **optional_params} self._model.add(layer_template['model'](**layer_params)) self._model.compile(**learning_params)
def to_target_value(cls, config: dict): val = ErrorChecker.key_check_and_load('value', config) if val < 0 or val > 1: raise ValueError( 'Target value should be specified as a relative number between 0.0 and 1.0' ) return val
def __init__(self, config: dict): super().__init__(config) self.exponent = ErrorChecker.key_check_and_load('exponent', config, default=1) if self.exponent <= 0 or self.exponent >= 2: raise ValueError('Exponent should be in range (0, 2)')