def __init__(self, deepnet, api=None): """The Deepnet constructor can be given as first argument: - a deepnet structure - a deepnet id - a path to a JSON file containing a deepnet structure """ self.resource_id = None self.regression = False self.network = None self.networks = None self.input_fields = [] self.class_names = [] self.preprocess = [] self.optimizer = None self.missing_numerics = False self.api = get_api_connection(api) self.resource_id, deepnet = get_resource_dict( \ deepnet, "deepnet", api=self.api) if 'object' in deepnet and isinstance(deepnet['object'], dict): deepnet = deepnet['object'] self.input_fields = deepnet['input_fields'] if 'deepnet' in deepnet and isinstance(deepnet['deepnet'], dict): status = get_status(deepnet) objective_field = deepnet['objective_fields'] deepnet = deepnet['deepnet'] if 'code' in status and status['code'] == FINISHED: self.fields = deepnet['fields'] missing_tokens = deepnet.get('missing_tokens') ModelFields.__init__( self, self.fields, objective_id=extract_objective(objective_field), terms=True, categories=True, missing_tokens=missing_tokens) self.regression = \ self.fields[self.objective_id]['optype'] == NUMERIC if not self.regression: self.class_names = [category for category, _ in \ self.fields[self.objective_id][ \ 'summary']['categories']] self.class_names.sort() # order matters self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] self.missing_numerics = deepnet.get('missing_numerics', False) if 'network' in deepnet: network = deepnet['network'] self.network = network self.networks = network.get('networks', []) self.preprocess = network.get('preprocess') self.optimizer = network.get('optimizer', {}) else: raise Exception("The deepnet isn't finished yet") else: raise Exception("Cannot create the Deepnet instance. Could not" " find the 'deepnet' key in the resource:\n\n%s" % deepnet)
def __init__(self, model, api=None): self.api = get_api_connection(api) resource_id, model = extract_id(model, api) resource_type = get_resource_type(resource_id) kwargs = {"api": self.api} local_model = COMPONENT_CLASSES[resource_type](model, **kwargs) self.__class__.__bases__ = local_model.__class__.__bases__ for attr, value in local_model.__dict__.items(): setattr(self, attr, value) self.local_model = local_model
def __init__(self, anomaly, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the Minomaly attributes self.__dict__ = load(get_anomaly_id(anomaly), cache_get) return self.resource_id = None self.sample_size = None self.input_fields = None self.default_numeric_value = None self.mean_depth = None self.expected_mean_depth = None self.iforest = None self.id_fields = [] api = get_api_connection(api) self.resource_id, anomaly = get_resource_dict( anomaly, "anomaly", api=api) if 'object' in anomaly and isinstance(anomaly['object'], dict): anomaly = anomaly['object'] self.sample_size = anomaly.get('sample_size') self.input_fields = anomaly.get('input_fields') self.default_numeric_value = anomaly.get('default_numeric_value') self.id_fields = anomaly.get('id_fields', []) if 'model' in anomaly and isinstance(anomaly['model'], dict): ModelFields.__init__( self, anomaly['model'].get('fields'), missing_tokens=anomaly['model'].get('missing_tokens')) self.mean_depth = anomaly['model'].get('mean_depth') self.normalization_factor = anomaly['model'].get( 'normalization_factor') self.nodes_mean_depth = anomaly['model'].get( 'nodes_mean_depth') status = get_status(anomaly) if 'code' in status and status['code'] == FINISHED: self.expected_mean_depth = None if self.mean_depth is None or self.sample_size is None: raise Exception("The anomaly data is not complete. " "Score will not be available") self.norm = self.normalization_factor if \ self.normalization_factor is not None else \ self.norm_factor() iforest = anomaly['model'].get('trees', []) if iforest: self.iforest = [ build_tree([anomaly_tree['root']]) for anomaly_tree in iforest] self.top_anomalies = anomaly['model']['top_anomalies'] else: raise Exception("The anomaly isn't finished yet")
def __init__(self, anomaly, api=None): self.resource_id = None self.sample_size = None self.input_fields = None self.mean_depth = None self.expected_mean_depth = None self.iforest = None self.top_anomalies = None self.id_fields = [] self.api = get_api_connection(api) self.resource_id, anomaly = get_resource_dict( \ anomaly, "anomaly", api=self.api) if 'object' in anomaly and isinstance(anomaly['object'], dict): anomaly = anomaly['object'] self.sample_size = anomaly.get('sample_size') self.input_fields = anomaly.get('input_fields') self.id_fields = anomaly.get('id_fields', []) if 'model' in anomaly and isinstance(anomaly['model'], dict): ModelFields.__init__( \ self, anomaly['model'].get('fields'), \ missing_tokens=anomaly['model'].get('missing_tokens')) if ('top_anomalies' in anomaly['model'] and isinstance(anomaly['model']['top_anomalies'], list)): self.mean_depth = anomaly['model'].get('mean_depth') status = get_status(anomaly) if 'code' in status and status['code'] == FINISHED: self.expected_mean_depth = None if self.mean_depth is None or self.sample_size is None: raise Exception("The anomaly data is not complete. " "Score will" " not be available") else: default_depth = self.mean_depth if \ self.sample_size == 1 else (2 * (DEPTH_FACTOR + \ math.log(self.sample_size - 1) - \ (float(self.sample_size - 1) / self.sample_size))) self.expected_mean_depth = min(self.mean_depth, default_depth) iforest = anomaly['model'].get('trees', []) if iforest: self.iforest = [ AnomalyTree(anomaly_tree['root'], self.fields) for anomaly_tree in iforest ] self.top_anomalies = anomaly['model']['top_anomalies'] else: raise Exception("The anomaly isn't finished yet") else: raise Exception("Cannot create the Anomaly instance. Could not" " find the 'top_anomalies' key in the" " resource:\n\n%s" % anomaly['model'].keys())
def __init__(self, execution, api=None): self.resource_id = None self.outputs = None self.output_types = None self.output_resources = None self.result = None self.status = None self.source_location = None self.error = None self.error_message = None self.error_location = None self.call_stack = None self.api = get_api_connection(api) try: self.resource_id, execution = get_resource_dict( \ execution, "execution", self.api) except ValueError as resource: try: execution = json.loads(str(resource)) self.resource_id = execution["resource"] except ValueError: raise ValueError("The execution resource was faulty: \n%s" % \ resource) if 'object' in execution and isinstance(execution['object'], dict): execution = execution['object'] self.status = execution["status"] self.error = self.status.get("error") if self.error is not None: self.error_message = self.status.get("message") self.error_location = self.status.get("source_location") self.call_stack = self.status.get("call_stack") else: self.source_location = self.status.get("source_location") if 'execution' in execution and \ isinstance(execution['execution'], dict): execution = execution.get('execution') self.result = execution.get("result") self.outputs = dict((output[0], output[1]) \ for output in execution.get("outputs")) self.output_types = dict((output[0], output[2]) \ for output in execution.get("outputs")) self.output_resources = dict((res["variable"], res["id"]) \ for res in execution.get("output_resources")) self.execution = execution
def __init__(self, linear_regression, api=None): self.resource_id = None self.input_fields = [] self.term_forms = {} self.tag_clouds = {} self.term_analysis = {} self.items = {} self.item_analysis = {} self.categories = {} self.coefficients = [] self.data_field_types = {} self.field_codings = {} self.bias = None self.xtx_inverse = [] self.mean_squared_error = None self.number_of_parameters = None self.number_of_samples = None self.api = get_api_connection(api) self.resource_id, linear_regression = get_resource_dict( \ linear_regression, "linearregression", api=self.api) if 'object' in linear_regression and \ isinstance(linear_regression['object'], dict): linear_regression = linear_regression['object'] try: self.input_fields = linear_regression.get("input_fields", []) self.dataset_field_types = linear_regression.get( "dataset_field_types", {}) self.weight_field = linear_regression.get("weight_field") objective_field = linear_regression['objective_fields'] if \ linear_regression['objective_fields'] else \ linear_regression['objective_field'] except KeyError: raise ValueError("Failed to find the linear regression expected " "JSON structure. Check your arguments.") if 'linear_regression' in linear_regression and \ isinstance(linear_regression['linear_regression'], dict): status = get_status(linear_regression) if 'code' in status and status['code'] == FINISHED: linear_regression_info = linear_regression[ \ 'linear_regression'] fields = linear_regression_info.get('fields', {}) if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(fields.items(), key=lambda x: x[1].get("column_number"))] self.coeff_ids = self.input_fields[:] self.coefficients = linear_regression_info.get( \ 'coefficients', []) self.bias = linear_regression_info.get('bias', True) self.field_codings = linear_regression_info.get( \ 'field_codings', {}) self.number_of_parameters = linear_regression_info.get( \ "number_of_parameters") missing_tokens = linear_regression_info.get("missing_tokens") objective_id = extract_objective(objective_field) ModelFields.__init__(self, fields, objective_id=objective_id, terms=True, categories=True, numerics=True, missing_tokens=missing_tokens) self.field_codings = linear_regression_info.get( \ 'field_codings', {}) self.format_field_codings() for field_id in self.field_codings: if field_id not in fields and \ field_id in self.inverted_fields: self.field_codings.update( \ {self.inverted_fields[field_id]: \ self.field_codings[field_id]}) del self.field_codings[field_id] stats = linear_regression_info["stats"] if STATS and stats is not None and \ stats.get("xtx_inverse") is not None: self.xtx_inverse = stats["xtx_inverse"][:] self.mean_squared_error = stats["mean_squared_error"] self.number_of_samples = stats["number_of_samples"] # to be used in predictions self.t_crit = student_t.interval( \ CONFIDENCE, self.number_of_samples - self.number_of_parameters)[1] self.xtx_inverse = list( \ np.linalg.inv(np.array(self.xtx_inverse))) else: raise Exception("The linear regression isn't finished yet") else: raise Exception("Cannot create the LinearRegression instance." " Could not find the 'linear_regression' key" " in the resource:\n\n%s" % linear_regression)
def __init__(self, time_series, api=None): self.resource_id = None self.input_fields = [] self.objective_fields = [] self.all_numeric_objectives = False self.period = 1 self.ets_models = {} self.error = None self.damped_trend = None self.seasonality = None self.trend = None self.time_range = {} self.field_parameters = {} self._forecast = {} self.api = get_api_connection(api) self.resource_id, time_series = get_resource_dict( \ time_series, "timeseries", api=self.api) if 'object' in time_series and \ isinstance(time_series['object'], dict): time_series = time_series['object'] try: self.input_fields = time_series.get("input_fields", []) self._forecast = time_series.get("forecast") self.objective_fields = time_series.get("objective_fields", []) objective_field = time_series['objective_field'] if \ time_series.get('objective_field') else \ time_series['objective_fields'] except KeyError: raise ValueError("Failed to find the time series expected " "JSON structure. Check your arguments.") if 'time_series' in time_series and \ isinstance(time_series['time_series'], dict): status = get_status(time_series) if 'code' in status and status['code'] == FINISHED: time_series_info = time_series['time_series'] fields = time_series_info.get('fields', {}) self.fields = fields if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(self.fields.items(), key=lambda x: x[1].get("column_number"))] self.all_numeric_objectives = time_series_info.get( \ 'all_numeric_objectives') self.period = time_series_info.get('period', 1) self.ets_models = time_series_info.get('ets_models', {}) self.error = time_series_info.get('error') self.damped_trend = time_series_info.get('damped_trend') self.seasonality = time_series_info.get('seasonality') self.trend = time_series_info.get('trend') self.time_range = time_series_info.get('time_range') self.field_parameters = time_series_info.get( \ 'field_parameters', {}) objective_id = extract_objective(objective_field) ModelFields.__init__(self, fields, objective_id=objective_id) else: raise Exception("The time series isn't finished yet") else: raise Exception("Cannot create the TimeSeries instance." " Could not find the 'time_series' key" " in the resource:\n\n%s" % time_series)
def __init__(self, topic_model, api=None): self.resource_id = None self.stemmer = None self.seed = None self.case_sensitive = False self.bigrams = False self.ntopics = None self.temp = None self.phi = None self.term_to_index = None self.topics = [] self.api = get_api_connection(api) self.resource_id, topic_model = get_resource_dict( \ topic_model, "topicmodel", api=self.api) if 'object' in topic_model and isinstance(topic_model['object'], dict): topic_model = topic_model['object'] if 'topic_model' in topic_model \ and isinstance(topic_model['topic_model'], dict): status = get_status(topic_model) if 'code' in status and status['code'] == FINISHED: self.input_fields = topic_model['input_fields'] model = topic_model['topic_model'] self.topics = model['topics'] if 'language' in model and model['language'] is not None: lang = model['language'] if lang in CODE_TO_NAME: self.stemmer = Stemmer.Stemmer(CODE_TO_NAME[lang]) self.term_to_index = {self.stem(term): index for index, term in enumerate(model['termset'])} self.seed = abs(model['hashed_seed']) self.case_sensitive = model['case_sensitive'] self.bigrams = model['bigrams'] self.ntopics = len(model['term_topic_assignments'][0]) self.alpha = model['alpha'] self.ktimesalpha = self.ntopics * self.alpha self.temp = [0] * self.ntopics assignments = model['term_topic_assignments'] beta = model['beta'] nterms = len(self.term_to_index) sums = [sum(n[index] for n in assignments) for index in range(self.ntopics)] self.phi = [[0 for _ in range(nterms)] for _ in range(self.ntopics)] for k in range(self.ntopics): norm = sums[k] + nterms * beta for w in range(nterms): self.phi[k][w] = (assignments[w][k] + beta) / norm missing_tokens = model.get("missing_tokens") ModelFields.__init__(self, model['fields'], missing_tokens=missing_tokens) else: raise Exception("The topic model isn't finished yet") else: raise Exception("Cannot create the topic model instance. Could not" " find the 'topic_model' key in the" " resource:\n\n%s" % topic_model)
def __init__(self, fusion, api=None, max_models=None): self.resource_id = None self.models_ids = None self.objective_id = None self.distribution = None self.models_splits = [] self.cache_get = None self.regression = False self.fields = None self.class_names = None self.importance = {} self.api = get_api_connection(api) self.resource_id, fusion = get_resource_dict( \ fusion, "fusion", api=self.api) if 'object' in fusion: fusion = fusion.get('object', {}) self.model_ids, self.weights = get_models_weight( \ fusion['models']) model_types = [get_resource_type(model) for model in self.model_ids] for model_type in model_types: if model_type not in LOCAL_SUPERVISED: raise ValueError("The resource %s has not an allowed" " supervised model type.") self.importance = fusion.get('importance', []) self.missing_numerics = fusion.get('missing_numerics', True) if fusion.get('fusion'): self.fields = fusion.get( \ 'fusion', {}).get("fields") self.objective_id = fusion.get("objective_field") self.input_fields = fusion.get("input_fields") number_of_models = len(self.model_ids) # Downloading the model information to cache it if self.api.storage is not None: for model_id in self.model_ids: if get_resource_type(model_id) == "fusion": Fusion(model_id, api=self.api) else: SupervisedModel(model_id, api=self.api) if max_models is None: self.models_splits = [self.model_ids] else: self.models_splits = [self.model_ids[index:(index + max_models)] for index in range(0, number_of_models, max_models)] if self.fields: summary = self.fields[self.objective_id]['summary'] if 'bins' in summary: distribution = summary['bins'] elif 'counts' in summary: distribution = summary['counts'] elif 'categories' in summary: distribution = summary['categories'] else: distribution = [] self.distribution = distribution self.regression = \ self.fields[self.objective_id].get('optype') == 'numeric' if not self.regression: objective_field = self.fields[self.objective_id] categories = objective_field['summary']['categories'] classes = [category[0] for category in categories] self.class_names = sorted(classes) self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] ModelFields.__init__( \ self, self.fields, objective_id=self.objective_id)
def __init__(self, model, api=None, fields=None): if check_model_structure(model): self.resource_id = model['resource'] else: # If only the model id is provided, the short version of the model # resource is used to build a basic summary of the model self.api = get_api_connection(api) self.resource_id = get_model_id(model) if self.resource_id is None: raise Exception( self.api.error_message(model, resource_type='model', method='get')) if fields is not None and isinstance(fields, dict): query_string = EXCLUDE_FIELDS else: query_string = ONLY_MODEL model = retrieve_resource(self.api, self.resource_id, query_string=query_string) # Stored copies of the model structure might lack some necessary # keys if not check_model_structure(model): model = self.api.get_model(self.resource_id, query_string=query_string) if 'object' in model and isinstance(model['object'], dict): model = model['object'] if 'model' in model and isinstance(model['model'], dict): status = get_status(model) if 'code' in status and status['code'] == FINISHED: if (fields is None and ('model_fields' in model['model'] or 'fields' in model['model'])): fields = model['model'].get( 'model_fields', model['model'].get('fields', [])) # model_fields doesn't contain the datetime fields fields.update( datetime_fields(model['model'].get('fields', {}))) # pagination or exclusion might cause a field not to # be in available fields dict if not all(key in model['model']['fields'] for key in fields.keys()): raise Exception("Some fields are missing" " to generate a local model." " Please, provide a model with" " the complete list of fields.") for field in fields: field_info = model['model']['fields'][field] if 'summary' in field_info: fields[field]['summary'] = field_info['summary'] fields[field]['name'] = field_info['name'] objective_field = model['objective_fields'] missing_tokens = model['model'].get('missing_tokens') ModelFields.__init__( self, fields, objective_id=extract_objective(objective_field), missing_tokens=missing_tokens) self.description = model['description'] self.field_importance = model['model'].get('importance', None) if self.field_importance: self.field_importance = [ element for element in self.field_importance if element[0] in fields ] self.locale = model.get('locale', DEFAULT_LOCALE) else: raise Exception("The model isn't finished yet") else: raise Exception("Cannot create the BaseModel instance. Could not" " find the 'model' key in the resource:\n\n%s" % model)
def __init__(self, ensemble, api=None, max_models=None, cache_get=None): self.model_splits = [] self.multi_model = None self.api = get_api_connection(api) self.fields = None self.class_names = None if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_ensemble_id(ensemble), cache_get) self.api = get_api_connection(api) if len(self.models_splits) == 1: # retrieve the models from a cache get function try: models = [ Model(model_id, cache_get=cache_get) for model_id in self.models_splits[0] ] except Exception as exc: raise Exception('Error while calling the user-given' ' function %s: %s' % (cache_get.__name__, str(exc))) self.multi_model = MultiModel(models, self.api, fields=self.fields, class_names=self.class_names, cache_get=cache_get) return self.resource_id = None self.objective_id = None self.distributions = None self.distribution = None self.boosting = None self.boosting_offsets = None self.cache_get = None self.regression = False self.importance = {} query_string = ONLY_MODEL no_check_fields = False self.input_fields = [] if isinstance(ensemble, list): if all([isinstance(model, Model) for model in ensemble]): models = ensemble self.model_ids = [ local_model.resource_id for local_model in models ] else: try: models = [get_model_id(model) for model in ensemble] self.model_ids = models except ValueError as exc: raise ValueError('Failed to verify the list of models.' ' Check your model id values: %s' % str(exc)) else: ensemble = self.get_ensemble_resource(ensemble) self.resource_id = get_ensemble_id(ensemble) if not check_local_but_fields(ensemble): # avoid checking fields because of old ensembles ensemble = retrieve_resource(self.api, self.resource_id, no_check_fields=True) if ensemble['object'].get('type') == BOOSTING: self.boosting = ensemble['object'].get('boosting') models = ensemble['object']['models'] self.distributions = ensemble['object'].get('distributions', []) self.importance = ensemble['object'].get('importance', []) self.model_ids = models # new ensembles have the fields structure if ensemble['object'].get('ensemble'): self.fields = ensemble['object'].get( \ 'ensemble', {}).get("fields") self.objective_id = ensemble['object'].get("objective_field") query_string = EXCLUDE_FIELDS no_check_fields = True self.input_fields = ensemble['object'].get('input_fields') number_of_models = len(models) if max_models is None: self.models_splits = [models] else: self.models_splits = [ models[index:(index + max_models)] for index in range(0, number_of_models, max_models) ] if len(self.models_splits) == 1: if not isinstance(models[0], Model): if use_cache(cache_get): # retrieve the models from a cache get function try: models = [ Model(model_id, cache_get=cache_get) for model_id in self.models_splits[0] ] self.cache_get = cache_get except Exception as exc: raise Exception('Error while calling the user-given' ' function %s: %s' % (cache_get.__name__, str(exc))) else: models = [retrieve_resource( \ self.api, model_id, query_string=query_string, no_check_fields=no_check_fields) for model_id in self.models_splits[0]] model = models[0] else: # only retrieving first model self.cache_get = cache_get if not isinstance(models[0], Model): if use_cache(cache_get): # retrieve the models from a cache get function try: model = Model(self.models_splits[0][0], cache_get=cache_get) self.cache_get = cache_get except Exception as exc: raise Exception('Error while calling the user-given' ' function %s: %s' % (cache_get.__name__, str(exc))) else: model = retrieve_resource( \ self.api, self.models_splits[0][0], query_string=query_string, no_check_fields=no_check_fields) models = [model] if self.distributions is None: try: self.distributions = [] for model in models: self.distributions.append( {'training': model.root_distribution}) except AttributeError: self.distributions = [ model['object']['model']['distribution'] for model in models ] if self.boosting is None: self._add_models_attrs(model, max_models) if self.fields is None: self.fields, self.objective_id = self.all_model_fields( max_models=max_models) if self.fields: add_distribution(self) self.regression = \ self.fields[self.objective_id].get('optype') == NUMERIC if self.boosting: self.boosting_offsets = ensemble['object'].get('initial_offset', 0) \ if self.regression else dict(ensemble['object'].get( \ 'initial_offsets', [])) if not self.regression: try: objective_field = self.fields[self.objective_id] categories = objective_field['summary']['categories'] classes = [category[0] for category in categories] except (AttributeError, KeyError): classes = set() for distribution in self.distributions: for category in distribution['training']['categories']: classes.add(category[0]) self.class_names = sorted(classes) self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] ModelFields.__init__( \ self, self.fields, objective_id=self.objective_id) if len(self.models_splits) == 1: self.multi_model = MultiModel(models, self.api, fields=self.fields, class_names=self.class_names)
def __init__(self, model, api=None, fields=None, cache_get=None): """The Model constructor can be given as first argument: - a model structure - a model id - a path to a JSON file containing a model structure """ if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_model_id(model), cache_get) return self.resource_id = None self.ids_map = {} self.terms = {} self.regression = False self.boosting = None self.class_names = None api = get_api_connection(api) # retrieving model information from self.resource_id, model = get_resource_dict( \ model, "model", api=api, no_check_fields=fields is not None) if 'object' in model and isinstance(model['object'], dict): model = model['object'] if 'model' in model and isinstance(model['model'], dict): status = get_status(model) if 'code' in status and status['code'] == FINISHED: # fill boosting info before creating modelfields if model.get("boosted_ensemble"): self.boosting = model.get('boosting', False) if self.boosting == {}: self.boosting = False self.input_fields = model["input_fields"] BaseModel.__init__(self, model, api=api, fields=fields) root = model['model']['root'] self.weighted = "weighted_objective_summary" in root if self.boosting: # build boosted tree self.tree = b.build_boosting_tree( \ model['model']['root']) elif self.regression: self.root_distribution = model['model'][ \ 'distribution']['training'] # build regression tree self.tree = r.build_regression_tree(root, \ distribution=self.root_distribution, \ weighted=self.weighted) else: # build classification tree self.root_distribution = model['model'][\ 'distribution']['training'] self.laplacian_term = laplacian_term( \ extract_distribution(self.root_distribution)[1], self.weighted) self.tree = c.build_classification_tree( \ model['model']['root'], \ distribution=self.root_distribution, \ weighted=self.weighted) self.class_names = sorted( \ [category[0] for category in \ self.root_distribution["categories"]]) self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] if self.boosting: self.tree_type = BOOSTING self.offsets = b.OFFSETS elif self.regression: self.tree_type = REGRESSION self.offsets = r.OFFSETS[str(self.weighted)] else: self.tree_type = CLASSIFICATION self.offsets = c.OFFSETS[str(self.weighted)] else: raise Exception("Cannot create the Model instance." " Only correctly finished models can be" " used. The model status is currently:" " %s\n" % STATUSES[status['code']]) else: raise Exception("Cannot create the Model instance. Could not" " find the 'model' key in the resource:" "\n\n%s" % model)
def __init__(self, pca, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_pca_id(pca), cache_get) return self.resource_id = None self.input_fields = [] self.default_numeric_value = None self.term_forms = {} self.tag_clouds = {} self.dataset_field_types = {} self.term_analysis = {} self.categories = {} self.categories_probabilities = {} self.items = {} self.fields = {} self.item_analysis = {} self.standardize = None self.famd_j = 1 api = get_api_connection(api) self.resource_id, pca = get_resource_dict( \ pca, "pca", api=api) if 'object' in pca and \ isinstance(pca['object'], dict): pca = pca['object'] try: self.input_fields = pca.get("input_fields", []) self.default_numeric_value = pca.get("default_numeric_value") self.dataset_field_types = pca.get("dataset_field_types", {}) self.famd_j = 1 if (self.dataset_field_types['categorical'] != \ self.dataset_field_types['total']) else \ self.dataset_field_types['categorical'] except KeyError: raise ValueError("Failed to find the pca expected " "JSON structure. Check your arguments.") if 'pca' in pca and \ isinstance(pca['pca'], dict): status = get_status(pca) if 'code' in status and status['code'] == FINISHED: pca_info = pca[ \ 'pca'] fields = pca_info.get('fields', {}) self.fields = fields if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(list(self.fields.items()), key=lambda x: x[1].get("column_number"))] missing_tokens = pca_info.get("missing_tokens") ModelFields.__init__(self, fields, objective_id=None, terms=True, categories=True, numerics=False, missing_tokens=missing_tokens) for field_id in self.categories: field = self.fields[field_id] probabilities = [probability for _, probability in \ field["summary"]["categories"]] if field["summary"].get("missing_count", 0) > 0: probabilities.append(field["summary"]["missing_count"]) total = float(sum(probabilities)) if total > 0: probabilities = [probability / total for probability \ in probabilities] self.categories_probabilities[field_id] = probabilities self.components = pca_info.get('components') self.eigenvectors = pca_info.get('eigenvectors') self.cumulative_variance = pca_info.get('cumulative_variance') self.text_stats = pca_info.get('text_stats') self.standardized = pca_info.get('standardized') self.variance = pca_info.get('variance') else: raise Exception("The pca isn't finished yet") else: raise Exception("Cannot create the PCA instance." " Could not find the 'pca' key" " in the resource:\n\n%s" % pca)
def __init__(self, ensemble, model_fns_dir, api=None): self.resource_id = None # to be deprecated self.ensemble_id = None self.objective_id = None self.distributions = None self.distribution = None self.models_splits = [] self.multi_model = None self.boosting = None self.boosting_offsets = None self.regression = False self.fields = None self.class_names = None self.importance = {} self.predict_functions = [] self.api = get_api_connection(api) ensemble = self.get_ensemble_resource(ensemble) self.resource_id = get_ensemble_id(ensemble) self.ensemble_id = self.resource_id if not check_local_info(ensemble): # avoid checking fields because of old ensembles ensemble = retrieve_resource(self.api, self.resource_id, no_check_fields=True) if ensemble['object'].get('type') == BOOSTING: self.boosting = ensemble['object'].get('boosting') models = ensemble['object']['models'] self.distributions = ensemble['object'].get('distributions', []) self.importance = ensemble['object'].get('importance', []) self.model_ids = models # new ensembles have the fields structure if ensemble['object'].get('ensemble'): self.fields = ensemble['object'].get( \ 'ensemble', {}).get("fields") self.objective_id = ensemble['object'].get("objective_field") self.input_fields = ensemble['object'].get("input_fields") if model_fns_dir: self.get_model_fns(model_fns_dir) else: raise ValueError("The EnsemblePredictor object expects as" " argument the directory where the models" " predict functions are stored. To generate " " them, please check the 'bigmler export'" " command.") if self.fields: add_distribution(self) self.regression = \ self.fields[self.objective_id].get('optype') == NUMERIC if self.boosting: self.boosting_offsets = ensemble['object'].get('initial_offset', 0) \ if self.regression else dict(ensemble['object'].get( \ 'initial_offsets', [])) if not self.regression and self.boosting is None: try: objective_field = self.fields[self.objective_id] categories = objective_field['summary']['categories'] classes = [category[0] for category in categories] except (AttributeError, KeyError): classes = set() for distribution in self.distributions: for category in distribution['training']['categories']: classes.add(category[0]) self.class_names = sorted(classes)