def __init__(self, anomaly, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the Minomaly attributes self.__dict__ = load(get_anomaly_id(anomaly), cache_get) return self.resource_id = None self.sample_size = None self.input_fields = None self.default_numeric_value = None self.mean_depth = None self.expected_mean_depth = None self.iforest = None self.id_fields = [] api = get_api_connection(api) self.resource_id, anomaly = get_resource_dict( anomaly, "anomaly", api=api) if 'object' in anomaly and isinstance(anomaly['object'], dict): anomaly = anomaly['object'] self.sample_size = anomaly.get('sample_size') self.input_fields = anomaly.get('input_fields') self.default_numeric_value = anomaly.get('default_numeric_value') self.id_fields = anomaly.get('id_fields', []) if 'model' in anomaly and isinstance(anomaly['model'], dict): ModelFields.__init__( self, anomaly['model'].get('fields'), missing_tokens=anomaly['model'].get('missing_tokens')) self.mean_depth = anomaly['model'].get('mean_depth') self.normalization_factor = anomaly['model'].get( 'normalization_factor') self.nodes_mean_depth = anomaly['model'].get( 'nodes_mean_depth') status = get_status(anomaly) if 'code' in status and status['code'] == FINISHED: self.expected_mean_depth = None if self.mean_depth is None or self.sample_size is None: raise Exception("The anomaly data is not complete. " "Score will not be available") self.norm = self.normalization_factor if \ self.normalization_factor is not None else \ self.norm_factor() iforest = anomaly['model'].get('trees', []) if iforest: self.iforest = [ build_tree([anomaly_tree['root']]) for anomaly_tree in iforest] self.top_anomalies = anomaly['model']['top_anomalies'] else: raise Exception("The anomaly isn't finished yet")
def __init__(self, deepnet, api=None, cache_get=None): """The Deepnet constructor can be given as first argument: - a deepnet structure - a deepnet id - a path to a JSON file containing a deepnet structure """ if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_deepnet_id(deepnet), cache_get) return self.resource_id = None self.regression = False self.network = None self.networks = None self.input_fields = [] self.class_names = [] self.preprocess = [] self.optimizer = None self.missing_numerics = False api = get_api_connection(api) self.resource_id, deepnet = get_resource_dict( \ deepnet, "deepnet", api=api) if 'object' in deepnet and isinstance(deepnet['object'], dict): deepnet = deepnet['object'] self.input_fields = deepnet['input_fields'] if 'deepnet' in deepnet and isinstance(deepnet['deepnet'], dict): status = get_status(deepnet) objective_field = deepnet['objective_fields'] deepnet = deepnet['deepnet'] if 'code' in status and status['code'] == FINISHED: self.fields = deepnet['fields'] missing_tokens = deepnet.get('missing_tokens') ModelFields.__init__( self, self.fields, objective_id=extract_objective(objective_field), terms=True, categories=True, missing_tokens=missing_tokens) self.regression = \ self.fields[self.objective_id]['optype'] == NUMERIC if not self.regression: self.class_names = [category for category, _ in \ self.fields[self.objective_id][ \ 'summary']['categories']] self.class_names.sort() # order matters self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] self.missing_numerics = deepnet.get('missing_numerics', False) if 'network' in deepnet: network = deepnet['network'] self.network = network self.networks = network.get('networks', []) self.preprocess = network.get('preprocess') self.optimizer = network.get('optimizer', {}) else: raise Exception("The deepnet isn't finished yet") else: raise Exception("Cannot create the Deepnet instance. Could not" " find the 'deepnet' key in the resource:\n\n%s" % deepnet)
def __init__(self, logistic_regression, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_logistic_regression_id( \ logistic_regression), cache_get) return self.resource_id = None self.class_names = None self.input_fields = [] self.term_forms = {} self.tag_clouds = {} self.term_analysis = {} self.items = {} self.item_analysis = {} self.categories = {} self.coefficients = {} self.data_field_types = {} self.field_codings = {} self.numeric_fields = {} self.bias = None self.missing_numerics = None self.c = None self.eps = None self.lr_normalize = None self.balance_fields = None self.regularization = None api = get_api_connection(api) old_coefficients = False self.resource_id, logistic_regression = get_resource_dict( \ logistic_regression, "logisticregression", api=api) if 'object' in logistic_regression and \ isinstance(logistic_regression['object'], dict): logistic_regression = logistic_regression['object'] try: self.input_fields = logistic_regression.get("input_fields", []) self.dataset_field_types = logistic_regression.get( "dataset_field_types", {}) self.weight_field = logistic_regression.get("weight_field") objective_field = logistic_regression['objective_fields'] if \ logistic_regression['objective_fields'] else \ logistic_regression['objective_field'] except KeyError: raise ValueError("Failed to find the logistic regression expected " "JSON structure. Check your arguments.") if 'logistic_regression' in logistic_regression and \ isinstance(logistic_regression['logistic_regression'], dict): status = get_status(logistic_regression) if 'code' in status and status['code'] == FINISHED: logistic_regression_info = logistic_regression[ \ 'logistic_regression'] fields = logistic_regression_info.get('fields', {}) if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(list(fields.items()), key=lambda x: x[1].get("column_number"))] self.coefficients.update(logistic_regression_info.get( \ 'coefficients', [])) if not isinstance( list(self.coefficients.values())[0][0], list): old_coefficients = True self.bias = logistic_regression_info.get('bias', True) self.c = logistic_regression_info.get('c') self.eps = logistic_regression_info.get('eps') self.lr_normalize = logistic_regression_info.get('normalize') self.balance_fields = logistic_regression_info.get( \ 'balance_fields') self.regularization = logistic_regression_info.get( \ 'regularization') self.field_codings = logistic_regression_info.get( \ 'field_codings', {}) # old models have no such attribute, so we set it to False in # this case self.missing_numerics = logistic_regression_info.get( \ 'missing_numerics', False) objective_id = extract_objective(objective_field) missing_tokens = logistic_regression_info.get("missing_tokens") ModelFields.__init__(self, fields, objective_id=objective_id, terms=True, categories=True, numerics=True, missing_tokens=missing_tokens) self.field_codings = logistic_regression_info.get( \ 'field_codings', {}) self.format_field_codings() for field_id in self.field_codings: if field_id not in self.fields and \ field_id in self.inverted_fields: self.field_codings.update( \ {self.inverted_fields[field_id]: \ self.field_codings[field_id]}) del self.field_codings[field_id] if old_coefficients: self.map_coefficients() categories = self.fields[self.objective_id].get( \ "summary", {}).get('categories') if len(list(self.coefficients.keys())) > len(categories): self.class_names = [""] else: self.class_names = [] self.class_names.extend( sorted([category[0] for category in categories])) # order matters self.objective_categories = [ category[0] for category in categories ] else: raise Exception("The logistic regression isn't finished yet") else: raise Exception("Cannot create the LogisticRegression instance." " Could not find the 'logistic_regression' key" " in the resource:\n\n%s" % logistic_regression)
def __init__(self, cluster, api=None, cache_get=None): self.api = get_api_connection(api) self.centroids = None if use_cache(cache_get): # using a cache to store the cluster attributes self.__dict__ = load(get_cluster_id(cluster), cache_get) for index, centroid in enumerate(self.centroids): self.centroids[index] = Centroid(centroid) self.cluster_global = Centroid(self.cluster_global) return self.resource_id = None self.cluster_global = None self.total_ss = None self.within_ss = None self.between_ss = None self.ratio_ss = None self.critical_value = None self.input_fields = [] self.summary_fields = [] self.default_numeric_value = None self.k = None self.summary_fields = [] self.scales = {} self.term_forms = {} self.tag_clouds = {} self.term_analysis = {} self.item_analysis = {} self.items = {} self.datasets = {} self.resource_id, cluster = get_resource_dict( \ cluster, "cluster", api=self.api) if 'object' in cluster and isinstance(cluster['object'], dict): cluster = cluster['object'] if 'clusters' in cluster and isinstance(cluster['clusters'], dict): status = get_status(cluster) if 'code' in status and status['code'] == FINISHED: self.default_numeric_value = cluster.get( \ "default_numeric_value") self.summary_fields = cluster.get("summary_fields", []) self.input_fields = cluster.get("input_fields", []) self.datasets = cluster.get("cluster_datasets", {}) the_clusters = cluster['clusters'] cluster_global = the_clusters.get('global') clusters = the_clusters['clusters'] self.centroids = [Centroid(centroid) for centroid in clusters] self.cluster_global = cluster_global if cluster_global: self.cluster_global = Centroid(cluster_global) # "global" has no "name" and "count" then we set them self.cluster_global.name = GLOBAL_CLUSTER_LABEL self.cluster_global.count = \ self.cluster_global.distance['population'] self.total_ss = the_clusters.get('total_ss') self.within_ss = the_clusters.get('within_ss') if not self.within_ss: self.within_ss = sum(centroid.distance['sum_squares'] for centroid in self.centroids) self.between_ss = the_clusters.get('between_ss') self.ratio_ss = the_clusters.get('ratio_ss') self.critical_value = cluster.get('critical_value', None) self.k = cluster.get('k') self.scales.update(cluster['scales']) self.term_forms = {} self.tag_clouds = {} self.term_analysis = {} fields = cluster['clusters']['fields'] summary_fields = cluster['summary_fields'] for field_id in summary_fields: try: del fields[field_id] except KeyError: # clusters retrieved from API will only contain # model fields pass for field_id, field in list(fields.items()): if field['optype'] == 'text': self.term_forms[field_id] = {} self.term_forms[field_id].update(field[ 'summary']['term_forms']) self.tag_clouds[field_id] = {} self.tag_clouds[field_id].update(field[ 'summary']['tag_cloud']) self.term_analysis[field_id] = {} self.term_analysis[field_id].update( field['term_analysis']) if field['optype'] == 'items': self.items[field_id] = {} self.items[field_id].update( dict(field['summary']['items'])) self.item_analysis[field_id] = {} self.item_analysis[field_id].update( field['item_analysis']) missing_tokens = cluster['clusters'].get('missing_tokens') ModelFields.__init__(self, fields, missing_tokens=missing_tokens) if not all([field_id in self.fields for field_id in self.scales]): raise Exception("Some fields are missing" " to generate a local cluster." " Please, provide a cluster with" " the complete list of fields.") else: raise Exception("The cluster isn't finished yet") else: raise Exception("Cannot create the Cluster instance. Could not" " find the 'clusters' key in the resource:\n\n%s" % cluster)
def __init__(self, association, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the association attributes self.__dict__ = load(get_association_id(association), cache_get) for index, item in enumerate(self.items): self.items[index] = Item(item["index"], item, self.fields) for index, rule in enumerate(self.rules): self.rules[index] = AssociationRule(rule) return self.resource_id = None self.complement = None self.discretization = {} self.default_numeric_value = None self.field_discretizations = {} self.items = [] self.max_k = None self.max_lhs = None self.min_confidence = None self.min_leverage = None self.min_support = None self.min_lift = None self.search_strategy = DEFAULT_SEARCH_STRATEGY self.rules = [] self.significance_level = None api = get_api_connection(api) self.resource_id, association = get_resource_dict( \ association, "association", api=api) if 'object' in association and isinstance(association['object'], dict): association = association['object'] if 'associations' in association and \ isinstance(association['associations'], dict): status = get_status(association) if 'code' in status and status['code'] == FINISHED: self.input_fields = association['input_fields'] self.default_numeric_value = association.get( \ 'default_numeric_value') associations = association['associations'] fields = associations['fields'] ModelFields.__init__( \ self, fields, \ missing_tokens=associations.get('missing_tokens')) self.complement = associations.get('complement', False) self.discretization = associations.get('discretization', {}) self.field_discretizations = associations.get( 'field_discretizations', {}) self.items = [Item(index, item, fields) for index, item in enumerate(associations.get('items', []))] self.max_k = associations.get('max_k', 100) self.max_lhs = associations.get('max_lhs', 4) self.min_confidence = associations.get('min_confidence', 0) self.min_leverage = associations.get('min_leverage', -1) self.min_support = associations.get('min_support', 0) self.min_lift = associations.get('min_lift', 0) self.search_strategy = associations.get('search_strategy', \ DEFAULT_SEARCH_STRATEGY) self.rules = [AssociationRule(rule) for rule in associations.get('rules', [])] self.significance_level = associations.get( 'significance_level', 0.05) else: raise Exception("The association isn't finished yet") else: raise Exception("Cannot create the Association instance. Could not" " find the 'associations' key in the " "resource:\n\n%s" % association)
def __init__(self, linear_regression, api=None, cache_get=None): self.xtx_inverse = [] if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_linear_regression_id(linear_regression), cache_get) for index, elem in enumerate(self.xtx_inverse): self.xtx_inverse[index] = np.array(elem) return self.resource_id = None self.input_fields = [] self.term_forms = {} self.tag_clouds = {} self.term_analysis = {} self.items = {} self.item_analysis = {} self.categories = {} self.coefficients = [] self.data_field_types = {} self.field_codings = {} self.bias = None self.xtx_inverse = [] self.mean_squared_error = None self.number_of_parameters = None self.number_of_samples = None api = get_api_connection(api) self.resource_id, linear_regression = get_resource_dict( \ linear_regression, "linearregression", api=api) if 'object' in linear_regression and \ isinstance(linear_regression['object'], dict): linear_regression = linear_regression['object'] try: self.input_fields = linear_regression.get("input_fields", []) self.dataset_field_types = linear_regression.get( "dataset_field_types", {}) self.weight_field = linear_regression.get("weight_field") objective_field = linear_regression['objective_fields'] if \ linear_regression['objective_fields'] else \ linear_regression['objective_field'] except KeyError: raise ValueError("Failed to find the linear regression expected " "JSON structure. Check your arguments.") if 'linear_regression' in linear_regression and \ isinstance(linear_regression['linear_regression'], dict): status = get_status(linear_regression) if 'code' in status and status['code'] == FINISHED: linear_regression_info = linear_regression[ \ 'linear_regression'] fields = linear_regression_info.get('fields', {}) if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(list(fields.items()), key=lambda x: x[1].get("column_number"))] self.coeff_ids = self.input_fields[:] self.coefficients = linear_regression_info.get( \ 'coefficients', []) self.bias = linear_regression_info.get('bias', True) self.field_codings = linear_regression_info.get( \ 'field_codings', {}) self.number_of_parameters = linear_regression_info.get( \ "number_of_parameters") missing_tokens = linear_regression_info.get("missing_tokens") objective_id = extract_objective(objective_field) ModelFields.__init__(self, fields, objective_id=objective_id, terms=True, categories=True, numerics=True, missing_tokens=missing_tokens) self.field_codings = linear_regression_info.get( \ 'field_codings', {}) self.format_field_codings() for field_id in self.field_codings: if field_id not in fields and \ field_id in self.inverted_fields: self.field_codings.update( \ {self.inverted_fields[field_id]: \ self.field_codings[field_id]}) del self.field_codings[field_id] stats = linear_regression_info["stats"] if STATS and stats is not None and \ stats.get("xtx_inverse") is not None: self.xtx_inverse = stats["xtx_inverse"][:] self.mean_squared_error = stats["mean_squared_error"] self.number_of_samples = stats["number_of_samples"] # to be used in predictions self.t_crit = student_t.interval( \ CONFIDENCE, self.number_of_samples - self.number_of_parameters)[1] self.xtx_inverse = list( \ np.linalg.inv(np.array(self.xtx_inverse))) else: raise Exception("The linear regression isn't finished yet") else: raise Exception("Cannot create the LinearRegression instance." " Could not find the 'linear_regression' key" " in the resource:\n\n%s" % linear_regression)
def __init__(self, ensemble, api=None, max_models=None, cache_get=None): self.model_splits = [] self.multi_model = None self.api = get_api_connection(api) self.fields = None self.class_names = None if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_ensemble_id(ensemble), cache_get) self.api = get_api_connection(api) if len(self.models_splits) == 1: # retrieve the models from a cache get function try: models = [ Model(model_id, cache_get=cache_get) for model_id in self.models_splits[0] ] except Exception as exc: raise Exception('Error while calling the user-given' ' function %s: %s' % (cache_get.__name__, str(exc))) self.multi_model = MultiModel(models, self.api, fields=self.fields, class_names=self.class_names, cache_get=cache_get) return self.resource_id = None self.objective_id = None self.distributions = None self.distribution = None self.boosting = None self.boosting_offsets = None self.cache_get = None self.regression = False self.importance = {} query_string = ONLY_MODEL no_check_fields = False self.input_fields = [] if isinstance(ensemble, list): if all([isinstance(model, Model) for model in ensemble]): models = ensemble self.model_ids = [ local_model.resource_id for local_model in models ] else: try: models = [get_model_id(model) for model in ensemble] self.model_ids = models except ValueError as exc: raise ValueError('Failed to verify the list of models.' ' Check your model id values: %s' % str(exc)) else: ensemble = self.get_ensemble_resource(ensemble) self.resource_id = get_ensemble_id(ensemble) if not check_local_but_fields(ensemble): # avoid checking fields because of old ensembles ensemble = retrieve_resource(self.api, self.resource_id, no_check_fields=True) if ensemble['object'].get('type') == BOOSTING: self.boosting = ensemble['object'].get('boosting') models = ensemble['object']['models'] self.distributions = ensemble['object'].get('distributions', []) self.importance = ensemble['object'].get('importance', []) self.model_ids = models # new ensembles have the fields structure if ensemble['object'].get('ensemble'): self.fields = ensemble['object'].get( \ 'ensemble', {}).get("fields") self.objective_id = ensemble['object'].get("objective_field") query_string = EXCLUDE_FIELDS no_check_fields = True self.input_fields = ensemble['object'].get('input_fields') number_of_models = len(models) if max_models is None: self.models_splits = [models] else: self.models_splits = [ models[index:(index + max_models)] for index in range(0, number_of_models, max_models) ] if len(self.models_splits) == 1: if not isinstance(models[0], Model): if use_cache(cache_get): # retrieve the models from a cache get function try: models = [ Model(model_id, cache_get=cache_get) for model_id in self.models_splits[0] ] self.cache_get = cache_get except Exception as exc: raise Exception('Error while calling the user-given' ' function %s: %s' % (cache_get.__name__, str(exc))) else: models = [retrieve_resource( \ self.api, model_id, query_string=query_string, no_check_fields=no_check_fields) for model_id in self.models_splits[0]] model = models[0] else: # only retrieving first model self.cache_get = cache_get if not isinstance(models[0], Model): if use_cache(cache_get): # retrieve the models from a cache get function try: model = Model(self.models_splits[0][0], cache_get=cache_get) self.cache_get = cache_get except Exception as exc: raise Exception('Error while calling the user-given' ' function %s: %s' % (cache_get.__name__, str(exc))) else: model = retrieve_resource( \ self.api, self.models_splits[0][0], query_string=query_string, no_check_fields=no_check_fields) models = [model] if self.distributions is None: try: self.distributions = [] for model in models: self.distributions.append( {'training': model.root_distribution}) except AttributeError: self.distributions = [ model['object']['model']['distribution'] for model in models ] if self.boosting is None: self._add_models_attrs(model, max_models) if self.fields is None: self.fields, self.objective_id = self.all_model_fields( max_models=max_models) if self.fields: add_distribution(self) self.regression = \ self.fields[self.objective_id].get('optype') == NUMERIC if self.boosting: self.boosting_offsets = ensemble['object'].get('initial_offset', 0) \ if self.regression else dict(ensemble['object'].get( \ 'initial_offsets', [])) if not self.regression: try: objective_field = self.fields[self.objective_id] categories = objective_field['summary']['categories'] classes = [category[0] for category in categories] except (AttributeError, KeyError): classes = set() for distribution in self.distributions: for category in distribution['training']['categories']: classes.add(category[0]) self.class_names = sorted(classes) self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] ModelFields.__init__( \ self, self.fields, objective_id=self.objective_id) if len(self.models_splits) == 1: self.multi_model = MultiModel(models, self.api, fields=self.fields, class_names=self.class_names)
def __init__(self, model, api=None, fields=None, cache_get=None): """The Model constructor can be given as first argument: - a model structure - a model id - a path to a JSON file containing a model structure """ if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_model_id(model), cache_get) return self.resource_id = None self.ids_map = {} self.terms = {} self.regression = False self.boosting = None self.class_names = None api = get_api_connection(api) # retrieving model information from self.resource_id, model = get_resource_dict( \ model, "model", api=api, no_check_fields=fields is not None) if 'object' in model and isinstance(model['object'], dict): model = model['object'] if 'model' in model and isinstance(model['model'], dict): status = get_status(model) if 'code' in status and status['code'] == FINISHED: # fill boosting info before creating modelfields if model.get("boosted_ensemble"): self.boosting = model.get('boosting', False) if self.boosting == {}: self.boosting = False self.input_fields = model["input_fields"] BaseModel.__init__(self, model, api=api, fields=fields) root = model['model']['root'] self.weighted = "weighted_objective_summary" in root if self.boosting: # build boosted tree self.tree = b.build_boosting_tree( \ model['model']['root']) elif self.regression: self.root_distribution = model['model'][ \ 'distribution']['training'] # build regression tree self.tree = r.build_regression_tree(root, \ distribution=self.root_distribution, \ weighted=self.weighted) else: # build classification tree self.root_distribution = model['model'][\ 'distribution']['training'] self.laplacian_term = laplacian_term( \ extract_distribution(self.root_distribution)[1], self.weighted) self.tree = c.build_classification_tree( \ model['model']['root'], \ distribution=self.root_distribution, \ weighted=self.weighted) self.class_names = sorted( \ [category[0] for category in \ self.root_distribution["categories"]]) self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] if self.boosting: self.tree_type = BOOSTING self.offsets = b.OFFSETS elif self.regression: self.tree_type = REGRESSION self.offsets = r.OFFSETS[str(self.weighted)] else: self.tree_type = CLASSIFICATION self.offsets = c.OFFSETS[str(self.weighted)] else: raise Exception("Cannot create the Model instance." " Only correctly finished models can be" " used. The model status is currently:" " %s\n" % STATUSES[status['code']]) else: raise Exception("Cannot create the Model instance. Could not" " find the 'model' key in the resource:" "\n\n%s" % model)
def __init__(self, pca, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_pca_id(pca), cache_get) return self.resource_id = None self.input_fields = [] self.default_numeric_value = None self.term_forms = {} self.tag_clouds = {} self.dataset_field_types = {} self.term_analysis = {} self.categories = {} self.categories_probabilities = {} self.items = {} self.fields = {} self.item_analysis = {} self.standardize = None self.famd_j = 1 api = get_api_connection(api) self.resource_id, pca = get_resource_dict( \ pca, "pca", api=api) if 'object' in pca and \ isinstance(pca['object'], dict): pca = pca['object'] try: self.input_fields = pca.get("input_fields", []) self.default_numeric_value = pca.get("default_numeric_value") self.dataset_field_types = pca.get("dataset_field_types", {}) self.famd_j = 1 if (self.dataset_field_types['categorical'] != \ self.dataset_field_types['total']) else \ self.dataset_field_types['categorical'] except KeyError: raise ValueError("Failed to find the pca expected " "JSON structure. Check your arguments.") if 'pca' in pca and \ isinstance(pca['pca'], dict): status = get_status(pca) if 'code' in status and status['code'] == FINISHED: pca_info = pca[ \ 'pca'] fields = pca_info.get('fields', {}) self.fields = fields if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(list(self.fields.items()), key=lambda x: x[1].get("column_number"))] missing_tokens = pca_info.get("missing_tokens") ModelFields.__init__(self, fields, objective_id=None, terms=True, categories=True, numerics=False, missing_tokens=missing_tokens) for field_id in self.categories: field = self.fields[field_id] probabilities = [probability for _, probability in \ field["summary"]["categories"]] if field["summary"].get("missing_count", 0) > 0: probabilities.append(field["summary"]["missing_count"]) total = float(sum(probabilities)) if total > 0: probabilities = [probability / total for probability \ in probabilities] self.categories_probabilities[field_id] = probabilities self.components = pca_info.get('components') self.eigenvectors = pca_info.get('eigenvectors') self.cumulative_variance = pca_info.get('cumulative_variance') self.text_stats = pca_info.get('text_stats') self.standardized = pca_info.get('standardized') self.variance = pca_info.get('variance') else: raise Exception("The pca isn't finished yet") else: raise Exception("Cannot create the PCA instance." " Could not find the 'pca' key" " in the resource:\n\n%s" % pca)
def __init__(self, topic_model, api=None, cache_get=None): self.lang = None self.stemmer = None if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_topic_model_id(topic_model), cache_get) if self.lang in CODE_TO_NAME: self.stemmer = Stemmer.Stemmer(CODE_TO_NAME[self.lang]) return self.resource_id = None self.seed = None self.case_sensitive = False self.bigrams = False self.ntopics = None self.temp = None self.phi = None self.term_to_index = None self.topics = [] api = get_api_connection(api) self.resource_id, topic_model = get_resource_dict( \ topic_model, "topicmodel", api=api) if 'object' in topic_model and isinstance(topic_model['object'], dict): topic_model = topic_model['object'] if 'topic_model' in topic_model \ and isinstance(topic_model['topic_model'], dict): status = get_status(topic_model) if 'code' in status and status['code'] == FINISHED: self.input_fields = topic_model['input_fields'] model = topic_model['topic_model'] self.topics = model['topics'] if 'language' in model and model['language'] is not None: self.lang = model['language'] if self.lang in CODE_TO_NAME: self.stemmer = Stemmer.Stemmer(CODE_TO_NAME[self.lang]) self.term_to_index = { self.stem(term): index for index, term in enumerate(model['termset']) } self.seed = abs(model['hashed_seed']) self.case_sensitive = model['case_sensitive'] self.bigrams = model['bigrams'] self.ntopics = len(model['term_topic_assignments'][0]) self.alpha = model['alpha'] self.ktimesalpha = self.ntopics * self.alpha self.temp = [0] * self.ntopics assignments = model['term_topic_assignments'] beta = model['beta'] nterms = len(self.term_to_index) sums = [ sum(n[index] for n in assignments) for index in range(self.ntopics) ] self.phi = [[0 for _ in range(nterms)] for _ in range(self.ntopics)] for k in range(self.ntopics): norm = sums[k] + nterms * beta for w in range(nterms): self.phi[k][w] = (assignments[w][k] + beta) / norm missing_tokens = model.get("missing_tokens") ModelFields.__init__(self, model['fields'], missing_tokens=missing_tokens) else: raise Exception("The topic model isn't finished yet") else: raise Exception("Cannot create the topic model instance. Could not" " find the 'topic_model' key in the" " resource:\n\n%s" % topic_model)
def __init__(self, time_series, api=None, cache_get=None): if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_time_series_id(time_series), cache_get) return self.resource_id = None self.input_fields = [] self.default_numeric_value = None self.objective_fields = [] self.all_numeric_objectives = False self.period = 1 self.ets_models = {} self.error = None self.damped_trend = None self.seasonality = None self.trend = None self.time_range = {} self.field_parameters = {} self._forecast = {} api = get_api_connection(api) self.resource_id, time_series = get_resource_dict( \ time_series, "timeseries", api=api) if 'object' in time_series and \ isinstance(time_series['object'], dict): time_series = time_series['object'] try: self.input_fields = time_series.get("input_fields", []) self.default_numeric_value = time_series.get( \ "default_numeric_value") self._forecast = time_series.get("forecast") self.objective_fields = time_series.get("objective_fields", []) objective_field = time_series['objective_field'] if \ time_series.get('objective_field') else \ time_series['objective_fields'] except KeyError: raise ValueError("Failed to find the time series expected " "JSON structure. Check your arguments.") if 'time_series' in time_series and \ isinstance(time_series['time_series'], dict): status = get_status(time_series) if 'code' in status and status['code'] == FINISHED: time_series_info = time_series['time_series'] fields = time_series_info.get('fields', {}) self.fields = fields if not self.input_fields: self.input_fields = [ \ field_id for field_id, _ in sorted(list(self.fields.items()), key=lambda x: x[1].get("column_number"))] self.all_numeric_objectives = time_series_info.get( \ 'all_numeric_objectives') self.period = time_series_info.get('period', 1) self.ets_models = time_series_info.get('ets_models', {}) self.error = time_series_info.get('error') self.damped_trend = time_series_info.get('damped_trend') self.seasonality = time_series_info.get('seasonality') self.trend = time_series_info.get('trend') self.time_range = time_series_info.get('time_range') self.field_parameters = time_series_info.get( \ 'field_parameters', {}) objective_id = extract_objective(objective_field) ModelFields.__init__(self, fields, objective_id=objective_id) else: raise Exception("The time series isn't finished yet") else: raise Exception("Cannot create the TimeSeries instance." " Could not find the 'time_series' key" " in the resource:\n\n%s" % time_series)
def __init__(self, fusion, api=None, max_models=None, cache_get=None): if use_cache(cache_get): # using a cache to store the model attributes self.__dict__ = load(get_fusion_id(fusion), cache_get) self.api = get_api_connection(api) return self.resource_id = None self.models_ids = None self.objective_id = None self.distribution = None self.models_splits = [] self.cache_get = None self.regression = False self.fields = None self.class_names = None self.importance = {} self.api = get_api_connection(api) self.resource_id, fusion = get_resource_dict( \ fusion, "fusion", api=self.api) if 'object' in fusion: fusion = fusion.get('object', {}) self.model_ids, self.weights = get_models_weight( \ fusion['models']) model_types = [get_resource_type(model) for model in self.model_ids] for model_type in model_types: if model_type not in LOCAL_SUPERVISED: raise ValueError("The resource %s has not an allowed" " supervised model type." % model_type) self.importance = fusion.get('importance', []) self.missing_numerics = fusion.get('missing_numerics', True) if fusion.get('fusion'): self.fields = fusion.get( \ 'fusion', {}).get("fields") self.objective_id = fusion.get("objective_field") self.input_fields = fusion.get("input_fields") number_of_models = len(self.model_ids) # Downloading the model information to cache it if self.api.storage is not None or cache_get is not None: for model_id in self.model_ids: if get_resource_type(model_id) == "fusion": Fusion(model_id, api=self.api, cache_get=cache_get) else: SupervisedModel(model_id, api=self.api, cache_get=cache_get) if max_models is None: self.models_splits = [self.model_ids] else: self.models_splits = [ self.model_ids[index:(index + max_models)] for index in range(0, number_of_models, max_models) ] if self.fields: add_distribution(self) summary = self.fields[self.objective_id]['summary'] if 'bins' in summary: distribution = summary['bins'] elif 'counts' in summary: distribution = summary['counts'] elif 'categories' in summary: distribution = summary['categories'] else: distribution = [] self.distribution = distribution self.regression = \ self.fields[self.objective_id].get('optype') == NUMERIC if not self.regression: objective_field = self.fields[self.objective_id] categories = objective_field['summary']['categories'] classes = [category[0] for category in categories] self.class_names = sorted(classes) self.objective_categories = [category for \ category, _ in self.fields[self.objective_id][ \ "summary"]["categories"]] ModelFields.__init__( \ self, self.fields, objective_id=self.objective_id)