def __init__(self, features=None, excluded_features=None, max_categories=None, output_column_name='encoded_features'): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(max_categories, [int, type(None)]) _raise_error_if_not_of_type(output_column_name, [str]) # Set up options opts = { 'max_categories': max_categories, 'output_column_name': output_column_name, } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._OneHotEncoder() proxy.init_transformer(opts) super(OneHotEncoder, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, max_categories=None, output_column_name = 'encoded_features'): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(max_categories, [int, _NoneType]) _raise_error_if_not_of_type(output_column_name, [str]) # Set up options opts = { 'max_categories': max_categories, 'output_column_name': output_column_name, } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._OneHotEncoder() proxy.init_transformer(opts) super(OneHotEncoder, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, num_bits=18, output_column_name='hashed_features'): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(num_bits, [int]) _raise_error_if_not_of_type(output_column_name, [str]) # Set up options opts = { 'num_bits': num_bits, 'output_column_name': output_column_name, } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._FeatureHasher() proxy.init_transformer(opts) super(FeatureHasher, self).__init__(proxy, self.__class__)
def query(self, query, num_results=10, expansion_k=5, expansion_epsilon=0.1, expansion_near_match_weight=.5): """ Search for text. Parameters ---------- query: str A string of text. num_results : int The number of results to return. expansion_k : int Maximum number of nearest words to include from query token. expansion_epsilon : float Maximum distance to allow between query token and nearby word when doing query expansion. Must be between 0 and 1. expansion_near_match_weight : float Multiplier to use on BM25 scores for documents indexed via an approximate match with a given token. This will be used for each of the `expansion_k` words that are considered an approximate match. Must be between 0 and 1. Returns ------- out: SFrame The rows of the original SFrame along with a `score` column which contains the BM25 score between this query and the row. Examples -------- >>> import graphlab as gl >>> sf = gl.SFrame({'text': ['Hello my friend', 'I love this burrito']}) >>> s = gl.search.create(sf, features=['text']) >>> s.query('burrito') """ if _sys.version_info.major == 2: _raise_error_if_not_of_type(query, [str, unicode]) else: _raise_error_if_not_of_type(query, [str]) q = query.split(' ') results = self.__proxy__.query_index( q, expansion_k=expansion_k, expansion_epsilon=expansion_epsilon, expansion_near_match_weight=expansion_near_match_weight) results = self.__proxy__.join_query_result(results, method='default', num_results=num_results) return results
def __init__(self, features=None, excluded_features=None, output_column_prefix=None, verbose=True): self._setup() _features, _exclude = process_features(features, excluded_features) #Type check _raise_error_if_not_of_type(output_column_prefix, [str, NoneType]) _raise_error_if_not_of_type(verbose, [bool]) state = {} state['output_column_prefix'] = output_column_prefix state['features'] = _features state['excluded_features'] = _exclude state['fitted'] = False state['verbose'] = verbose if _exclude: self._exclude = True self._features = _exclude else: self._exclude = False self._features = _features self.__proxy__.update(state)
def __init__(self, features=None, excluded_features=None, threshold=1, output_category_name=None, output_column_prefix=None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(threshold, [int, _NoneType]) # Set up options opts = { 'threshold': threshold, 'output_category_name': output_category_name, 'output_column_prefix': output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._CountThresholder() proxy.init_transformer(opts) super(CountThresholder, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, separator = ".", none_tag = "__none__", output_column_prefix = None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features(features, excluded_features) # Type checking _raise_error_if_not_of_type(output_column_prefix, [str, type(None)]) if output_column_prefix is None: output_column_prefix = '' opts = { 'separator' : separator, 'none_tag' : none_tag, 'output_column_prefix' : output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._TransformToFlatDictionary() proxy.init_transformer(opts) super(TransformToFlatDictionary, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, num_bits=18, output_column_name='hashed_features'): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(num_bits, [int]) _raise_error_if_not_of_type(output_column_name, [str]) # Set up options opts = { 'num_bits': num_bits, 'output_column_name': output_column_name, } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._FeatureHasher() proxy.init_transformer(opts) super(FeatureHasher, self).__init__(proxy, self.__class__)
def get_prediction_score(self, node_id): """ Return the prediction score (if leaf node) or None if its an intermediate node. Parameters ---------- node_id: id of the node to get the prediction value. Returns ------- float or None: returns float value of predictio if leaf node and None if not. Examples -------- .. sourcecode:: python >>> tree.get_prediction_score(120) # Leaf node 0.251092 >>> tree.get_prediction_score(120) # Not a leaf node None """ _raise_error_if_not_of_type(node_id, [int,long], "node_id") _numeric_param_check_range("node_id", node_id, 0, self.num_nodes - 1) node = self.nodes[node_id] return None if node.is_leaf == False else node.value
def _validate_job_create_args(function, name, environment): """ Validate the arguments for job.create and map_job.create """ __LOGGER__.info("Validating job.") _raise_error_if_not_of_type(environment, [type(None), str, _environment._Environment], 'environment') _raise_error_if_not_of_type(name, [type(None), str], 'name') if name is not None and not _job_name_checker.match(name): raise ValueError('Job name can only contain digits, characters, "-" and "_".') # Setup the env if not environment: try: environment = _gl.deploy.environments['async'] except KeyError: __LOGGER__.info("Creating a LocalAsync environment called 'async'.") try: environment = _environment.LocalAsync('async') except KeyError: environment = _gl.deploy.environments['async'] else: if isinstance(environment, str): __LOGGER__.debug("Loading environment: %s" % environment) environment = _gl.deploy.environments[environment] # Clone to prevent the user's environment to reflect changes. return function, name, environment
def _validate_job_create_args(function, name, environment): """ Validate the arguments for job.create and map_job.create """ __LOGGER__.info("Validating job.") _raise_error_if_not_of_type(environment, [type(None), str, _environment._Environment], 'environment') _raise_error_if_not_of_type(name, [type(None), str], 'name') if name is not None and not _job_name_checker.match(name): raise ValueError( 'Job name can only contain digits, characters, "-" and "_".') # Setup the env if not environment: try: environment = _gl.deploy.environments['async'] except KeyError: __LOGGER__.info( "Creating a LocalAsync environment called 'async'.") try: environment = _environment.LocalAsync('async') except KeyError: environment = _gl.deploy.environments['async'] else: if isinstance(environment, str): __LOGGER__.debug("Loading environment: %s" % environment) environment = _gl.deploy.environments[environment] # Clone to prevent the user's environment to reflect changes. return function, name, environment
def __init__(self, features=None, excluded_features=None, strategy='logarithmic', num_bins=10, output_column_prefix=None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(num_bins, [int]) _raise_error_if_not_of_type(strategy, [str]) # Set up options opts = { 'strategy': strategy, 'num_bins': num_bins, 'output_column_prefix': output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._FeatureBinner() proxy.init_transformer(opts) super(FeatureBinner, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, threshold=1, output_category_name=None, output_column_prefix=None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) # Type checking _raise_error_if_not_of_type(threshold, [int, type(None)]) # Set up options opts = { 'threshold': threshold, 'output_category_name': output_category_name, 'output_column_prefix': output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._CountThresholder() proxy.init_transformer(opts) super(CountThresholder, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, strategy='logarithmic', num_bins=10, output_column_prefix=None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features(features, excluded_features) # Type checking _raise_error_if_not_of_type(num_bins, [int]) _raise_error_if_not_of_type(strategy, [str]) # Set up options opts = { 'strategy': strategy, 'num_bins': num_bins, 'output_column_prefix': output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._FeatureBinner() proxy.init_transformer(opts) super(FeatureBinner, self).__init__(proxy, self.__class__)
def __init__(self, features=None, excluded_features=None, output_column_prefix=None, transform_function=lambda x: x, transform_function_name="none"): self._setup() # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features( features, excluded_features) #Type check _raise_error_if_not_of_type(output_column_prefix, [str, type(None)]) state = {} state['output_column_prefix'] = output_column_prefix state['features'] = _features state['excluded_features'] = _exclude state['fitted'] = False state['transform_function'] = transform_function state['transform_function_name'] = transform_function_name if _exclude: self._exclude = True self._features = _exclude else: self._exclude = False self._features = _features self.__proxy__.update(state)
def fit(self, data): """ Fits a transformer using the SFrame `data`. The `fit` phase does not train a deep learning model, it only checks that the trained model is comptable with the data provided. If the `auto` model is chosen, then the fit phase choses the right model to extract features from. Parameters ---------- data : SFrame The data used to fit the transformer. Returns ------- self (A fitted object) See Also -------- transform, fit_transform Examples -------- # Create data. >>> import graphlab as gl # Import data from MNIST >>> data = gl.SFrame('http://s3.amazonaws.com/dato-datasets/mnist/sframe/train6k') # Create a DeepFeatureExtractorObject >>> extractor = gl.feature_engineering.DeepFeatureExtractor( features = 'image') # Fit the encoder for a given dataset. >>> extractor = extractor.fit(data) # Return the model used for the deep feature extraction. >>> extractor['model'] """ _mt._get_metric_tracker().track(self.__class__.__module__ + '.fit') # Check that the column is in the SFrame. _raise_error_if_not_of_type(data, [_SFrame]) _raise_error_if_column_exists(data, self._state["features"]) # Make sure the output column_name exists. count = 1 old_output_column_name = self._state["output_column_name"] output_column_name = old_output_column_name while output_column_name in data.column_names(): output_column_name = "%s.%s" % (old_output_column_name, count) count = count + 1 self._state["output_column_name"] = output_column_name if data[self._state["features"]].dtype() != _Image: raise ToolkitError( "Feature `%s` must be of type Image." % self._state["features"]) return self
def item_history_count(self, item_id): ''' Returns the number of interactions per item. ''' _raise_error_if_not_of_type(item_id, self._allowed_item_types, 'item_id') user = _gl.SFrame({self.item_id_column: [item_id]}) return user.join(self._item_counts, on=self.item_id_column)
def user_history_count(self, user_id): ''' Returns the number of interactions per user. ''' _raise_error_if_not_of_type(user_id, self._allowed_user_types, 'user_id') user = _gl.SFrame({self.user_id_column: [user_id]}) return user.join(self._user_counts, on=self.user_id_column)
def user_details(self, user_id): ''' Returns the row of the user table given a user id. ''' _raise_error_if_not_of_type(user_id, self._allowed_user_types, 'user_id') user = _gl.SFrame({self.user_id_column: [user_id]}) return user.join(self._users, on=self.user_id_column)
def _set_one_input(self, name='input', value=None, from_task=None, delete=False): """ Set/Update an input for this Task. Parameters ---------- name : str Name for this input. This will be how the code refers to this input at runtime. Default is 'input'. value : obj (supported by GL Pickle) Value for the object refered to using 'name'. from_task : Task|str Dependent Task to set as input, specifying the tuple with: (Task, output_name). Tasks can be referred to either by name or by reference. The output_name needs to be a string. For example, if the following is specified: >>> task._set_one_input(name='in', from_task='dep') then an input named 'in' will be defined on this Task, which has a dependency on the output of the Task named 'dep'. delete : bool, optional If delete is set to True then the name input is removed. """ _raise_error_if_not_of_type(name, str, "name") _raise_error_if_not_of_type(from_task, [type(None), Task], "from_task") # Delete the input. if delete is True and name in self._data['inputs']: del self._data['inputs'][name] return self # Early binding: Set the input if from_task is None: self._data['inputs'][name] = value self._set_dirty_bit() return self # Late binding: Set an input from a task. elif isinstance(from_task, Task): task = from_task self._data['inputs'][name] = task self._set_dirty_bit() return self
def set_inputs(self, names): """ Set input(s) for this Task. Inputs can be any object that can be pickled using GL-Pickle but cannot come from the output of another task. For that, use the set_inputs_from_task function. Parameters ---------- names : list [str] | dict [str, obj] If a dict is provided, then each key is considered a name for an input in this Task, and each value is considered the definition of the input. When a list is provided, then each entry is considered a name for an input in this Task, and the value for that slot is set to None. Returns ------- self : Task See Also -------- set_output Examples -------- To define only input names for a task, use a list of strings: >>> # For late binding >>> t1 = graphlab.deploy._task.Task(my_func, 'set_inputs_ex1') >>> t1.set_inputs(['one', 'two', 'three']) >>> # For early binding >>> t3 = graphlab.deploy._task.Task(my_func, 'set_inputs_ex3') >>> t3.set_inputs({ ... 'b' : 'set_inputs_ex2', ... 'c' : 'foo', ... 'd' : ('foo', 'bar')}) """ if names is None: raise TypeError('Names are required while binding two tasks.') _raise_error_if_not_of_type(names, [list, dict], 'names') if isinstance(names, list): for name in set(names): self._set_one_input(name=name, delete=False) elif isinstance(names, dict): for key, value in names.items(): self._set_one_input(name=key, value=value, delete=False) return self
def _set_one_input(self, name='input', value=None, from_task=None, delete=False): """ Set/Update an input for this Task. Parameters ---------- name : str Name for this input. This will be how the code refers to this input at runtime. Default is 'input'. value : obj (supported by GL Pickle) Value for the object refered to using 'name'. from_task : Task|str Dependent Task to set as input, specifying the tuple with: (Task, output_name). Tasks can be referred to either by name or by reference. The output_name needs to be a string. For example, if the following is specified: >>> task._set_one_input(name='in', from_task='dep') then an input named 'in' will be defined on this Task, which has a dependency on the output of the Task named 'dep'. delete : bool, optional If delete is set to True then the name input is removed. """ _raise_error_if_not_of_type(name, str, "name") _raise_error_if_not_of_type(from_task, [type(None), Task], "from_task") # Delete the input. if delete is True and name in self._data['inputs']: del self._data['inputs'][name] return self # Early binding: Set the input if from_task is None: self._data['inputs'][name] = value self._set_dirty_bit() return self # Late binding: Set an input from a task. elif isinstance(from_task, Task): task = from_task self._data['inputs'][name] = task self._set_dirty_bit() return self
def set_inputs(self, names): """ Set input(s) for this Task. Inputs can be any object that can be pickled using GL-Pickle but cannot come from the output of another task. For that, use the set_inputs_from_task function. Parameters ---------- names : list [str] | dict [str, obj] If a dict is provided, then each key is considered a name for an input in this Task, and each value is considered the definition of the input. When a list is provided, then each entry is considered a name for an input in this Task, and the value for that slot is set to None. Returns ------- self : Task See Also -------- set_output Examples -------- To define only input names for a task, use a list of strings: >>> # For late binding >>> t1 = graphlab.deploy._task.Task(my_func, 'set_inputs_ex1') >>> t1.set_inputs(['one', 'two', 'three']) >>> # For early binding >>> t3 = graphlab.deploy._task.Task(my_func, 'set_inputs_ex3') >>> t3.set_inputs({ ... 'b' : 'set_inputs_ex2', ... 'c' : 'foo', ... 'd' : ('foo', 'bar')}) """ if names is None: raise TypeError('Names are required while binding two tasks.') _raise_error_if_not_of_type(names, [list, dict], 'names') if isinstance(names, list): for name in set(names): self._set_one_input(name=name, delete=False) elif isinstance(names, dict): for key, value in names.iteritems(): self._set_one_input(name=key, value=value, delete=False) return self
def __init__(self, func, name=None, description=None): """ Create a new Task specifying its name and optionally a description. """ # Must be a function _raise_error_if_not_function(func, "func") # Set the name name = func.__name__ if not name else name _raise_error_if_not_of_type(name, str, "name") self.name = name self._data = dict() self._data['code'] = None self._data['codestr'] = None self._data['inputs'] = dict() self._data['output'] = None self._data['packages'] = set() self._data['description'] = '' self._modified_since_last_saved = None if description is not None: self.set_description(description) # Inspect the function. specs = _inspect.getargspec(func) varargs = specs.varargs defaults = _copy.copy(specs.defaults) args = _copy.copy(specs.args) # Set the code to function arguments + *args + **kwargs self.set_code(func) # Set the inputs all_args = _copy.copy(args) if varargs: all_args.append(varargs) self.set_inputs(all_args) # Bind default values if defaults: for index, arg in enumerate(args[-len(defaults):]): self.set_inputs({arg : defaults[index]}) # Set required packages if _sys.version_info.major == 3: func_dict = func.__dict__ else: func_dict = func.func_dict
def fit(self, data): """ Fits a transformer using the SFrame `data`. The `fit` phase does not train a deep learning model, it only checks that the trained model is comptable with the data provided. If the `auto` model is chosen, then the fit phase choses the right model to extract features from. Parameters ---------- data : SFrame The data used to fit the transformer. Returns ------- self (A fitted object) See Also -------- transform, fit_transform Examples -------- # Create data. >>> import graphlab as gl # Import data from MNIST >>> data = gl.SFrame('https://static.turi.com/datasets/mnist/sframe/train6k') # Create a DeepFeatureExtractorObject >>> extractor = gl.feature_engineering.DeepFeatureExtractor(features = 'image') # Fit the encoder for a given dataset. >>> extractor = extractor.fit(data) # Return the model used for the deep feature extraction. >>> extractor['model'] """ _mt._get_metric_tracker().track(self.__class__.__module__ + '.fit') # Check that the column is in the SFrame. _raise_error_if_not_of_type(data, [_SFrame]) for feature in self._state["features"]: _raise_error_if_column_exists(data, feature) if data[feature].dtype() != _Image: raise ToolkitError("Feature `%s` must be of type Image." % feature) return self
def __init__(self, name, session_aware=True): """ Constructor for base Environment, should not be instantiated directly. """ if not name: raise TypeError("Name is required when creating an Environment.") _raise_error_if_not_of_type(name, [str, unicode], 'name') self._session = _gl.deploy._default_session self.name = name self._env_type = type(self).__name__ self._modified_since_last_saved = None if session_aware: self._session.register(self)
def __init__(self, func, name=None, description=None): """ Create a new Task specifying its name and optionally a description. """ # Must be a function _raise_error_if_not_function(func, "func") # Set the name name = func.__name__ if not name else name _raise_error_if_not_of_type(name, str, "name") self.name = name self._data = dict() self._data['code'] = None self._data['codestr'] = None self._data['inputs'] = dict() self._data['output'] = None self._data['packages'] = set() self._data['description'] = '' self._modified_since_last_saved = None if description is not None: self.set_description(description) # Inspect the function. specs = _inspect.getargspec(func) varargs = specs.varargs defaults = _copy.copy(specs.defaults) args = _copy.copy(specs.args) # Set the code to function arguments + *args + **kwargs self.set_code(func) # Set the inputs all_args = _copy.copy(args) if varargs: all_args.append(varargs) self.set_inputs(all_args) # Bind default values if defaults: for index, arg in enumerate(args[-len(defaults):]): self.set_inputs({arg: defaults[index]}) # Set required packages if 'required_packages' in func.func_dict: self.set_required_packages(func.func_dict['required_packages'])
def users_in_common(self, item_a, item_b, num_results=None): """ Get data on the users in common between two items. Parameters ---------- item_a : The id of one item. item_b : The id of the other item. num_results : int, optional The number of users in common to return. Returns ------- out : SFrame A SFrame with the two item columns given above, the number of users that rated each, and a dictionary mapping the user to a pair of the ratings, with the first rating being the rating of the first item and the second being the rating of the second item. If no ratings are provided, these values are always 1.0. Returns the observation data relevant to the provided user. """ _raise_error_if_not_of_type(item_a, self._allowed_item_types, 'item_id') _raise_error_if_not_of_type(item_b, self._allowed_item_types, 'item_id') item_a_history = self.item_history(item_a) item_b_history = self.item_history(item_b) item_a_users = item_a_history[self.user_id_column].unique() item_b_users = item_b_history[self.user_id_column].unique() users_in_common = set(item_a_users).intersection(set(item_b_users)) users = _gl.SFrame({self.user_id_column: list(users_in_common)}) users = users.join(self._users, on=self.user_id_column) if num_results is not None: users = users.head(num_results) result = { 'item_a_count': len(item_a_users), 'item_b_count': len(item_b_users), 'in_common_count': len(users_in_common), 'in_common_users': users } return result
def process_features(features, exclude): """ Parameters ---------- features : list[str] | str | None, optional Column names of features to be transformed. If None, all columns are selected. If string, that column is transformed. If list of strings, this list of column names is selected. exclude : list[str] | str | None, optional Column names of features to be ignored in transformation. Can be string or list of strings. Either 'exclude' or 'features' can be passed, but not both. Returns ------- (features, exclude) that are processed. """ # Check types _raise_error_if_not_of_type(features, [NoneType, str, list], 'features') _raise_error_if_not_of_type(exclude, [NoneType, str, list], 'exclude') # Make a copy of the parameters. _features = _copy.copy(features) _exclude = _copy.copy(exclude) # Check of both are None or empty. if _features and _exclude: raise ValueError( "The parameters 'features' and 'exclude' cannot both be set." " Please set one or the other.") if _features == [] and not _exclude: raise ValueError("Features cannot be an empty list.") # Allow a single list _features = [_features] if type(_features) == str else _features _exclude = [_exclude] if type(_exclude) == str else _exclude # Type check each feature/exclude if _features: for f in _features: _raise_error_if_not_of_type(f, str, "Feature names") if _exclude: for e in _exclude: _raise_error_if_not_of_type(e, str, "Excluded feature names") if _exclude is not None and _features is not None: feature_set = set(_features) for col_name in _exclude: if col_name in feature_set: raise ValueError( "'%s' appears in both features and excluded_features." % col_name) return _features, _exclude
def set_description(self, description): """ Set the description for this Task. Parameters ---------- description : str A description for the Task. Returns ------- self : Task """ _raise_error_if_not_of_type(description, str, "description") self._data['description'] = description self._set_dirty_bit() return self
def set_description(self, description): """ Set the description for this Task. Parameters ---------- description : str A description for the Task. Returns ------- self : Task """ _raise_error_if_not_of_type(description, str, "description") self._data['description'] = description self._set_dirty_bit() return self
def user_history(self, user_id, num_results=None): """ Returns the observation data relevant to the provided user. """ _raise_error_if_not_of_type(user_id, self._allowed_user_types, 'user_id') try: result = self._observations_by_user.get_group(user_id) except RuntimeError as e: result = self._empty_observation_data if self.item_name_column: result = result.join(self._items, on=self.item_id_column) if num_results is not None: result = result.head(num_results) return result
def set_name(self, name): """ Set the name of the Task, which must be unique. Parameters ---------- name : str Name of the Task. Returns ------- self : Task """ _raise_error_if_not_of_type(name, str, "name") self.name = str(name) self._set_dirty_bit() return self
def __init__(self, name, session_aware=True): """ Constructor for base Environment, should not be instantiated directly. """ if not name: raise TypeError("Name is required when creating an Environment.") if _sys.version_info.major == 3: _raise_error_if_not_of_type(name, [str], 'name') else: _raise_error_if_not_of_type(name, [str, unicode], 'name') self._session = _gl.deploy._default_session self.name = name self._env_type = type(self).__name__ self._modified_since_last_saved = None if session_aware: self._session.register(self)
def set_name(self, name): """ Set the name of the Task, which must be unique. Parameters ---------- name : str Name of the Task. Returns ------- self : Task """ _raise_error_if_not_of_type(name, str, "name") self.name = str(name) self._set_dirty_bit() return self
def __init__(self, steps): """ Parameters ---------- steps: list[Transformer] | list[tuple(name, Transformer)] List of Transformers or (name, Transformer) tuples. These are chained in the order in which they are provided in the list. """ # Basic type checking. _raise_error_if_not_of_type(steps, [list]) # Split into (name, transformer) pairs. If the name is not present # then use the index as name. transformers = [] index = 0 for step in steps: if isinstance(step, tuple): name, tr = step else: tr = step name = index if isinstance(tr, list): tr = TransformerChain(tr) if not issubclass(tr.__class__, _TransformerBase): raise TypeError( "Each step in the chain must be a Transformer.") transformers.append((name, tr)) index = index + 1 # Save into a dictionary for lookups by name and index. self._state = {} self._state["steps"] = steps self._state["steps_by_name"] = {} index = 0 for name, tr in transformers: self._state["steps_by_name"][name] = tr index = index + 1 # The transformers as (name, obj) tuple (used here for fitting # and transforming). self._transformers = transformers
def __init__(self, steps): """ Parameters ---------- steps: list[Transformer] | list[tuple(name, Transformer)] List of Transformers or (name, Transformer) tuples. These are chained in the order in which they are provided in the list. """ # Basic type checking. _raise_error_if_not_of_type(steps, [list]) # Split into (name, transformer) pairs. If the name is not present # then use the index as name. transformers = [] index = 0 for step in steps: if isinstance(step, tuple): name, tr = step else: tr = step name = index if isinstance(tr, list): tr = TransformerChain(tr) if not issubclass(tr.__class__, _TransformerBase): raise TypeError("Each step in the chain must be a Transformer.") transformers.append((name, tr)) index = index + 1 # Save into a dictionary for lookups by name and index. self._state = {} self._state["steps"] = steps self._state["steps_by_name"] = {} index = 0 for name, tr in transformers: self._state["steps_by_name"][name] = tr index = index + 1 # The transformers as (name, obj) tuple (used here for fitting # and transforming). self._transformers = transformers
def __init__(self, reference_features=None, feature="feature", verbose=False): # Process and make a copy of the reference_features _reference_features, _exclude = _internal_utils.process_features(reference_features, None) # Type checking _raise_error_if_not_of_type(feature, [str]) # Set up options opts = { 'reference_features': reference_features, 'feature': feature, 'verbose': verbose } opts['reference_features'] = _reference_features # Initialize object proxy = _gl.extensions._CategoricalImputer() proxy.init_transformer(opts) super(CategoricalImputer, self).__init__(proxy, self.__class__)
def delete_inputs(self, names): """ Set input(s) for this Task. Inputs can be any object that can be pickled using GL-Pickle but cannot come from the output of another task. For that, use the set_inputs_from_task function. Parameters ---------- names : list [str] When a list is provided, then each entry is considered a name for an input in this Task, and is hence removed. Returns ------- self : Task See Also -------- delete_output Examples -------- To define only input names for a task, use a list of strings: >>> # For late binding >>> t1 = graphlab.deploy._task.Task(my_func, 'set_inputs_ex1') >>> t1.delete_inputs(['one', 'two', 'three']) """ if names is None: return self _raise_error_if_not_of_type(names, [list], 'names') for name in set(names): self._set_one_input(name=name, delete=True) return self
def delete_inputs(self, names): """ Set input(s) for this Task. Inputs can be any object that can be pickled using GL-Pickle but cannot come from the output of another task. For that, use the set_inputs_from_task function. Parameters ---------- names : list [str] When a list is provided, then each entry is considered a name for an input in this Task, and is hence removed. Returns ------- self : Task See Also -------- delete_output Examples -------- To define only input names for a task, use a list of strings: >>> # For late binding >>> t1 = graphlab.deploy._task.Task(my_func, 'set_inputs_ex1') >>> t1.delete_inputs(['one', 'two', 'three']) """ if names is None: return self _raise_error_if_not_of_type(names, [list], 'names') for name in set(names): self._set_one_input(name=name, delete=True) return self
def item_details(self, item_id): """ Obtain data for a given item. Parameters ---------- item_id : int, str The id of the desired item. Returns ------- out : SFrame Data for the desired item. If no row has the desired item_id, then an empty SFrame is returned. """ _raise_error_if_not_of_type(item_id, self._allowed_item_types, 'item_id') item = _gl.SFrame({self.item_id_column: [item_id]}) return item.join(self._items, on=self.item_id_column)
def fit(self, dataset): """ Fits a transformer using the SFrame `dataset`. Parameters ---------- data : SFrame The data used to fit the transformer. Returns ------- self (A fitted object) See Also -------- transform, fit_transform """ _mt._get_metric_tracker().track(self.__class__.__module__ + '.fit') _raise_error_if_not_of_type(dataset, [_SFrame]) fitted_state = {} feature_columns = get_column_names(dataset, self._exclude, self._features) feature_columns = select_valid_features(dataset, feature_columns, [str, list]) fitted_state['features'] = feature_columns validate_feature_columns(dataset.column_names(), feature_columns) fitted_state['col_type_map'] = { col_name: col_type for (col_name, col_type ) in zip(dataset.column_names(), dataset.column_types()) } fitted_state['fitted'] = True self.__proxy__.update(fitted_state) return self
def __init__( self, features=None, excluded_features=None, min_document_frequency=0.0, max_document_frequency=1.0, output_column_prefix=None, ): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features(features, excluded_features) # Type checking _raise_error_if_not_of_type(min_document_frequency, [float, int]) _raise_error_if_not_of_type(max_document_frequency, [float, int]) _raise_error_if_not_of_type(output_column_prefix, [str, _NoneType]) # Set up options opts = { "min_document_frequency": min_document_frequency, "max_document_frequency": max_document_frequency, "output_column_prefix": output_column_prefix, } if _exclude: opts["exclude"] = True opts["features"] = _exclude else: opts["exclude"] = False opts["features"] = _features # Initialize object proxy = _gl.extensions._TFIDF() proxy.init_transformer(opts) super(TFIDF, self).__init__(proxy, self.__class__)
def __init__(self, features, model='auto', output_column_prefix=None): """ Parameters ---------- """ _raise_error_if_not_of_type(features, [str, list, type(None)]) _raise_error_if_not_of_type(model, [str, _NeuralNetClassifier]) _raise_error_if_not_of_type(output_column_prefix, [str, type(None)]) if isinstance(features, str): features = [features] # Set the model. self._state = {} self._state["features"] = features if not output_column_prefix: output_column_prefix = "deep_features" self._state["output_column_prefix"] = output_column_prefix self._state['model'] = model if self._state["model"] == 'auto': model_path = \ "https://static.turi.com/products/graphlab-create/resources/models/python2.7/imagenet_model_iter45" import graphlab as gl self._state['model'] = gl.load_model(model_path) if type(self._state['model']) is not _NeuralNetClassifier: raise ValueError( "Model parameters must be of type NeuralNetClassifier " + "or string literal 'auto'")
def __init__(self, feature, model = 'auto', output_column_name=None): """ Parameters ---------- """ _raise_error_if_not_of_type(feature, [str]) _raise_error_if_not_of_type(model, [str, _NeuralNetClassifier]) _raise_error_if_not_of_type(output_column_name, [str, _NoneType]) # Set the model. self._state = {} self._state["features"] = feature if not output_column_name: self._state["output_column_name"] = "deep_features_%s" % feature else: self._state["output_column_name"] = output_column_name self._state['model'] = model if self._state["model"] == 'auto': model_path = \ "http://s3.amazonaws.com/dato-datasets/deeplearning/imagenet_model_iter45" import graphlab as gl self._state['model'] = gl.load_model(model_path) if type(self._state['model']) is not _NeuralNetClassifier: raise ValueError("Model parameters must be of type NeuralNetClassifier " + "or string literal 'auto'")
def __init__(self, features=None, excluded_features=None, min_document_frequency=0.0, max_document_frequency=1.0, output_column_prefix=None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features(features, excluded_features) # Type checking _raise_error_if_not_of_type(min_document_frequency, [float, int]) _raise_error_if_not_of_type(max_document_frequency, [float, int]) _raise_error_if_not_of_type(output_column_prefix, [str, type(None)]) # Set up options opts = { 'min_document_frequency': min_document_frequency, 'max_document_frequency': max_document_frequency, 'output_column_prefix' : output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._TFIDF() proxy.init_transformer(opts) super(TFIDF, self).__init__(proxy, self.__class__)
def from_model(cls, model, tree_id = 0): import graphlab as _gl from graphlab.toolkits import _supervised_learning as _sl import json as _json _raise_error_if_not_of_type(tree_id, [int,long], "tree_id") _numeric_param_check_range("tree_id", tree_id, 0, model.num_trees - 1) tree = DecisionTree() nodes = {} tree_str = _gl.extensions._xgboost_get_tree(model.__proxy__, tree_id) metadata_mapping = _gl.extensions._get_metadata_mapping(model.__proxy__) trees_json = _json.loads(tree_str) # Parse the tree from the JSON. tree._make_tree(trees_json, metadata_mapping) tree.root_id = 0 # Keep track of the attributes. for key in {"num_examples", "num_features", "num_unpacked_features", "max_depth"}: setattr(tree, key, model[key]) return tree
def process_features(features, exclude): """ Parameters ---------- features : list[str] | str | None, optional Column names of features to be transformed. If None, all columns are selected. If string, that column is transformed. If list of strings, this list of column names is selected. exclude : list[str] | str | None, optional Column names of features to be ignored in transformation. Can be string or list of strings. Either 'exclude' or 'features' can be passed, but not both. Returns ------- (features, exclude) that are processed. """ # Make a copy of the parameters. _features = _copy.copy(features) _exclude = _copy.copy(exclude) # Check of both are None or empty. if _features and _exclude: raise ValueError("The parameters 'features' and 'exclude' cannot both be set." " Please set one or the other.") if _features == [] and not _exclude: raise ValueError("Features cannot be an empty list.") # Check types _raise_error_if_not_of_type(_features, [NoneType, str, list], 'features') _raise_error_if_not_of_type(_exclude, [NoneType, str, list], 'exclude') # Allow a single list _features = [_features] if type(_features) == str else _features _exclude = [_exclude] if type(_exclude) == str else _exclude # Type check each feature/exclude if _features: for f in _features: _raise_error_if_not_of_type(f, str, "Feature names") if _exclude: for e in _exclude: _raise_error_if_not_of_type(e, str, "Excluded feature names") return _features, _exclude
def __init__(self, feature, min_document_frequency=0.0, max_document_frequency=1.0, output_column_name=None): # Type checking _raise_error_if_not_of_type(feature, [str]) _raise_error_if_not_of_type(min_document_frequency, [float, int]) _raise_error_if_not_of_type(max_document_frequency, [float, int]) _raise_error_if_not_of_type(output_column_name, [str, _NoneType]) # Set up options opts = { "features": [feature], "min_document_frequency": min_document_frequency, "max_document_frequency": max_document_frequency, "output_column_name": output_column_name, } # Initialize object proxy = _gl.extensions._TFIDF() proxy.init_transformer(opts) super(TFIDF, self).__init__(proxy, self.__class__)
def __init__(self, feature, query, k1 = 1.5, b = 0.75, min_document_frequency = 0.0, max_document_frequency=1.0, output_column_name=None): # Convert query to list if necessary if isinstance(query, _gl.SArray): query = list(query) if isinstance(query, set): query = list(query) # Type checking _raise_error_if_not_of_type(feature, [str]) for q in query: _raise_error_if_not_of_type(q, [str]) # query must be list of strings _raise_error_if_not_of_type(k1, [float, int]) _raise_error_if_not_of_type(b, [float, int]) _raise_error_if_not_of_type(min_document_frequency, [float, int]) _raise_error_if_not_of_type(max_document_frequency, [float, int]) _raise_error_if_not_of_type(output_column_name, [str, _NoneType]) # Set up options opts = { 'features': [feature], 'query': query, 'k1': k1, 'b': b, 'min_document_frequency': min_document_frequency, 'max_document_frequency': max_document_frequency, 'output_column_name' : output_column_name } # Initialize object proxy = _gl.extensions._BM25() proxy.init_transformer(opts) super(BM25, self).__init__(proxy, self.__class__)
def create(dataset, item, features=None, min_support=1, max_patterns=100, min_length=1): """ Create a :class:`~graphlab.frequent_pattern_mining.FrequentPatternMiner` to extract the set of frequently occurring items in an event-series. Parameters ---------- dataset : SFrame Dataset for training the model. item: string Name of the column containing the item. The values in this column must be of string or integer type. features : list[string], optional Names of the columns containing features. 'None' (the default) indicates that all columns except the target variable should be used as features. The feature columns are the ones that together identify a unique transaction ID for the item. min_support : int, optional The minimum number of times that a pattern must occur in order for it to be considered `frequent`. max_patterns : int, optional The maximum number of frequent patterns to be mined. min_length: int, optional The minimum size (number of elements in the set) of each pattern being mined. Returns ------- out : FrequentPatternMiner A trained model of type :class:`~graphlab.frequent_pattern_mining.FrequentPatternMiner`. Notes ----- Frequent closed itemests are mined using the `top-k FP growth` algorithm. Mining occurs until the top max_patterns closed itemsets of size min_length and support greater than min_support are found. See Also -------- FrequentPatternMiner References ---------- - Wikipedia - Association Rule Learning <https://en.wikipedia.org/wiki/Association_rule_learning> - Han, Jiawei, et al. "Mining top-k frequent closed patterns without minimum support." Data Mining, 2002. ICDM 2003. - Wang, Jianyong, et al. "TFP: An efficient algorithm for mining top-k frequent closed itemsets." Knowledge and Data Engineering, IEEE Transactions on 17.5 (2005): 652-663. Examples -------- .. sourcecode:: python >>> import graphlab as gl >>> bakery_sf = gl.SFrame("http://s3.amazonaws.com/dato-datasets/bakery.sf") >>> bakery_sf Data: +---------+-------------+-------+----------+----------+-----------------+ | Receipt | SaleDate | EmpId | StoreNum | Quantity | Item | +---------+-------------+-------+----------+----------+-----------------+ | 1 | 12-JAN-2000 | 20 | 20 | 1 | GanacheCookie | | 1 | 12-JAN-2000 | 20 | 20 | 5 | ApplePie | | 2 | 15-JAN-2000 | 35 | 10 | 1 | CoffeeEclair | | 2 | 15-JAN-2000 | 35 | 10 | 3 | ApplePie | | 2 | 15-JAN-2000 | 35 | 10 | 4 | AlmondTwist | | 2 | 15-JAN-2000 | 35 | 10 | 3 | HotCoffee | | 3 | 8-JAN-2000 | 13 | 13 | 5 | OperaCake | | 3 | 8-JAN-2000 | 13 | 13 | 3 | OrangeJuice | | 3 | 8-JAN-2000 | 13 | 13 | 3 | CheeseCroissant | | 4 | 24-JAN-2000 | 16 | 16 | 1 | TruffleCake | +---------+-------------+-------+----------+----------+-----------------+ [266209 rows x 6 columns] >>> model = gl.frequent_pattern_mining.create(train, 'Item', features=['Receipt'], min_length=4, max_patterns=500) Model fields ------------ Min support : 1 Max patterns : 500 Min pattern length : 4 Most frequent patterns ---------------------- ['CoffeeEclair', 'HotCoffee', 'AlmondTwist', 'ApplePie']: 1704 ['LemonLemonade', 'LemonCookie', 'RaspberryLemonade', 'RaspberryCookie']: 1565 ['LemonLemonade', 'LemonCookie', 'RaspberryLemonade', 'GreenTea']: 1290 ['LemonLemonade', 'RaspberryLemonade', 'RaspberryCookie', 'GreenTea']: 1289 ['LemonLemonade', 'LemonCookie', 'RaspberryCookie', 'GreenTea']: 1279 ['LemonCookie', 'RaspberryLemonade', 'RaspberryCookie', 'GreenTea']: 1279 ['AppleTart', 'AppleDanish', 'AppleCroissant', 'CherrySoda']: 1253 ['LemonLemonade', 'LemonCookie', 'RaspberryLemonade', 'RaspberryCookie', 'GreenTea']: 1221 ['CherryTart', 'ApricotDanish', 'OperaCake', 'ApricotTart']: 61 ['CherryTart', 'ApricotDanish', 'OperaCake', 'RaspberryLemonade']: 55 """ _mt._get_metric_tracker().track('toolkit.frequent_pattern_mining.create') # Type checking. _raise_error_if_not_sframe(dataset, "dataset") _raise_error_if_not_of_type(item, str, "item") _raise_error_if_not_of_type(features, [list, _types.NoneType], "features") _raise_error_if_not_of_type(min_support, [int, float], "min_support") _raise_error_if_not_of_type(max_patterns, [int, float], "max_patterns") _raise_error_if_not_of_type(min_length, [int, float], "min_length") # Value checking. column_names = dataset.column_names() # If features is None, then use all other column names than item if features is None: features = column_names features.remove(item) # Call the C++ create function. proxy = _gl.extensions._pattern_mining_create( dataset, item, features, min_support, max_patterns, min_length) return FrequentPatternMiner(proxy)
def create(data, row_label=None, features=None, feature_model='auto', method='lsh', verbose=True): """ Create a similarity search model, which can be used to quickly retrieve items similar to a query observation. In the case of images, this model automatically performs the appropriate feature engineering steps. NOTE: If you are using a CPU for the creation step with feature_model='auto', creation time may take a while. This is because extracting features for images on a CPU is expensive. With a GPU, one can expect large speedups. .. warning:: The similarity search toolkit is currently in beta, and feedback is welcome! Please send comments to [email protected]. Parameters ---------- dataset : SFrame The SFrame that represents the training data for the model, including at least one column of images. row_label : str, optional Name of the SFrame column with row id's. If 'row_label' is not specified, row numbers are used to identify reference dataset rows when the model is queried. features : str, optional The name of an image column in the input 'dataset' SFrame. feature_model : 'auto' | A model of type NeuralNetClassifier, optional A trained model for extracting features from raw data objects. By default ('auto'), we choose an appropriate model from our set of pre-trained models. See :class:`~graphlab.toolkits.feature_engineering.DeepFeatureExtractor` for more information. method : {'lsh', 'brute_force'}, optional The method used for nearest neighbor search. The 'lsh' option uses locality-sensitive hashing to find approximate results more quickly. verbose : bool, optional If True, print verbose output during model creation. Returns ------- out : SimilaritySearchModel See Also -------- SimilaritySearchModel graphlab.toolkits.nearest_neighbors graphlab.toolkits.feature_engineering Notes ----- The similarity search toolkit currently uses cosine distance to evaluate the similarity between each query and candidate results. Examples -------- First, split data into reference and query. >>> import graphlab as gl >>> data = gl.SFrame('http://s3.amazonaws.com/dato-datasets/mnist/sframe/train6k') >>> reference, query = data.random_split(0.8) Build neuralnet feature extractor for images: >>> nn_model = gl.neuralnet_classifier.create(reference, target='label') Construct SimilaritySearchModel: >>> model = gl.similarity_search.create(reference, features= 'image', ... feature_model=nn_model) Find the most similar items in the reference set for each item in the query set: >>> model.search(query) """ _mt._get_metric_tracker().track(__name__ + '.create') _raise_error_if_not_of_type(data, [_SFrame]) _raise_error_if_not_of_type(features, [str]) _raise_error_if_column_exists(data, features) if data[features].dtype() != _Image: raise _ToolkitError("Feature `%s` must be of type Image" \ % features) return SimilaritySearchModel(data, row_label=row_label, feature=features, feature_model=feature_model, method=method, verbose=verbose)
def search(self, data, row_label=None, k=5): """ Search for the nearest neighbors from the reference set for each element of the query set. The query SFrame must include columns with the same names as the row_label and feature columns used to create the SimilaritySearchModel. Parameters ---------- data : SFrame Query data. Must contain columns with the same names and types as the features used to train the model. Additional columns are allowed, but ignored. row_label : string, optional Name of the query SFrame column with row id's. If 'row_label' is not specified, row numbers are used to identify query dataset rows in the output SFrame. k : int, optional Number of nearest neighbors to return from the reference set for each query observation. The default is 5 neighbors. Returns ------- out A SFrame that contains all the nearest neighbors. Examples -------- First, split data into reference and query: >>> import graphlab as gl >>> data = gl.SFrame('http://s3.amazonaws.com/dato-datasets/mnist/sframe/train6k') >>> reference, query = data.random_split(0.8) Build a neural net feature extractor for images: >>> nn_model = gl.neuralnet_classifier.create(reference, target='label') Construct the SimilaritySearchModel: >>> model = gl.similarity_search.create(reference, features='image', ... feature_model=nn_model) Find the most similar items in the reference set for each query: >>> model.search(query) """ _raise_error_if_not_of_type(row_label, [str, _NoneType]) feature = self._state['features'] _raise_error_if_column_exists(data, feature) if (data[feature].dtype() != self._feature_type): raise ValueError('Feature columns must have same data type in both reference and query set') if row_label != None: _raise_error_if_column_exists(data, row_label) if data[feature].dtype() == _Image: transformed_data = self._extractor.transform(data) else: transformed_data = data transformed_data[self._state['output_column_name']] = transformed_data[feature] return self._neighbors_model.query(transformed_data, label=row_label, k=k)
def __init__(self, name, stages=[[]], final_stage=None, environment=None, _exec_dir=None, _task_output_paths=None, _job_type = 'PIPELINE'): """ Construct a job. Parameters ---------- name : str Name of this Job, must be unique. stages: list[list[Task]] Collection of task(s) to be executed. final_stage : list[task] | task Collection of task(s) whose outputs are to be returned._ environment : Environment, optional Environment used for this execution. See :py:class:`~graphlab.deploy.environment.LocalAsync` for an example environment. """ _raise_error_if_not_of_type(name, [str], 'name') _raise_error_if_not_of_type(stages, [list], 'stages') _raise_error_if_not_of_type(final_stage, [list, _Task, type(None)], 'final_stage') self.name = name self.environment = environment self._stages = stages self._num_tasks = 0 self._status = 'Pending' self._start_time = None self._end_time = None self._error = None self._job_type = _job_type # Set the packages self._packages = set() for task in self._stages: for t in task: self._num_tasks += 1 self._packages.update(t.get_required_packages()) self._final_stage = final_stage self._task_status = {} self._session = _gl.deploy._default_session if not _exec_dir: relative_path = "job-results-%s" % str(_uuid()) self._exec_dir = self.get_path_join_method()(self._session.results_dir, relative_path) else: self._exec_dir = _exec_dir # Location where all the outputs for the tasks are saved. if not _task_output_paths: Job._update_exec_dir(self, self._exec_dir) else: self._task_output_paths = _task_output_paths
def __init__(self, features=None, excluded_features=None, output_column_name='quadratic_features'): #Type checking _raise_error_if_not_of_type(output_column_name, [str]) # set up options opts = { 'output_column_name': output_column_name } # Make a copy of the parameters. _features = _copy.copy(features) _exclude = _copy.copy(excluded_features) # Check of both are None or empty. if _features and _exclude: raise ValueError("The parameters 'features' and 'exclude' cannot both be set." " Please set one or the other.") if _features == [] and not _exclude: raise ValueError("Features cannot be an empty list.") # Check types _raise_error_if_not_of_type(_features, [NoneType, list, str, tuple], 'features') _raise_error_if_not_of_type(_exclude, [NoneType, list, str, tuple], 'exclude') # Allow a single list _features = [_features] if type(_features) == str or type(_features) == tuple else _features _exclude = [_exclude] if type(_exclude) == str or type(_exclude) == tuple else _exclude # Type check each feature/exclude if _features: for f in _features: _raise_error_if_not_of_type(f, [str, tuple], "Feature names") if _exclude: for e in _exclude: _raise_error_if_not_of_type(e, [str, tuple], "Excluded feature names") if _exclude: opts['exclude'] = True unprocessed_features = _exclude else: opts['exclude'] = False unprocessed_features = _features pair_list = set() if unprocessed_features is not None: if type(unprocessed_features[0]) is tuple: for t in unprocessed_features: pair_list.add(tuple(sorted(t))) elif type(unprocessed_features[0]) is str: if _exclude: for t in unprocessed_features: pair_list.add(t) else: for t in unprocessed_features: for k in unprocessed_features: pair_list.add(tuple(sorted((t, k)))) if type(output_column_name) is not str: raise ValueError("'output_column_name' must be of type str") if unprocessed_features is not None: if type(unprocessed_features[0]) is str: opts['features'] = unprocessed_features if _exclude: opts['feature_pairs'] = list(pair_list) else: opts['feature_pairs'] = [list(x) for x in pair_list] else: opts['feature_pairs'] = [list(x) for x in pair_list ] opts['features'] = [list(x) for x in unprocessed_features] else: opts['feature_pairs'] = None opts['features'] = None # initialize object proxy = _gl.extensions._QuadraticFeatures() proxy.init_transformer(opts) super(QuadraticFeatures, self).__init__(proxy, self.__class__)
def sample(self, k, diversity=0.5, method=None, side_data=None, **kwargs): """ After constructing a diverse sampler, sample a diverse set stochastically. The stochastic algorithm depends on the sampling method itself. Parameters ---------- k : int The number of items to sample. diversity : double in [0, 1], optional This is a tunable parameter that trades off between quality and diversity. A diversity factor of 0 will only consider quality when building a set (equivalent to using the method "quality_only"), while a diversity factor of 1 will only consider item similarity and will ignore quality. A value between 0 and 1 will force the algorithm to trade off between quality and diversity. The actual effect of the diversity factor depends on the algorithm: - When method="weighted_vertex_cover", the diversity factor changes the number of nearest-neighbors to remove when sampling an item. Specifically, the number of neighbors is set to the value floor( (N-1)/(k-1) * diversity_factor). - When method="ipsen", the diversity factor will scale the similarity values by the value of diversity, and the quality values by (1-diversity). method : {'random', 'quality_only', 'weighted_vertex_cover', 'ipsen'}, optional The sampling method to use. The options available are: - *"random"*: Returns a completely random set of items, with no reference to item qualities or similarities. Note that the greedy method is undefined for a random sampler. - *"quality_only"*: Form a sampling distribution with the item qualities, and return a set from this distribution. The sample() method will sample a set from this distribution, while the greedy() method will return the top-k items according to item quality. Requirements: The column quality_feature must be present. - *"ipsen"*: Sample a diverse set using an approximation to the log- determinant. One method of sampling diverse sets is to use determinantal point process (DPP) sampling (see: http://arxiv.org/abs/1207.6083). Given any set of items, one measure of diversity is the log-determinant of the items' similarity matrix L. The diagonal entry L_{ii} corresponds to the quality of item i, while the off-diagonal entries L_{ij} correspond to the similarity between items i and j. The log-determinant of this matrix corresponds directly to the joint quality-diversity of the items that define L (high-qualities lead to a larger value of the determinant, while large similarities diminish the log-determinant). However, DPP sampling does not scale well, so we can instead approximate the log- determinant of a similarity matrix using more scalable methods. The Ipsen sampler uses the block-approximation of the determinant given in http://arxiv.org/pdf/1105.0437v1.pdf in order to mimic DPP sampling in a scalable fashion. Requirements: The columns quality_feature and similarity_features must be present. - *"weighted_vertex_cover"*: Sample a set of items with high quality, and with no (or a minimum) of nearest-neighbors also in the set. Given a graph with a quality field on the vertices and edges connecting similar items, for each item, this algorithm either samples from a distribution formed by item qualities or selects the item with the maximum quality, and then "covers" (or removes from consideration) that item's neighbors. There are two options depending on whether you pass in an SGraph or an SFrame. You can: 1. Define similarity by passing in an SGraph, where an edge between two vertices denotes the fact that those items are neighbors. Any time a point is sampled, all of its neighbors in the graph are removed. 2. Or, you can pass in an SFrame and the additional keyword argument "num_neighbors". Then when a point is sampled, its num_neighbors nearest-neighbors will be removed from consideration. Requirements: The parameters quality_feature and similarity_features must be defined (where they either match column names or vertex and edge fields). If no sampling_method is given, then the default option is to use the weighted_vertex_cover algorithm with a diversity factor of 0.1. side_data: sframe, optional An ID-based subset of the original data to sample from. Sometimes you may wish to sample from only a subset of the original data - e.g., only provide a diverse sample of movies from a particular user's top recommendations. In addition, some features may not be initially available when creating the sampler object. In order to sample from a subset of IDs, with the option to add additional features, set side_data to an SFrame with a column of IDs and (optionally) additional features. Note that the model must be aware of these initial features when creating it by adding the column names for the side-quality or side-similarity features. The sampler will first subset the groundset by the list of IDs passed. Then the sampler will use any updated or additional quality or similarity features in side_data. If some feature is not available in side_data, the sampler uses the original features in the SFrame or SGraph passed in with create(). If side_data is empty, then the sampler will return subsets from the original SFrame or SGraph passed in with the data parameter used in create(). similarity_function : string TODO: I haven't added this yet **kwargs : optional Additional method-specific parameters for fine-tuning. - *wvc_neighbors*: For sampling_method=weighted_vertex_cover and a sampler constructed with an SFrame, remove num_neighbors when a point is sampled. Examples -------- Sample k items directly from the ground set passed in via create() with the default sampling methods: >>> sf = graphlab.SFrame.read_csv( 'https://s3.amazonaws.com/dato-datasets/auto-mpg/auto-mpg.csv') >>> sampler = graphlab.diversity.diverse_sampler.create(data=sf, item_id='name', quality_feature='accel', similarity_features=['mpg', 'displ', 'hp', 'weight']) >>> sampler.sample(k=5) +-----+-----+-------+-----+--------+-------+----+--------+----------------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+-----+--------+-------+----+--------+----------------------+ | 15 | 8 | 318.0 | 150 | 3777 | 12.5 | 73 | 1 | dodge coronet custom | | 15 | 6 | 258.0 | 110 | 3730 | 19.0 | 75 | 1 | amc matador | | 30 | 4 | 97.0 | 67 | 1985 | 16.4 | 77 | 3 | subaru dl | | 34 | 4 | 86.0 | 65 | 1975 | 15.2 | 79 | 3 | maxda glc deluxe | | 32 | 4 | 98.0 | 70 | 2120 | 15.5 | 80 | 1 | chevrolet chevette | +-----+-----+-------+-----+--------+-------+----+--------+----------------------+ This method returns an SFrame (or SGraph, depending on what was used to create the sampler) containing the sampled items. If the diverse sampler was created with an SGraph, the sampler will return an SFrame containing the sampled vertices and their associated fields. You can change the sampling method with the "method" keyword. The default algorithm is weighted vertex cover. >>> sf = sampler.sample(k=5, method='ipsen') +-----+-----+-------+-----+--------+-------+----+--------+-----------------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+-----+--------+-------+----+--------+-----------------------+ | 15 | 8 | 350.0 | 165 | 3693 | 11.5 | 70 | 1 | buick skylark 320 | | 17 | 8 | 302.0 | 140 | 3449 | 10.5 | 70 | 1 | ford torino | | 15 | 8 | 400.0 | 150 | 3761 | 9.5 | 70 | 1 | chevrolet monte carlo | | 22 | 6 | 198.0 | 95 | 2833 | 15.5 | 70 | 1 | plymouth duster | | 19 | 6 | 232.0 | 100 | 2634 | 13.0 | 71 | 1 | amc gremlin | +-----+-----+-------+-----+--------+-------+----+--------+-----------------------+ Instead of stochastic sampling, you can also force the algorithm to try to form the best possible set by using the greedy method: >>> sf = sampler.sample(k=5, greedy=True) It's possible to tune the methods with the "diversity" keyword, which can range between 0 and 1. Larger values will favor reducing inter-item similarity (increasing diversity), while smaller values will favor high- quality items (decreasing diversity). >>> sf = sampler.sample(k=5, diversity=0.0, method='ipsen') +-----+-----+-------+-----+--------+-------+----+--------+--------------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+-----+--------+-------+----+--------+--------------------+ | 14 | 8 | 440.0 | 215 | 4312 | 8.5 | 70 | 1 | plymouth fury iii | | 15 | 8 | 390.0 | 190 | 3850 | 8.5 | 70 | 1 | amc ambassador dpl | | 18 | 6 | 199.0 | 97 | 2774 | 15.5 | 70 | 1 | amc hornet | | 18 | 6 | 232.0 | 100 | 3288 | 15.5 | 71 | 1 | amc matador | | 11 | 8 | 429.0 | 208 | 4633 | 11.0 | 72 | 1 | mercury marquis | +-----+-----+-------+-----+--------+-------+----+--------+--------------------+ >>> sf = sampler.sample(k=5, diversity=1.0, method='ipsen') +-----+-----+-------+-----+--------+-------+----+--------+---------------------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+-----+--------+-------+----+--------+---------------------------+ | 18 | 8 | 307.0 | 130 | 3504 | 12.0 | 70 | 1 | chevrolet chevelle malibu | | 15 | 8 | 350.0 | 165 | 3693 | 11.5 | 70 | 1 | buick skylark 320 | | 18 | 8 | 318.0 | 150 | 3436 | 11.0 | 70 | 1 | plymouth satellite | | 16 | 8 | 304.0 | 150 | 3433 | 12.0 | 70 | 1 | amc rebel sst | | 18 | 6 | 171.0 | 97 | 2984 | 14.5 | 75 | 1 | ford pinto | +-----+-----+-------+-----+--------+-------+----+--------+---------------------------+ Finally, if you want to restrict the ground set to a smaller subset, you can pass in a list of IDs with the "side_data" keyword: >>> ford_sf = sf[sf['name'].apply(lambda x: 'ford' in x)]['name'] >>> sampler.sample(k=5, side_data=graphlab.SFrame({'name':names_sf})) +-----------------------+-----+-----+-------+-----+--------+-------+----+--------+ | name | mpg | cyl | displ | hp | weight | accel | yr | origin | +-----------------------+-----+-----+-------+-----+--------+-------+----+--------+ | ford pinto runabout | 21 | 4 | 122.0 | 86 | 2226 | 16.5 | 72 | 1 | | ford maverick | 18 | 6 | 250.0 | 88 | 3021 | 16.5 | 73 | 1 | | ford gran torino (sw) | 14 | 8 | 302.0 | 140 | 4638 | 16.0 | 74 | 1 | | ford fairmont (auto) | 20 | 6 | 200.0 | 85 | 2965 | 15.8 | 78 | 1 | | ford ltd landau | 17 | 8 | 302.0 | 129 | 3725 | 13.4 | 79 | 1 | +-----------------------+-----+-----+-------+-----+--------+-------+----+--------+ You can also add updated features, or even features that weren't passed in when creating the model (as long as they are one of the features specified in "quality_feature" or "similarity_featurs"). These new features will be joined to the original dataset. However, if any new features were not specified in the "similarity_features" parameter during sampler creation, they will not be included when computing similarity between items. """ _raise_error_if_not_of_type(k, int) if side_data is not None: _raise_error_if_not_of_type(side_data, _gl.SFrame) opts = dict() if method is not None: opts["method"] = method if diversity < 0.0 or diversity > 1.0: raise ValueError("The diversity parameter must be between 0.0 and 1.0.") opts["diversity"] = diversity if "wvc_neighbors" in kwargs.keys(): opts["num_neighbors"] = kwargs["wvc_neighbors"] if "greedy" in kwargs.keys(): opts["greedy"] = kwargs["greedy"] if side_data is None: return self.__proxy__.sample_from_ground_set(k, opts) else: return self.__proxy__.sample_from_frame_ref_data(k, side_data, opts)
def create(graph, label_field, threshold=1e-3, weight_field='', self_weight=1.0, undirected=False, max_iterations=None, _single_precision=False, _distributed='auto', verbose=True): """ Given a weighted graph with observed class labels of a subset of vertices, infer the label probability for the unobserved vertices using the "label propagation" algorithm. The algorithm iteratively updates the label probability of current vertex as a weighted sum of label probability of self and the neighboring vertices until converge. See :class:`graphlab.label_propagation.LabelPropagationModel` for the details of the algorithm. Parameters ---------- graph : SGraph The graph on which to compute the label propagation. label_field: str Vertex field storing the initial vertex labels. The values in must be [0, num_classes). None values indicate unobserved vertex labels. threshold : float, optional Threshold for convergence, measured in the average L2 norm (the sum of squared values) of the delta of each vertex's label probability vector. max_iterations: int, optional The max number of iterations to run. Default is unlimited. If set, the algorithm terminates when either max_iterations or convergence threshold is reached. weight_field: str, optional Vertex field for edge weight. If empty, all edges are assumed to have unit weight. self_weight: float, optional The weight for self edge. undirected: bool, optional If true, treat each edge as undirected, and propagates label in both directions. _single_precision : bool, optional If true, running label propagation in single precision. The resulting probability values may less accurate, but should run faster and use less memory. _distributed : distributed environment, internal verbose : bool, optional If True, print progress updates. Returns ------- out : LabelPropagationModel References ---------- - Zhu, X., & Ghahramani, Z. (2002). `Learning from labeled and unlabeled data with label propagation <http://www.cs.cmu.edu/~zhuxj/pub/CMU-CALD-02-107.pdf>`_. Examples -------- If given an :class:`~graphlab.SGraph` ``g``, we can create a :class:`~graphlab.label_propagation.LabelPropagationModel` as follows: >>> g = graphlab.load_graph('http://snap.stanford.edu/data/email-Enron.txt.gz', ... format='snap') # Initialize random classes for a subset of vertices # Leave the unobserved vertices with None label. >>> import random >>> def init_label(vid): ... x = random.random() ... if x < 0.2: ... return 0 ... elif x > 0.9: ... return 1 ... else: ... return None >>> g.vertices['label'] = g.vertices['__id'].apply(init_label, int) >>> m = graphlab.label_propagation.create(g, label_field='label') We can obtain for each vertex the predicted label and the probability of each label in the graph ``g`` using: >>> labels = m['labels'] # SFrame >>> labels +------+-------+-----------------+-------------------+----------------+ | __id | label | predicted_label | P0 | P1 | +------+-------+-----------------+-------------------+----------------+ | 5 | 1 | 1 | 0.0 | 1.0 | | 7 | None | 0 | 0.8213214997 | 0.1786785003 | | 8 | None | 1 | 5.96046447754e-08 | 0.999999940395 | | 10 | None | 0 | 0.534984718273 | 0.465015281727 | | 27 | None | 0 | 0.752801638549 | 0.247198361451 | | 29 | None | 1 | 5.96046447754e-08 | 0.999999940395 | | 33 | None | 1 | 5.96046447754e-08 | 0.999999940395 | | 47 | 0 | 0 | 1.0 | 0.0 | | 50 | None | 0 | 0.788279032657 | 0.211720967343 | | 52 | None | 0 | 0.666666666667 | 0.333333333333 | +------+-------+-----------------+-------------------+----------------+ [36692 rows x 5 columns] See Also -------- LabelPropagationModel """ _mt._get_metric_tracker().track('toolkit.graph_analytics.label_propagation.create') _raise_error_if_not_of_type(label_field, str) _raise_error_if_not_of_type(weight_field, str) if not isinstance(graph, _SGraph): raise TypeError('graph input must be a SGraph object.') if graph.vertices[label_field].dtype() != int: raise TypeError('label_field %s must be integer typed.' % label_field) opts = {'label_field': label_field, 'threshold': threshold, 'weight_field': weight_field, 'self_weight': self_weight, 'undirected': undirected, 'max_iterations': max_iterations, 'single_precision': _single_precision, 'graph': graph.__proxy__} distributed_context = _get_distributed_execution_environment() if distributed_context is None: params = _main.run('label_propagation', opts, verbose) model = params['model'] else: model = _distributed_run('distributed_labelprop', opts, env=_distributed, verbose=verbose) return LabelPropagationModel(model)
def sample(self, k, diversity=0.1, subset_ids=None, **kwargs): """ After constructing a diverse sampler, sample a diverse set stochastically. The stochastic algorithm depends on the sampling method itself. Parameters ---------- k : int The number of items to sample. diversity : double in [0, 1], optional This is a tunable parameter that trades off between quality and diversity. A diversity factor of 0 will only consider quality when building a set, while a diversity factor of 1 will only consider item similarity and will ignore quality. A value between 0 and 1 will force the algorithm to trade off between quality and diversity. Note that this keyword argument is only applicable if both quality and similarity features were passed to create(). The actual effect of the diversity factor depends on the algorithm: - When the method is vertex cover or weighted vertex cover, the diversity factor changes the number of nearest-neighbors to remove when sampling an item. Specifically, the number of neighbors is set to the value floor( (N-1)/(k-1) * diversity_factor). subset_ids: SArray, optional A list of IDs to sample from. Sometimes you may wish to sample from only a subset of the original data - e.g., only provide a diverse sample of movies from a particular user's top recommendations. If subset_ids is empty, then the sampler will return subsets from the original SFrame or SGraph passed in with the data parameter used in create(). **kwargs : optional Additional method-specific parameters for fine-tuning. - *greedy*: Use the greedy algorithm to generate a set. Instead of stochastically building a set based on a distribution, for each item, take the mode of the current distribution. For instance, if only quality features are being considered, using the greedy option will return the top-k items. Usually the greedy algorithm provides the highest-quality and most-diverse set, but for each set of items and algorithm, there is only one set that greedy can generate. Based on which features were given to create(), different sampling methods will be used. One of the four following algorithms are chosen based on the initial feature set. - *"random"*: If no quality or similarity features are given. Returns a completely random set of items, with no reference to item qualities or similarities. Note that the greedy method is undefined for a random sampler, so it is ignored. - *"quality-only"*: If only a quality feature are given. Generate a distribution over items based on their quality, and sample from this distribution. If greedy is specified, the top-k items in terms of quality are returned. - *"vertex-cover"*: If only similarity features are given. An internal graph is generated if an SFrame is given, and each item is connected to it's k-nearest neighbors, where k is determined by the diversity factor. When an item is sampled at random, its neighbors are removed from the candidate set. If an SGraph is given initially, all vertices connected to a sampled point are removed. Note that the greedy method for this algorithm is undefined, so it is ignored. - *"weighted_vertex_cover"*: The same as vertex cover, except each vertex has an associated quality field. When selecting the next point, it is sampled from a distribution over the remaining points' qualities. If greedy is specified, then the next point is the point with the highest quality in the remaining points. Examples -------- Sample k items directly from the reference set passed in via create() with the default sampling methods: >>> cars = graphlab.SFrame.read_csv('https://s3.amazonaws.com/dato-datasets/auto-mpg/auto-mpg.csv') >>> sampler = graphlab.diverse_sampler.create(data=cars, item_id='name', quality_feature='accel', similarity_features=['mpg', 'displ', 'hp', 'weight', 'origin']) >>> sampler.sample(k=5) +-----+-----+-------+-----+--------+-------+----+--------+----------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+-----+--------+-------+----+--------+----------------+ | 26 | 4 | 121.0 | 113 | 2234 | 12.5 | 70 | 2 | bmw 2002 | | 18 | 6 | 232.0 | 100 | 2945 | 16.0 | 73 | 1 | amc hornet | | 24 | 4 | 116.0 | 75 | 2158 | 15.5 | 73 | 2 | opel manta | | 36 | 4 | 98.0 | 70 | 2125 | 17.3 | 82 | 1 | mercury lynx l | | 44 | 4 | 97.0 | 52 | 2130 | 24.6 | 82 | 2 | vw pickup | +-----+-----+-------+-----+--------+-------+----+--------+----------------+ This method returns an SFrame (or SGraph, depending on what was used to create the sampler) containing the sampled items. If the diverse sampler was created with an SGraph, the sampler will return an SFrame containing the sampled vertices and their associated fields. Instead of stochastic sampling, you can also force the algorithm to try to form the best possible set by using the greedy method: >>> sampler.sample(k=5, greedy=True) +-----+-----+-------+----+--------+-------+----+--------+-------------------------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+----+--------+-------+----+--------+-------------------------------+ | 19 | 4 | 120.0 | 88 | 3270 | 21.9 | 76 | 2 | peugeot 504 | | 27 | 4 | 141.0 | 71 | 3190 | 24.8 | 79 | 2 | peugeot 504 | | 23 | 8 | 260.0 | 90 | 3420 | 22.2 | 79 | 1 | oldsmobile cutlass salon b... | | 43 | 4 | 90.0 | 48 | 2335 | 23.7 | 80 | 2 | vw dasher (diesel) | | 44 | 4 | 97.0 | 52 | 2130 | 24.6 | 82 | 2 | vw pickup | +-----+-----+-------+----+--------+-------+----+--------+-------------------------------+ In this example, two Peugeot cars were selected. Although they were somewhat different based on the original similarity features we specified, it's possible to get an even more diverse sample. To increase diversity, the "diversity" keyword (which can range between 0 and 1) can be increased. Larger values will favor reducing inter-item similarity (increasing diversity), while smaller values will favor high- quality items (decreasing diversity). >>> sampler.sample(k=5, diversity=0.8, greedy=True) +-----+-----+-------+-----+--------+-------+----+--------+-------------------------------+ | mpg | cyl | displ | hp | weight | accel | yr | origin | name | +-----+-----+-------+-----+--------+-------+----+--------+-------------------------------+ | 27 | 4 | 97.0 | 60 | 1834 | 19.0 | 71 | 2 | volkswagen model 111 | | 32 | 4 | 71.0 | 65 | 1836 | 21.0 | 74 | 3 | toyota corolla 1200 | | 17 | 6 | 231.0 | 110 | 3907 | 21.0 | 75 | 1 | buick century | | 27 | 4 | 141.0 | 71 | 3190 | 24.8 | 79 | 2 | peugeot 504 | | 23 | 8 | 260.0 | 90 | 3420 | 22.2 | 79 | 1 | oldsmobile cutlass salon b... | +-----+-----+-------+-----+--------+-------+----+--------+-------------------------------+ Finally, if you want to restrict the reference set to a smaller subset, you can pass in a list of IDs with the "subset_ids" keyword: >>> ford_names = gl.SArray([n for n in cars['name'] if 'ford' in n]) >>> sampler.sample(5, diversity=1.0, subset_ids=ford_names) +-----------------------+-----+-----+-------+-----+--------+-------+----+--------+ | name | mpg | cyl | displ | hp | weight | accel | yr | origin | +-----------------------+-----+-----+-------+-----+--------+-------+----+--------+ | ford gran torino (sw) | 13 | 8 | 302.0 | 140 | 4294 | 16.0 | 72 | 1 | | ford maverick | 15 | 6 | 250.0 | 72 | 3158 | 19.5 | 75 | 1 | | ford fiesta | 36 | 4 | 98.0 | 66 | 1800 | 14.4 | 78 | 1 | | ford escort 2h | 29 | 4 | 98.0 | 65 | 2380 | 20.7 | 81 | 1 | | ford fairmont futura | 24 | 4 | 140.0 | 92 | 2865 | 16.4 | 82 | 1 | +-----------------------+-----+-----+-------+-----+--------+-------+----+--------+ """ _raise_error_if_not_of_type(k, int) if subset_ids is not None: _raise_error_if_not_of_type(subset_ids, _gl.SArray) if diversity < 0.0 or diversity > 1.0: raise ValueError("The diversity parameter must be between 0.0 and 1.0.") if k <= 0: raise ValueError("k must be greater than 0.") opts = dict() opts["diversity"] = diversity if "wvc_neighbors" in kwargs.keys(): opts["num_neighbors"] = kwargs["wvc_neighbors"] if "greedy" in kwargs.keys(): opts["greedy"] = kwargs["greedy"] if subset_ids is None: return self.__proxy__.sample_from_ground_set(k, opts) else: return self.__proxy__.sample_from_id_subset(k, subset_ids, opts)
def create(data, features=None, bm25_k1=1.5, bm25_b=0.75, tfidf_threshold=0.01): """ Create a searchable index of text columns in an SFrame. .. warning:: This toolkit is currently in beta, and feedback is welcome! Please send comments to [email protected]. Parameters ---------- data : SFrame An SFrame containing at least one str column containing text that should be indexed. features : list of str A list of column names that contain text that should be indexed. Default: all str columns in the provided dataset. bm25_k1 : float Tuning parameter for the relative importance of term frequencies when computing the BM25 score between a query token and a document. bm25_b : float Tuning parameter to downweight scores of long documents when computing the BM25 score between a query token and a document. tfidf_threshold : float Tuning parameter to skip indexing words that have a TF-IDF score below this value. query_expansion_k : int Maximum number of nearest words to include from query token. query_expansion_epsilon : float Maximum distance to allow between query token and nearby word when doing query expansion. Must be between 0 and 1. query_expansion_near_match_weight : float Multiplier to use on BM25 scores for documents indexed via an approximate match with a given token. Must be between 0 and 1. Returns ------- out SearchModel See Also -------- SearchModel.query References ---------- Christopher D. Manning, Hinrich Schutze, and Prabhakar Raghavan. Introduction to information retrieval. http://nlp.stanford.edu/IR-book/pdf/irbookonlinereading.pdf Examples -------- >>> import graphlab as gl >>> sf = gl.SFrame({'text': ['Hello my friend', 'I love this burrito']}) >>> m = gl._internal.search.create(sf) >>> print m.query('burrito') """ # Input validation on data and features if features is None: features = _get_str_columns(data) if not isinstance(features, list): raise ValueError("Expected features to be a list.") _raise_error_if_not_of_type(data, [_gl.SFrame]) _raise_error_if_not_of_type(features, [list]) for f in features: if data[f].dtype() != str: raise _ToolkitError("Feature `%s` must be of type str" % f) # Store options options = {} options['bm25_b'] = bm25_b options['bm25_k1'] = bm25_k1 options['tfidf_threshold'] = tfidf_threshold # Construct model proxy = _gl.extensions._SearchIndex() proxy.init_options(options) proxy.init_indexer(data) for f in features: proxy.index(f) return SearchModel(proxy)
def __init__(self, features=None, excluded_features=None, n=2, method="word", to_lower=True, ignore_punct=True, ignore_space=True, delimiters=["\r", "\v", "\n", "\f", "\t", " ", "!", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~"], output_column_prefix=None): # Process and make a copy of the features, exclude. _features, _exclude = _internal_utils.process_features(features, excluded_features) # Type checking _raise_error_if_not_of_type(features, [list, str, _NoneType]) _raise_error_if_not_of_type(excluded_features, [list, str, _NoneType]) _raise_error_if_not_of_type(n, [int]) _raise_error_if_not_of_type(method, [str]) _raise_error_if_not_of_type(to_lower, [bool]) _raise_error_if_not_of_type(ignore_punct, [bool]) _raise_error_if_not_of_type(ignore_space, [bool]) _raise_error_if_not_of_type(delimiters, [list, _NoneType]) _raise_error_if_not_of_type(output_column_prefix, [str, _NoneType]) if delimiters != None: for delim in delimiters: _raise_error_if_not_of_type(delim, str, "delimiters") if (len(delim) != 1): raise ValueError("Delimiters must be single-character strings") if n < 1: raise ValueError("Input 'n' must be greater than 0") if n > 5 and method == 'word': warnings.warn("It is unusual for n-grams to be of size larger than 5.") if method != "word" and method != "character": raise ValueError("Invalid 'method' input value. Please input " + "either 'word' or 'character' ") # Set up options opts = { 'n': n, 'features': features, 'ngram_type': method, 'to_lower': to_lower, 'ignore_punct': ignore_punct, 'ignore_space': ignore_space, 'delimiters': delimiters, 'output_column_prefix' : output_column_prefix } if _exclude: opts['exclude'] = True opts['features'] = _exclude else: opts['exclude'] = False opts['features'] = _features # Initialize object proxy = _gl.extensions._NGramCounter() proxy.init_transformer(opts) super(NGramCounter, self).__init__(proxy, self.__class__)