def rake(
string: str,
model=None,
vectorizer=None,
top_k: int = 5,
atleast: int = 1,
stopwords=get_stopwords,
**kwargs,
):
"""
Extract keywords using Rake algorithm.
Parameters
----------
string: str
model: Object, optional (default=None)
Transformer model or any model has `attention` method.
vectorizer: Object, optional (default=None)
Prefer `sklearn.feature_extraction.text.CountVectorizer` or,
`malaya.text.vectorizer.SkipGramCountVectorizer`.
If None, will generate ngram automatically based on `stopwords`.
top_k: int, optional (default=5)
return top-k results.
ngram: tuple, optional (default=(1,1))
n-grams size.
atleast: int, optional (default=1)
at least count appeared in the string to accept as candidate.
stopwords: List[str], (default=malaya.texts.function.get_stopwords)
A callable that returned a List[str], or a List[str], or a Tuple[str]
For automatic Ngram generator.
Returns
-------
result: Tuple[float, str]
"""
stopwords = validator.validate_stopwords(stopwords)
if model is not None:
if not hasattr(model, 'attention'):
raise ValueError('model must have `attention` method')
if top_k < 1:
raise ValueError('top_k must bigger than 0')
if atleast < 1:
raise ValueError('atleast must bigger than 0')
if not vectorizer:
auto_ngram = True
else:
auto_ngram = False
if not hasattr(vectorizer, 'fit'):
raise ValueError('vectorizer must have `fit` method')
if auto_ngram and not len(stopwords):
raise ValueError('insert stopwords if auto_ngram')
if model:
string = transformer_textcleaning(string)
attention = model.attention([string])[0]
d = defaultdict(float)
for k, v in attention:
d[k] += v
else:
d = None
if auto_ngram:
vocab = _auto_ngram(string, stopwords)
else:
vocab = _base(string, vectorizer=vectorizer, **kwargs)
phrase_list = list(vocab.keys())
scores = rake_function.calculate_word_scores(phrase_list, attentions=d)
keywordcandidates = rake_function.generate_candidate_keyword_scores(
phrase_list, scores)
sortedKeywords = sorted(keywordcandidates.items(),
key=operator.itemgetter(1),
reverse=True)
total = sum([i[1] for i in sortedKeywords])
ranked_sentences = [(i[1] / total, i[0]) for i in sortedKeywords
if vocab[i[0]] >= atleast]
return ranked_sentences[:top_k]
def rake(string: str,
model=None,
top_k: int = 5,
auto_ngram: bool = True,
ngram_method: str = 'bow',
ngram: Tuple[int, int] = (1, 1),
atleast: int = 1,
stop_words: List[str] = STOPWORDS,
**kwargs):
"""
Extract keywords using Rake algorithm.
Parameters
----------
string: str
model: Object, optional (default='None')
Transformer model or any model has `attention` method.
top_k: int, optional (default=5)
return top-k results.
auto_ngram: bool, optional (default=True)
If True, will generate keyword candidates using N suitable ngram. Else use `ngram_method`.
ngram_method: str, optional (default='bow')
Only usable if `auto_ngram` is False. supported ngram generator:
* ``'bow'`` - bag-of-word.
* ``'skipgram'`` - bag-of-word with skip technique.
ngram: tuple, optional (default=(1,1))
n-grams size.
atleast: int, optional (default=1)
at least count appeared in the string to accept as candidate.
stop_words: list, (default=malaya.text.function.STOPWORDS)
list of stop words to remove.
Returns
-------
result: Tuple[float, str]
"""
if model is not None:
if not hasattr(model, 'attention'):
raise ValueError('model must has or `attention` method')
if top_k < 1:
raise ValueError('top_k must bigger than 0')
if atleast < 1:
raise ValueError('atleast must bigger than 0')
if ngram_method not in ('bow', 'skipgram'):
raise ValueError("ngram_method must be in ['bow', 'skip-gram']")
if auto_ngram and not len(stop_words):
raise ValueError('insert stop_words if auto_ngram')
if model:
string = transformer_textcleaning(string)
attention = model.attention([string])[0]
d = defaultdict(float)
for k, v in attention:
d[k] += v
else:
d = None
if auto_ngram:
vocab = _auto_ngram(string, stop_words)
else:
vocab = _base(string,
ngram_method=ngram_method,
ngram=ngram,
stop_words=stop_words,
**kwargs)
phrase_list = list(vocab.keys())
scores = rake_function.calculate_word_scores(phrase_list, attentions=d)
keywordcandidates = rake_function.generate_candidate_keyword_scores(
phrase_list, scores)
sortedKeywords = sorted(keywordcandidates.items(),
key=operator.itemgetter(1),
reverse=True)
total = sum([i[1] for i in sortedKeywords])
ranked_sentences = [(i[1] / total, i[0]) for i in sortedKeywords
if vocab[i[0]] >= atleast]
return ranked_sentences[:top_k]