def predict_phishing_words(model_name, model_store_type, email_subject, email_body, min_text_length, label_threshold,
                           word_threshold, top_word_limit, is_return_error):
    model_data = get_model_data(model_name, model_store_type)
    model = demisto_ml.decode_model(model_data)
    text = "%s %s" % (email_subject, email_body)
    res = demisto.executeCommand('WordTokenizerNLP', {'value': text,
                                                      'hashWordWithSeed': demisto.args().get('hashSeed')})
    if is_error(res[0]):
        return_error(res[0]['Contents'])
    tokenized_text_result = res[0]['Contents']
    input_text = tokenized_text_result['hashedTokenizedText'] if tokenized_text_result.get('hashedTokenizedText') else \
        tokenized_text_result['tokenizedText']
    filtered_text, filtered_text_number_of_words = demisto_ml.filter_model_words(input_text, model)
    if filtered_text_number_of_words == 0:
        handle_error("The model does not contains any of the input text words", is_return_error)
    if filtered_text_number_of_words < min_text_length:
        handle_error("The model contains less then %d words" % min_text_length, is_return_error)

    explain_result = demisto_ml.explain_model_words(model,
                                                    input_text,
                                                    0,
                                                    word_threshold,
                                                    top_word_limit)
    if explain_result["Probability"] < label_threshold:
        handle_error("Label probability is %.2f and it's below the input threshold", is_return_error)

    if tokenized_text_result.get('hashedTokenizedText'):
        hash_word_to_plain = dict(
            zip(tokenized_text_result['hashedTokenizedText'].split(" "),
                tokenized_text_result['tokenizedText'].split(" ")))
        explain_result['PositiveWords'] = map(lambda x: hash_word_to_plain[x], explain_result['PositiveWords'])
        explain_result['NegativeWords'] = map(lambda x: hash_word_to_plain[x], explain_result['NegativeWords'])
    explain_result['OriginalText'] = tokenized_text_result['originalText']
    explain_result['TextTokensHighlighted'] = tokenized_text_result['tokenizedText']

    res = demisto.executeCommand('HighlightWords', {'text': tokenized_text_result['tokenizedText'],
                                                    'terms': ",".join(explain_result['PositiveWords'])})
    res = res[0]
    if not is_error(res):
        highlighted_text_markdown = res['Contents']
        explain_result['TextTokensHighlighted'] = highlighted_text_markdown
    explain_result_hr = dict(explain_result)
    explain_result_hr['PositiveWords'] = ", ".join(explain_result_hr['PositiveWords'])
    explain_result_hr['NegativeWords'] = ", ".join(explain_result_hr['NegativeWords'])
    explain_result_hr['Probability'] = "%.2f" % explain_result_hr['Probability']
    return {
        'Type': entryTypes['note'],
        'Contents': explain_result,
        'ContentsFormat': formats['json'],
        'HumanReadable': tableToMarkdown('DBot Predict Phishing Words', explain_result_hr,
                                         headers=['TextTokensHighlighted', 'Label', 'Probability',
                                                  'PositiveWords', 'NegativeWords'],
                                         removeNull=True),
        'HumanReadableFormat': formats['markdown'],
        'EntryContext': {
            'DBotPredictPhishingWords': explain_result
        }
    }
def predict_phishing_words(model_name, model_store_type, email_subject, email_body):
    model_data = get_model_data(model_name, model_store_type)
    model = demisto_ml.decode_model(model_data)
    text = "%s %s" % (email_subject, email_body)
    res = demisto.executeCommand('WordTokenizerNLP', {'value': text,
                                                      'hashWordWithSeed': demisto.args().get('hashSeed')})
    if is_error(res[0]):
        return_error(res[0]['Contents'])
    tokenized_text_result = res[0]['Contents']
    input_text = tokenized_text_result['hashedTokenizedText'] if tokenized_text_result.get('hashedTokenizedText') else \
        tokenized_text_result['tokenizedText']
    filtered_text, filtered_text_number_of_words = demisto_ml.filter_model_words(input_text, model)
    if filtered_text_number_of_words == 0:
        return_error("The model does not contains any of the input text words")

    explain_result = demisto_ml.explain_model_words(model,
                                                    input_text,
                                                    float(demisto.args().get('labelProbabilityThreshold', 0)),
                                                    float(demisto.args().get('wordThreshold', 0)),
                                                    int(demisto.args()['topWordsLimit']))

    if tokenized_text_result.get('hashedTokenizedText'):
        hash_word_to_plain = dict(
            zip(tokenized_text_result['hashedTokenizedText'].split(" "),
                tokenized_text_result['tokenizedText'].split(" ")))
        explain_result['PositiveWords'] = map(lambda x: hash_word_to_plain[x], explain_result['PositiveWords'])
        explain_result['NegativeWords'] = map(lambda x: hash_word_to_plain[x], explain_result['NegativeWords'])
    explain_result['OriginalText'] = tokenized_text_result['originalText']
    explain_result['TextTokensHighlighted'] = tokenized_text_result['tokenizedText']

    res = demisto.executeCommand('HighlightWords', {'text': tokenized_text_result['tokenizedText'],
                                                    'terms': ",".join(explain_result['PositiveWords'])})
    res = res[0]
    if not is_error(res):
        highlighted_text_markdown = res['Contents']
        explain_result['TextTokensHighlighted'] = highlighted_text_markdown

    return {
        'Type': entryTypes['note'],
        'Contents': explain_result,
        'ContentsFormat': formats['json'],
        'HumanReadable': tableToMarkdown('DBot Predict Phihsing Words', explain_result,
                                         headers=['TextTokensHighlighted', 'Label', 'Probability',
                                                  'PositiveWords', 'NegativeWords'],
                                         removeNull=True),
        'HumanReadableFormat': formats['markdown'],
        'EntryContext': {
            'DBotPredictPhishingWords': explain_result
        }
    }
def predict_phishing_words(model_name,
                           model_store_type,
                           email_subject,
                           email_body,
                           min_text_length,
                           label_threshold,
                           word_threshold,
                           top_word_limit,
                           is_return_error,
                           set_incidents_fields=False):
    model_data = get_model_data(model_name, model_store_type, is_return_error)
    model = demisto_ml.decode_model(model_data)
    text = "%s %s" % (email_subject, email_body)
    res = demisto.executeCommand(
        'WordTokenizerNLP', {
            'value': text,
            'hashWordWithSeed': demisto.args().get('hashSeed')
        })
    if is_error(res[0]):
        handle_error(res[0]['Contents'], is_return_error)
    tokenized_text_result = res[0]['Contents']
    input_text = tokenized_text_result['hashedTokenizedText'] if tokenized_text_result.get('hashedTokenizedText') else \
        tokenized_text_result['tokenizedText']
    filtered_text, filtered_text_number_of_words = demisto_ml.filter_model_words(
        input_text, model)
    if filtered_text_number_of_words == 0:
        handle_error("The model does not contain any of the input text words",
                     is_return_error)
    if filtered_text_number_of_words < min_text_length:
        handle_error(
            "The model contains fewer than %d words" % min_text_length,
            is_return_error)

    explain_result = demisto_ml.explain_model_words(model, input_text, 0,
                                                    word_threshold,
                                                    top_word_limit)
    predicted_prob = explain_result["Probability"]
    if predicted_prob < label_threshold:
        handle_error(
            "Label probability is {:.2f} and it's below the input confidence threshold"
            .format(predicted_prob), is_return_error)

    if tokenized_text_result.get('hashedTokenizedText'):
        words_to_token_maps = tokenized_text_result['wordsToHashedTokens']
    else:
        words_to_token_maps = tokenized_text_result['originalWordsToTokens']
    positive_tokens = set([
        ''.join(c for c in word if c.isalnum())
        for word in explain_result['PositiveWords']
    ])
    negative_tokens = set([
        ''.join(c for c in word if c.isalnum())
        for word in explain_result['NegativeWords']
    ])
    positive_words = find_words_contain_tokens(positive_tokens,
                                               words_to_token_maps)
    negative_words = find_words_contain_tokens(negative_tokens,
                                               words_to_token_maps)
    positive_words = [s.strip(punctuation) for s in positive_words]
    negative_words = [s.strip(punctuation) for s in negative_words]

    if len(positive_words) > 0:
        res = demisto.executeCommand(
            'HighlightWords', {
                'text': tokenized_text_result['originalText'],
                'terms': ",".join(positive_words)
            })
        res = res[0]
        if not is_error(res):
            highlighted_text_markdown = res['Contents']
        else:
            highlighted_text_markdown = tokenized_text_result[
                'originalText'].strip()
    else:
        highlighted_text_markdown = tokenized_text_result[
            'originalText'].strip()

    explain_result['PositiveWords'] = positive_words
    explain_result['NegativeWords'] = negative_words
    explain_result['OriginalText'] = tokenized_text_result[
        'originalText'].strip()
    explain_result['TextTokensHighlighted'] = highlighted_text_markdown
    predicted_label = explain_result["Label"]

    explain_result_hr = dict()
    explain_result_hr['TextTokensHighlighted'] = highlighted_text_markdown
    explain_result_hr['Label'] = predicted_label
    explain_result_hr['Probability'] = "%.2f" % predicted_prob
    explain_result_hr['Confidence'] = "%.2f" % predicted_prob
    explain_result_hr['PositiveWords'] = ", ".join(positive_words)
    explain_result_hr['NegativeWords'] = ", ".join(negative_words)
    incident_context = demisto.incidents()[0]
    if not incident_context['isPlayground'] and set_incidents_fields:
        demisto.executeCommand(
            "setIncident", {
                'dbotprediction': predicted_label,
                'dbotpredictionprobability': predicted_prob,
                'dbottextsuggestionhighlighted': highlighted_text_markdown
            })
    return {
        'Type':
        entryTypes['note'],
        'Contents':
        explain_result,
        'ContentsFormat':
        formats['json'],
        'HumanReadable':
        tableToMarkdown('DBot Predict Phishing Words',
                        explain_result_hr,
                        headers=[
                            'TextTokensHighlighted', 'Label', 'Confidence',
                            'PositiveWords', 'NegativeWords'
                        ],
                        removeNull=True),
        'HumanReadableFormat':
        formats['markdown'],
        'EntryContext': {
            'DBotPredictPhishingWords': explain_result
        }
    }
def predict_phishing_words(model_name, model_store_type, email_subject,
                           email_body):
    model_data = get_model_data(model_name, model_store_type)
    model = demisto_ml.decode_model(model_data)
    text = "%s %s" % (email_subject, email_body)
    res = demisto.executeCommand(
        'WordTokenizerNLP', {
            'value': text,
            'hashWordWithSeed': demisto.args().get('hashSeed')
        })
    if is_error(res[0]):
        return_error(res[0]['Contents'])
    tokenized_text_result = res[0]['Contents']
    input_text = tokenized_text_result['hashedTokenizedText'] if tokenized_text_result.get('hashedTokenizedText') else \
        tokenized_text_result['tokenizedText']
    filtered_text, filtered_text_number_of_words = demisto_ml.filter_model_words(
        input_text, model)
    if filtered_text_number_of_words == 0:
        return_error("The model does not contains any of the input text words")

    explain_result = demisto_ml.explain_model_words(
        model, input_text,
        float(demisto.args().get('labelProbabilityThreshold', 0)),
        float(demisto.args().get('wordThreshold', 0)),
        int(demisto.args()['topWordsLimit']))

    if tokenized_text_result.get('hashedTokenizedText'):
        hash_word_to_plain = dict(
            zip(tokenized_text_result['hashedTokenizedText'].split(" "),
                tokenized_text_result['tokenizedText'].split(" ")))
        explain_result['PositiveWords'] = map(lambda x: hash_word_to_plain[x],
                                              explain_result['PositiveWords'])
        explain_result['NegativeWords'] = map(lambda x: hash_word_to_plain[x],
                                              explain_result['NegativeWords'])
    explain_result['OriginalText'] = tokenized_text_result['originalText']
    explain_result['TextTokensHighlighted'] = tokenized_text_result[
        'tokenizedText']

    res = demisto.executeCommand(
        'HighlightWords', {
            'text': tokenized_text_result['tokenizedText'],
            'terms': ",".join(explain_result['PositiveWords'])
        })
    res = res[0]
    if not is_error(res):
        highlighted_text_markdown = res['Contents']
        explain_result['TextTokensHighlighted'] = highlighted_text_markdown

    return {
        'Type':
        entryTypes['note'],
        'Contents':
        explain_result,
        'ContentsFormat':
        formats['json'],
        'HumanReadable':
        tableToMarkdown('DBot Predict Phihsing Words',
                        explain_result,
                        headers=[
                            'TextTokensHighlighted', 'Label', 'Probability',
                            'PositiveWords', 'NegativeWords'
                        ],
                        removeNull=True),
        'HumanReadableFormat':
        formats['markdown'],
        'EntryContext': {
            'DBotPredictPhishingWords': explain_result
        }
    }