def classifier_status(url, username, password, classifier_ids):
n = NaturalLanguageClassifier(url=url,
username=username,
password=password)
for classifier_id in classifier_ids:
status = n.status(classifier_id)
print("%s: %s" % (status["status"], status["status_description"]))
def __initialize(self, credential_file_path):
if not credential_file_path:
credential_file_path = os.path.expanduser(DEFAULT_CREDENTIAL_PATH)
with open(credential_file_path, 'r') as credential_file:
credential = json.load(credential_file)
self.__nlc = NaturalLanguageClassifier(url=credential['url'], username=credential['username'], password=credential['password'])
def train_nlc(url, username, password, truth, name):
logger.info("Train model %s with %d instances" % (name, len(truth)))
with tempfile.TemporaryFile() as training_file:
# NLC cannot handle newlines.
truth[QUESTION] = truth[QUESTION].str.replace("\n", " ")
to_csv(training_file, truth[[QUESTION, ANSWER_ID]], header=False, index=False)
training_file.seek(0)
nlc = NaturalLanguageClassifier(url=url, username=username, password=password)
r = nlc.create(training_data=training_file, name=name)
logger.info(pretty_print_json(r))
return r["classifier_id"]
def nlc_router_train(url, username, password, oracle_out, path, all_correct):
"""
NLC Training on the oracle experiment output to determine which system(NLC or Solr) should
answer particular question.
1. Splitting up the oracle experiment output data into 8 equal training records and testing records. This is to
ensure 8-fold cross validation of the data-set. All training and Testing files will be stored
at the "path"
2. Perform NLC training on the all 8 training set simultaneously and returns list of classifier
ids as json file in the working directory
:param url: URL of NLC instance
:param username: NLC Username
:param password: NLC password
:param oracle_out: file created by oracle experiment
:param path: directory path to save intermediate results
:param all_correct: optional boolean parameter to train with only correct QA pairs
:return: list of classifier ids by NLC training
"""
ensure_directory_exists(path)
sys_name = oracle_out[SYSTEM][0]
oracle_out[QUESTION] = oracle_out[QUESTION].str.replace("\n", " ")
kfold_split(oracle_out, path, NLC_ROUTER_FOLDS, True)
classifier_list = []
list = []
for x in range(0, NLC_ROUTER_FOLDS):
train = pandas.read_csv(os.path.join(path, "Train{0}.csv".format(str(x))))
if all_correct:
logger.info("Training only on CORRECT examples.")
# Ignore records from training which are not correct
train = train[train[CORRECT]]
train = train[train[IN_PURVIEW]]
train = train[[QUESTION, ANSWERING_SYSTEM]]
logger.info("Training set size = {0}".format(str(len(train))))
with tempfile.TemporaryFile() as training_file:
to_csv(training_file, train[[QUESTION, ANSWERING_SYSTEM]], header=False, index=False)
training_file.seek(0)
nlc = NaturalLanguageClassifier(url=url, username=username, password=password)
classifier_id = nlc.create(training_data=training_file, name="{0}_fold_{1}".format(str(sys_name), str(x)))
classifier_list.append(classifier_id["classifier_id"].encode("utf-8"))
list.append({classifier_id["name"].encode("utf-8"): classifier_id["classifier_id"].encode("utf-8")})
logger.info(pretty_print_json(classifier_id))
pretty_print_json(classifier_id)
with open(os.path.join(path, 'classifier.json'), 'wb') as f:
json.dump(list, f)
return classifier_list
def __init__(self):
self.fname = ""
self.modelSearchList = ModelSearchList()
self.text_data = []
self.target_label = []
self.watson_crediantial = watson_key()
#self.watson_classifier = self.watson_crediantial.classifier_twitter_classfier
#self.watson_classifier = self.watson_crediantial.classifier_twitter_hash_classfier
#self.watson_classifier = self.watson_crediantial.classifier_twitter_unblance_keyword_classfier
#self.watson_classifier = self.watson_crediantial.twitter_priority_classfier
self.watson_classifier = self.watson_crediantial.twitter_category_classfier
self.natural_language_classifier = NaturalLanguageClassifier(
username=self.watson_crediantial.username,
password=self.watson_crediantial.password)
def classifier_list(url, username, password):
connection = NaturalLanguageClassifier(url=url, username=username, password=password)
return connection.list()["classifiers"]
def __init__(self, url, username, password, classifier_id, corpus):
self.nlc = NaturalLanguageClassifier(url=url, username=username, password=password)
self.classifier_id = classifier_id
self.corpus = corpus
def remove_classifiers(url, username, password, classifier_ids):
n = NaturalLanguageClassifier(url=url, username=username, password=password)
for classifier_id in classifier_ids:
n.remove(classifier_id)
def classifier_status(url, username, password, classifier_ids):
n = NaturalLanguageClassifier(url=url, username=username, password=password)
for classifier_id in classifier_ids:
status = n.status(classifier_id)
print(" Instance name: %s with classifier id %s is %s; Description: %s" % (status["name"],status["classifier_id"],status["status"], status["status_description"]))
import sys
import operator
import requests
import json
import twitter
from watson_developer_cloud import NaturalLanguageClassifierV1 as NaturalLanguageClassifier
#The IBM Bluemix credentials
nlc_username = '******'
nlc_password = '******'
natural_language_classifier = NaturalLanguageClassifier(username=nlc_username,
password=nlc_password)
classifierId = 'e4be4cx148-nlc-16'
classes = natural_language_classifier.remove(classifierId)
print(json.dumps(classes, indent=2))
def __init__(self, app):
super(NaturalLanguageClassifierUtils, self).__init__()
self.app = app
self.service = NaturalLanguageClassifier(
username='******',
password='******')
def __init__(self):
super(NaturalLanguageClassifierUtils, self).__init__()
self.service = NaturalLanguageClassifier(
username='******',
password='******')
def __init__(self):
self.classifier = NaturalLanguageClassifier(
username='******',
password='******')
def buildNLC(credentials):
return NaturalLanguageClassifier(
username=credentials['credentials']['username'],
password=credentials['credentials']['password'])
def __init__(self):
super(NaturalLanguageClassifierUtils,
self).__init__("natural_language_classifier")
self.service = NaturalLanguageClassifier(username=self.getUser(),
password=self.getPassword())
opts, args = getopt.getopt(sys.argv[1:], "hdv", [])
except getopt.GetoptError as err:
print(str(err))
print(usage())
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
usage()
sys.exit()
elif opt == '-d':
DEBUG = True
try:
# list classifiers to get classifier_id
natural_language_classifier = NaturalLanguageClassifier(
url=nlcConstants.getUrl(),
username=nlcConstants.getUsername(),
password=nlcConstants.getPassword())
res = natural_language_classifier.list()
sys.stdout.write('Response: \n%s\n' % json.dumps(res, indent=2))
# sys.stdout.write('Classifiers:\n')
# sys.stdout.write('classifier_id,\tname,\tlanguage,\tcreated,\turl')
# classifiers = res['classifiers']
# for classifier in classifiers:
# sys.stdout.write('%s,\t%s,\t%s,\t%s,\t%s\n' % (classifier['classifier_id'], classifier['name'], classifier['language'], classifier['created'], classifier['url']))
except Exception as e:
sys.stdout.write(str(e))
exit(1)
import json
from os.path import join, dirname
from watson_developer_cloud import NaturalLanguageClassifierV1 as NaturalLanguageClassifier
natural_language_classifier = NaturalLanguageClassifier(
username='******', password='******')
classifiers = natural_language_classifier.list()
print(json.dumps(classifiers, indent=2))
# create a classifier
# with open('../resources/weather_data_train.csv', 'rb') as training_data:
# print(json.dumps(natural_language_classifier.create(training_data=training_data, name='weather2'), indent=2))
# replace 47C164-nlc-243 with your classifier id
status = natural_language_classifier.status('47C164-nlc-243')
print(json.dumps(status, indent=2))
classes = natural_language_classifier.classify('47C164-nlc-243',
'How hot will it be tomorrow?')
print(json.dumps(classes, indent=2))
def __init__(self, app):
super(NaturalLanguageClassifierUtils, self).__init__()
self.app = app
self.service = \
NaturalLanguageClassifier(username=username, password=password)
