def classifier_status(url, username, password, classifier_ids): n = NaturalLanguageClassifier(url=url, username=username, password=password) for classifier_id in classifier_ids: status = n.status(classifier_id) print("%s: %s" % (status["status"], status["status_description"]))
def __initialize(self, credential_file_path): if not credential_file_path: credential_file_path = os.path.expanduser(DEFAULT_CREDENTIAL_PATH) with open(credential_file_path, 'r') as credential_file: credential = json.load(credential_file) self.__nlc = NaturalLanguageClassifier(url=credential['url'], username=credential['username'], password=credential['password'])
def train_nlc(url, username, password, truth, name): logger.info("Train model %s with %d instances" % (name, len(truth))) with tempfile.TemporaryFile() as training_file: # NLC cannot handle newlines. truth[QUESTION] = truth[QUESTION].str.replace("\n", " ") to_csv(training_file, truth[[QUESTION, ANSWER_ID]], header=False, index=False) training_file.seek(0) nlc = NaturalLanguageClassifier(url=url, username=username, password=password) r = nlc.create(training_data=training_file, name=name) logger.info(pretty_print_json(r)) return r["classifier_id"]
def nlc_router_train(url, username, password, oracle_out, path, all_correct): """ NLC Training on the oracle experiment output to determine which system(NLC or Solr) should answer particular question. 1. Splitting up the oracle experiment output data into 8 equal training records and testing records. This is to ensure 8-fold cross validation of the data-set. All training and Testing files will be stored at the "path" 2. Perform NLC training on the all 8 training set simultaneously and returns list of classifier ids as json file in the working directory :param url: URL of NLC instance :param username: NLC Username :param password: NLC password :param oracle_out: file created by oracle experiment :param path: directory path to save intermediate results :param all_correct: optional boolean parameter to train with only correct QA pairs :return: list of classifier ids by NLC training """ ensure_directory_exists(path) sys_name = oracle_out[SYSTEM][0] oracle_out[QUESTION] = oracle_out[QUESTION].str.replace("\n", " ") kfold_split(oracle_out, path, NLC_ROUTER_FOLDS, True) classifier_list = [] list = [] for x in range(0, NLC_ROUTER_FOLDS): train = pandas.read_csv(os.path.join(path, "Train{0}.csv".format(str(x)))) if all_correct: logger.info("Training only on CORRECT examples.") # Ignore records from training which are not correct train = train[train[CORRECT]] train = train[train[IN_PURVIEW]] train = train[[QUESTION, ANSWERING_SYSTEM]] logger.info("Training set size = {0}".format(str(len(train)))) with tempfile.TemporaryFile() as training_file: to_csv(training_file, train[[QUESTION, ANSWERING_SYSTEM]], header=False, index=False) training_file.seek(0) nlc = NaturalLanguageClassifier(url=url, username=username, password=password) classifier_id = nlc.create(training_data=training_file, name="{0}_fold_{1}".format(str(sys_name), str(x))) classifier_list.append(classifier_id["classifier_id"].encode("utf-8")) list.append({classifier_id["name"].encode("utf-8"): classifier_id["classifier_id"].encode("utf-8")}) logger.info(pretty_print_json(classifier_id)) pretty_print_json(classifier_id) with open(os.path.join(path, 'classifier.json'), 'wb') as f: json.dump(list, f) return classifier_list
def __init__(self): self.fname = "" self.modelSearchList = ModelSearchList() self.text_data = [] self.target_label = [] self.watson_crediantial = watson_key() #self.watson_classifier = self.watson_crediantial.classifier_twitter_classfier #self.watson_classifier = self.watson_crediantial.classifier_twitter_hash_classfier #self.watson_classifier = self.watson_crediantial.classifier_twitter_unblance_keyword_classfier #self.watson_classifier = self.watson_crediantial.twitter_priority_classfier self.watson_classifier = self.watson_crediantial.twitter_category_classfier self.natural_language_classifier = NaturalLanguageClassifier( username=self.watson_crediantial.username, password=self.watson_crediantial.password)
def classifier_list(url, username, password): connection = NaturalLanguageClassifier(url=url, username=username, password=password) return connection.list()["classifiers"]
def __init__(self, url, username, password, classifier_id, corpus): self.nlc = NaturalLanguageClassifier(url=url, username=username, password=password) self.classifier_id = classifier_id self.corpus = corpus
def remove_classifiers(url, username, password, classifier_ids): n = NaturalLanguageClassifier(url=url, username=username, password=password) for classifier_id in classifier_ids: n.remove(classifier_id)
def classifier_status(url, username, password, classifier_ids): n = NaturalLanguageClassifier(url=url, username=username, password=password) for classifier_id in classifier_ids: status = n.status(classifier_id) print(" Instance name: %s with classifier id %s is %s; Description: %s" % (status["name"],status["classifier_id"],status["status"], status["status_description"]))
import sys import operator import requests import json import twitter from watson_developer_cloud import NaturalLanguageClassifierV1 as NaturalLanguageClassifier #The IBM Bluemix credentials nlc_username = '******' nlc_password = '******' natural_language_classifier = NaturalLanguageClassifier(username=nlc_username, password=nlc_password) classifierId = 'e4be4cx148-nlc-16' classes = natural_language_classifier.remove(classifierId) print(json.dumps(classes, indent=2))
def __init__(self, app): super(NaturalLanguageClassifierUtils, self).__init__() self.app = app self.service = NaturalLanguageClassifier( username='******', password='******')
def __init__(self): super(NaturalLanguageClassifierUtils, self).__init__() self.service = NaturalLanguageClassifier( username='******', password='******')
def __init__(self): self.classifier = NaturalLanguageClassifier( username='******', password='******')
def buildNLC(credentials): return NaturalLanguageClassifier( username=credentials['credentials']['username'], password=credentials['credentials']['password'])
def __init__(self): super(NaturalLanguageClassifierUtils, self).__init__("natural_language_classifier") self.service = NaturalLanguageClassifier(username=self.getUser(), password=self.getPassword())
opts, args = getopt.getopt(sys.argv[1:], "hdv", []) except getopt.GetoptError as err: print(str(err)) print(usage()) sys.exit(2) for opt, arg in opts: if opt == '-h': usage() sys.exit() elif opt == '-d': DEBUG = True try: # list classifiers to get classifier_id natural_language_classifier = NaturalLanguageClassifier( url=nlcConstants.getUrl(), username=nlcConstants.getUsername(), password=nlcConstants.getPassword()) res = natural_language_classifier.list() sys.stdout.write('Response: \n%s\n' % json.dumps(res, indent=2)) # sys.stdout.write('Classifiers:\n') # sys.stdout.write('classifier_id,\tname,\tlanguage,\tcreated,\turl') # classifiers = res['classifiers'] # for classifier in classifiers: # sys.stdout.write('%s,\t%s,\t%s,\t%s,\t%s\n' % (classifier['classifier_id'], classifier['name'], classifier['language'], classifier['created'], classifier['url'])) except Exception as e: sys.stdout.write(str(e)) exit(1)
import json from os.path import join, dirname from watson_developer_cloud import NaturalLanguageClassifierV1 as NaturalLanguageClassifier natural_language_classifier = NaturalLanguageClassifier( username='******', password='******') classifiers = natural_language_classifier.list() print(json.dumps(classifiers, indent=2)) # create a classifier # with open('../resources/weather_data_train.csv', 'rb') as training_data: # print(json.dumps(natural_language_classifier.create(training_data=training_data, name='weather2'), indent=2)) # replace 47C164-nlc-243 with your classifier id status = natural_language_classifier.status('47C164-nlc-243') print(json.dumps(status, indent=2)) classes = natural_language_classifier.classify('47C164-nlc-243', 'How hot will it be tomorrow?') print(json.dumps(classes, indent=2))
def __init__(self, app): super(NaturalLanguageClassifierUtils, self).__init__() self.app = app self.service = \ NaturalLanguageClassifier(username=username, password=password)