def get_classify_data(usr): user = usr['display_name'] print user options, remainder = parse_args() classifier = Classifier(options.server_ip,options.server_port, options.name, 10.0) #train data_reader = select_Interrest_Blog(user) for row in data_reader: label = row['CATEGORY'] dat = row['TITLE'] datum = Datum({"message": dat}) classifier.train([LabeledDatum(label, datum)]) url_list = [] url_list = get_rss_data_from_catlist(usr,['social','fun','entertainment','game']) for data in url_list: title = data["title"] datum = Datum({"message": title}) classifier.train([LabeledDatum('no', datum)]) # print classifier.get_status() # print classifier.save("tutorial") # print classifier.load("tutorial") # print classifier.get_config() url_list = [] ret1 = [] ret2 = [] url_list = get_rss_data_from_catlist(usr,['it','popular','life','knowledge']) for data in url_list: title = data["title"] datum = Datum({"message": title}) ans = classifier.classify([datum]) if ans != None: estm = get_most_likely(ans[0]) if estm[0] == 'yes': ret1.append(data) else: ret2.append(data) print ret1 print "" print ret2 return ret1,ret2
def main(): args = parse_options() client = Classifier('127.0.0.1', args.port, 'test', 0) for i in range(0, 1000000): d = Datum() # Learn same data rand = random.randint(0, 1) d.add_number('key', 1.0 if rand else 2.0) ld = LabeledDatum('Pos' if rand else 'Neg', d) client.train([ld]) if not i % 10000: print 'train ' + str(i) + ' data'
if __name__ == '__main__': options, remainder = parse_args() classifier = Classifier(options.server_ip,options.server_port, options.name, 10.0) print classifier.get_config() print classifier.get_status() for line in open('train.dat'): label, file = line[:-1].split(',') dat = open(file).read() datum = Datum({"message": dat}) classifier.train([LabeledDatum(label, datum)]) print classifier.get_status() print classifier.save("tutorial") print classifier.load("tutorial") print classifier.get_config() count_ok = 0 count_ng = 0 for line in open('test.dat'): label, file = line[:-1].split(',') dat = open(file).read() datum = Datum({"message": dat})
class ClassifierTest(unittest.TestCase): def setUp(self): self.config = { "method": "AROW", "converter": { "string_filter_types": {}, "string_filter_rules": [], "num_filter_types": {}, "num_filter_rules": [], "string_types": {}, "string_rules": [{"key": "*", "type": "str", "sample_weight": "bin", "global_weight": "bin"}], "num_types": {}, "num_rules": [{"key": "*", "type": "num"}], }, "parameter": {"regularization_weight": 1.001}, } TestUtil.write_file("config_classifier.json", json.dumps(self.config)) self.srv = TestUtil.fork_process("classifier", port, "config_classifier.json") try: self.cli = Classifier(host, port, "name") except: TestUtil.kill_process(self.srv) raise def tearDown(self): if self.cli: self.cli.get_client().close() TestUtil.kill_process(self.srv) def test_get_client(self): self.assertTrue(isinstance(self.cli.get_client(), msgpackrpc.client.Client)) def test_get_config(self): config = self.cli.get_config() self.assertEqual(json.dumps(json.loads(config), sort_keys=True), json.dumps(self.config, sort_keys=True)) def test_train(self): d = Datum({"skey1": "val1", "skey2": "val2", "nkey1": 1.0, "nkey2": 2.0}) data = [["label", d]] self.assertEqual(self.cli.train(data), 1) def test_classify(self): d = Datum({"skey1": "val1", "skey2": "val2", "nkey1": 1.0, "nkey2": 2.0}) data = [d] result = self.cli.classify(data) def test_set_label(self): self.assertEqual(self.cli.set_label("label"), True) def test_get_labels(self): self.cli.set_label("label") self.assertEqual(self.cli.get_labels(), {"label": 0}) def test_delete_label(self): self.cli.set_label("label") self.assertEqual(self.cli.delete_label("label"), True) def test_save(self): self.assertEqual(len(self.cli.save("classifier.save_test.model")), 1) def test_load(self): model_name = "classifier.load_test.model" self.cli.save(model_name) self.assertEqual(self.cli.load(model_name), True) def test_get_status(self): self.cli.get_status() def test_str(self): self.assertEqual("estimate_result{label: label, score: 1.0}", str(EstimateResult("label", 1.0)))
#!/usr/bin/env python import random import time from jubatus.classifier.client import Classifier from jubatus.classifier.types import LabeledDatum from jubatus.common import Datum data = [] for i in xrange(0, 100000): d = Datum() for j in xrange(0, 20): d.add_number(str(j) + '-' + str(i), random.random() + 1.0) ld = LabeledDatum('Pos' if random.randint(0, 1) else 'Neg', d) data.append(ld) client = Classifier('127.0.0.1', 9199, 'test', 0) start_time = time.time() client.train(data) end_time = time.time() print str(len(data)) + ' ... ' + str((end_time - start_time) * 1000) + ' msec'
if __name__ == '__main__': options, remainder = parse_args() classifier = Classifier(options.server_ip, options.server_port, options.name, 10.0) print classifier.get_config() print classifier.get_status() for line in open('train.dat'): label, file = line[:-1].split(',') dat = open(file).read() datum = Datum({"message": dat}) classifier.train([LabeledDatum(label, datum)]) print classifier.get_status() print classifier.save("tutorial") print classifier.load("tutorial") print classifier.get_config() count_ok = 0 count_ng = 0 for line in open('test.dat'): label, file = line[:-1].split(',') dat = open(file).read() datum = Datum({"message": dat})
#!/usr/bin/env python import random import time from jubatus.classifier.client import Classifier from jubatus.classifier.types import LabeledDatum from jubatus.common import Datum data = [] for i in xrange(0, 100000): d = Datum() for j in xrange(0, 20): d.add_number(str(j) + "-" + str(i), random.random() + 1.0) ld = LabeledDatum("Pos" if random.randint(0, 1) else "Neg", d) data.append(ld) client = Classifier("127.0.0.1", 9199, "test", 0) start_time = time.time() client.train(data) end_time = time.time() print str(len(data)) + " ... " + str((end_time - start_time) * 1000) + " msec"
def main(): client = Classifier("127.0.0.1", port, "sleeping", timeout) client.train([])
class ClassifierTest(unittest.TestCase): def setUp(self): self.config = { "method": "AROW", "converter": { "string_filter_types": {}, "string_filter_rules": [], "num_filter_types": {}, "num_filter_rules": [], "string_types": {}, "string_rules": [{ "key": "*", "type": "str", "sample_weight": "bin", "global_weight": "bin" }], "num_types": {}, "num_rules": [{ "key": "*", "type": "num" }] }, "parameter": { "regularization_weight": 1.001 } } TestUtil.write_file('config_classifier.json', json.dumps(self.config)) self.srv = TestUtil.fork_process('classifier', port, 'config_classifier.json') try: self.cli = Classifier(host, port, "name") except: TestUtil.kill_process(self.srv) raise def tearDown(self): if self.cli: self.cli.get_client().close() TestUtil.kill_process(self.srv) def test_get_client(self): self.assertTrue( isinstance(self.cli.get_client(), msgpackrpc.client.Client)) def test_get_config(self): config = self.cli.get_config() self.assertEqual(json.dumps(json.loads(config), sort_keys=True), json.dumps(self.config, sort_keys=True)) def test_train(self): d = Datum({ "skey1": "val1", "skey2": "val2", "nkey1": 1.0, "nkey2": 2.0 }) data = [["label", d]] self.assertEqual(self.cli.train(data), 1) def test_classify(self): d = Datum({ "skey1": "val1", "skey2": "val2", "nkey1": 1.0, "nkey2": 2.0 }) data = [d] result = self.cli.classify(data) def test_set_label(self): self.assertEqual(self.cli.set_label("label"), True) def test_get_labels(self): self.cli.set_label("label") self.assertEqual(self.cli.get_labels(), {"label": 0}) def test_delete_label(self): self.cli.set_label("label") self.assertEqual(self.cli.delete_label("label"), True) def test_save(self): self.assertEqual(len(self.cli.save("classifier.save_test.model")), 1) def test_load(self): model_name = "classifier.load_test.model" self.cli.save(model_name) self.assertEqual(self.cli.load(model_name), True) def test_get_status(self): self.cli.get_status() def test_str(self): self.assertEqual("estimate_result{label: label, score: 1.0}", str(EstimateResult("label", 1.0)))