def test_array_stream(self): arr = stream_array(tokenize(StringIO('[]'))) self.assertListEqual([i for i in arr], []) arr = stream_array(tokenize(StringIO('["People", "Places", "Things"]'))) self.assertListEqual([i for i in arr], ["People", "Places", "Things"]) arr = stream_array(tokenize(StringIO('["Apples", "Bananas", ["Pears", "Limes"]]'))) self.assertListEqual([i for i in arr], ["Apples", "Bananas", ["Pears", "Limes"]]) arr = stream_array(tokenize(StringIO('["Apples", ["Pears", "Limes"], "Bananas"]'))) self.assertListEqual([i for i in arr], ["Apples", ["Pears", "Limes"], "Bananas"])
def test_array_stream(self): arr = stream_array(tokenize(StringIO('[]'))) self.assertListEqual([i for i in arr], []) arr = stream_array(tokenize(StringIO('["People", "Places", "Things"]'))) self.assertListEqual([i for i in arr], ["People", "Places", "Things"]) arr = stream_array(tokenize(StringIO('["Apples", "Bananas", ["Pears", "Limes"]]'))) self.assertListEqual([i for i in arr], ["Apples", "Bananas", ["Pears", "Limes"]]) arr = stream_array(tokenize(StringIO('["Apples", ["Pears", "Limes"], "Bananas"]'))) self.assertListEqual([i for i in arr], ["Apples", ["Pears", "Limes"], "Bananas"]) arr = stream_array(tokenize(StringIO('["Apples", {"key":"value"}, "Bananas"]'))) self.assertListEqual([i for i in arr], ["Apples", {"key": "value"}, "Bananas"])
def test_array_stream_of_documents_with_incomplete_json(self): arr = [] with self.assertRaises(ValueError): messages = stream_array( tokenize( StringIO( '[{"key": "value"}, {"key": "value"}, {"INCOMPLETE'))) for message in messages: arr.append(message) self.assertListEqual(arr, [{"key": "value"}, {"key": "value"}])
def load_data(filename): print('Loading data from:', filename) x = [] y = [] nlp = spacy.load('en_core_web_sm') # noinspection SpellCheckingInspection def handle_message(item): x.append(item['question']) y.append(1) doc = nlp(item['answer']) first_sent = next(doc.sents) x.append(first_sent.string.strip()) y.append(0) with open(filename, 'r') as f: print('Processing stream...') messages = stream_array(tokenize(f)) for message in messages: handle_message(message) x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.1, shuffle=True) x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.1, shuffle=True) train_df = pd.DataFrame({'x': x_train, 'y': y_train}) val_df = pd.DataFrame({'x': x_val, 'y': y_val}) test_df = pd.DataFrame({'x': x_test, 'y': y_test}) classes = np.array([0, 1]) print('Lengths Train: {}, Val: {}, Test: {}, Classes: {}'.format( len(train_df), len(val_df), len(test_df), len(classes))) return train_df, val_df, test_df, classes
def test_array_stream_of_values_with_extra_invalid_json(self): arr = stream_array(tokenize(StringIO('["People", "Places"], EXTRA'))) self.assertListEqual(list(arr), ["People", "Places"])
def test_array_stream_of_values_with_incomplete_json(self): arr = stream_array( tokenize(StringIO('["People", "Places", "INCOMPLETE'))) self.assertListEqual(list(arr), ["People", "Places"])
def test_array_stream_of_documents_with_extra_invalid_json(self): arr = stream_array( tokenize(StringIO('[{"key": "value"}, {"key": "value"}] EXTRA'))) self.assertListEqual(list(arr), [{"key": "value"}, {"key": "value"}])
def test_array_stream_of_documents(self): arr = stream_array( tokenize(StringIO('[{"key": "value"}, {"key": "value"}]'))) self.assertListEqual(list(arr), [{"key": "value"}, {"key": "value"}])