def get_data(f):
lines = io.open(f, 'r', encoding='utf-8').readlines()
examples = []
for i in lines:
s1 = i.split("\t")[0].lower()
s2 = i.split("\t")[1].lower()
e = (example(s1), example(s2))
examples.append(e)
shuffle(examples)
return examples
def get_data(f):
lines = json.load(open(f, 'r'))
examples = []
for i in lines:
#s1 = i.split("\t")[0].lower()
#s2 = i.split("\t")[1].lower()
e = (example(i["orig"]), example(i["para"]), example(i["neg_orign"]),
example(i["neg_para"]))
examples.append(e)
shuffle(examples)
return examples
def get_seqs(p1, p2, words, params):
p1 = example(p1)
p2 = example(p2)
if params.wordtype == "words":
p1.populate_embeddings(words, True)
p2.populate_embeddings(words, True)
else:
p1.populate_embeddings_ngrams(words, 3, True)
p2.populate_embeddings_ngrams(words, 3, True)
return p1.embeddings, p2.embeddings
def get_data(f):
data = open(f, 'r')
lines = data.readlines()
examples = []
for i in lines:
i = i.strip()
if (len(i) > 0):
i = i.split('\t')
if len(i) == 2:
e = (example(i[0]), example(i[1]))
examples.append(e)
else:
print i
return examples
def getSimiDataset(filename):
data = open(filename, 'r')
lines = data.readlines()
examples = []
for i in lines:
i = i.strip()
if (len(i) > 0):
i = i.split('\t')
if len(i) == 3:
e = (example(i[0]), example(i[1]), float(i[2]))
examples.append(e)
else:
print i
return examples
def jsonFile():
data = request.get_json()
Index = data.get("Review")
my_list = []
for person in data["Review"]:
my_list.append(person["review"])
# for i in my_list:
# print("HELLO")
# print(i)
ex = example()
score, review_index = ex.predict(my_list, data_text)
# score_list = [score, my_list[review_index]]
appended_string = str(score[0]) + "Txix+T-xixT" + str(
my_list[review_index])
# score.append(review)
print(score[0])
print(type(score))
# return str(score_list)
return appended_string
def get_data(params):
lines = io.open(params.data, 'r', encoding='utf-8').readlines()
scores = {}
random.shuffle(lines)
for i in lines:
i = i.lower()
i = i.split("\t")
score = float(i[2])
idx = i[3]
if idx not in scores and score <= params.max_value and score >= params.min_value:
ex = (example(i[0]), example(i[1]))
scores[idx] = ex
data = []
for i in scores:
data.append(scores[i])
return data
def input_examples(dataset):
examples = []
i = 0
for x_s, y_s in zip(dataset.features, dataset.labels):
exm = example.example(id=i, x_s=x_s, y_s=y_s)
examples.append(exm)
i += 1
return examples
def generate_examples(Statements, Queries):
examples = []
for story_id in xrange(max(Statements.story_id)):
ss = Statements[Statements.story_id == story_id]
qs = Queries[Queries.story_id == story_id]
for _, q in qs.iterrows():
statements = ss[ss.line_id < q.line_id].statement.values
question = q.query
answer = q.answer
hints = map(int, q.support_line.split())
examples.append(example.example(statements, question, answer, hints))
return examples
def generate_examples(Statements, Queries):
examples = []
for story_id in xrange(max(Statements.story_id)):
ss = Statements[Statements.story_id == story_id]
qs = Queries[Queries.story_id == story_id]
for _, q in qs.iterrows():
statements = ss[ss.line_id < q.line_id].statement.values
question = q.query
answer = q.answer
hints = map(int, q.support_line.split())
examples.append(
example.example(statements, question, answer, hints))
return examples
def get_data(params):
lines = io.open(params.data, 'r', encoding='utf-8').readlines()
d = {}
data = []
idx = 0
for i in lines:
i = i.strip()
if len(i) == 0:
continue
i = i.split('\t')
if len(i) == 1:
d[idx] = (i[0], [])
idx += 1
else:
d[idx - 1] = (d[idx - 1][0], d[idx - 1][1] + [i[0]])
for i in d.items():
r = i[1][0]
trans = i[1][1]
r = example(r)
trans = [example(j) for j in trans]
data.append((i[0], r, trans))
train = []
test = []
val = []
for i in data:
idx, r, t = i
if idx < 40000:
train.append((i[1], i[2]))
elif idx < 45000:
val.append((i[1], i[2]))
elif idx < 50000:
test.append((i[1], i[2]))
return train, val, test
def test_example_fail(self):
e = example()
self.assertEqual(e.myfunc(), 4)
def test_example_pass(self):
e = example()
self.assertEqual(e.myfunc(), 5)
import numpy as np
import example
from six import print_
if __name__ == "__main__":
x = np.random.rand(20).reshape(5, 4)
print_(example.example(x))
import numpy as np
import example
if __name__ == "__main__":
x = np.random.rand(20).reshape(5, 4)
print example.example(x)
def response():
# fname = request.form.get("q")
fname = request.args.get("q")
ex1 = example(fname)
# return render_template('index.html', message = ex1)
return ex1
def main(targets):
'''
Runs the main project pipeline logic, given the targets.
targets must contain: 'data', 'analysis', 'model'.
`main` runs the targets in order of data=>analysis=>model.
'''
if 'all' in targets:
with open('config/data-params.json') as fh:
data_cfg = json.load(fh)
get_data(**data_cfg)
with open('config/eda-params.json') as fh:
eda_cfg = json.load(fh)
do_eda(**eda_cfg)
with open('config/auto-params.json') as fh:
auto_cfg = json.load(fh)
autophrase(**auto_cfg)
with open('config/visual-params.json') as fh:
visual_cfg = json.load(fh)
visual(**visual_cfg)
with open('config/example-params.json') as fh:
example_cfg = json.load(fh)
example(**example_cfg)
if 'test' in targets:
with open('config/data-params-test.json') as fh:
data_cfg = json.load(fh)
get_data(**data_cfg)
with open('config/eda-params-test.json') as fh:
eda_cfg = json.load(fh)
do_eda(**eda_cfg)
with open('config/auto-params-test.json') as fh:
auto_cfg = json.load(fh)
autophrase(**auto_cfg)
with open('config/visual-params-test.json') as fh:
visual_cfg = json.load(fh)
visual(**visual_cfg)
if 'data' in targets:
with open('config/data-params.json') as fh:
data_cfg = json.load(fh)
get_data(**data_cfg)
if 'eda' in targets:
with open('config/eda-params.json') as fh:
eda_cfg = json.load(fh)
do_eda(**eda_cfg)
if 'auto' in targets:
with open('config/auto-params.json') as fh:
auto_cfg = json.load(fh)
autophrase(**auto_cfg)
if 'visual' in targets:
with open('config/visual-params.json') as fh:
visual_cfg = json.load(fh)
visual(**visual_cfg)
if 'example' in targets:
with open('config/example-params.json') as fh:
visual_cfg = json.load(fh)
visual(**visual_cfg)
return
import example data_count = 3 example.example(data_count)
def test_main(self, mock_main_show_raw_image):
assert example() is None
def get_seqs(p1, p2, ngram_words, word_words, params):
if params.combination_type == "ngram-word":
np1 = example(p1)
np2 = example(p2)
wp1 = example(p1)
wp2 = example(p2)
np1.populate_embeddings_ngrams(ngram_words, 3, True)
np2.populate_embeddings_ngrams(ngram_words, 3, True)
wp1.populate_embeddings(word_words, True)
wp2.populate_embeddings(word_words, True)
return np1.embeddings, wp1.embeddings, np2.embeddings, wp2.embeddings
elif params.combination_type == "ngram-lstm":
np1 = example(p1)
np2 = example(p2)
wp1 = example(p1)
wp2 = example(p2)
np1.populate_embeddings_ngrams(ngram_words, 3, True)
np2.populate_embeddings_ngrams(ngram_words, 3, True)
wp1.populate_embeddings(word_words, True)
wp2.populate_embeddings(word_words, True)
return np1.embeddings, wp1.embeddings, np2.embeddings, wp2.embeddings
elif params.combination_type == "word-lstm":
np1 = example(p1)
np2 = example(p2)
wp1 = example(p1)
wp2 = example(p2)
np1.populate_embeddings(word_words, True)
np2.populate_embeddings(word_words, True)
wp1.populate_embeddings(word_words, True)
wp2.populate_embeddings(word_words, True)
return np1.embeddings, wp1.embeddings, np2.embeddings, wp2.embeddings
elif params.combination_type == "ngram-word-lstm":
np1 = example(p1)
np2 = example(p2)
wp1 = example(p1)
wp2 = example(p2)
lp1 = example(p1)
lp2 = example(p2)
np1.populate_embeddings_ngrams(ngram_words, 3, True)
np2.populate_embeddings_ngrams(ngram_words, 3, True)
wp1.populate_embeddings(word_words, True)
wp2.populate_embeddings(word_words, True)
lp1.populate_embeddings(word_words, True)
lp2.populate_embeddings(word_words, True)
return np1.embeddings, wp1.embeddings, lp1.embeddings, np2.embeddings, wp2.embeddings, lp2.embeddings
from pylab import *
from example import example
sample_curve_standard = lambda: example(hidden=100, examples=500, epochs=50, eta=0.04, binary=False, embedded=True)[1]
sample_curve_binary = lambda: example(hidden=100, examples=500, epochs=50, eta=1, binary=True, embedded=True)[1]
N = 10
s_collection = []
b_collection = []
for i in range(N):
s_collection.append(sample_curve_standard())
b_collection.append(sample_curve_binary())
import numpy as np
import example
if __name__ == "__main__":
x = np.random.rand(20).reshape(5,4)
print example.example(x)
def test_example_useoption(self):
e = example()
self.assertEqual(str(e.myfunc()), self.options['option-answer'])
def StartUp(self,event):
a = example.example("asdf")
a.Func()
import numpy as np
import example
from six import print_
if __name__ == "__main__":
x = np.random.rand(20).reshape(5,4)
print_(example.example(x))
from pylab import *
from example import example
sample_curve_standard = lambda: example(
hidden=100, examples=500, epochs=50, eta=0.04, binary=False, embedded=True
)[1]
sample_curve_binary = lambda: example(
hidden=100, examples=500, epochs=50, eta=1, binary=True, embedded=True)[1]
N = 10
s_collection = []
b_collection = []
for i in range(N):
s_collection.append(sample_curve_standard())
b_collection.append(sample_curve_binary())
def test_should_issue_example_message(self):
out = mock()
example(out)
verify(out).write("Python")
def test_example():
'''test output of example'''
assert example() == 'this is just an example'
def get_data(lines):
examples = []
for i in lines:
e = (example(i[0]), example(i[1]))
examples.append(e)
return examples
def test_variations(variable):
assert example(variable)
from pylab import *
import example
binary = example.example(100, 500, 50, 1, None, True, True, True)[1]
standard = {}
eta_vals = 10.0**(-arange(25.0)/5.0)
for eta in eta_vals:
standard[eta] = example.example(100, 500, 50, eta, None, False, True, True)[1]
from pylab import *
import example
binary = example.example(100, 500, 50, 1, None, True, True, True)[1]
standard = {}
eta_vals = 10.0**(-arange(25.0) / 5.0)
for eta in eta_vals:
standard[eta] = example.example(100, 500, 50, eta, None, False, True,
True)[1]
