def get(self, program_id=None): program = Program.get_by_id(int(program_id)) rendered = parse.flatten(parse.parse_workout(program.program_code)) template_values = { 'json': rendered } template = template_file('program_run.html') self.response.out.write(template.render(template_values))
def batch_to_input(batch, word2index): contexts, questions, answer_I, answer_II, gold_output = list(zip(*batch)) max_ctxt = max([len(ctxt) for ctxt in contexts]) max_len = max([ctxt.size(1) for ctxt in parse.flatten(contexts)]) max_q = max([qq.size(1) for qq in questions]) max_a_I = max([aa.size(1) for aa in answer_I]) max_a_II = max([aa.size(1) for aa in answer_II]) c_p, c_mask, q_p, a_I_p, a_II_p = [], [], [], [], [] for i in range(len(batch)): temp_c_p = [] for j in range(len(contexts[i])): if contexts[i][j].size(1) == max_len: temp_c_p.append(contexts[i][j]) else: temp_c_p.append(torch.cat([contexts[i][j], Variable(LongTensor([word2index['<PAD>']]*(max_len-contexts[i][j].size(1)))).view(1,-1)], 1)) while(len(temp_c_p) < max_ctxt): temp_c_p.append(Variable(LongTensor([word2index['<PAD>']]*max_len)).view(1,-1)) temp_c_p = torch.cat(temp_c_p) c_p.append(temp_c_p) c_mask.append(torch.cat([Variable(ByteTensor(tuple(map(lambda s: s ==0, t.data))),volatile=False) for t in temp_c_p]).view(temp_c_p.size(0),-1)) if questions[i].size(1) == max_q: q_p.append(questions[i]) else: q_p.append(torch.cat([questions[i],Variable(LongTensor([word2index['<PAD>']]*(max_q-questions[i].size(1)))).view(1,-1)],1)) if answer_I[i].size(1) == max_a_I: a_I_p.append(answer_I[i]) else: a_I_p.append(torch.cat([answer_I[i],Variable(LongTensor([word2index['<PAD>']]*(max_a_I-answer_I[i].size(1)))).view(1,-1)],1)) if answer_II[i].size(1) == max_a_II: a_II_p.append(answer_II[i]) else: a_II_p.append(torch.cat([answer_II[i],Variable(LongTensor([word2index['<PAD>']]*(max_a_II-answer_II[i].size(1)))).view(1,-1)],1)) q_p = torch.cat(q_p) a_I_p = torch.cat(a_I_p) a_II_p = torch.cat(a_II_p) q_mask = torch.cat([Variable(ByteTensor(tuple(map(lambda s: s ==0, t.data))),volatile=False) for t in q_p]).view(q_p.size(0),-1) a_I_mask = torch.cat([Variable(ByteTensor(tuple(map(lambda s: s ==0, t.data))),volatile=False) for t in a_I_p]).view(a_I_p.size(0),-1) a_II_mask = torch.cat([Variable(ByteTensor(tuple(map(lambda s: s ==0, t.data))),volatile=False) for t in a_II_p]).view(a_II_p.size(0),-1) return c_p, c_mask, q_p, q_mask, a_I_p, a_I_mask, a_II_p, a_II_mask, Variable(LongTensor(gold_output))
import parse from functools import reduce import numpy import matplotlib.pyplot morpheme_data = parse.get_morpheme_from_neko() counter = parse.get_counter_from_words(parse.flatten(morpheme_data)).most_common(10) xticks = numpy.array(reduce(lambda result, count: result + [count[0][0]], counter, [])) x = numpy.array(list(range(1, 11))) y = numpy.array(reduce(lambda result, count: result + [count[1]], counter, [])) matplotlib.pyplot.bar(x, y, align = 'center') matplotlib.pyplot.xticks(x, xticks) matplotlib.pyplot.show()
def flatten(s): return p.flatten(s)
import parse from functools import reduce import numpy import matplotlib.pyplot from collections import Counter morpheme_data = parse.get_morpheme_from_neko() counter_by_word = parse.get_counter_from_words( parse.flatten(morpheme_data)).items() counter_by_number = list( Counter(list(map(lambda count: count[1], counter_by_word))).items()) sorted_by_number = sorted(counter_by_number, key=lambda value: value[0]) x = numpy.array( reduce(lambda result, count: result + [count[0]], sorted_by_number, [])) y = numpy.array( reduce(lambda result, count: result + [count[1]], sorted_by_number, [])) matplotlib.pyplot.bar(x, y, align='center') matplotlib.pyplot.show()
import parse from functools import reduce import numpy import matplotlib.pyplot morpheme_data = parse.get_morpheme_from_neko() counter_by_word = list(parse.get_counter_from_words(parse.flatten(morpheme_data)).items()) sorted_by_number = sorted(list(map(lambda count: count[1], counter_by_word)), reverse = True) matplotlib.pyplot.plot(sorted_by_number) matplotlib.pyplot.xscale('log') matplotlib.pyplot.yscale('log') matplotlib.pyplot.show()
def test_parse_workout(self): r = parse.parse_workout(""" - 10m easy warmup - 5x: - 1m gear up - 1m gear even further up - 2x: - 15s insane - 15s slow - 10m cool down """) self.assertEquals( r, [{'d1': 'easy warmup', 'd2': '', 'dur': 600}, (('REPEAT', 5), [{'d1': 'gear up', 'd2': '', 'dur': 60}, {'d1': 'gear even further up', 'd2': '', 'dur': 60}, (('REPEAT', 2), [{'d1': 'insane', 'd2': '', 'dur': 15}, {'d1': 'slow', 'd2': '', 'dur': 15}])]), {'d1': 'cool down', 'd2': '', 'dur': 600}]) flt = parse.flatten(r) self.assertEquals( flt, [{'d1': 'easy warmup', 'd2': '', 'dur': 600, 'index': 0, 'level': 0}, {'d1': 'gear up', 'd2': '', 'dur': 60, 'index': 600, 'level': 1}, {'d1': 'gear even further up', 'd2': '', 'dur': 60, 'index': 660, 'level': 1}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 720, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 735, 'level': 2}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 750, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 765, 'level': 2}, {'d1': 'gear up', 'd2': '', 'dur': 60, 'index': 780, 'level': 1}, {'d1': 'gear even further up', 'd2': '', 'dur': 60, 'index': 840, 'level': 1}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 900, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 915, 'level': 2}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 930, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 945, 'level': 2}, {'d1': 'gear up', 'd2': '', 'dur': 60, 'index': 960, 'level': 1}, {'d1': 'gear even further up', 'd2': '', 'dur': 60, 'index': 1020, 'level': 1}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 1080, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 1095, 'level': 2}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 1110, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 1125, 'level': 2}, {'d1': 'gear up', 'd2': '', 'dur': 60, 'index': 1140, 'level': 1}, {'d1': 'gear even further up', 'd2': '', 'dur': 60, 'index': 1200, 'level': 1}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 1260, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 1275, 'level': 2}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 1290, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 1305, 'level': 2}, {'d1': 'gear up', 'd2': '', 'dur': 60, 'index': 1320, 'level': 1}, {'d1': 'gear even further up', 'd2': '', 'dur': 60, 'index': 1380, 'level': 1}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 1440, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 1455, 'level': 2}, {'d1': 'insane', 'd2': '', 'dur': 15, 'index': 1470, 'level': 2}, {'d1': 'slow', 'd2': '', 'dur': 15, 'index': 1485, 'level': 2}, {'d1': 'cool down', 'd2': '', 'dur': 600, 'index': 1500, 'level': 0}])
import parse from functools import reduce import numpy import matplotlib.pyplot morpheme_data = parse.get_morpheme_from_neko() counter_by_word = list( parse.get_counter_from_words(parse.flatten(morpheme_data)).items()) sorted_by_number = sorted(list(map(lambda count: count[1], counter_by_word)), reverse=True) matplotlib.pyplot.plot(sorted_by_number) matplotlib.pyplot.xscale('log') matplotlib.pyplot.yscale('log') matplotlib.pyplot.show()
import parse def get_nouns_from_sentence(sentence): result = [] nouns = '' for i in range(0, len(sentence)): if sentence[i]['品詞'] == '名詞': nouns = nouns + sentence[i]['表層形'] elif len(nouns) != 0: result.append(nouns) nouns = '' if len(nouns) != 0: result.append(nouns) return result morpheme_data = parse.get_morpheme_from_neko() print(parse.flatten(list(map(lambda sentence: get_nouns_from_sentence(sentence), morpheme_data))))
import parse morpheme_data = parse.get_morpheme_from_neko() print(parse.get_counter_from_words(parse.flatten(morpheme_data)))
import parse from functools import reduce import numpy import matplotlib.pyplot morpheme_data = parse.get_morpheme_from_neko() counter = parse.get_counter_from_words( parse.flatten(morpheme_data)).most_common(10) xticks = numpy.array( reduce(lambda result, count: result + [count[0][0]], counter, [])) x = numpy.array(list(range(1, 11))) y = numpy.array(reduce(lambda result, count: result + [count[1]], counter, [])) matplotlib.pyplot.bar(x, y, align='center') matplotlib.pyplot.xticks(x, xticks) matplotlib.pyplot.show()
import parse def get_noun_phrase_from_sentence(sentence): result = [] for i in range(1, len(sentence) - 1): word = sentence[i] if word['表層形'] == 'の' and word['品詞'] == '助詞' and word[ '品詞細分類1'] == '連体化' and sentence[ i - 1]['品詞'] == '名詞' and sentence[i + 1]['品詞'] == '名詞': result.append(sentence[i - 1]['表層形'] + 'の' + sentence[i + 1]['表層形']) return result morpheme_data = parse.get_morpheme_from_neko() print( parse.flatten( list( map(lambda sentence: get_noun_phrase_from_sentence(sentence), morpheme_data))))