def test2(self): lexerlist = Lexer('Q').tokenize(1) parserlist = Parser().parse(lexerlist) self.assertEqual(parserlist[0].kind, "propositions") self.assertEqual(parserlist[1].kind, "proposition") self.assertEqual(parserlist[2].kind, "atomic") self.assertEqual(parserlist[3].kind, "ID") self.assertEqual(parserlist[4].kind, "more-proposition") self.assertEqual(parserlist[5].kind, "epsilon")
import numpy as np import tensorflow as tf import random import sys, os import json import argparse from parserr import Parser from datamanager import DataManager from actor import ActorNetwork from LSTM_critic import LSTM_CriticNetwork tf.logging.set_verbosity(tf.logging.ERROR) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' #get parse argv = sys.argv[1:] parser = Parser().getParser() args, _ = parser.parse_known_args(argv) random.seed(args.seed) #get data dataManager = DataManager(args.dataset) train_data, dev_data, test_data = dataManager.getdata(args.grained, args.maxlenth) word_vector = dataManager.get_wordvector(args.word_vector) if args.fasttest == 1: train_data = train_data[:100] dev_data = dev_data[:20] test_data = test_data[:20] print("train_data ", len(train_data)) print("dev_data", len(dev_data)) print("test_data", len(test_data))
import unittest from lexer import Lexer from parserr import Parser file = open(raw_input("Enter Filename: "),'r') data = file.readlines() file.close() currentline = 1 for lines in data: print "Proposition: " + lines.rstrip() tokenlist = Lexer(lines.rstrip()).tokenize(currentline) print "Lexer: ", tokenlist grammarlist = Parser().parse(tokenlist) print "Parser:", grammarlist print currentline += 1 ''' class Test(unittest.TestCase): def test1(self): tokenlist = Lexer('Q').tokenize(1) self.assertEqual(tokenlist[0].kind, "ID") self.assertEqual(tokenlist[0].loc.col, 1) self.assertEqual(tokenlist[0].loc.line, 1) def test2(self): tokenlist = Lexer('Q').tokenize(1) grammarlist = Parser().parse(tokenlist)
from lexer import Lexer from parserr import Parser from smtbuilder import SMTbuilder import os, sys, unittest file = open(sys.argv[1], "r") data = file.readlines() file.close() currentline = 1 for lines in data: lexerlist = Lexer(lines.rstrip()).tokenize(currentline) parserlist = Parser().parse(lexerlist) if not "Syntax Error" in parserlist: # no grammar error found SMTbuilder("output.py").build(lexerlist) import output else: print parserlist # prints error currentline += 1 class Test(unittest.TestCase): def test1(self): lexerlist = Lexer('Q').tokenize(1) self.assertEqual(lexerlist[0].kind, "ID") self.assertEqual(lexerlist[0].loc.col, 1) self.assertEqual(lexerlist[0].loc.line, 1)
from grammar import Grammar from grammar_menu import UI from parserr import Parser def read_pif(file_path): lst = [] with open(file_path, 'r') as file: for line in file.readlines(): elem = line.split("->")[0].strip() lst.append(elem) return lst if __name__ == "__main__": # grammar = Grammar("g_test.in") # ui = UI(grammar) # ui.run() lst = read_pif("p1.in") print(lst) grammar = Grammar("g1.in") parser = Parser(grammar) w = ["a", "a", "c", "b", "c"] parser.run(w)
def __init__(self): self.parser = Parser()
class Summarizer: def __init__(self): self.parser = Parser() def summarize(self, text, title, source, category): sentences = self.parser.splitSentences(text) titleWords = self.parser.removePunctations(title) titleWords = self.parser.splitWords(title) (keywords, wordCount) = self.parser.getKeywords(text) topKeywords = self.getTopKeywords(keywords[:10], wordCount, source, category) result = self.computeScore(sentences, titleWords, topKeywords) result = self.sortScore(result) return result def getTopKeywords(self, keywords, wordCount, source, category): # Add getting top keywords in the database here for keyword in keywords: articleScore = 1.0 * keyword['count'] / wordCount keyword['totalScore'] = articleScore * 1.5 return keywords def sortScore(self, dictList): return sorted(dictList, key=lambda x: -x['totalScore']) def sortSentences(self, dictList): return sorted(dictList, key=lambda x: x['order']) def computeScore(self, sentences, titleWords, topKeywords): keywordList = [keyword['word'] for keyword in topKeywords] summaries = [] for i, sentence in enumerate(sentences): sent = self.parser.removePunctations(sentence) words = self.parser.splitWords(sent) sbsFeature = self.sbs(words, topKeywords, keywordList) dbsFeature = self.dbs(words, topKeywords, keywordList) titleFeature = self.parser.getTitleScore(titleWords, words) sentenceLength = self.parser.getSentenceLengthScore(words) sentencePosition = self.parser.getSentencePositionScore( i, len(sentences)) keywordFrequency = (sbsFeature + dbsFeature) / 2.0 * 10.0 totalScore = (titleFeature * 1.5 + keywordFrequency * 2.0 + sentenceLength * 0.5 + sentencePosition * 1.0) / 4.0 summaries.append({ # 'titleFeature': titleFeature, # 'sentenceLength': sentenceLength, # 'sentencePosition': sentencePosition, # 'keywordFrequency': keywordFrequency, 'totalScore': totalScore, 'sentence': sentence, 'order': i }) return summaries def sbs(self, words, topKeywords, keywordList): score = 0.0 if len(words) == 0: return 0 for word in words: word = word.lower() index = -1 if word in keywordList: index = keywordList.index(word) if index > -1: score += topKeywords[index]['totalScore'] return 1.0 / abs(len(words)) * score def dbs(self, words, topKeywords, keywordList): k = len(list(set(words) & set(keywordList))) + 1 summ = 0.0 firstWord = {} secondWord = {} for i, word in enumerate(words): if word in keywordList: index = keywordList.index(word) if firstWord == {}: firstWord = { 'i': i, 'score': topKeywords[index]['totalScore'] } else: secondWord = firstWord firstWord = { 'i': i, 'score': topKeywords[index]['totalScore'] } distance = firstWord['i'] - secondWord['i'] summ += (firstWord['score'] * secondWord['score']) / (distance**2) return (1.0 / k * (k + 1.0)) * summ