def __init__(self, modelname=None):
self.MAX_SEQUENCE_LENGTH = 1000
self.MAX_NB_WORDS = 20000
self.EMBEDDING_DIM = 100
self.VALIDATION_SPLIT = 0.25
self.EPOCHS = 64
self.BATCH_SIZE = 32
self.POOL_SIZE = 5
self.FILTERS = 64
self.LSTM_OUTPUT_SIZE = 70
if modelname == None:
self.MODEL_NAME = "clause_model"
else:
self.MODEL_NAME = modelname
self._document = LawDocument()
self._clause = Clause()
self.texts = [] # list of text samples
self.labels_index = {} # dictionary mapping label name to numeric id
self.labels = [] # list of label ids
self.label_name = []
self._debug = 1
self._save_model = False
def __init__(self, modelname=None):
self._document = LawDocument()
self.clause_model = clause_training.ClauseTraining()
self.clause_model.load_model_label()
pass
def __init__(self, filename=None):
"""
"""
self.law_document = LawDocument()
self.important_word = []
self.top_n_scored = []
self.mean_scored = []
def get_document_type(self, dictname) :
textfname = "../dictionary/text/" + dictname + ".txt"
law_document = LawDocument()
law_document.analyze(filename=textfname)
text = law_document.document_type;
return text
def get_document(self, dictname) :
textfname = "../dictionary/text/" + dictname + ".txt"
law_document = LawDocument()
law_document.analyze(filename=textfname)
text = "\n".join(law_document.document_title);
return text
def file_clean(self, filename):
from docutone.core.document import LawDocument
if (filename.endswith(".txt")):
ofile = filename
else:
ofile = docutonelocate.convert_file(filename)
lawdoc = LawDocument()
document = lawdoc.get_fusion_document(ofile)
for sentence in document:
print(' '.join(sentence))
def __init__(self):
self.texts = [] # list of text samples
self.labels_index = {} # dictionary mapping label name to numeric id
self.labels_files = {} # dictionary mapping label name to numeric id
self.labels_name = {} # dictionary mapping label name to numeric id
self.file_label = [] # file label id
self.labels = [] # list of label ids
self.classifiers = [] # list of classifier
self.law_doc = LawDocument()
self.folder_structure = {}
self.folder_order = []
pass
def file_named_tag(self, filename):
from docutone.core.document import LawDocument
if (filename.endswith(".txt")):
ofile = filename
else:
ofile = docutonelocate.convert_file(filename)
lawdoc = LawDocument()
document = lawdoc.get_fusion_document(ofile)
self.new_ner = {}
for sentence in document:
self.get_sentence_named_tag(sentence)
self.write_ner()
def __init__(self):
self.contract = Contract(0)
self.verified_terms = {}
self._filetime = None
self.fullname = None
self.filename = None
self._title = None
self._contract_date = None
self.keywords = []
self.segment = Segmentation()
self.document = LawDocument()
def __init__(self):
self.law_doc = LawDocument()
self.file_index = 1
self.folder_structure = {}
self.folder_order = []
self.corpus_document = []
instance = Terms()
self.categories = instance.get_all_term_items()
pass
def __init__(self, debug=0, crf_model=True):
self.texts = [] # list of legal terms tests
self.terms_index = {} # mapping legal term name to numeric id
self.terms_name = {} # legal term name
self.terms_label = [] # mapping legal term name to label
self.labels = [] # list of legal term label ids
self._debug = debug
self.seg = Segmentation()
self.seg.load_suggest_words()
self.lawdocument = LawDocument()
self.clause = Clause()
self.doc_type = None
self.doc_path = None
self.labor_model = True
self.crf_model = crf_model
def get_document_chapiter(self, sims, dictname) :
textfname = "../dictionary/text/" + dictname + ".txt"
law_document = LawDocument()
law_document.analyze(filename=textfname)
text = "";
n_line = 1
for sim in sims :
doc_no, simil = sim[0], sim [1]
if (simil > 0.4) :
text += "******** " + str(n_line) + " ********\n"
text += law_document.get_document_chapiter(doc_no) + "\n"
n_line += 1
if n_line > 2:
break;
else :
break
return text
def __init__(self, stopwords_file=None):
"""
Keyword arguments:
stopwords_file : stopwords file name
"""
self.pagerank_config = {
'alpha': 0.85,
}
self.seg = Segmentation(stopwords_file=stopwords_file)
self.law_document = LawDocument()
self.sentences = None
self.words_no_filter = None # 2维列表
self.words_no_stop_words = None
self.words_all_filters = None
self.key_sentences = None
def __init__(self, filename=None):
self.MAX_SEQUENCE_LENGTH = 1000
self.MAX_NB_WORDS = 20000
self.EMBEDDING_DIM = 100
self.VALIDATION_SPLIT = 0.25
self.embeddings_index = self.load_embedding_base()
self._document = LawDocument()
self.label_name = []
self.texts = [] # list of text samples
self.labels_index = {} # dictionary mapping label name to numeric id
self.labels = [] # list of label ids
self._debug = 1
pass
def search_document(self, textpath, filename) :
ld = LawDocument()
ld.analyze(filename=filename)
doc_tab = []
names = os.listdir("../dictionary/dict")
n_file = 1
for filename in os.listdir("../dictionary/dict") :
if filename.endswith(".dict") :
dictname = filename.replace('.dict', '')
total = 0.0
sentences = []
for sentence in ld.table_contents :
if len(sentence) > 1 :
sims = self.text_search_lsi(textpath, sentence[1])
total += self.get_similarity_value(sims)
doc_tab.append([dictname, total])
doc_tab = sorted(doc_tab, key=lambda total: total[1], reverse=True)
return self.get_document_type(doc_tab[0][0])
def __init__(self):
"""
"""
self.law_document = LawDocument()
self.all_keywords = util.load_legalterm_type()
def test_imdb(self):
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
law_document = LawDocument()
fname = os.path.join(
variables.BASE_DIR,
'data/Corpus/TEXT/合同、协议/劳动合同/1. 劳动合同- 最终版.DOC.txt')
sentences = law_document.get_sentences(fname)
# tokenizer
tokenizer = Tokenizer(nb_words=self.MAX_NB_WORDS)
tokenizer.fit_on_texts([sentences])
self.sequences = tokenizer.texts_to_sequences([sentences])
self.word_index = tokenizer.word_index
seq_length = 10
data = [m for m in self.word_index.values()]
index_word = {}
for w, id in list(self.word_index.items()):
index_word[id] = w
dataX = []
dataY = []
length = len(data) - seq_length
for i in range(0, length, seq_length):
seq_in = data[i:i + seq_length - 1]
seq_out = data[i + seq_length]
dataX.append(seq_in)
dataY.append(seq_out)
"""
raw_text = sentences
chars = sorted(list(set("word telphone main")))
# create mapping of unique chars to integers
chars = sorted(list(set(raw_text)))
char_to_int = dict((c, i) for i, c in enumerate(chars))
int_to_char = dict((i, c) for i, c in enumerate(chars))
# summarize the loaded data
n_chars = len(raw_text)
n_vocab = len(chars)
print ("Total Characters: ", n_chars)
print ("Total Vocab: ", n_vocab)
# prepare the dataset of input to output pairs encoded as integers
seq_length = 100
dataX = []
dataY = []
for i in range(0, n_chars - seq_length, 1):
seq_in = raw_text[i:i + seq_length]
seq_out = raw_text[i + seq_length]
dataX.append([char_to_int[char] for char in seq_in])
dataY.append(char_to_int[seq_out])
"""
n_patterns = len(dataX)
print("Total Patterns: ", n_patterns)
# reshape X to be [samples, time steps, features]
X = np.reshape(dataX, (n_patterns, seq_length - 1, 1))
n_vocab = len(index_word)
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
# define the LSTM model
model = Sequential()
model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
# define the checkpoint
filepath = "weights-improvement-{epoch:02d}-{loss:.4f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks_list = [checkpoint]
# fit the model
#model.fit(X, y, nb_epoch=20, batch_size=128, callbacks=callbacks_list)
start = np.random.randint(0, len(dataX) - 1)
pattern = dataX[start]
print("Seed:")
print("\"", ''.join([index_word[value] for value in pattern]), "\"")
# generate characters
for i in range(1000):
x = np.reshape(pattern, (1, len(pattern), 1))
x = x / float(n_vocab)
prediction = model.predict(x, verbose=0)
index = np.argmax(prediction)
result = index_word[index]
seq_in = [index_word[value] for value in pattern]
sys.stdout.write(result)
pattern.append(index)
pattern = pattern[1:len(pattern)]
print("\nDone.")