def exc():
# 5. Test
tmp = store.loadTermData()
termList = {'X': [i[0] for i in tmp], 'y': [int(i[1]) for i in tmp]}
print('=======================================================')
print('=> Term Classifying...')
if (file_model):
clf = store.loadClassifier(file=file_model)
else:
clf = store.loadClassifier()
results = []
for i in range(len(termList['X'])):
preprocessd_term = preprocess(termList['X'][i])
X = np.asarray([extractFeatureText(termList['X'][i], getFeatureNames())])
y_hat = clf.predict(X)[0].tolist()[0]
results.append(clf.predict(X)[0].tolist() + clf.predict_proba(X)[0].tolist() +
[1 if (y_hat != termList['y'][i]) else 0, preprocessd_term] + X[0].tolist())
titles = ['TestCase', 'Term', 'Label', 'Predicted Label', 'Name Score', 'Address Score', 'Phone Score', 'Error', 'Preprocessed_Term'] + \
getFeatureNames()
tacc = sum([1 for (y1, y2) in zip(termList['y'], [result[0] for result in results]) if (y1 == y2)]) / len(termList['y'])
if (file_model):
store.saveTermTestResults(tacc, titles, termList, results, file=file_model + '_' + file_term_classify_result)
else:
store.saveTermTestResults(tacc, titles, termList, results, file=timeManage.getTime() + '_' + file_term_classify_result)
return tacc
def exc(file='testdata.txt', label_file=folder_datasource + '/' + 'fa_labels'):
# 5. Test & Labels
textList = store.loadTextData(file=file)
labels = store.loadCSV(label_file)
if (file_model):
_time, templateList = sg.parseAddress(textList, file_model)
else:
_time, templateList = sg.parseAddress(textList)
acc, dist = checkAddressSegmentAccuracy(labels, templateList)
titles = [['#', 'Text', 'Runtime'],
['TestCase', 'Top', 'Name', 'Address', 'Phone', 'NameScore', 'AddressScore', 'PhoneScore', 'Score',
'PrepName'] + ['Name_' + ft for ft in getFeatureNames()] + \
['PrepAddress'] + ['Address_' + ft for ft in getFeatureNames()] + \
['PrepPhone'] + ['Phone_' + ft for ft in getFeatureNames()]]
if (file_model):
store.saveResults(titles, (textList, _time),
templateList,
file=file_model + '_' + file_segment_address_result,
acc=acc)
else:
store.saveResults(titles, (textList, _time),
templateList,
file=timeManage.getTime() + '_' +
file_segment_address_result,
acc=acc)
return acc
def exc(file='testdata.txt', label_file=folder_datasource + '/' + 'fa_labels'):
# 5. Test & Labels
textList = store.loadTextData(file=file)
labels = store.loadCSV(label_file)
if (file_model):
_time, templateList = sg.parseAddress(textList, file_model)
else:
_time, templateList = sg.parseAddress(textList)
if (sum([len(itmp) for itmp in templateList]) > 0):
acc, dist = checkAddressSegmentAccuracy(labels, templateList)
else:
acc = 0
titles = [['#', 'Text', 'Runtime'],
['TestCase', 'Top', 'Name', 'Address', 'Phone', 'NameScore', 'AddressScore', 'PhoneScore', 'Score',
'PrepName'] + ['Name_' + ft for ft in getFeatureNames()] + \
['PrepAddress'] + ['Address_' + ft for ft in getFeatureNames()] + \
['PrepPhone'] + ['Phone_' + ft for ft in getFeatureNames()]]
if (file_model):
store.saveResults(titles, (textList, _time), templateList, file=file_model + '_' + file_segment_address_result,
acc=acc)
else:
store.saveResults(titles, (textList, _time), templateList, file=timeManage.getTime() + '_' + file_segment_address_result,
acc=acc)
return acc
def templateFiler(clf, ptemplates):
templates = []
m = {0: 'name', 1: 'address', 2: 'phone'}
ft = fe.getFeatureNames()
dft = {key: value for (key, value) in zip(ft, range(len(ft)))}
for terms in ptemplates:
# preprocessedTerms = [fe.extractFeatureText(term) for term in terms]
preprocessedTerms = []
for term in terms:
if (bpreprocessing):
preprocessedTerms.append(fe.preprocess(term))
btmp = False
_X = []
for term in preprocessedTerms:
if (len(term) > 0):
_X.append(fe.feature(term, fe.getFeatureNames()))
else:
btmp = True
if (btmp):
continue
X = np.asarray(_X)
cls = clf.predict(X)
tmp = copy.deepcopy(cls.reshape((1, cls.shape[0])).tolist()[0])
tmp.sort()
if (tmp == list(range(len(terms)))):
dct = {}
probs = clf.predict_proba(X)
for (term, cl, prob, prepTerm, _x) in zip(terms, cls, probs,
preprocessedTerms, _X):
try:
dct[m[int(cl)]] = {
'term': term,
'score': prob[int(cl)],
'preprocessed': prepTerm,
'features': _x
}
except ValueError:
dct[m[int(cl)]] = {
'term': term,
'score': prob[cl],
'preprocessed': prepTerm,
'features': _x
}
dct['score'] = sum([log(dct[key]['score']) for key in dct])
if (checkTemplate(dct, dft)):
templates.append(dct)
if (len(templates) > 0):
templates = sorted(templates, key=lambda k: k['score'], reverse=True)
return templates
def exc():
# 5. Test
tmp = store.loadTermData()
termList = {'X': [i[0] for i in tmp], 'y': [int(i[1]) for i in tmp]}
print('=======================================================')
print('=> Term Classifying...')
if (file_model):
clf = store.loadClassifier(file=file_model)
else:
clf = store.loadClassifier()
results = []
ft = getFeatureNames()
for i in range(len(termList['X'])):
preprocessd_term = preprocess(termList['X'][i])
X = np.asarray([extractFeatureText(termList['X'][i], ft)])
y_hat = clf.predict(X)[0].tolist()[0]
results.append(clf.predict(X)[0].tolist() + clf.predict_proba(X)[0].tolist() +
[1 if (y_hat != termList['y'][i]) else 0, preprocessd_term] + X[0].tolist())
titles = ['TestCase', 'Term', 'Label', 'Predicted Label', 'Name Score', 'Address Score', 'Phone Score', 'Error', 'Preprocessed_Term'] + ft
tacc = sum([1 for (y1, y2) in zip(termList['y'], [result[0] for result in results]) if (y1 == y2)]) / len(termList['y'])
if (file_model):
store.saveTermTestResults(tacc, titles, termList, results, file=file_model + '_' + file_term_classify_result)
else:
store.saveTermTestResults(tacc, titles, termList, results, file=timeManage.getTime() + '_' + file_term_classify_result)
return tacc
def getModelConfig(testAcc=0, valAcc=0):
dct = {}
dct['name'] = timeManage.getTime()
dct['dictionary'] = {
'folder': folder_dictionary,
'file': [files_dictionary]
}
dct['database'] = {
'folder': folder_data,
'name': files_data[0],
'address': files_data[1],
'phone': files_data[2]
}
dct['preprocessing'] = {
'flag': bpreprocessing,
'type': list(preprocessing_name.keys())
}
dct['features'] = getFeatureNames()
dct['model'] = {
'class': model_type,
'target': model_target,
'config': model_config
}
dct['results'] = {'test-accuracy': testAcc, 'validate-accuracy': valAcc}
return dct
def templateFiler(clf, ptemplates):
templates = []
m = {0: 'name', 1: 'address', 2: 'phone'}
ft = fe.getFeatureNames()
dft = {key: value for (key, value) in zip(ft, range(len(ft)))}
for terms in ptemplates:
# preprocessedTerms = [fe.extractFeatureText(term) for term in terms]
preprocessedTerms = []
for term in terms:
if (bpreprocessing):
preprocessedTerms.append(fe.preprocess(term))
btmp = False
_X = []
for term in preprocessedTerms:
if (len(term) > 0):
_X.append(fe.feature(term, fe.getFeatureNames()))
else:
btmp = True
if (btmp):
continue
X = np.asarray(_X)
cls = clf.predict(X)
tmp = copy.deepcopy(cls.reshape((1, cls.shape[0])).tolist()[0])
tmp.sort()
if (tmp == list(range(len(terms)))):
dct = {}
probs = clf.predict_proba(X)
for (term, cl, prob, prepTerm, _x) in zip(terms, cls, probs, preprocessedTerms, _X):
try:
dct[m[int(cl)]] = {'term': term, 'score': prob[int(cl)], 'preprocessed': prepTerm, 'features': _x}
except ValueError:
dct[m[int(cl)]] = {'term': term, 'score': prob[cl], 'preprocessed': prepTerm, 'features': _x}
dct['score'] = sum([log(dct[key]['score']) for key in dct])
if (checkTemplate(dct, dft)):
templates.append(dct)
if (len(templates) > 0):
templates = sorted(templates, key=lambda k: k['score'], reverse=True)
return templates
def exc(file='testdata.txt'):
# 5. Test
textList = store.loadTextData(file=file)
if (file_model):
_time, templateList = sg.parseAddress(textList, file_model)
else:
_time, templateList = sg.parseAddress(textList)
titles = [['#', 'Text', 'Runtime'],
['TestCase', 'Top', 'Name', 'Address', 'Phone', 'NameScore', 'AddressScore', 'PhoneScore', 'Score',
'PrepName'] + ['Name_' + ft for ft in getFeatureNames()] + \
['PrepAddress'] + ['Address_' + ft for ft in getFeatureNames()] + \
['PrepPhone'] + ['Phone_' + ft for ft in getFeatureNames()]]
if (file_model):
store.saveResults(titles, (textList, _time), templateList, file=file_model + '_' + file_segment_address_result)
else:
store.saveResults(titles, (textList, _time), templateList, file=timeManage.getTime() + '_' + file_segment_address_result)
def exc():
# 1. Read txt data
tupleData = store.loadTxtData()
# 2. Preprocessing
preprocessedData = tupleData
if (bpreprocessing):
preprocessedData = dataPreprocess(tupleData)
store.savePreprocessedDataCSV(preprocessedData, getFeatureNames())
def exc():
# 1. Read txt data
tupleData = store.loadTxtData()
# 2. Preprocessing
preprocessedData = tupleData
if (bpreprocessing):
preprocessedData = dataPreprocess(tupleData)
store.savePreprocessedDataCSV(preprocessedData, getFeatureNames())
def exc(file='testdata.txt'):
# 5. Test
textList = store.loadTextData(file=file)
if (file_model):
_time, templateList = sg.parseAddress(textList, file_model)
else:
_time, templateList = sg.parseAddress(textList)
titles = [['#', 'Text', 'Runtime'],
['TestCase', 'Top', 'Name', 'Address', 'Phone', 'NameScore', 'AddressScore', 'PhoneScore', 'Score',
'PrepName'] + ['Name_' + ft for ft in getFeatureNames()] + \
['PrepAddress'] + ['Address_' + ft for ft in getFeatureNames()] + \
['PrepPhone'] + ['Phone_' + ft for ft in getFeatureNames()]]
if (file_model):
store.saveResults(titles, (textList, _time),
templateList,
file=file_model + '_' + file_segment_address_result)
else:
store.saveResults(titles, (textList, _time),
templateList,
file=timeManage.getTime() + '_' +
file_segment_address_result)
def getModelConfig(testAcc=0, valAcc=0):
dct = {}
dct['name'] = timeManage.getTime()
dct['dictionary'] = {
'folder': folder_datasource,
'file': [files_dictionary]
}
dct['database'] = {
'folder': folder_data,
'name': files_data[0],
'address': files_data[1],
'phone': files_data[2]
}
dct['preprocessing'] = {
'flag': bpreprocessing,
'type': list(preprocessing_name.keys())
}
dct['features'] = getFeatureNames()
dct['model'] = {
'class': model_type,
'target': model_target,
'config': model_config
}
dct['results'] = {
'test-accuracy': testAcc,
'validate-accuracy': valAcc
}
return dct
# 2. Extract Features
extract_feature.exc()
# 3. Random Data
random_data.exc()
# 4. Train Data
tacc = train.exc()
# 5. Test
# 5.1. Test Term Classification
ttacc = tt.exc()
# 5.2. Test Address Segmentation
tas.exc()
millis_E = int(round(time.time() * 1000))
logs.append([timeManage.getTime(),
(millis_E - millis_S)/1000,
', '.join([key for key in preprocessing_name if preprocessing_name[key]])
if bpreprocessing else '',
str(len(getFeatureNames())) + ' features: ' + ', '.join(getFeatureNames()),
model_type,
modelDetails(),
tacc,
ttacc]
)
workbook = xlsxwriter.Workbook(folder_running_logs + '/' + file_log)
writeSheet(workbook.add_worksheet('logs'), logs)
workbook.close()
def exc():
# store.loadFeatureCSV()
# 4.1 Random Training and Testing Data
x = randomSample()
# store.saveTrainingTestingData(x)
store.saveTrainingTestingDataCSV(x, getFeatureNames())
def exc():
preprocessedData = store.loadPreprocessedDataCSV()
# 3. Extract features
featureTuples = extractFeature(preprocessedData)
store.saveFeatureCSV(featureTuples, getFeatureNames())
def exc():
preprocessedData = store.loadPreprocessedDataCSV()
# 3. Extract features
featureTuples = extractFeature(preprocessedData)
store.saveFeatureCSV(featureTuples, getFeatureNames())
# 3. Random Data
random_data.exc()
# 4. Train Data
tacc = train.exc()
# 5. Test
# 5.1. Test Term Classification
ttacc = tt.exc()
# 5.2. Test Address Segmentation
acc = tas.exc()
millis_E = int(round(time.time() * 1000))
features = getFeatureNames()
tfilters = getTemplateRemoveFilters()
logs.append([timeManage.getTime(),
(millis_E - millis_S)/1000,
', '.join([key for key in preprocessing_name if preprocessing_name[key]])
if bpreprocessing else '',
str(len(features)) + ' features: ' + ', '.join(features),
model_type,
modelDetails(),
tacc,
ttacc,
str(len(tfilters)) + ' filters: ' + ', '.join(tfilters),
acc]
)
def printData(text):
print('Text : ', text)
for i, j in zip(getFeatureNames(), feature(preprocess(text))):
print(i, ' : ', j)
print('---------------------------------')
extract_feature.exc()
# 3. Random Data
random_data.exc()
# 4. Train Data
tacc = train.exc()
# 5. Test
# 5.1. Test Term Classification
ttacc = tt.exc()
# 5.2. Test Address Segmentation
acc = tas.exc()
millis_E = int(round(time.time() * 1000))
features = getFeatureNames()
tfilters = getTemplateRemoveFilters()
logs.append([
timeManage.getTime(), (millis_E - millis_S) / 1000, ', '.join(
[key for key in preprocessing_name
if preprocessing_name[key]]) if bpreprocessing else '',
str(len(features)) + ' features: ' + ', '.join(features), model_type,
modelDetails(), tacc, ttacc,
str(len(tfilters)) + ' filters: ' + ', '.join(tfilters), acc
])
workbook = xlsxwriter.Workbook(folder_running_logs + '/' + file_log)
writeSheet(workbook.add_worksheet('logs'), logs)
workbook.close()