def _compute_classification(self, line: List[str], text_index: int,
class_index: int) -> Classification:
text = line[text_index]
class_type_actual = line[class_index]
scores, class_type_predicted = self._classify(text)
return Classification(text, line, class_type_actual,
class_type_predicted, scores)
def test_optimise(self):
clsfy = Classification(self.X, self.y)
theta_init = pd.Series(np.ones(len(self.X.columns)))
opt = clsfy.train(theta_init)
expected_vector = np.array([-25.161, 0.206, 0.201])
for exp_el, result_el in zip(opt, expected_vector):
self.assertAlmostEqual(result_el, exp_el, places=3)
def test_predict(self):
clsfy = Classification(self.X, self.y)
theta_init = pd.Series(np.ones(len(self.X.columns)))
opt = clsfy.train(theta_init)
prediction = clsfy._predict(np.dot(opt, self.X.T))
print(prediction)
print(clsfy.f_score(self.y, prediction))
def measure(filename, searchLang='polish', metric=Cosine):
manager = LanguageManager.getManager()
nGram2List = defaultdict(list)
baseList = []
with open(filename, 'r') as file:
for line in file:
line = line.lstrip()
if not line:
continue
baseLang, text = line.split(' ', 1)
baseList.append(baseLang == searchLang)
guess = manager.guessLanguage(text)
guess = flipDict(guess[0][metric])
for nGram, lang in guess.items():
bestLang = min(lang.items(), key=itemgetter(1))[0]
nGram2List[nGram].append(bestLang == searchLang)
for nGram, data in nGram2List.items():
print(f"{nGram}-Gram")
c = Classification(baseList, data)
print(f"Precision: {c.getPrecision()}")
print(f"Recall: {c.getRecall()}")
print(f"F1: {c.getF1()}")
print(f"Accuracy: {c.getAccuracy()}\n")
def write_geometries(self):
from classification import Classification
c = Classification(self, None)
cs = self.load_results(nozeros=False)
no_zeros = c.remove_impossible_angles(cs)
np.savetxt(self.folder+"/nozero_results.txt", no_zeros.values())
if not os.path.exists(self.folder+"/original/"):
os.makedirs(self.folder+"/original/")
write_results_to_file(self, self.oxygen_coordinates, no_zeros.values(), self.cell, self.nearest_neighbors_nos, folder=self.folder+'/nozeros/')
def classify(self, frame):
img_h = len(frame)
img_w = len(frame[0])
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb, (self.width, self.height))
input_data = np.expand_dims(frame_resized, axis=0)
self.interpreter.set_tensor(self.index, input_data)
self.interpreter.invoke()
output_details = self.interpreter.get_output_details()
boxes = self.interpreter.get_tensor(output_details[0]['index'])[
0] # Bounding box coordinates of detected objects
classes = self.interpreter.get_tensor(
output_details[1]['index'])[0] # Class index of detected objects
scores = self.interpreter.get_tensor(
output_details[2]['index'])[0] # Confidence of detected objects
date_time = str(datetime.now())
cv2.putText(frame, date_time, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 0, 0), 2)
classification = Classification(frame)
for i in range(len(scores)):
if ((scores[i] > self.min_conf_threshold) and (scores[i] <= 1.0)):
#Get bounding box coordinates and draw box
ymin = int(max(1, (boxes[i][0] * img_h)))
xmin = int(max(1, (boxes[i][1] * img_w)))
ymax = int(min(img_h, (boxes[i][2] * img_h)))
xmax = int(min(img_w, (boxes[i][3] * img_w)))
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (10, 255, 0),
4)
obj_name = self.labels[int(classes[i])]
classification.items += [Item(obj_name)]
#Drawing label (?)
labelSize, baseLine = cv2.getTextSize(obj_name,
cv2.FONT_HERSHEY_SIMPLEX,
1, 2)
label_ymin = max(ymin, labelSize[1] + 10)
cv2.rectangle(
frame, (xmin, label_ymin - labelSize[1] - 10),
(xmin + labelSize[0], label_ymin + baseLine - 10),
(255, 255, 255), cv2.FILLED)
cv2.putText(frame, obj_name, (xmin, label_ymin - 7),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)
return classification
def main():
# Parameters
data_directory = '../data/data-real-r-3'
features_path = '../data/features-data-real-r-3'
results_file = './results/silhouette_rf.csv'
classification = Classification(data_directory, features_path, algorithm='knn', feature_agglomeration=True)
classification.transform(results_file=results_file)
classification.evaluate(text_to_add='RF, silhueta')
exit()
# Parameters
data_directory = '../data/data-real-r-3'
features_path = '../data/features-data-real-r-3'
results_file = './results-text/all_rf.csv'
classification = Classification(data_directory, features_path, algorithm='knn', feature_agglomeration=True)
classification.transform(results_file=results_file)
classification.evaluate(text_to_add='RF')
def scoreModels():
approvedIndexes = noteResultIds[noteTestId]
for model in getAllModels():
simVec = getSimilarityVector(noteTestId, model.vectors)
simInd = getBestNIndexes(simVec, firstNResults)
classified = [ind in approvedIndexes for ind in simInd]
cl = Classification([True] * len(simInd), classified)
print(
f"{model} precission:{cl.getPrecision()}, recall:{cl.getRecall()}, F1:{cl.getF1()}"
)
def __init__(self, dataset_lists, feature="edge_hist",
learning_model="dnn", score_type="all"):
self.feature = feature
self.learning_model = learning_model
self.score_type = score_type
self.best_accuracy = 0
self.best_recall = 0
self.best_params = {}
self.best_confusion_matrix = 0
self.clf = Classification(dataset_lists, feature, learning_model,
output_every_scores=False)
def training(self, name, data, target):
X_train, X_test, y_train, y_test = train_test_split(data,
target,
test_size=0.1)
clf = Classification(X_train, y_train, name)
# clf.knn_model()
clf.nb_model()
# clf.mnb_model()
clf.rfc_model()
clf.svm_model()
clf.find_best_estimator(X_test, y_test)
def main(dataset_lists, feature, model, save_model, output_failed_data):
params = DNN_PARAMS if model == "dnn" else SVM_PARAMS
clf_params = {
"dataset_lists": dataset_lists,
"feature": feature,
"learning_model": model,
"save_model": save_model,
"output_failed_data": output_failed_data,
}
clf = Classification(**clf_params)
clf.run(**params)
def __init__(self):
super().__init__()
self.defaultmap = QtGui.QPixmap("default.png")
self.processmap = QtGui.QMovie("procession.gif")
self.images = []
self.emotions = []
self.labelfont = QtGui.QFont('Times', 12)
self.analysis = Classification()
self.mainWindow = None
# for storing data
self.capturePixels = None
def classify(self):
print("\nclassify")
self.process_classification_data()
self.split_data(0.7)
c = Classification(self.training_data, self.test_data)
c.decision_tree_classifier()
c.random_forest_classifier()
c.naive_bayes()
c.logistic_regression()
c.gbtc()
c.lsvc()
def display_page(pathname):
if pathname == '' or pathname == '/' or pathname == config.login_url:
# print('--> login')
return Login()
elif pathname == config.home_url:
# print('--> home')
return Homepage()
elif pathname == config.training_url:
# print('--> training')
return Training()
elif pathname == config.classification_url:
# print('--> classification')
return Classification()
def create(spec):
"""Creates ImageSpec.
:rtype: ImageSpec
"""
image_type = spec.get('imageType', 'MOSAIC')
if image_type == 'MOSAIC':
return _createMosaic(spec)
if image_type == 'CLASSIFICATION':
return Classification(spec, create)
if image_type == 'CHANGE_DETECTION':
return ChangeDetection(spec, create)
if image_type == 'ASSET':
return Asset(spec)
else:
raise Exception('Unexpected image image_type: ' + str(image_type))
def handle_options():
from water_algorithm import add_options
parser = get_parser()
add_options(parser)
add_options_ih(parser)
add_options_limit(parser)
(options, args) = parser.parse_args()
wa = IceIhLimited(n_x = options.n_x, n_y = options.n_y, n_z = options.n_z, slab = options.slab, orth = options.orth, misc = options.misc, limit = options.limit)
wa.classification = Classification(wa)
hr = HandleResults(wa)
wa.invariant_count = options.invariant_count
wa.execute_commands(options, args)
wa.execute_local_commands(options, args)
hr.execute_commands(options, args)
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SentenceClassification.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load(embedding_size, dm.vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath, args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Classification(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
else:
raise ValueError("Unknown mode")
def try_class():
wa = Dodecahedron(folder="dodecahedron")
classi = Classification(wa).group_by_aadd_and_aad(nozeros=False)
hr = HandleResults(0, 30026, wa)
smdip = hr.get_small_dipole_moment_estimates()
#print smdip
print classi[6].keys()
count = 0
for i in classi[6][6]:
if i in smdip:
energy = hr.read_energy(i)
count += 1
if energy != None:
print "Geometry %i has 0 add and a small dipole moment (Energy %f eV)" % (i, energy)
else:
print "Geometry %i has 0 add and a small dipole moment" % i
print "Count: %i vs %i" % (count, len(classi[6][6]))
def __init__(self):
self.dictionary = Dictionary()
self.dictionary.init()
final_train_doc_index = 995
self.classification = Classification(final_train_doc_index,
knn_mode=True)
# train_data = []
# for i in range(0, final_train_doc_index):
# train_data.append(self.dictionary.doc2vec[i].toarray()[0])
# self.classification.learn(train_data)
# print('complete learned ...')
# final_test_doc_index = 158200
# counter = 1
# while 1:
# test_data = []
# doc_ids = []
# if (final_test_doc_index - final_train_doc_index) < 1000:
# for i in range(final_train_doc_index, final_test_doc_index):
# test_data.append(self.dictionary.doc2vec[i].toarray()[0])
# doc_ids.append(i)
# self.classification.classify_test_data(test_data, doc_ids)
# break
# else:
# for i in range(final_train_doc_index, final_train_doc_index + 1000):
# test_data.append(self.dictionary.doc2vec[i].toarray()[0])
# doc_ids.append(i)
# self.classification.classify_test_data(test_data, doc_ids)
# print('complete phase: ' + str(final_train_doc_index))
# counter += 1
# final_train_doc_index += 1000
#
# # TODO: SAVE CLASS LABELS
# self.classification.save_labels()
# TODO: LOAD CLASS LABELS
self.classification.load_labels()
print('complete classification ...')
KMeans.init()
# KMeans.create(self.dictionary.doc2vec)
print('complete clustering ...')
self.clustering_mode = True
def implement(self):
# output: the index of classification /int
soildetector = Detector(self.img_arr)
detected_img_arr = soildetector.soil_img_arr()
if self.saveProImg == True: soildetector.save_pro_img()
Data.processed_img_arr = soildetector.processed_rgb_arr_3d
print("土壤检测耗时(s): ", soildetector.used_time)
imgprocessor = PreprocessImg(detected_img_arr)
X_data = imgprocessor.get_X()
Data.grayhist_img_arr = imgprocessor.thresh_gray_img
print("图片处理耗时(s): ", imgprocessor.used_time)
classifier = Classification(X_data)
class_index = classifier.predict()
print("图片分类耗时(s): ", classifier.used_time)
return class_index
def main(gazetteer, datasetFile, annotatedEntities, vocabularyFile):
print("Geração dos módulos...")
classification = Classification(gazetteer, datasetFile, annotatedEntities,
vocabularyFile)
#classification.teste()
#return
while (True):
print("Esperando por tweets...")
try:
auth = OAuthHandler(consumerKey, consumerSecret)
auth.set_access_token(accessToken, accessSecret)
api = tweepy.API(auth)
twitterStream = Stream(auth, MyListener(classification))
#twitterStream.filter(follow=['8802752']) #G1 - teste
twitterStream.filter(follow=['524349796']) #BHTrans
except ValueError:
print("Erro: " + ValueError)
def __init__(self,
O_O_distance=2.8,
O_H_distance=1.0,
intermediate_saves=[],
folder="wales_21+",
group_saves=[]):
WaterAlgorithm()
self.initialize(O_H_distance=O_H_distance,
O_O_distance=O_O_distance,
intermediate_saves=intermediate_saves,
group_saves=group_saves,
folder=folder,
charge=1,
do_symmetry_check=False,
order=[
3, 14, 9, 2, 10, 19, 16, 15, 7, 18, 4, 0, 17, 1,
12, 6, 5, 8, 20, 11, 13
])
self.N = 21
self.classification = Classification(self)
def __init__(self,
O_O_distance=2.7,
O_H_distance=1.0,
intermediate_saves=[],
folder="pepa_up",
group_saves=[]):
WaterAlgorithm()
cell = np.array([[+20.5739789056, +0.0000000000, +0.0000000000],
[+0.0000000000, +7.2740000000, +0.0000000000],
[+0.0000000000, +0.0000000000, +45.0000000000]])
periodic = [True, True, False]
self.initialize(O_H_distance=O_H_distance,
O_O_distance=O_O_distance,
intermediate_saves=intermediate_saves,
group_saves=group_saves,
folder=folder,
do_symmetry_check=False,
periodic=periodic,
cell=cell)
self.classification = Classification(self)
def main():
# Preselection example
# Parameters
data_directory = '../data/generated-data-r-2-n-6-4'
features_path = '../data/features-generated-data-r-2-n-6-4'
results_file = '/results-preselection/generated-data-r-2-n-6-4.csv'
#true_objects_indexes = [0, 1, 2, 3, 4, 5, 6, 7]
#false_objects_indexes = [8, 9]
true_objects_indexes = [0, 1, 2, 3, 4, 5]
false_objects_indexes = [6, 7, 8, 9]
preselection = Preselection(data_directory, features_path, true_objects_indexes, false_objects_indexes)
preselection.transform(results_file=results_file)
#preselection.evaluate()
exit()
# Texts example
# Parameters
data_directory = '../data/data-real-r-3-text'
results_file = '/results-text/all_knn.csv'
classification = ClassificationText(data_directory, algorithm='knn', feature_agglomeration=False, selection='none')
classification.transform(results_file=results_file)
classification.evaluate()
exit()
# Images example
# Parameters
data_directory = '../data/data-real-r-3'
features_path = '../data/features-data-real-r-3'
results_file = '/results/kmeans_knn.csv'
classification = Classification(data_directory, features_path, algorithm='knn', feature_agglomeration=False,
selection='kmeans')
classification.transform(results_file=results_file)
classification.evaluate()
def get(self):
global gl
# return gl
file_id = request.args.get('id')
gl = {'server_start_ts': dt.microsecond, 'id': file_id}
if not file_id:
file_id = 'admin_hkr_se'
print('ML Default FCS admin_hkr_se')
else:
print('ML RECEIVED FCS:', file_id)
files = ['a1_24h_noc200.fcs', 'RFP_Well_A3.fcs', 'RFP_Well_A6.fcs', 'RFP_Well_B3.fcs']
response = {}
for file in files:
cls = Classification(file_id, False)
response[file] = cls.predict(file)
gl.update(response)
return response
def _classify_measurements(self, measurements):
classifications = {}
for socket in self._available_hardware:
classifications[socket] = Classification()
for cpu in self._available_hardware[socket]:
measurement = measurements[cpu]
is_cpu_bound = True
is_inter = measurement.inter_socket_coherence > config.INTER_SOCKET_COHERENCE_THRESHOLD_PER_TASK
if is_inter:
classifications[socket].is_inter.append(cpu)
is_cpu_bound = False
if not is_inter and \
measurement.intra_socket_coherence > config.INTRA_SOCKET_COHERENCE_THRESHOLD_PER_TASK:
classifications[socket].is_intra.append(cpu)
is_cpu_bound = False
if measurement.remote_dram > config.REMOTE_MEMORY_ACCESS_THRESHOLD_PER_TASK:
classifications[socket].is_remote_dram.append(cpu)
is_cpu_bound = False
if measurement.memory_bandwidth > config.MEMORY_UTILIZATION_THRESHOLD_PER_TASK:
classifications[socket].is_memory_bound.append(cpu)
is_cpu_bound = False
# if measurement.cycles == 0:
if measurement.cycles > config.IDLE_THRESHOLD_PER_TASK:
classifications[socket].is_idle.append(cpu)
is_cpu_bound = False
if is_cpu_bound:
classifications[socket].is_cpu_bound.append(cpu)
return classifications
def run(filename, min_size, max_size, dist):
noun_dict = noun_extractor.get_nouns(filename)
(synset_list, synset_dict) = cluster.get_synset_list(noun_dict)
matrix = cluster.gen_sim_matrix(synset_list)
clustering = cluster.format_clustering(cluster.cluster(matrix),
synset_list)
clusters = cluster.get_clusters(clustering,
synset_list,
min_size,
max_size,
dist=dist)
clusters = filter(lambda x: x[0] is not None, clusters)
cluster_counts = cluster.get_cluster_counts(clusters, synset_dict)
# sort clusters by noun counts, most frequent first
sorted_clusters = [
x[1] for x in sorted(enumerate(clusters),
key=lambda x: cluster_counts[x[0]],
reverse=True)
]
hypernyms = filter(lambda x: x is not None,
map(lambda x: lca(x), sorted_clusters))
return Classification(noun_dict, synset_list, synset_dict, matrix,
clustering, sorted_clusters, hypernyms)
def get_accuracy_data(self):
classification = Classification()
data = []
data = doc_features_glob
for i in range(80, 90):
document = self.brat_reader.documents['essay' + str(i)]
self.extract_doc_features(document, 'essay' + str(i))
# data.append(doc_features_glob[-1])
classification.set_data(data)
arguments = classification.divided_args
links = classification.divided_links
arguments_features = classification.getFeatures(arguments)
links_features = classification.getFeatures(links)
# delete ?
def argument_extract_features(document):
argument_features = {}
for word in arguments_features:
argument_features['hold(%s)' % word] = (word in set(document))
return argument_features
def extract_features(document):
features = {}
if 'Support' and 'Attack' in document:
src = links_features
else:
src = arguments_features
for word in src:
features['hold(%s)' % word] = (word in set(document))
return features
arguments_test_set = nltk.classify.apply_features(
extract_features, arguments)
links_test_set = []
# links_test_set = nltk.classify.apply_features(extract_features, links)
return [arguments_test_set, links_test_set]
def __init__(self,
dialog_acts_filename,
restaurant_matrix_filename,
machine_learning_model=""):
"""
initialize the dialog agent
Parameters
----------
dialog_acts_filename : str
filename of dialog acts. Dialog acts should be in format of [label utterance] with whitespace as separator.
restaurant_matrix_filename : str
csv file containing restaurants, their contacts and information.
machine_learning_model: str
nn for neural net, standard LR
Returns
-------
None.
"""
self.statelog = []
self.clf_agent = Classification()
self.clf_agent.initialize_data(dialog_acts_filename)
if machine_learning_model == "nn":
self.clf_agent.train_nn()
else:
self.clf_agent.train_lr()
#preparation for preference extraction
file = pd.read_csv(restaurant_matrix_filename)
#extracting columns
self.restaurant_names = list(file['restaurantname'])
self.price_range = list(file['pricerange'])
self.area = list(file['area'])
self.food_types = list(file['food'])
self.phone = list(file['phone'])
self.address = list(file['addr'])
self.post_code = list(file['postcode'])
self.good_food = list(file['good food'])
self.open_kitchen = list(file['open kitchen'])
self.hygiene = list(file['hygiene'])
self.suggestions = []
self.delay = 1
self.responses_formal = {
"Welcome": [
"Hello! I can recommend restaurants to you. To start, please enter your request. You can ask for restaurants by area, price range, or food type\n",
"Hello and welcome to our restaurant system. You can ask for restaurants by area, price range, or food type. To start, please enter your request\n",
"Hello! You can ask for restaurants by area, price range, or food type. How may I help you?\n"
],
"Area": ['What part of town do you have in mind?\n'],
'Price': [
'What is your desired price range? Cheap, moderate, or expensive?\n',
'Would you prefer a cheap, moderate, or expensive restaurant?\n'
],
'Food': [
"What kind of food would you like? \n",
"What type of food do you prefer?\n"
],
"AffirmPreferences": [
'So, you are looking for a restaurant in the {0} part of town, with {1} price range, serving {2} food, correct?\n'
],
"Answer": [
"Okay, here is your recommendation: '{}'. Is it fine? \n",
"I have found a nice restaurant matching your preferences: ‘{}’. Do you like it? \n",
"I can recommend a good place that fits your criteria: ‘{}’. Is it fine? \n"
],
"NoOptions": [
"Sorry, there are no recommendations matching your demands.Let's try finding something else\n",
"Unfortunately, I couldn’t find a restaurant that matches your expectations. Let’s try to find something else \n"
],
'Goodbye': [
"Thank you for using our restaurant system. Come back! \n",
"Thank you, I hope I was useful. Do come back! \n"
]
}
self.responses_informal = {
"Welcome": [
"Hi, let’s choose a restaurant! Where do you want to eat? Area, price range, food type?\n"
],
"Area": ["What part of town do you have in mind?\n"],
"Price": ["What’s your budget? Cheap, moderate, or expensive?\n"],
"Food": ["What sort of food would you like? \n"],
"AffirmPreferences": [
"So, you want a restaurant in the {0} part of town, with {1} price range, serving {2} food, am I right?\n"
],
"Answer": [
"Okay, I came up with a recommendation: '{}'. Sounds good? \n",
"I have found a cool place matching your preferences: ‘{}’. You like it? \n"
],
"NoOptions": [
"Sorry, there’s nothing matching your needs. Wanna try something else? \n"
],
"Goodbye": ["Thanks, hope it was useful. See you! \n"]
}
self.responses = self.responses_informal
self.implication_rules = {
"cheap,good food": ["busy"],
"spanish": ["long time"],
'busy': ['long time', 'not romantic'],
'long time': ['not children', 'romantic'],
'not hygiene,open kitchen': ['not romantic'],
'not good food,not hygiene': ['not busy'],
'open kitchen': ['children'],
'long time,not open kitchen': ['boring'],
'boring,expensive': ['not busy'],
'boring': ['not romantic']
}
#-*- coding:utf-8 -*-
# AUTHOR: yaolili
# FILE: runClassification.py
# ROLE: run classifier in Classification and get the result of prediction
# CREATED: 2015-12-15 09:28:02
# MODIFIED: 2015-12-15 09:28:03
import sys
import os
from classification import Classification
if __name__ == "__main__":
if len(sys.argv) < 5:
print "sys.argv[1]: classifier"
print "sys.argv[2]: trainFile"
print "sys.argv[3]: devFile"
print "sys.argv[4]: outputFile"
exit()
cfInstance = Classification(sys.argv[1], sys.argv[2], sys.argv[3])
cfInstance.getPreResult(sys.argv[4])