def filter(self, data):
"""
Filters the dataset(s). When providing a list, this can be used to create compatible train/test sets,
since the filter only gets initialized with the first dataset and all subsequent datasets get transformed
using the same setup.
NB: inputformat(Instances) must have been called beforehand.
:param data: the Instances to filter
:type data: Instances or list of Instances
:return: the filtered Instances object(s)
:rtype: Instances or list of Instances
"""
if isinstance(data, list):
result = []
for d in data:
result.append(
Instances(
javabridge.static_call(
"Lweka/filters/Filter;", "useFilter",
"(Lweka/core/Instances;Lweka/filters/Filter;)Lweka/core/Instances;",
d.jobject, self.jobject)))
return result
else:
return Instances(
javabridge.static_call(
"Lweka/filters/Filter;", "useFilter",
"(Lweka/core/Instances;Lweka/filters/Filter;)Lweka/core/Instances;",
data.jobject, self.jobject))
def split_data(data, test_size): # split the data
# create placeholder for train split
data_train = Instances.copy_instances(data)
# remove all instances from the placeholder
for i in reversed(range(len(data_train))):
data_train.delete(i)
# create placeholder for test split
data_test = Instances.copy_instances(data)
# remove all instances from the placeholder
for i in reversed(range(len(data_test))):
data_test.delete(i)
# create list of indices
indices = list(range(len(data)))
# shuffle indices
random.shuffle(indices)
# calculate number of indices in the test split
num_test = int(round(len(indices) * test_size, 0))
# get indices for the test split
test_ids = indices[:num_test]
# fill test split with instances
for idx in test_ids:
data_test.add_instance(data.get_instance(idx))
# get indices for the train split
train_ids = indices[num_test:]
# fill train split with instances
for idx in train_ids:
data_train.add_instance(data.get_instance(idx))
return data_train, data_test
def load_file(self, dfile, incremental=False):
"""
Loads the specified file and returns the Instances object.
In case of incremental loading, only the structure.
:param dfile: the file to load
:type dfile: str
:param incremental: whether to load the dataset incrementally
:type incremental: bool
:return: the full dataset or the header (if incremental)
:rtype: Instances
"""
self.enforce_type(self.jobject,
"weka.core.converters.FileSourcedConverter")
self.incremental = incremental
if not javabridge.is_instance_of(dfile, "Ljava/io/File;"):
dfile = javabridge.make_instance(
"Ljava/io/File;", "(Ljava/lang/String;)V",
javabridge.get_env().new_string_utf(str(dfile)))
javabridge.call(self.jobject, "reset", "()V")
javabridge.call(self.jobject, "setFile", "(Ljava/io/File;)V", dfile)
if incremental:
self.structure = Instances(
javabridge.call(self.jobject, "getStructure",
"()Lweka/core/Instances;"))
return self.structure
else:
return Instances(
javabridge.call(self.jobject, "getDataSet",
"()Lweka/core/Instances;"))
def splitTrainSet(data,m_numLabledData=10) :
total = data.num_instances
labeled_amount = int(m_numLabledData * total / 100)
unlabeled_amount = total - labeled_amount
rand = Random(1)
data.randomize(rand)
labledDataSet = Instances.create_instances(data.relationname,data.attributes(),labeled_amount)
UnlabledDataSet = Instances.create_instances(data.relationname,data.attributes(),unlabeled_amount)
for i in range(labeled_amount) :
labledDataSet.add_instance(data.get_instance(i))
labledDataSet.randomize(rand)
for i in range(unlabeled_amount) :
UnlabledDataSet.add_instance(data.get_instance(labeled_amount + i))
# labledDataSet.randomize(rand)
labledDataSet.class_is_last()
# UnlabledDataSet.randomize(rand)
UnlabledDataSet.class_is_last()
return labledDataSet,UnlabledDataSet
def train_internal(self):
best_weights_arr = []
#create an empty F with source as template
F = Instances.template_instances(self.source[0])
withF = False
print("Find weight for each source data set")
for source in self.source:
bestWeight, bestError = self.process_source(source, F, withF)
best_weights_arr.append(bestWeight)
#sort the data based on the weights
self.source = [
source for _, source in sorted(zip(best_weights_arr, self.source),
reverse=True,
key=operator.itemgetter(0))
]
print("Train for final stage")
withF = True
while len(self.source) > 0: #self.max_source_dataset):
weight, _ = self.process_source(self.source[0], F, withF)
for inst in self.source[0]:
inst.weight = weight
F = Instances.append_instances(F, self.source[0])
F.class_is_last()
self.source.pop(0)
return F
def load_file(self, dfile, incremental=False):
"""
Loads the specified file and returns the Instances object.
In case of incremental loading, only the structure.
:param dfile: the file to load
:type dfile: str
:param incremental: whether to load the dataset incrementally
:type incremental: bool
:return: the full dataset or the header (if incremental)
:rtype: Instances
:raises Exception: if the file does not exist
"""
self.enforce_type(self.jobject,
"weka.core.converters.FileSourcedConverter")
self.incremental = incremental
if not javabridge.is_instance_of(dfile, "Ljava/io/File;"):
dfile = javabridge.make_instance(
"Ljava/io/File;", "(Ljava/lang/String;)V",
javabridge.get_env().new_string_utf(str(dfile)))
javabridge.call(self.jobject, "reset", "()V")
# check whether file exists, otherwise previously set file gets loaded again
sfile = javabridge.to_string(dfile)
if not os.path.exists(sfile):
raise Exception("Dataset file does not exist: " + str(sfile))
javabridge.call(self.jobject, "setFile", "(Ljava/io/File;)V", dfile)
if incremental:
self.structure = Instances(
javabridge.call(self.jobject, "getStructure",
"()Lweka/core/Instances;"))
return self.structure
else:
return Instances(
javabridge.call(self.jobject, "getDataSet",
"()Lweka/core/Instances;"))
def load_url(self, url, incremental=False):
"""
Loads the specified URL and returns the Instances object.
In case of incremental loading, only the structure.
:param url: the URL to load the data from
:type url: str
:param incremental: whether to load the dataset incrementally
:type incremental: bool
:return: the full dataset or the header (if incremental)
:rtype: Instances
"""
self.enforce_type(self.jobject,
"weka.core.converters.URLSourcedLoader")
self.incremental = incremental
javabridge.call(self.jobject, "reset", "()V")
javabridge.call(self.jobject, "setURL", "(Ljava/lang/String;)V",
str(url))
if incremental:
self.structure = Instances(
javabridge.call(self.jobject, "getStructure",
"()Lweka/core/Instances;"))
return self.structure
else:
return Instances(
javabridge.call(self.jobject, "getDataSet",
"()Lweka/core/Instances;"))
def testNB(training_data, testing_data):
train_data = Instances.copy_instances(training_data)
test_data = Instances.copy_instances(testing_data)
evaluation = Evaluation(train_data)
classifier = Classifier(classname="weka.classifiers.bayes.NaiveBayes")
classifier.build_classifier(
train_data) # build classifier on the training data
evaluation.test_model(classifier,
test_data) # test and evaluate model on the test set
print("")
print("")
print(
evaluation.summary(
"--------------Naive Bayes Evaluation--------------"))
print("Accuracy: " + str(evaluation.percent_correct))
print("")
print("Label\tPrecision\t\tRecall\t\t\tF-Measure")
print("<=50K\t" + str(evaluation.precision(0)) + "\t" +
str(evaluation.recall(0)) + "\t" + str(evaluation.f_measure(0)))
print(">50K\t" + str(evaluation.precision(1)) + "\t" +
str(evaluation.recall(1)) + "\t" + str(evaluation.f_measure(1)))
print("Mean\t" + str(((evaluation.precision(1)) +
(evaluation.precision(0))) / 2) + "\t" +
str(((evaluation.recall(1)) + (evaluation.recall(0))) / 2) + "\t" +
str(((evaluation.f_measure(1)) + (evaluation.f_measure(0))) / 2))
def main():
"""
Just runs some example code.
"""
# load a dataset
iris_file = helper.get_data_dir() + os.sep + "iris.arff"
helper.print_info("Loading dataset: " + iris_file)
loader = Loader("weka.core.converters.ArffLoader")
iris_data = loader.load_file(iris_file)
iris_data.class_is_last()
# train classifier
classifier = Classifier("weka.classifiers.trees.J48")
classifier.build_classifier(iris_data)
# save and read object
helper.print_title("I/O: model (using serialization module)")
outfile = tempfile.gettempdir() + os.sep + "j48.model"
serialization.write(outfile, classifier)
model = Classifier(jobject=serialization.read(outfile))
print(model)
# save classifier and dataset header (multiple objects)
helper.print_title("I/O: model and header (using serialization module)")
serialization.write_all(
outfile,
[classifier, Instances.template_instances(iris_data)])
objects = serialization.read_all(outfile)
for i, obj in enumerate(objects):
helper.print_info("Object #" + str(i + 1) + ":")
if javabridge.get_env().is_instance_of(
obj,
javabridge.get_env().find_class("weka/core/Instances")):
obj = Instances(jobject=obj)
elif javabridge.get_env().is_instance_of(
obj,
javabridge.get_env().find_class(
"weka/classifiers/Classifier")):
obj = Classifier(jobject=obj)
print(obj)
# save and read object
helper.print_title("I/O: just model (using Classifier class)")
outfile = tempfile.gettempdir() + os.sep + "j48.model"
classifier.serialize(outfile)
model, _ = Classifier.deserialize(outfile)
print(model)
# save classifier and dataset header (multiple objects)
helper.print_title("I/O: model and header (using Classifier class)")
classifier.serialize(outfile, header=iris_data)
model, header = Classifier.deserialize(outfile)
print(model)
if header is not None:
print(header)
def load_file(self, dfile, incremental=False, class_index=None):
"""
Loads the specified file and returns the Instances object.
In case of incremental loading, only the structure.
:param dfile: the file to load
:type dfile: str
:param incremental: whether to load the dataset incrementally
:type incremental: bool
:param class_index: the class index string to use ('first', 'second', 'third', 'last-2', 'last-1', 'last' or 1-based index)
:type class_index: str
:return: the full dataset or the header (if incremental)
:rtype: Instances
:raises Exception: if the file does not exist
"""
self.enforce_type(self.jobject,
"weka.core.converters.FileSourcedConverter")
self.incremental = incremental
if not javabridge.is_instance_of(dfile, "Ljava/io/File;"):
dfile = javabridge.make_instance(
"Ljava/io/File;", "(Ljava/lang/String;)V",
javabridge.get_env().new_string_utf(str(dfile)))
javabridge.call(self.jobject, "reset", "()V")
# check whether file exists, otherwise previously set file gets loaded again
sfile = javabridge.to_string(dfile)
if not os.path.exists(sfile):
raise Exception("Dataset file does not exist: " + str(sfile))
javabridge.call(self.jobject, "setFile", "(Ljava/io/File;)V", dfile)
if incremental:
self.structure = Instances(
javabridge.call(self.jobject, "getStructure",
"()Lweka/core/Instances;"))
result = self.structure
else:
result = Instances(
javabridge.call(self.jobject, "getDataSet",
"()Lweka/core/Instances;"))
if class_index is not None:
if class_index == 'first':
result.class_index = 0
elif class_index == 'second':
result.class_index = 1
elif class_index == 'third':
result.class_index = 2
elif class_index == 'last-2':
result.class_index = result.num_attributes - 3
elif class_index == 'last-1':
result.class_index = result.num_attributes - 2
elif class_index == 'last':
result.class_index = result.num_attributes - 1
else:
result.class_index = int(class_index)
return result
def LabeledUnlabeldata(data, unlabeled, tree, y, cal_method=None):
data1 = Instances.copy_instances(data)
labeling = Instances.copy_instances(unlabeled)
tree.build_classifier(data1)
j = i = s = l = 0
while i < labeling.num_instances:
clsLabel = tree.classify_instance(labeling.get_instance(i))
##### probability calculation #####
# dist = tree.distribution_for_instance(labeling.get_instance(i))
dist = calculate_probability_distribution(tree, labeling, i,
cal_method)
for k, dk in enumerate(dist):
if dk >= y:
j = i
while j < labeling.num_instances:
clsLabel = tree.classify_instance(labeling.get_instance(j))
##### probability calculation #####
# dist = tree.distribution_for_instance(labeling.get_instance(j))
dist = calculate_probability_distribution(
tree, labeling, j, cal_method)
for dp in dist:
if dp >= y:
inst = labeling.get_instance(i)
inst.set_value(inst.class_index, clsLabel)
data1.add_instance(inst)
labeling.delete(i)
l += 1
j -= 1
j += 1
if k == (len(dist) - 1) and (l != 0):
tree.build_classifier(data1)
i = -1
s += l
l = 0
i += 1
data1.compactify()
return data1
def folds(self, nfolds=10, seed=None):
"""
Get (training,testing) datasets for cross-validation.
Arguments:
nfolds (int, optional): Number of folds. Default value is
10.
seed (int, optional): Seed value for shuffling
dataset. Default value is random int 0 <= x <= 10000.
Returns:
list of (Instances,Instances) tuples: Each list element is
a pair of (training,testing) datasets, respectively.
"""
seed = seed or randint(0, 10000)
rnd = WekaRandom(seed)
fold_size = labmath.ceil(self.instances.num_instances / nfolds)
# Shuffle the dataset.
instances = WekaInstances.copy_instances(self.instances)
instances.randomize(rnd)
folds = []
for i in range(nfolds):
offset = i * fold_size
testing_end = min(offset + fold_size, instances.num_instances - 1)
# Calculate dataset indices for testing and training data.
testing_range = (offset, testing_end - offset)
left_range = (0, offset)
right_range = (testing_end, instances.num_instances - testing_end)
# If there's nothing to test, move on.
if testing_range[1] < 1: continue
# Create testing and training folds.
testing = WekaInstances.copy_instances(instances, *testing_range)
left = WekaInstances.copy_instances(instances, *left_range)
right = WekaInstances.copy_instances(instances, *right_range)
training = WekaInstances.append_instances(left, right)
# Add fold to collection.
folds.append((training, testing))
return folds
def main():
"""
Just runs some example code.
"""
# load a dataset
iris_file = helper.get_data_dir() + os.sep + "iris.arff"
helper.print_info("Loading dataset: " + iris_file)
loader = Loader("weka.core.converters.ArffLoader")
iris_data = loader.load_file(iris_file)
iris_data.class_is_last()
# train classifier
classifier = Classifier("weka.classifiers.trees.J48")
classifier.build_classifier(iris_data)
# save and read object
helper.print_title("I/O: single object")
outfile = tempfile.gettempdir() + os.sep + "j48.model"
serialization.write(outfile, classifier)
model = Classifier(jobject=serialization.read(outfile))
print(model)
# save classifier and dataset header (multiple objects)
helper.print_title("I/O: single object")
serialization.write_all(outfile, [classifier, Instances.template_instances(iris_data)])
objects = serialization.read_all(outfile)
for i, obj in enumerate(objects):
helper.print_info("Object #" + str(i+1) + ":")
if javabridge.get_env().is_instance_of(obj, javabridge.get_env().find_class("weka/core/Instances")):
obj = Instances(jobject=obj)
elif javabridge.get_env().is_instance_of(obj, javabridge.get_env().find_class("weka/classifiers/Classifier")):
obj = Classifier(jobject=obj)
print(obj)
def load_model(filename):
""" Load the model from cache.
Args:
filename(str): The target file name (without extension) to load. Example: LMT
Returns:
The classifier and data object if the target caching is saved, otherwise None.
"""
# Path to the cashed model (example: caches/model/LMT.cache)
path = os.path.join(os.path.join('caches', 'model'), filename + '.cache')
print("Path to the cashed model to load:", path)
if os.path.isfile(path):
cached_model, cached_data_used_for_training = serialization.read_all(
path)
print("Loading cached classifier")
trained_classifier = Classifier(jobject=cached_model)
print("Loading cached data")
training_data = Instances(jobject=cached_data_used_for_training)
localizer_log.msg("Loaded model: {filename}".format(filename=filename))
return [trained_classifier, training_data]
localizer_log.msg("Failed to load cache of 'model'.")
return None
def df_to_instances(self):
'''
transform pandas data frame to arff style data
:param df: panda data frame
:param relation: relation, string
:param attr_label: label attribute, string
:return: arff style data
'''
atts = []
for col in self.df.columns:
if col != self.attr_label:
att = Attribute.create_numeric(col)
else:
att = Attribute.create_nominal(col, ['0', '1'])
atts.append(att)
nrow = len(self.df)
result = Instances.create_instances(self.relation, atts, nrow)
# data
for i in range(nrow):
inst = Instance.create_instance(
self.df.iloc[i].astype('float64').to_numpy().copy(order='C'))
result.add_instance(inst)
return result
def create_subsample(data, percent, seed=1):
"""
Generates a subsample of the dataset.
:param data: the data to create the subsample from
:type data: Instances
:param percent: the percentage (0-100)
:type percent: float
:param seed: the seed value to use
:type seed: int
"""
if percent <= 0 or percent >= 100:
return data
data = Instances.copy_instances(data)
data.randomize(Random(seed))
data = Instances.copy_instances(data, 0, int(round(data.num_instances() * percent / 100.0)))
return data
def main():
try:
jvm.start()
loader = Loader(classname="weka.core.converters.CSVLoader")
data = loader.load_file("./data/adult.csv")
data.class_is_last() # set class attribute
# randomize data
folds = k
seed = 1
rnd = Random(seed)
rand_data = Instances.copy_instances(data)
rand_data.randomize(rnd)
if rand_data.class_attribute.is_nominal:
rand_data.stratify(folds)
NaiveBayes(rand_data, folds, seed, data)
DecisionTree(rand_data, folds, seed, data)
except Exception as e:
raise e
finally:
jvm.stop()
def predBtn_clicked(self):
gender = self.gender_entry.get()
age = int(self.age_entry.get())
height = int(self.height_entry.get())
weight = int(self.weight_entry.get())
sociability = self.sociability_entry.get()
stability = self.stability_entry.get()
'''Create the model'''
objects = serialization.read_all("J48.model")
cls = Classifier(jobject=objects[0])
data = Instances(jobject=objects[1])
'''Create the test set to be classified'''
gender_values = ["Man", "Woman"]
sociability_values = ["Introvert", "Extrovert"]
stability_values = ["Stable", "Unstable"]
values = [
gender_values.index(gender), age, height, weight,
self.BMI(weight, height),
stability_values.index(stability),
sociability_values.index(sociability),
Instance.missing_value()
]
inst = Instance.create_instance(values)
inst.dataset = data
'''Classification'''
prediction = int(cls.classify_instance(inst))
self.controller.show_frame("Result").show(prediction)
self.clear()
def save_model(model, data, filename):
"""Save the model to the target caching file.
The caches should be defined in the config file. See README and
config.sample for reference.
Args:
model(obj): The model to be saved. Should be a
weka.classifier.Classifier object.
data(obj): The training set to be cached.
target(str): The target option in '[cached]' section in the config
file.
filename(str): The target file to save.
Returns:
True if the target caching is saved, otherwise False.
"""
folder = os.path.join('caches', 'model')
path = os.path.join(folder, filename + '.cache')
build_if_not_exist(folder)
serialization.write_all(path, [model, Instances.template_instances(data)])
localizer_log.msg(
"Saved cache of {target_name}.".format(target_name='model'))
return True
def create_dataset_header():
"""
Creates the dataset header.
:return: the header
:rtype: Instances
"""
att_msg = Attribute.create_string("Message")
att_cls = Attribute.create_nominal("Class", ["miss", "hit"])
result = Instances.create_instances("MessageClassificationProblem", [att_msg, att_cls], 0)
return result
def get_outputformat(self):
"""
Returns the output format.
:return: the output format
:rtype: Instances
"""
inst = javabridge.call(self.jobject, "getOutputFormat", "()Lweka/core/Instances;")
if inst is None:
return None
else:
return Instances(inst)
def dataset_format(self):
"""
Returns the dataset format.
:return: the format
:rtype: Instances
"""
data = javabridge.call(self.jobject, "getDatasetFormat", "()Lweka/core/Instances;")
if data is None:
return None
else:
return Instances(data)
def generate_examples(self):
"""
Returns complete dataset.
:return: the generated dataset
:rtype: Instances
"""
data = javabridge.call(self.jobject, "generateExamples", "()Lweka/core/Instances;")
if data is None:
return None
else:
return Instances(data)
def outputformat(self):
"""
Returns the output format.
:return: the output format
:rtype: Instances
"""
inst = self.__outputformat()
if inst is None:
return None
else:
return Instances(inst)
def load(self):
"""
Loads the text files from the specified directory and returns the Instances object.
In case of incremental loading, only the structure.
:return: the full dataset or the header (if incremental)
:rtype: Instances
"""
javabridge.call(self.jobject, "reset", "()V")
return Instances(
javabridge.call(self.jobject, "getDataSet",
"()Lweka/core/Instances;"))
def instances(self):
"""
Returns the data used in the analysis.
:return: the data in use
:rtype: Instances
"""
inst = javabridge.call(self.jobject, "getInstances",
"()Lweka/core/Instances;")
if inst is None:
return None
else:
return Instances(inst)
def filter(self, data):
"""
Filters the dataset.
:param data: the Instances to filter
:type data: Instances
:return: the filtered Instances object
:rtype: Instances
"""
return Instances(javabridge.static_call(
"Lweka/filters/Filter;", "useFilter",
"(Lweka/core/Instances;Lweka/filters/Filter;)Lweka/core/Instances;",
data.jobject, self.jobject))
def DecisionTree(rnd_data, folds, seed, data):
data_size = rnd_data.num_instances
fold_size = math.floor(data_size / folds)
# cross-validation
evaluation = Evaluation(rnd_data)
for i in range(folds):
this_fold = fold_size
test_start = i * fold_size
test_end = (test_start + fold_size)
if ((data_size - test_end) / fold_size < 1):
this_fold = data_size - test_start
test = Instances.copy_instances(rnd_data, test_start,
this_fold) # generate validation fold
if i == 0:
train = Instances.copy_instances(rnd_data, test_end,
data_size - test_end)
else:
train_1 = Instances.copy_instances(rnd_data, 0, test_start)
train_2 = Instances.copy_instances(rnd_data, test_end,
data_size - test_end)
train = Instances.append_instances(
train_1, train_2) # generate training fold
# build and evaluate classifier
cls = Classifier(classname="weka.classifiers.trees.J48")
cls.build_classifier(train) # build classifier on training set
evaluation.test_model(cls,
test) # test classifier on validation/test set
print("")
print("=== Decision Tree ===")
print("Classifier: " + cls.to_commandline())
print("Dataset: " + data.relationname)
print("Folds: " + str(folds))
print("Seed: " + str(seed))
print("")
print(
evaluation.summary("=== " + str(folds) + "-fold Cross-Validation ==="))
def LabeledUnlabeldata(data, unlabeled, tree, y, cal_method=None ) :
data1 = Instances.copy_instances(data)
labeling = Instances.copy_instances(unlabeled)
tree.build_classifier(data1)
update=False
it=0
labeling_num_instances = labeling.num_instances
while labeling.num_instances > 3 and it < labeling_num_instances:
it+=1
update = False
removed_index=set()
print("labeling.num_instances ===>> " , labeling.num_instances)
for i,xi in enumerate(labeling) :
clsLabel= tree.classify_instance(xi)
dist = calculate_probability_distribution(tree , labeling , i , cal_method)
for dp in dist :
if dp >= y :
update = True
xi.set_value(xi.class_index,clsLabel)
data1.add_instance(xi)
removed_index.add(i)
print("labeling ==================>>", labeling.num_instances)
print("removed_index ==================>>", len(removed_index))
removed_index_list = sorted(removed_index)
for i,ii in enumerate(removed_index_list) :
labeling.delete(ii-i)
print("labeling ==================>>", labeling.num_instances)
if update:
tree.build_classifier(data1)
data1.compactify()
return data1
def __init__(self, model=None, header=None):
"""
Initializes the container.
:param model: the model to store (eg Classifier or Clusterer)
:type model: object
:param header: the header instances
:type header: Instances
"""
super(ModelContainer, self).__init__()
self.set("Model", model)
if header is not None:
header = Instances.template_instances(header)
self.set("Header", header)
self._allowed = ["Model", "Header"]
def __init__(self, model=None, header=None):
"""
Initializes the container.
:param model: the model to store (eg Classifier or Clusterer)
:type model: object
:param header: the header instances
:type header: Instances
"""
super(ModelContainer, self).__init__()
self.set("Model", model)
if header is not None:
header = Instances.template_instances(header)
self.set("Header", header)
self._allowed = ["Model", "Header"]
def generate_thresholdcurve_data(evaluation, class_index):
"""
Generates the threshold curve data from the evaluation object's predictions.
:param evaluation: the evaluation to obtain the predictions from
:type evaluation: Evaluation
:param class_index: the 0-based index of the class-label to create the plot for
:type class_index: int
:return: the generated threshold curve data
:rtype: Instances
"""
jtc = JavaObject.new_instance("weka.classifiers.evaluation.ThresholdCurve")
pred = javabridge.call(evaluation.jobject, "predictions", "()Ljava/util/ArrayList;")
result = Instances(
javabridge.call(jtc, "getCurve", "(Ljava/util/ArrayList;I)Lweka/core/Instances;", pred, class_index))
return result
def create_dataset(tweets):
text_att = Attribute.create_string('TEXT')
nom_att = Attribute.create_nominal('CLASS', class_values)
dataset = Instances.create_instances("tweets", [text_att, nom_att],
len(tweets))
for tweet in tweets:
values = []
values.append(dataset.attribute(0).add_string_value(tweet))
values.append(Instance.missing_value())
inst = Instance.create_instance(values)
dataset.add_instance(inst)
dataset.class_is_last()
return dataset
def addNominals(self, dataset):
# Add the nominal values for all columns, in case a column has none
ignoreAttributes = ['readmitted']
atts = []
for a in dataset.attributes():
if (not (a.is_nominal)) or (a.name in ignoreAttributes) :
atts.append(a)
else:
newValues = list(a.values)
pvalue = 'DefaultNominal'
if(pvalue not in newValues):
newValues.append(pvalue)
atts.append(Attribute.create_nominal(a.name, newValues))
newDataset = Instances.create_instances("Dataset", atts, 0)
newDataset.class_is_last()
return newDataset
def main():
"""
Creates a dataset from scratch using random data and outputs it.
"""
atts = []
for i in xrange(5):
atts.append(Attribute.create_numeric("x" + str(i)))
data = Instances.create_instances("data", atts, 10)
for n in xrange(10):
values = []
for i in xrange(5):
values.append(n*100 + i)
inst = Instance.create_instance(values)
data.add_instance(inst)
print(data)
def ndarray_to_instances(array, relation, att_template="Att-#", att_list=None):
"""
Converts the numpy matrix into an Instances object and returns it.
:param array: the numpy ndarray to convert
:type array: numpy.darray
:param relation: the name of the dataset
:type relation: str
:param att_template: the prefix to use for the attribute names, "#" is the 1-based index,
"!" is the 0-based index, "@" the relation name
:type att_template: str
:param att_list: the list of attribute names to use
:type att_list: list
:return: the generated instances object
:rtype: Instances
"""
if len(numpy.shape(array)) != 2:
raise Exception("Number of array dimensions must be 2!")
rows, cols = numpy.shape(array)
# header
atts = []
if att_list is not None:
if len(att_list) != cols:
raise Exception(
"Number columns and provided attribute names differ: " + str(cols) + " != " + len(att_list))
for name in att_list:
att = Attribute.create_numeric(name)
atts.append(att)
else:
for i in xrange(cols):
name = att_template.replace("#", str(i+1)).replace("!", str(i)).replace("@", relation)
att = Attribute.create_numeric(name)
atts.append(att)
result = Instances.create_instances(relation, atts, rows)
# data
for i in xrange(rows):
inst = Instance.create_instance(array[i])
result.add_instance(inst)
return result
def do_execute(self):
"""
The actual execution of the actor.
:return: None if successful, otherwise error message
:rtype: str
"""
result = None
data = self.input.payload
if isinstance(self._input.payload, Instance):
inst = self.input.payload
data = inst.dataset
elif isinstance(self.input.payload, Instances):
data = self.input.payload
inst = None
append = True
if self._header is None or (self._header.equal_headers(data) is not None):
self._header = Instances.template_instances(data, 0)
outstr = str(data)
append = False
elif inst is not None:
outstr = str(inst)
else:
outstr = str(data)
f = None
try:
if append:
f = open(str(self.resolve_option("output")), "a")
else:
f = open(str(self.resolve_option("output")), "w")
f.write(outstr)
f.write("\n")
except Exception as e:
result = self.full_name + "\n" + traceback.format_exc()
finally:
if f is not None:
f.close()
return result
def addPatientNominals(self, patient, dataset):
# Add the nominal values for the patient to the master header, in case they aren't already there
# Loop and add patient's nominal values in case they aren't in masterDataset
# newDataset will be the new master header
# Waiting on prediction patient to be defined
# Should be like {sex_cd: "m", ...}
ignoreAttributes = ['readmitted']
atts = []
for a in dataset.attributes():
if (not (a.is_nominal)) or (a.name in ignoreAttributes) :
atts.append(a)
else:
newValues = list(a.values)
#print a.name
pvalue = patient[a.name]
if(pvalue not in newValues):
newValues.append(pvalue)
atts.append(Attribute.create_nominal(a.name, newValues))
newDataset = Instances.create_instances("Dataset", atts, 0)
newDataset.class_is_last()
return newDataset
def copy(self, from_row=None, num_rows=None):
return WekaInstances.copy_instances(self.instances,
from_row=from_row,
num_rows=num_rows)
print("\nLoading dataset: " + fname + "\n")
data = loader.load_file(fname)
data.class_is_last()
# define classifiers
classifiers = ["weka.classifiers.rules.OneR", "weka.classifiers.trees.J48"]
# cross-validate original dataset
for classifier in classifiers:
cls = Classifier(classname=classifier)
evl = Evaluation(data)
evl.crossvalidate_model(cls, data, 10, Random(1))
print("%s (original): %0.0f%%" % (classifier, evl.percent_correct))
# replace 'outlook' in first 4 'no' instances with 'missing'
modified = Instances.copy_instances(data)
count = 0
for i in xrange(modified.num_instances):
if modified.get_instance(i).get_string_value(modified.class_index) == "no":
count += 1
modified.get_instance(i).set_missing(0)
if count == 4:
break
# cross-validate modified dataset
for classifier in classifiers:
cls = Classifier(classname=classifier)
evl = Evaluation(modified)
evl.crossvalidate_model(cls, modified, 10, Random(1))
print("%s (modified): %0.0f%%" % (classifier, evl.percent_correct))
import os
import weka.core.jvm as jvm
from weka.core.converters import Loader, Saver
from weka.core.dataset import Instances
from weka.filters import Filter
jvm.start()
# load weather.nominal
fname = data_dir + os.sep + "weather.nominal.arff"
print("\nLoading dataset: " + fname + "\n")
loader = Loader(classname="weka.core.converters.ArffLoader")
data = loader.load_file(fname)
# output header
print(Instances.template_instances(data))
# remove attribute no 3
print("\nRemove attribute no 3")
fltr = Filter(classname="weka.filters.unsupervised.attribute.Remove", options=["-R", "3"])
fltr.set_inputformat(data)
filtered = fltr.filter(data)
# output header
print(Instances.template_instances(filtered))
# save modified dataset
saver = Saver(classname="weka.core.converters.ArffSaver")
saver.save_file(filtered, data_dir + os.sep + "weather.nominal-filtered.arff")
jvm.stop()
def main():
"""
Just runs some example code.
"""
# load a dataset
iris_file = helper.get_data_dir() + os.sep + "iris.arff"
helper.print_info("Loading dataset: " + iris_file)
loader = Loader("weka.core.converters.ArffLoader")
iris_data = loader.load_file(iris_file)
iris_data.class_is_last()
helper.print_title("Iris dataset")
print(iris_data)
helper.print_title("Iris dataset (incrementally output)")
for i in iris_data:
print(i)
helper.print_title("Iris summary")
print(Instances.summary(iris_data))
helper.print_title("Iris attributes")
for a in iris_data.attributes():
print(a)
helper.print_title("Instance at #0")
print(iris_data.get_instance(0))
print(iris_data.get_instance(0).values)
print("Attribute stats (first):\n" + str(iris_data.attribute_stats(0)))
print("total count (first attribute):\n" + str(iris_data.attribute_stats(0).total_count))
print("numeric stats (first attribute):\n" + str(iris_data.attribute_stats(0).numeric_stats))
print("nominal counts (last attribute):\n"
+ str(iris_data.attribute_stats(iris_data.num_attributes - 1).nominal_counts))
helper.print_title("Instance values at #0")
for v in iris_data.get_instance(0):
print(v)
# append datasets
helper.print_title("append datasets")
data1 = Instances.copy_instances(iris_data, 0, 2)
data2 = Instances.copy_instances(iris_data, 2, 2)
print("Dataset #1:\n" + str(data1))
print("Dataset #2:\n" + str(data2))
msg = data1.equal_headers(data2)
print("#1 == #2 ? " + "yes" if msg is None else msg)
combined = Instances.append_instances(data1, data2)
print("Combined:\n" + str(combined))
# merge datasets
helper.print_title("merge datasets")
data1 = Instances.copy_instances(iris_data, 0, 2)
data1.class_index = -1
data1.delete_attribute(1)
data1.delete_first_attribute()
data2 = Instances.copy_instances(iris_data, 0, 2)
data2.class_index = -1
data2.delete_attribute(4)
data2.delete_attribute(3)
data2.delete_attribute(2)
print("Dataset #1:\n" + str(data1))
print("Dataset #2:\n" + str(data2))
msg = data1.equal_headers(data2)
print("#1 == #2 ? " + ("yes" if msg is None else msg))
combined = Instances.merge_instances(data2, data1)
print("Combined:\n" + str(combined))
# load dataset incrementally
iris_file = helper.get_data_dir() + os.sep + "iris.arff"
helper.print_info("Loading dataset incrementally: " + iris_file)
loader = Loader("weka.core.converters.ArffLoader")
iris_data = loader.load_file(iris_file, incremental=True)
iris_data.class_is_last()
helper.print_title("Iris dataset")
print(iris_data)
for inst in loader:
print(inst)
# create attributes
helper.print_title("Creating attributes")
num_att = Attribute.create_numeric("num")
print("numeric: " + str(num_att))
date_att = Attribute.create_date("dat", "yyyy-MM-dd")
print("date: " + str(date_att))
nom_att = Attribute.create_nominal("nom", ["label1", "label2"])
print("nominal: " + str(nom_att))
# create dataset
helper.print_title("Create dataset")
dataset = Instances.create_instances("helloworld", [num_att, date_att, nom_att], 0)
print(str(dataset))
# create an instance
helper.print_title("Create and add instance")
values = [3.1415926, date_att.parse_date("2014-04-10"), 1.0]
inst = Instance.create_instance(values)
print("Instance #1:\n" + str(inst))
dataset.add_instance(inst)
values = [2.71828, date_att.parse_date("2014-08-09"), Instance.missing_value()]
inst = Instance.create_instance(values)
dataset.add_instance(inst)
print("Instance #2:\n" + str(inst))
inst.set_value(0, 4.0)
print("Instance #2 (updated):\n" + str(inst))
print("Dataset:\n" + str(dataset))
dataset.delete_with_missing(2)
print("Dataset (after delete of missing):\n" + str(dataset))
values = [(1, date_att.parse_date("2014-07-11"))]
inst = Instance.create_sparse_instance(values, 3, classname="weka.core.SparseInstance")
print("sparse Instance:\n" + str(inst))
dataset.add_instance(inst)
print("dataset with mixed dense/sparse instance objects:\n" + str(dataset))
# create dataset (lists)
helper.print_title("Create dataset from lists")
x = [[randint(1, 10) for _ in range(5)] for _ in range(10)]
y = [randint(0, 1) for _ in range(10)]
dataset2 = ds.create_instances_from_lists(x, y, "generated from lists")
print(dataset2)
x = [[randint(1, 10) for _ in range(5)] for _ in range(10)]
dataset2 = ds.create_instances_from_lists(x, name="generated from lists (no y)")
print(dataset2)
# create dataset (matrices)
helper.print_title("Create dataset from matrices")
x = np.random.randn(10, 5)
y = np.random.randn(10)
dataset3 = ds.create_instances_from_matrices(x, y, "generated from matrices")
print(dataset3)
x = np.random.randn(10, 5)
dataset3 = ds.create_instances_from_matrices(x, name="generated from matrices (no y)")
print(dataset3)
# create more sparse instances
diabetes_file = helper.get_data_dir() + os.sep + "diabetes.arff"
helper.print_info("Loading dataset: " + diabetes_file)
loader = Loader("weka.core.converters.ArffLoader")
diabetes_data = loader.load_file(diabetes_file)
diabetes_data.class_is_last()
helper.print_title("Create sparse instances using template dataset")
sparse_data = Instances.template_instances(diabetes_data)
for i in range(diabetes_data.num_attributes - 1):
inst = Instance.create_sparse_instance(
[(i, float(i+1) / 10.0)], sparse_data.num_attributes, classname="weka.core.SparseInstance")
sparse_data.add_instance(inst)
print("sparse dataset:\n" + str(sparse_data))
# simple scatterplot of iris dataset: petalwidth x petallength
iris_data = loader.load_file(iris_file)
iris_data.class_is_last()
pld.scatter_plot(
iris_data, iris_data.attribute_by_name("petalwidth").index,
iris_data.attribute_by_name("petallength").index,
percent=50,
wait=False)
# line plot of iris dataset (without class attribute)
iris_data = loader.load_file(iris_file)
iris_data.class_is_last()
pld.line_plot(iris_data, atts=range(iris_data.num_attributes - 1), percent=50, title="Line plot iris", wait=False)
# matrix plot of iris dataset
iris_data = loader.load_file(iris_file)
iris_data.class_is_last()
pld.matrix_plot(iris_data, percent=50, title="Matrix plot iris", wait=True)
def saveClassifier(self, filename, path='/home/sbiastoch/Schreibtisch/classifiers/'): serialization.write_all(path+filename, [self.classifier, Instances.template_instances(self.data)])
# Discretize
print("Discretize numeric attributes (supervised)")
fltr = Filter(classname="weka.filters.supervised.attribute.Discretize")
fltr.inputformat(data)
filtered = fltr.filter(data)
print(filtered)
# PCA
print("Principal components analysis")
fltr = Filter(classname="weka.filters.unsupervised.attribute.PrincipalComponents")
fltr.inputformat(data)
filtered = fltr.filter(data)
print(filtered)
# load anneal
loader = Loader(classname="weka.core.converters.ArffLoader")
fname = data_dir + os.sep + "anneal.arff"
print("\nLoading dataset: " + fname + "\n")
data = loader.load_file(fname)
data.class_is_last()
# RemoveUseless
print("RemoveUseless")
fltr = Filter(classname="weka.filters.unsupervised.attribute.RemoveUseless")
fltr.inputformat(data)
filtered = fltr.filter(data)
print("Original header (#att=" + str(data.num_attributes) + "):\n" + str(Instances.template_instances(data)))
print("Filtered header (#att=" + str(filtered.num_attributes) + "):\n" + str(Instances.template_instances(filtered)))
jvm.stop()
with open(data_dir + os.sep + infile, "rb") as csvfile:
print(infile)
outfile = os.path.splitext(infile)[0] + ".arff"
reader = csv.reader(csvfile)
data = None
ref_present = True
for index, row in enumerate(reader):
if index == 0:
atts = []
ref_present = ("Reference value" in row) or ("Reference Value" in row)
for idx, col in enumerate(row):
col = col.lower()
atts.append(Attribute.create_numeric(col))
if not ref_present and (idx == 0):
atts.append(Attribute.create_numeric("reference value"))
data = Instances.create_instances("irdc", atts, 0)
else:
values = []
for idx, col in enumerate(row):
values.append(float(col))
if not ref_present and (idx == 0):
values.append(float('NaN'))
inst = Instance.create_instance(values)
data.add_instance(inst)
saver = Saver(classname="weka.core.converters.ArffSaver")
saver.save_file(data, data_dir + os.sep + outfile)
# train/test/predict
print("Train/test/predict...")
def plot_learning_curve(classifiers, train, test=None, increments=100, metric="percent_correct",
title="Learning curve", label_template="[#] @ $", key_loc="lower right",
outfile=None, wait=True):
"""
Plots
:param classifiers: list of Classifier template objects
:type classifiers: list of Classifier
:param train: dataset to use for the building the classifier, used for evaluating it test set None
:type train: Instances
:param test: optional dataset to use for the testing the built classifiers
:type test: Instances
:param increments: the increments (>= 1: # of instances, <1: percentage of dataset)
:type increments: float
:param metric: the name of the numeric metric to plot (Evaluation.<metric>)
:type metric: str
:param title: the title for the plot
:type title: str
:param label_template: the template for the label in the plot
(#: 1-based index, @: full classname, !: simple classname, $: options)
:type label_template: str
:param key_loc: the location string for the key
:type key_loc: str
:param outfile: the output file, ignored if None
:type outfile: str
:param wait: whether to wait for the user to close the plot
:type wait: bool
"""
if not plot.matplotlib_available:
logger.error("Matplotlib is not installed, plotting unavailable!")
return
if not train.has_class():
logger.error("Training set has no class attribute set!")
return
if (test is not None) and (train.equal_headers(test) is not None):
logger.error("Training and test set are not compatible: " + train.equal_headers(test))
return
if increments >= 1:
inc = increments
else:
inc = round(train.num_instances * increments)
steps = []
cls = []
evls = {}
for classifier in classifiers:
cl = Classifier.make_copy(classifier)
cls.append(cl)
evls[cl] = []
if test is None:
tst = train
else:
tst = test
for i in xrange(train.num_instances):
if (i > 0) and (i % inc == 0):
steps.append(i+1)
for cl in cls:
# train
if cl.is_updateable:
if i == 0:
tr = Instances.copy_instances(train, 0, 1)
cl.build_classifier(tr)
else:
cl.update_classifier(train.get_instance(i))
else:
if (i > 0) and (i % inc == 0):
tr = Instances.copy_instances(train, 0, i + 1)
cl.build_classifier(tr)
# evaluate
if (i > 0) and (i % inc == 0):
evl = Evaluation(tst)
evl.test_model(cl, tst)
evls[cl].append(getattr(evl, metric))
fig, ax = plt.subplots()
ax.set_xlabel("# of instances")
ax.set_ylabel(metric)
ax.set_title(title)
fig.canvas.set_window_title(title)
ax.grid(True)
i = 0
for cl in cls:
evl = evls[cl]
i += 1
plot_label = label_template.\
replace("#", str(i)).\
replace("@", cl.classname).\
replace("!", cl.classname[cl.classname.rfind(".") + 1:]).\
replace("$", join_options(cl.config))
ax.plot(steps, evl, label=plot_label)
plt.draw()
plt.legend(loc=key_loc, shadow=True)
if outfile is not None:
plt.savefig(outfile)
if wait:
plt.show()
def main():
"""
Just runs some example code.
"""
# load a dataset
data_file = helper.get_data_dir() + os.sep + "vote.arff"
helper.print_info("Loading dataset: " + data_file)
loader = Loader("weka.core.converters.ArffLoader")
data = loader.load_file(data_file)
data.class_is_last()
# classifier
classifier = Classifier(classname="weka.classifiers.trees.J48")
# randomize data
folds = 10
seed = 1
rnd = Random(seed)
rand_data = Instances.copy_instances(data)
rand_data.randomize(rnd)
if rand_data.class_attribute.is_nominal:
rand_data.stratify(folds)
# perform cross-validation and add predictions
predicted_data = None
evaluation = Evaluation(rand_data)
for i in xrange(folds):
train = rand_data.train_cv(folds, i)
# the above code is used by the StratifiedRemoveFolds filter,
# the following code is used by the Explorer/Experimenter
# train = rand_data.train_cv(folds, i, rnd)
test = rand_data.test_cv(folds, i)
# build and evaluate classifier
cls = Classifier.make_copy(classifier)
cls.build_classifier(train)
evaluation.test_model(cls, test)
# add predictions
addcls = Filter(
classname="weka.filters.supervised.attribute.AddClassification",
options=["-classification", "-distribution", "-error"])
# setting the java object directory avoids issues with correct quoting in option array
addcls.set_property("classifier", Classifier.make_copy(classifier))
addcls.inputformat(train)
addcls.filter(train) # trains the classifier
pred = addcls.filter(test)
if predicted_data is None:
predicted_data = Instances.template_instances(pred, 0)
for n in xrange(pred.num_instances):
predicted_data.add_instance(pred.get_instance(n))
print("")
print("=== Setup ===")
print("Classifier: " + classifier.to_commandline())
print("Dataset: " + data.relationname)
print("Folds: " + str(folds))
print("Seed: " + str(seed))
print("")
print(evaluation.summary("=== " + str(folds) + " -fold Cross-Validation ==="))
print("")
print(predicted_data)
