def __init__(self, train, label, extra):
Classifier.__init__(self, train, label, extra)
self.aIn = extra.aIn
self.aOut = extra.aOut
self.bIn = extra.bIn
self.bOut = extra.bOut
self.epsilon = extra.epsilon
self.simple = extra.simple
def __init__(self, train, label, extra):
Classifier.__init__(self, train, label, extra)
nfile = path(extra.newsgroups)
print >> sys.stderr, "dump file:", nfile
ff = file(nfile)
self.voc = pickle.load(ff)
self.revVoc = dict([(v, k) for k, v in self.voc.items()])
gps = pickle.load(ff)
self.wfr = pickle.load(ff)
self.dfs = pickle.load(ff)
self.classifier = Megam(mode="binomial")
def __init__(self, train, label, extra):
Classifier.__init__(self, train, label, extra)
nfile = path(extra.newsgroups)
print >>sys.stderr, "dump file:", nfile
ff = file(nfile)
self.voc = pickle.load(ff)
self.revVoc = dict([(v,k) for k,v in self.voc.items()])
gps = pickle.load(ff)
self.wfr = pickle.load(ff)
self.dfs = pickle.load(ff)
self.classifier = Megam(mode="binomial")
def objective(points, mat, log=False):
"""The total weight of cross edges plus the total weight of in-cluster
anti-edges. Larger is worse."""
obj = 0
if log:
weightFn = safeLog
else:
weightFn = lambda x: x
for same,p1,p2 in Classifier.instances(points):
weight = mat[p1.index][p2.index]
if same:
obj += weightFn(1 - weight)
else:
obj += weightFn(weight)
return obj
def genClassifiers(self):
"""Generate a set of simulated classifiers with
appropriate skill settings.
Returns
-------
None
Sets the `classifiers` instance attribute directly.
"""
# Models are simple placeholders
skillModel = ClassifierSkillModelBinary()
skillPriorModel = ClassifierSkillPriorBinary()
self.classifiers = Classifiers([
Classifier(id=classifierId,
skillModel=skillModel,
skillPriorModel=skillPriorModel)
for classifierId in range(self.numClassfiers)
])
def objective(points, mat, log=False):
"""The total weight of cross edges plus the total weight of in-cluster
anti-edges. Larger is worse."""
obj = 0
if log:
weightFn = safeLog
else:
weightFn = lambda x: x
for same, p1, p2 in Classifier.instances(points):
weight = mat[p1.index][p2.index]
if same:
obj += weightFn(1 - weight)
else:
obj += weightFn(weight)
return obj
def extracts(self, **extraArgs):
""" Receive new annotations and return a new Subjects list
Subjects implicitly encapsulate a list of annotations and annotations
implicity encapsulate classifiers. Processing of raw extracted
annotations is delegated to instances of the concrete AnnotationBase
subclass.
The intention is that the list of subjects is used to update
annotations for known subjects or add new subjects with a single
annotation to the set of known subjects.
Arguments:
-- extraArgs - Arguments forwarded to the conrete AnnotationBase
subclass's constructor and the Classifier constructor.
"""
uniqueMessages = self.sqsReceive()[0]
extractSummaries = [
self.parseExtractSummary(uniqueMessage)
for uniqueMessage in uniqueMessages
]
# NOTE: Current design passes extracted annotations data to the
# AnnotationBase subclass's constructor for processing.
subjects = Subjects([
Subject(id=subjectId,
annotations=Annotations([
self.annotationType(
id=classificationId,
classifier=Classifier(id=classifierId,
**extraArgs),
zooniverseAnnotations=zooniverseAnnotations,
**extraArgs)
])) for classificationId, subjectId, classifierId,
zooniverseAnnotations in extractSummaries
])
return subjects
from Classifiers import Classifier
from Classifiers import get_keras_mnist
from Classifiers import build_MyNetAlpha
if __name__ == '__main__':
train_X, train_y, test_X, test_y, data_shape, labels = get_keras_mnist()
model = build_MyNetAlpha(data_shape, len(labels))
classifier = Classifier()
classifier.train_epochs = 1
classifier.output = 'output/mnist/MyNetAlpha-alpha'
classifier.train(model, train_X, train_y, test_X, test_y, labels)
vec_freq = load(open("Preprocessing/preproc-freq.pkl", "rb"))
vec_binary = load(open("Preprocessing/preproc-binary.pkl", "rb"))
vec_keywords = load(open("Preprocessing/preproc-keywords.pkl", "rb"))
train_vec_freq = hstack([vec_freq, vec_keywords])
train_vec_binary = hstack([vec_binary, vec_keywords])
train_labels = load(open("Preprocessing/train_labels.pkl", "rb"))
del vec_freq
del vec_binary
del vec_keywords
classifiers = [
"DecisionTrees5", "DecisionTrees10", "DecisionTrees20",
"RandomForestClassifier5", "RandomForestClassifier10",
"RandomForestClassifier20", "MultinomialNaiveBayes1",
"MultinomialNaiveBayes0.1", "MultinomialNaiveBayes5",
"LogisticRegressionSagaL1", "LogisticRegressionSagaL2", "AdaBoost",
"RidgeClassifier", "LinearSVC", "SGDClassifier", "Perceptron",
"PassiveAggressiveClassifier"
]
clf = None
for classifier in classifiers:
print("Evaluating using " + classifier + " at " + str(datetime.now()))
try:
clf = Classifier(classifier, train_labels, train_vec_freq,
train_vec_binary)
accuracy = clf.cross_validate()
print(classifier + " : " + accuracy)
except Exception as e:
print(classifier + " : " + str(e))
from Classifiers import Classifier
from Classifiers import KerasMNISTDataset
from Classifiers import build_LeNet_5
if __name__ == '__main__':
dataset = KerasMNISTDataset()
train_X, train_y, validation_X, validation_y, test_X, test_y, data_shape, set_of_labels = dataset.get(
)
model = build_LeNet_5(data_shape, len(set_of_labels))
classifier = Classifier()
classifier.train_epochs = 1
classifier.output = 'output/mnist/LeNet_5-alpha'
classifier.train(model,
train_X,
train_y,
test_X,
test_y,
set_of_labels,
initial_alpha=0.001,
factor=0.5)
def __init__(self, train, label, extra):
Classifier.__init__(self, train, label, extra)
self.data = { 0 : {}, 1 : {} }
vec_freq = vectorizer.fit_transform(summary)
test_vec_freq = hstack([vec_freq, vec_keywords])
test_vec_binary = hstack([vec_binary, vec_keywords])
del vectorizer
del vec_freq
del vec_binary
del vec_keywords
train_classes_dict = load(open("Preprocessing/train_classes_dict.pkl", "rb"))
print("Revenue Classes:")
for c, value in train_classes_dict.items():
print('\t' + str(c) + ' : ' + value)
predictions = []
for classifier in classifiers:
if args.v:
print("Calculating " + classifier)
try:
clf = Classifier(classifier, train_labels, train_vec_freq,
train_vec_binary, test_vec_freq, test_vec_binary)
pred = clf.predict_class(args.v)
if args.v:
print("- Class " + str(pred))
predictions.append(pred)
except Exception as e:
print(classifier + ": " + str(e))
pred = int(round(mean(predictions)))
print("Prediction: class " + str(pred))
def __init__(self, input_dim, k=2, tol=10, max_iter=300):
Classifier.__init__(self, input_dim)
self.k = k
self.tol = tol
self.max_iter = max_iter
self.interia = None
from Classifiers import Classifier
from Classifiers import get_keras_mnist
from Classifiers import build_MyNetCharlie
if __name__ == '__main__':
train_X, train_y, test_X, test_y, data_shape, labels = get_keras_mnist()
model = build_MyNetCharlie(
data_shape,
len(labels),
filters=[64, 128],
enable_dropout=False,
enable_batch_normalization=True
)
classifier = Classifier()
classifier.train_epochs = 1
classifier.output = 'output/mnist/charlienet-alpha'
classifier.train(model, train_X, train_y, test_X, test_y, labels)
def __init__(self, train, label, extra):
Classifier.__init__(self, train, label, extra)
self.classifier = Megam()
import numpy as np
from Classifiers import Classifier, get_keras_mnist, build_AlexNet
if __name__ == '__main__':
train_X, train_y, test_X, test_y, data_shape, labels = get_keras_mnist()
model = build_AlexNet((28, 28, 1), len(labels))
classifier = Classifier()
classifier.train_epochs = 500
classifier.output = 'output/mnist/AlexNet-alpha'
classifier.train(model, train_X, train_y, test_X, test_y, labels)
from Classifiers import Classifier
from Classifiers import ReshapedKerasMNISTDataset
from Classifiers import build_MNIST_KerasNet
# import pdb
if __name__ == '__main__':
dataset = ReshapedKerasMNISTDataset()
train_X, train_y, validation_X, validation_y, test_X, test_y, data_shape, set_of_labels = dataset.get(
)
model = build_MNIST_KerasNet(data_shape, len(set_of_labels))
classifier = Classifier()
classifier.train_epochs = 500
classifier.output = 'output/mnist/KerasNet-alpha'
# pdb.set_trace()
classifier.build(model,
train_X,
train_y,
test_X,
test_y,
set_of_labels,
validation_X=validation_X,
validation_y=validation_y)
images = dataset.images
onehots = dataset.onehots
set_of_labels = dataset.set_of_labels
print('images', len(images))
print('onehots', len(onehots))
print('set_of_labels', str(set_of_labels))
print('train_X.shape', train_X.shape)
print('train_y.shape', train_y.shape)
print('validation_X.shape', validation_X.shape)
print('validation_y.shape', validation_y.shape)
print('test_X.shape', test_X.shape)
print('test_y.shape', test_y.shape)
print('train_y[0]', train_y[0,:], 'validation_y[1]', validation_y[1,:], 'test_y[2]', test_y[2,:])
print(np.sum(train_y,axis=0))
print(np.sum(validation_y,axis=0))
print(np.sum(test_y,axis=0))
model = build_AlexNet(data_shape, len(set_of_labels))
classifier = Classifier()
classifier.train_epochs = 100
classifier.output = 'output/mnist/AlexNet-bravo'
classifier.build(model,
train_X, train_y, test_X, test_y, set_of_labels,
validation_X=validation_X, validation_y=validation_y,
learning_rate_decay=StepDecay(alpha_zero=0.0002, factor=0.5, drop_every=5),
initial_weights="~/Data/Code/keras-effective-adventure/bravo/output/mnist/AlexNet-alpha/weights-epoch_0029.h5")