def loadFile(self):
if self.fileIndex:
fn = self.recentFiles[self.fileIndex]
self.recentFiles.remove(fn)
self.recentFiles.insert(0, fn)
self.fileIndex = 0
else:
fn = self.recentFiles[0]
self.filecombo.clear()
for file in self.recentFiles:
self.filecombo.addItem(os.path.split(file)[1])
#self.filecombo.updateGeometry()
self.matrix = None
self.labels = None
self.data = None
pb = OWGUI.ProgressBar(self, 100)
self.error()
try:
self.matrix, self.labels, self.data = readMatrix(fn, pb)
except Exception, ex:
self.error("Error while reading the file: '%s'" % str(ex))
return
def run(self):
plan = []
estimate_index = 0
for i, (selected, method, count,
offsets) in enumerate(self.estimators):
if self.results[i] is None and getattr(self, selected):
plan.append((i, method,
[estimate_index + offset for offset in offsets]))
estimate_index += count
estimators = [method() for _, method, _ in plan]
if not estimators:
return
pb = OWGUI.ProgressBar(self, len(self._test_data()))
estimates = self.run_estimation(estimators, pb.advance)
pb.finish()
self.predictions = [v for v, _ in estimates]
estimates = [prob.reliability_estimate for _, prob in estimates]
for i, (index, method, estimate_indices) in enumerate(plan):
self.results[index] = [[e[est_index] for e in estimates] \
for est_index in estimate_indices]
def computeMatrix(self):
if not self.data:
return
data = self.data
dist = self.metrics[self.Metrics][1](data)
self.error(0)
try:
self.matrix = orange.SymMatrix(len(data))
except orange.KernelException as ex:
self.error(0, "Could not create distance matrix! %s" % str(ex))
self.matrix = None
self.send("Distance Matrix", None)
return
self.matrix.setattr('items', data)
pb = OWGUI.ProgressBar(self, 100)
milestones = orngMisc.progressBarMilestones(
len(data) * (len(data) - 1) / 2, 100)
count = 0
for i in range(len(data)):
for j in range(i + 1):
self.matrix[i, j] = dist(data[i], data[j])
if count in milestones:
pb.advance()
count += 1
pb.finish()
self.send("Distance Matrix", self.matrix)
def getLearningCurve(self, learners):
pb = OWGUI.ProgressBar(self, iterations=self.steps * self.folds)
curve = orngTest.learningCurveN(learners,
self.data,
folds=self.folds,
proportions=self.curvePoints,
callback=pb.advance)
pb.finish()
return curve
def getLearningCurve(self, learners):
pb = OWGUI.ProgressBar(self, iterations=self.steps * self.folds)
curve = Orange.evaluation.testing.learning_curve_n(
learners,
self.data,
folds=self.folds,
proportions=self.curvePoints,
callback=pb.advance)
pb.finish()
return curve
def loadFile(self):
if not hasattr(self, "originalData"):
return
if self.fileIndex:
fn = self.recentFiles[self.fileIndex]
self.recentFiles.remove(fn)
self.recentFiles.insert(0, fn)
self.fileIndex = 0
else:
if len(self.recentFiles) > 0:
fn = self.recentFiles[0]
else:
return
self.filecombo.clear()
for file in self.recentFiles:
self.filecombo.addItem(os.path.split(file)[1])
#self.filecombo.updateGeometry()
self.error()
try:
self.matrix = None
self.labels = None
self.data = None
pb = OWGUI.ProgressBar(self, 100)
self.matrix, self.labels, self.data = readMatrix(fn, pb)
dstFile, ext = os.path.splitext(fn)
warning = ""
self.warning()
if os.path.exists(dstFile + ".tab"):
self.data = orange.ExampleTable(dstFile + ".tab")
self.matrix.items = self.data
else:
warning += "ExampleTable %s not found!\n" % (dstFile + ".tab")
if os.path.exists(dstFile + ".res"):
self.matrix.results = pickle.load(open(dstFile + ".res", 'rb'))
else:
warning += "Results pickle %s not found!\n" % (dstFile + ".res")
self.matrix.originalData = self.originalData
if warning != "":
self.warning(warning.rstrip())
self.relabel()
except Exception as e:
self.error("Error while reading the file\n\n%s" % e.message)
def setData(self, data):
self.data = self.isDataWithClass(
data, orange.VarTypes.Discrete, checkMissing=True) and data or None
#self.setLearner()
if self.data:
learner = self.constructLearner()
pb = OWGUI.ProgressBar(self, iterations=self.trees)
learner.callback = pb.advance
try:
self.classifier = learner(self.data)
self.classifier.name = self.name
except Exception, (errValue):
self.error(str(errValue))
self.classifier = None
pb.finish()
def setData(self, data):
self.data = self.isDataWithClass(
data, orange.VarTypes.Discrete, checkMissing=True) and data or None
#self.setLearner()
if self.data:
learner = self.constructLearner()
pb = OWGUI.ProgressBar(self, iterations=self.trees)
self.progressBarInit()
learner.callback = lambda v: self.progressBarSet(100.0 * v)
try:
self.classifier = learner(self.data)
self.classifier.name = self.name
except Exception, (errValue):
self.error(str(errValue))
self.classifier = None
self.progressBarFinished()
def setData(self, data):
self.data = self.isDataWithClass(data, orange.VarTypes.Discrete, checkMissing=True) and data or None
#self.setLearner()
if self.data:
learner = self.constructLearner()
pb = OWGUI.ProgressBar(self, iterations=self.trees)
learner.callback = pb.advance
try:
self.classifier = learner(self.data)
self.classifier.name = self.name
self.streeEnabled(True)
except Exception as errValue:
self.error(str(errValue))
self.classifier = None
self.streeEnabled(False)
pb.finish()
else:
self.classifier = None
self.streeEnabled(False)
self.send("Random Forest Classifier", self.classifier)
class OWExampleDistance(OWWidget):
settingsList = ["Metrics", "Normalize"]
contextHandlers = {"": DomainContextHandler("", ["Label"])}
def __init__(self, parent=None, signalManager = None):
OWWidget.__init__(self, parent, signalManager, 'ExampleDistance', wantMainArea = 0, resizingEnabled = 0)
self.inputs = [("Data", ExampleTable, self.dataset)]
self.outputs = [("Distances", orange.SymMatrix)]
self.Metrics = 0
self.Normalize = True
self.Label = ""
self.loadSettings()
self.data = None
self.matrix = None
self.metrics = [
("Euclidean", orange.ExamplesDistanceConstructor_Euclidean),
("Pearson Correlation", orngClustering.ExamplesDistanceConstructor_PearsonR),
("Spearman Rank Correlation", orngClustering.ExamplesDistanceConstructor_SpearmanR),
("Manhattan", orange.ExamplesDistanceConstructor_Manhattan),
("Hamming", orange.ExamplesDistanceConstructor_Hamming),
("Relief", orange.ExamplesDistanceConstructor_Relief),
]
cb = OWGUI.comboBox(self.controlArea, self, "Metrics", box="Distance Metrics",
items=[x[0] for x in self.metrics],
tooltip="Choose metrics to measure pairwise distance between examples.",
callback=self.distMetricChanged, valueType=str)
cb.setMinimumWidth(170)
OWGUI.separator(self.controlArea)
box = OWGUI.widgetBox(self.controlArea, "Normalization",
addSpace=True)
self.normalizeCB = OWGUI.checkBox(box, self, "Normalize", "Normalize data",
callback=self.computeMatrix)
self.normalizeCB.setEnabled(self.Metrics in [0, 3])
self.labelCombo = OWGUI.comboBox(self.controlArea, self, "Label", box="Example Label",
items=[],
tooltip="Attribute used for example labels",
callback=self.setLabel, sendSelectedValue = 1)
self.labelCombo.setDisabled(1)
OWGUI.rubber(self.controlArea)
def sendReport(self):
self.reportSettings("Settings",
[("Metrics", self.metrics[self.Metrics][0]),
("Label", self.Label)])
self.reportData(self.data)
def distMetricChanged(self):
self.normalizeCB.setEnabled(self.Metrics in [0, 3])
self.computeMatrix()
def computeMatrix(self):
if not self.data:
return
data = self.data
constructor = self.metrics[self.Metrics][1]()
constructor.normalize = self.Normalize
dist = constructor(data)
self.error(0)
try:
self.matrix = orange.SymMatrix(len(data))
except orange.KernelException, ex:
self.error(0, "Could not create distance matrix! %s" % str(ex))
self.matrix = None
self.send("Distances", None)
return
self.matrix.setattr('items', data)
pb = OWGUI.ProgressBar(self, 100)
milestones = orngMisc.progressBarMilestones(len(data)*(len(data)-1)/2, 100)
count = 0
for i in range(len(data)):
for j in range(i+1):
self.matrix[i, j] = dist(data[i], data[j])
if count in milestones:
pb.advance()
count += 1
pb.finish()
self.send("Distances", self.matrix)
def showWidget(self):
self.information()
if self.ow is not None:
self.ow.topWidgetPart.hide()
self.ow.setLayout(self.layout())
elif self.layout() is not None:
sip.delete(self.layout())
self.ow = None
if self.data is None:
self.information("No learning data given.")
return
if self.model is None: return
if "model" not in self.model.domain: return
if "label" in self.model.domain:
attr = self.model["label"].value.split(', ')
modelType = self.model["model"].value.upper()
projWidget = None
if modelType == "SCATTERPLOT" or modelType == "SCATTTERPLOT":
projWidget = self.setWidget(OWScatterPlot.OWScatterPlot)
if modelType == "RADVIZ":
projWidget = self.setWidget(OWRadviz.OWRadviz)
if modelType == "POLYVIZ":
projWidget = self.setWidget(OWPolyviz.OWPolyviz)
if projWidget is not None:
self.ow.setData(self.data)
self.ow.setShownAttributes(attr)
self.ow.handleNewSignals()
################################
### add new model types here ###
################################
if modelType == "SPCA" or modelType == "LINPROJ":
self.setWidget(OWLinProj.OWLinProj)
self.ow.setData(self.data)
self.ow.setShownAttributes(attr)
self.ow.handleNewSignals()
xAnchors, yAnchors = self.model["anchors"].value
self.ow.updateGraph(None, setAnchors=1, XAnchors=xAnchors, YAnchors=yAnchors)
if modelType == "TREE":
self.setWidget(OWClassificationTreeGraph.OWClassificationTreeGraph)
classifier = self.model["classifier"].value
self.ow.ctree(classifier)
if modelType == "BAYES":
self.setWidget(OWNomogram.OWNomogram)
classifier = self.model["classifier"].value
self.ow.classifier(classifier)
if modelType == "KNN":
exclude = [att for att in self.data.domain if att.name not in attr + [self.data.domain.classVar.name]]
data2 = orange.Preprocessor_ignore(self.data, attributes = exclude)
dist = orange.ExamplesDistanceConstructor_Euclidean(data2)
smx = orange.SymMatrix(len(data2))
smx.setattr('items', data2)
pb = OWGUI.ProgressBar(self, 100)
milestones = orngMisc.progressBarMilestones(len(data2)*(len(data2)-1)/2, 100)
count = 0
for i in range(len(data2)):
for j in range(i+1):
smx[i, j] = dist(data2[i], data2[j])
if count in milestones:
pb.advance()
count += 1
pb.finish()
self.setWidget(OWMDS.OWMDS)
self.ow.cmatrix(smx)
if self.ow is not None:
self.ow.send = self.send
if self.layout() is not None: sip.delete(self.layout())
self.setLayout(self.ow.layout())
self.ow.topWidgetPart.show()
self.update()
def score(self, ids):
"""compute scores for the list of learners"""
if (not self.data):
for id in ids:
self.learners[id].results = None
return
# test which learners can accept the given data set
# e.g., regressions can't deal with classification data
learners = []
n = len(self.data.domain.attributes) * 2
indices = orange.MakeRandomIndices2(
p0=min(n, len(self.data)),
stratified=orange.MakeRandomIndices2.StratifiedIfPossible)
new = self.data.selectref(indices(self.data))
# new = self.data.selectref([1]*min(n, len(self.data)) +
# [0]*(len(self.data) - min(n, len(self.data))))
self.warning(0)
for l in [self.learners[id] for id in ids]:
learner = l.learner
if self.preprocessor:
learner = self.preprocessor.wrapLearner(learner)
try:
predictor = learner(new)
if predictor(new[0]).varType == new.domain.classVar.varType:
learners.append(learner)
else:
l.scores = []
except Exception as ex:
self.warning(
0,
"Learner %s ends with exception: %s" % (l.name, str(ex)))
l.scores = []
if not learners:
return
# computation of results (res, and cm if classification)
pb = None
if self.resampling == 0:
pb = OWGUI.ProgressBar(self, iterations=self.nFolds)
res = orngTest.crossValidation(
learners,
self.data,
folds=self.nFolds,
strat=orange.MakeRandomIndices.StratifiedIfPossible,
callback=pb.advance,
storeExamples=True)
pb.finish()
elif self.resampling == 1:
pb = OWGUI.ProgressBar(self, iterations=len(self.data))
res = orngTest.leaveOneOut(learners,
self.data,
callback=pb.advance,
storeExamples=True)
pb.finish()
elif self.resampling == 2:
pb = OWGUI.ProgressBar(self, iterations=self.pRepeat)
res = orngTest.proportionTest(learners,
self.data,
self.pLearning / 100.,
times=self.pRepeat,
callback=pb.advance,
storeExamples=True)
pb.finish()
elif self.resampling == 3:
pb = OWGUI.ProgressBar(self, iterations=len(learners))
res = orngTest.learnAndTestOnLearnData(learners,
self.data,
storeExamples=True,
callback=pb.advance)
pb.finish()
elif self.resampling == 4:
if not self.testdata:
for l in self.learners.values():
l.scores = []
return
pb = OWGUI.ProgressBar(self, iterations=len(learners))
res = orngTest.learnAndTestOnTestData(learners,
self.data,
self.testdata,
storeExamples=True,
callback=pb.advance)
pb.finish()
if self.isclassification():
cm = orngStat.computeConfusionMatrices(res,
classIndex=self.targetClass)
if self.preprocessor: # Unwrap learners
learners = [l.wrappedLearner for l in learners]
res.learners = learners
for l in [self.learners[id] for id in ids]:
if l.learner in learners:
l.results = res
self.error(list(range(len(self.stat))))
scores = []
for i, s in enumerate(self.stat):
try:
scores.append(eval("orngStat." + s.f))
except Exception as ex:
self.error(i, "An error occurred while evaluating orngStat." + s.f + "on %s due to %s" % \
(" ".join([l.name for l in learners]), ex))
scores.append([None] * len(self.learners))
for (i, l) in enumerate(learners):
self.learners[l.id].scores = [s[i] if s else None for s in scores]
self.sendResults()
def sendData(self):
"""Compute result of widget processing and send to output"""
# Skip if title list is empty:
if self.titleLabels == list():
return
# Check that something has been selected...
if len(self.selectedTitles) == 0:
self.infoBox.setText(u'Please select one or more titles.',
'warning')
self.send(u'Text data', None, self)
return
# Clear created Inputs.
self.clearCreatedInputs()
# Initialize progress bar.
progressBar = OWGUI.ProgressBar(self,
iterations=len(self.selectedTitles))
# Attempt to connect to Theatre-classique and retrieve plays...
xml_contents = list()
annotations = list()
try:
for title in self.selectedTitles:
response = urllib2.urlopen(
self.document_base_url +
self.filteredTitleSeg[title].annotations[u'url'])
xml_contents.append(unicode(response.read(), u'utf8'))
annotations.append(
self.filteredTitleSeg[title].annotations.copy())
progressBar.advance() # 1 tick on the progress bar...
# If an error occurs (e.g. http error, or memory error)...
except:
# Set Info box and widget to 'error' state.
self.infoBox.setText(
u"Couldn't download data from theatre-classique website.",
'error')
# Reset output channel.
self.send(u'Text data', None, self)
return
# Store downloaded XML in input objects...
for xml_content_idx in xrange(len(xml_contents)):
newInput = Input(xml_contents[xml_content_idx], self.captionTitle)
self.createdInputs.append(newInput)
# If there's only one play, the widget's output is the created Input.
if len(self.createdInputs) == 1:
self.segmentation = self.createdInputs[0]
# Otherwise the widget's output is a concatenation...
else:
self.segmentation = Segmenter.concatenate(
self.createdInputs,
self.captionTitle,
import_labels_as=None,
)
# Annotate segments...
for idx, segment in enumerate(self.segmentation):
segment.annotations.update(annotations[idx])
self.segmentation[idx] = segment
# Store imported URLs as setting.
self.importedURLs = [
self.filteredTitleSeg[self.selectedTitles[0]].annotations[u'url']
]
# Set status to OK and report data size...
message = u'%i segment@p sent to output ' % len(self.segmentation)
message = pluralize(message, len(self.segmentation))
numChars = 0
for segment in self.segmentation:
segmentLength = len(Segmentation.get_data(segment.str_index))
numChars += segmentLength
message += u'(%i character@p).' % numChars
message = pluralize(message, numChars)
self.infoBox.setText(message)
progressBar.finish()
# Clear progress bar.
progressBar.finish()
# Send token...
self.send(u'Text data', self.segmentation, self)
self.sendButton.resetSettingsChangedFlag()
def sendData(self):
# Si le lien vers treetagger n'est pas trouve
if self.NoLink:
self.infoBox.setText(u"Sorry, TreeTagger's link not found.",
"error")
self.send('Text data', None)
# Important: if input data is None, propagate this value to output...
elif not self.inputData:
self.infoBox.setText(u"Widget needs input", "warning")
self.send('Text data', None)
# affiche que quelque chose se passe...
else:
self.infoBox.setText(u'TreeTagger is running...', "warning")
# Initialisation de variables
total_tagged_text = list()
new_segmentations = list()
i = 0
# Initialize progress bar.
self.progressBar = OWGUI.ProgressBar(self, iterations=5)
# Copie de la segmentation avec ajout d'une annotation...
copy_of_input_seg = Segmentation()
copy_of_input_seg.label = self.inputData.label
for seg_idx, segment in enumerate(self.inputData):
attr = " ".join(
["%s='%s'" % item for item in segment.annotations.items()])
segment.annotations["tt_xb"] = attr
copy_of_input_seg.append(segment)
# avancer la progressBar d'un cran
self.progressBar.advance()
concatenated_text = copy_of_input_seg.to_string(
formatting="<xb_tt %(tt_xb)s>%(__content__)s</xb_tt>",
display_all=True,
)
# avancer la progressBar d'un cran
self.progressBar.advance()
tagged_text = self.tag(concatenated_text)
tagged_input = Input(tagged_text)
tagged_segmentation = Segmenter.import_xml(tagged_input, "xb_tt")
# avancer la progressBar d'un cran
self.progressBar.advance()
# Si checkBox xml active
if self.activer_xml == True:
xml_segmentation = Segmenter.recode(
tagged_segmentation,
substitutions=[
(re.compile(r"<unknown>"), '[unknown]'),
(re.compile(r"(.+)\t(.+)\t(.+)"),
'<w lemma="&3" type="&2">&1</w>'),
(re.compile(r'"""'), '"""'),
],
)
final_segmentation = xml_segmentation
# Si checkBox xml desactive
else:
xml_segmentation = Segmenter.recode(
tagged_segmentation,
substitutions=[
(re.compile(r"<unknown>"), '[unknown]'),
(re.compile(r"(.+)\t(.+)\t(.+)"),
'<w lemma="&3" type="&2">&1</w>'),
(re.compile(r'"""'), '"""'),
],
)
final_segmentation = Segmenter.import_xml(
xml_segmentation, "w")
self.infoBox.dataSent('')
# Enregistrer le lien de treetagger...
if self.system == "nt":
file = open("treetagger_link.txt", 'w')
else:
file = open(
os.path.normpath("/Users/" + self.user +
"/treetagger_link.txt"), 'w')
file.write(self.treetagger_link)
file.close()
# Clear progress bar.
self.progressBar.finish()
# envoyer la seguementation
self.send('Text data', final_segmentation, self)
self.compteur += 1
self.sendButton.resetSettingsChangedFlag()
def sendData(self):
"""Compute result of widget processing and send to output"""
# Clear created Inputs
self.clearCreatedInputs()
if self.service == u'Twitter':
try:
self.createdInputs = self.get_tweets(
self.word_to_search, self.nb_tweet, self.include_RT,
self.useTwitterLicenseKey,
(self.twitterLicenseKeysConsumerKey,
self.twitterLicenseKeysConsumerSecret,
(self.twitterLicenseKeysAccessToken,
self.twitterLicenseKeysAccessTokenSecret)))
except (HTTP401Authentication, HTTP400BadRequest):
self.infoBox.setText(
u'Please enter valid Twitter api keys.',
u'error',
)
self.send(u'Text data', None, self)
return False
except SearchEngineLimitError:
self.infoBox.setText(
u'Twitter search limit has been exceeded.',
u'error',
)
self.send(u'Text data', None, self)
return False
elif self.service == u'Wikipedia':
self.createdInputs = self.get_wiki_article(self.word_to_search,
self.wiki_section,
self.wiki_type_of_text)
elif self.service == u'Bing':
self.createdInputs = self.get_bing_entries(self.word_to_search,
self.nb_bing_entry)
if len(self.createdInputs) == 0:
self.infoBox.setText(
u'Please try to change query or settings.',
u'warning',
)
self.send(u'Text data', None, self)
return False
# Initialize progress bar
progressBar = OWGUI.ProgressBar(self, iterations=50)
output_segmentation = Segmenter.concatenate(self.createdInputs,
self.captionTitle,
import_labels_as=None)
message = u'%i segment@p sent to output ' % len(output_segmentation)
message = pluralize(message, len(output_segmentation))
numChars = 0
for segment in output_segmentation:
segmentLength = len(Segmentation.get_data(segment.str_index))
numChars += segmentLength
message += u'(%i character@p).' % numChars
message = pluralize(message, numChars)
self.infoBox.setText(message)
for _ in xrange(50):
progressBar.advance()
# Clear progress bar.
progressBar.finish()
self.send('Text data', output_segmentation, self)
self.sendButton.resetSettingsChangedFlag()