class OWTextableTextTree(OWTextableBaseWidget):
"""Orange widget for loading text folders"""
name = "Text Tree"
description = "Import data from raw text trees"
icon = "icons/Textfolders.png"
icon = "icons/textTree.svg"
priority = 2
# Input and output channels...
inputs = [('Message', JSONMessage, "inputMessage", widget.Single)]
outputs = [('Text data', Segmentation)]
settingsHandler = VersionedSettingsHandler(
version=__version__.rsplit(".", 1)[0])
# Settings...
autoSend = settings.Setting(True)
folders = settings.Setting([])
encoding = settings.Setting('iso-8859-1')
operation = settings.Setting('nothing')
sampling = settings.Setting(100)
autoNumber = settings.Setting(False)
autoNumberKey = settings.Setting(u'num')
importFilenames = settings.Setting(True)
importFolderName = settings.Setting(True)
lastLocation = settings.Setting('.')
displayAdvancedSettings = settings.Setting(False)
folder = settings.Setting(u'')
want_main_area = False
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Other attributes...
self.segmentation = None
self.operation = "no"
self.applyInclusion = False
self.applyExclusion = False
self.applySampling = True
self.samplingRate = 100
self.createdInputs = list()
self.folderLabels = list()
self.selectedFolderLabels = list()
self.rootFolderPath = u''
self.inclusionsUser = u''
self.exclusionsUser = u''
self.newAnnotationKey = u''
self.newAnnotationValue = u''
# self.folder is a dictionary whose keys are :'rootPath', 'maxDepth','inclusionsUser','exclusionsUser', ...
# ... 'samplingRate' and 'fileList'
self.folder = dict()
# self.folders is a list of previously defined "self.folder" dictionaries
self.folders = list()
# self.inclusionList is the default inclusion list (used in minimal mode, ...
# ... and in advanced mode when no inclusion has been selected)
self.inclusionList = [".txt", ".html", ".xml", ".csv", ".rtf"]
# self.inclusionList is the default null inclusion list (used in minimal mode, ...
# ... and in advanced mode when no inclusion has been selected)
self.exclusionList = []
self.infoBox = InfoBox(widget=self.controlArea)
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute='infoBox',
sendIfPreCallback=self.updateGUI,
)
self.advancedSettings = AdvancedSettings(
widget=self.controlArea,
master=self,
callback=self.sendButton.settingsChanged,
)
# GUI...
# Advanced settings checkbox...
self.advancedSettings.draw()
# BASIC GUI...
# Basic folder box
basicFolderBox = gui.widgetBox(
widget=self.controlArea,
box=u'Source',
orientation='vertical',
addSpace=False,
)
basicFolderBoxLine1 = gui.widgetBox(
widget=basicFolderBox,
box=False,
orientation='horizontal',
)
gui.lineEdit(
widget=basicFolderBoxLine1,
master=self,
value='rootFolderPath',
orientation='horizontal',
label=u'Folder path:',
labelWidth=101,
callback=self.add,
tooltip=(u"The path of the folder."),
)
gui.separator(widget=basicFolderBoxLine1, width=5)
gui.button(
widget=basicFolderBoxLine1,
master=self,
label=u'Browse',
callback=self.browse,
tooltip=(u"Open a dialog for selecting a top folder."),
)
gui.separator(widget=basicFolderBox, width=3)
self.advancedSettings.basicWidgets.append(basicFolderBox)
self.advancedSettings.basicWidgetsAppendSeparator()
# ADVANCED GUI...
# folder box
folderBox = gui.widgetBox(
widget=self.controlArea,
box=u'Sources',
orientation='vertical',
addSpace=False,
)
folderBoxLine1 = gui.widgetBox(
widget=folderBox,
box=False,
orientation='horizontal',
addSpace=True,
)
self.folderListbox = gui.listBox(
widget=folderBoxLine1,
master=self,
value='selectedFolderLabels',
labels='folderLabels',
callback=self.updatefolderBoxButtons,
tooltip=(u"The list of folders whose content will be imported.\n"
u"\nIn the output segmentation, the content of each\n"
u"folder appears in the same position as in the list.\n"
u"\nColumn 1 shows the folder's name.\n"
u"Column 2 shows the folder's depth.\n"
u"Column 3 shows the inclusions filter.\n"
u"Column 4 shows the exclusions filter.\n"
u"Column 5 shows the folder's level of sampling."),
)
font = QFont()
font.setFamily('Courier')
font.setStyleHint(QFont.Courier)
font.setPixelSize(12)
self.folderListbox.setFont(font)
folderBoxCol2 = gui.widgetBox(
widget=folderBoxLine1,
orientation='vertical',
)
self.moveUpButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Move Up',
callback=self.moveUp,
tooltip=(u"Move the selected folder upward in the list."),
)
self.moveDownButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Move Down',
callback=self.moveDown,
tooltip=(u"Move the selected folder downward in the list."),
)
self.removeButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Remove',
callback=self.remove,
tooltip=(u"Remove the selected folder from the list."),
)
self.clearAllButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Clear All',
callback=self.clearAll,
tooltip=(u"Remove all folders from the list."),
)
self.exportButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'',
callback=self.exportList,
disabled=True,
tooltip=(u"Open a dialog for selecting a folder where the folder\n"
u"list can be exported in JSON format."),
)
self.importButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'',
callback=self.importList,
disabled=True,
tooltip=(u"Open a dialog for selecting a folder list to\n"
u"import (in JSON format). folders from this list\n"
u"will be added to those already imported."),
)
folderBoxLine2 = gui.widgetBox(
widget=folderBox,
box=False,
orientation='vertical',
)
# Add folder box
addFolderBox = gui.widgetBox(
widget=folderBoxLine2,
box=True,
orientation='vertical',
)
addFolderBoxLine1 = gui.widgetBox(
widget=addFolderBox,
orientation='horizontal',
)
# Folder path input
gui.lineEdit(
widget=addFolderBoxLine1,
master=self,
value='rootFolderPath',
orientation='horizontal',
label=u'Folder path:',
labelWidth=101,
callback=self.updateGUI,
tooltip=(u"The paths of the folders that will be added to the\n"
u"list when button 'Add' is clicked.\n\n"
u"Successive paths must be separated with ' / ' \n"
u"(whitespace + slash + whitespace). Their order in\n"
u"the list will be the same as in this field."),
)
gui.separator(widget=addFolderBoxLine1, width=5)
# Button Browse
gui.button(
widget=addFolderBoxLine1,
master=self,
label=u'Browse',
callback=self.browse,
tooltip=(u"Open a dialog for selecting a top folder.\n\n"
u"Selected folder paths will appear in the field to\n"
u"the left of this button afterwards, ready to be\n"
u"added to the list when button 'Add' is clicked."),
)
gui.separator(widget=addFolderBox, width=10)
# Filter box to input include
gui.separator(widget=addFolderBox, width=3)
includeBoxLine1 = gui.widgetBox(
widget=addFolderBox,
box=False,
orientation='horizontal',
)
# Include box
gui.checkBox(
widget=includeBoxLine1,
master=self,
value='applyInclusion',
label=u'Include',
labelWidth=100,
callback=lambda: includeLineEdit.setDisabled(not self.
applyInclusion),
tooltip=(u"Choose the inclusion(s)"),
)
includeLineEdit = gui.lineEdit(
widget=includeBoxLine1,
master=self,
value='inclusionsUser',
orientation='horizontal',
label=u'',
disabled=True,
labelWidth=101,
tooltip=(u"This field lets you specify a custom filter\n"
u"to select the folders to be\n"
u"added to the list."),
)
# Filter box to exclude
gui.separator(widget=addFolderBox, width=3)
excludeBoxLine1 = gui.widgetBox(
widget=addFolderBox,
box=False,
orientation='horizontal',
)
# Exclude box
gui.checkBox(
widget=excludeBoxLine1,
master=self,
value='applyExclusion',
label=u'Exclude',
labelWidth=100,
disabled=False,
callback=lambda: includeLineEdit2.setDisabled(not self.
applyExclusion),
tooltip=(u"Exclude the inclusion(s)"),
)
includeLineEdit2 = gui.lineEdit(
widget=excludeBoxLine1,
master=self,
value='exclusionsUser',
orientation='horizontal',
label=u'',
disabled=True,
labelWidth=101,
tooltip=(u"This field lets you specify a custom filter\n"
u"to select the folders to be\n"
u"added to the list."),
)
# Sampling box to input the level of sampling
gui.separator(widget=addFolderBox, width=3)
samplingBoxLine1 = gui.widgetBox(
widget=addFolderBox,
box=False,
orientation='horizontal',
)
# Check box for sampling
gui.checkBox(
widget=samplingBoxLine1,
master=self,
value='applySampling',
label=u'Sampling',
labelWidth=100,
disabled=False,
callback=lambda: samplingSpin.setDisabled(not self.applySampling),
tooltip=(u"Choose the sampling level"),
)
samplingSpin = gui.spin(
widget=samplingBoxLine1,
master=self,
value='samplingRate',
minv=10,
maxv=100,
labelWidth=50,
orientation='horizontal',
tooltip=(u"sampling level"),
)
gui.separator(widget=addFolderBox, width=3)
self.addButton = gui.button(
widget=addFolderBox,
master=self,
label=u'Add',
callback=self.add,
tooltip=(u"Add the folder(s) currently displayed in the\n"
u"'folders' text field to the list.\n\n"
u"Each of these folders will be associated with the\n"
u"specified encoding and annotation (if any).\n\n"
u"Other folders may be selected afterwards and\n"
u"assigned a different encoding and annotation."),
)
self.advancedSettings.advancedWidgets.append(folderBox)
self.advancedSettings.advancedWidgetsAppendSeparator()
# Options box...
optionsBox = gui.widgetBox(
widget=self.controlArea,
box=u'Options',
orientation='vertical',
addSpace=False,
)
optionsBoxLine1 = gui.widgetBox(
widget=optionsBox,
box=False,
orientation='horizontal',
)
gui.separator(widget=optionsBox, width=3)
optionsBoxLine2 = gui.widgetBox(
widget=optionsBox,
box=False,
orientation='horizontal',
)
gui.checkBox(
widget=optionsBoxLine2,
master=self,
value='autoNumber',
label=u'Auto-number with key:',
labelWidth=180,
callback=self.sendButton.settingsChanged,
tooltip=(u"Annotate folders with increasing numeric indices."),
)
self.autoNumberKeyLineEdit = gui.lineEdit(
widget=optionsBoxLine2,
master=self,
value='autoNumberKey',
orientation='horizontal',
callback=self.sendButton.settingsChanged,
tooltip=(u"Annotation key for folder auto-numbering."),
)
gui.separator(widget=optionsBox, width=3)
self.advancedSettings.advancedWidgets.append(optionsBox)
self.advancedSettings.advancedWidgetsAppendSeparator()
gui.rubber(self.controlArea)
# Send button...
self.sendButton.draw()
# Info box...
self.infoBox.draw()
self.adjustSizeWithTimer()
QTimer.singleShot(0, self.sendButton.sendIf)
def inputMessage(self, message):
"""Handle JSON message on input connection"""
if not message:
return
self.displayAdvancedSettings = True
self.advancedSettings.setVisible(True)
self.clearAll()
self.infoBox.inputChanged()
try:
jsonData = json.loads(message.content)
tempFolders = list()
for entry in jsonData:
path = entry.get('path', '')
encoding = entry.get('encoding', '')
annotationKey = entry.get('annotation_key', '')
annotationValue = entry.get('annotation_value', '')
if path == '' or encoding == '':
self.infoBox.setText(
u"Please verify keys and values of incoming "
u"JSON message.", 'error')
self.send('Text data', None, self)
return
depth = "0"
options = "[i]:{unicorn}"
tempFolders.append((
name,
path,
depth,
options,
))
self.folders.extend(tempFolders)
self.sendButton.settingsChanged()
except ValueError:
self.infoBox.setText(
u"Please make sure that incoming message is valid JSON.",
'error')
self.send('Text data', None, self)
return
def sendData(self):
"""Load folders, create and send segmentation"""
# Check that there's something on input...
if (self.displayAdvancedSettings
and not self.folders) or not (self.rootFolderPath
or self.displayAdvancedSettings):
self.infoBox.setText(u'Please select input folder.', 'warning')
self.send('Text data', None, self)
return
# Check that autoNumberKey is not empty (if necessary)...
if self.displayAdvancedSettings and self.autoNumber:
if self.autoNumberKey:
autoNumberKey = self.autoNumberKey
else:
self.infoBox.setText(
u'Please enter an annotation key for auto-numbering.',
'warning')
self.send('Text data', None, self)
return
else:
autoNumberKey = None
# Clear created Inputs...
self.clearCreatedInputs()
annotations = list()
counter = 1
if self.displayAdvancedSettings:
myFolders = self.folders
else:
myFolders = [self.folder]
# Annotations...
allFileListContent = list()
for myFolder in myFolders:
myFiles = myFolder['fileList']
for myFile in myFiles:
annotation = dict()
annotation['file name'] = myFile['fileName']
annotation['file depth level'] = myFile['depthLvl']
annotation['file path'] = myFile['absoluteFilePath']
try:
annotation['file encoding, confidence'] = myFile[
'encoding'] + ", " + str(myFile['encodingConfidence'])
except TypeError:
annotation['file encoding, confidence'] = "unknown"
depths = [k for k in myFile.keys() if k.startswith('depth_')]
for depth in depths:
annotation[depth] = myFile[depth]
annotations.append(annotation)
allFileListContent.append(myFile['fileContent'])
# Create an LTTL.Input for each files...
if len(allFileListContent) == 1:
label = self.captionTitle
else:
label = None
for index in range(len(allFileListContent)):
myInput = Input(allFileListContent[index], label)
segment = myInput[0]
segment.annotations.update(annotations[index])
myInput[0] = segment
self.createdInputs.append(myInput)
# If there's only one file, the widget's output is the created Input.
if len(allFileListContent) == 1:
self.segmentation = self.createdInputs[0]
# Otherwise the widget's output is a concatenation...
else:
self.segmentation = Segmenter.concatenate(
segmentations=self.createdInputs,
label=self.captionTitle,
copy_annotations=True,
import_labels_as=None,
sort=False,
auto_number_as=None,
merge_duplicates=False,
progress_callback=None,
)
message = u'%i segment@p sent to output ' % len(self.segmentation)
message = pluralize(message, len(self.segmentation))
numChars = 0
for segment in self.segmentation:
segmentLength = len(Segmentation.get_data(segment.str_index))
numChars += segmentLength
message += u'(%i character@p).' % numChars
message = pluralize(message, numChars)
self.infoBox.setText(message)
self.send('Text data', self.segmentation, self)
self.sendButton.resetSettingsChangedFlag()
def clearCreatedInputs(self):
for i in self.createdInputs:
Segmentation.set_data(i[0].str_index, None)
del self.createdInputs[:]
def importList(self):
"""Display a folderDialog and import folder list"""
folderPath = QFileDialog.getOpenFileName(self, u'Import folder List',
self.lastLocation,
u'Text folders (*)')
if not folderPath:
return
self.rootFolderPath = os.path.normpath(folderPath)
self.lastLocation = os.path.dirname(folderPath)
self.error()
try:
folderHandle = codecs.open(folderPath, encoding='utf8')
folderContent = folderHandle.read()
folderHandle.close()
except IOError:
QMessageBox.warning(None, 'Textable', "Couldn't open folder.",
QMessageBox.Ok)
return
try:
jsonData = json.loads(folderContent)
tempFolders = list()
for entry in jsonData:
path = entry.get('path', '')
encoding = entry.get('encoding', '')
annotationKey = entry.get('annotation_key', '')
annotationValue = entry.get('annotation_value', '')
if path == '' or encoding == '':
QMessageBox.warning(
None, 'Textable',
"Selected JSON folder doesn't have the right keys "
"and/or values.", QMessageBox.Ok)
return
tempFolders.append((
path,
encoding,
annotationKey,
annotationValue,
))
self.folders.extend(tempFolders)
if tempFolders:
self.sendButton.settingsChanged()
except ValueError:
QMessageBox.warning(None, 'Textable', "JSON parsing error.",
QMessageBox.Ok)
return
def exportList(self):
"""Display a folderDialog and export folder list"""
toDump = list()
myFolders = self.folders
for myFolder in myFolders:
toDump.append({
'path': myFolder[0],
'encoding': myFolder[1],
})
if myFolder[2] and myFolder[3]:
toDump[-1]['annotation_key'] = myFolder[2]
toDump[-1]['annotation_value'] = myFolder[3]
folderPath = QFileDialog.getSaveFileName(
self,
u'Export folder List',
self.lastLocation,
)
if folderPath:
self.lastLocation = os.path.dirname(folderPath)
outputfolder = codecs.open(
folderPath,
encoding='utf8',
mode='w',
errors='xmlcharrefreplace',
)
outputfolder.write(
normalizeCarriageReturns(
json.dumps(toDump, sort_keys=True, indent=4)))
outputfolder.close()
QMessageBox.information(None, 'Textable',
'folder list correctly exported',
QMessageBox.Ok)
def getFileList(self):
initialRootParentPath, _ = os.path.split(
# self.rootFolderPath is the initially selected's folder parent
self.rootFolderPath)
fileList = list()
# fileListExt is a list of files matching default extension
fileListExt = list()
depthList = list()
progressBarZero = gui.ProgressBar(self, iterations=1)
# Using os.walk to walk through directories :
# Variables descriptions :
# currPath is a STRING, the path to the directory.
# dirNames is a LIST of the names of subdirectories.
# fileNames is a LIST of the names of the files in currPath
# symlink are not considered in this analysis
for currPath, dirNames, fileNames in os.walk(self.rootFolderPath):
currRelPath = currPath[
len(initialRootParentPath) +
1:] # defines current relative path by similar initial parent path part
currRelPathList = os.path.normpath(currRelPath).split(
os.sep) # splits current relative path by os separator
for fileName in fileNames:
# file dict is a dictionary of the file's informations will get following keys :
# file = {
# "absoluteFilePath",
# "fileName",
# "depth_0",
# "depth_X"
# depthLvl",
# "fileContent"
# }
# 'fileContent','encoding' and 'encodingConfidence' keys are defined when function "openFileList" is called
file = dict()
# Initial annotations correspond different subfolders browsed by each depth level (used for depth_X annot.)
annotations = currRelPathList[:]
currDepth = len(annotations) - 1
depthList.append(currDepth)
file['absoluteFilePath'] = os.path.join(currPath, fileName)
file['fileName'] = fileName
file['depthLvl'] = currDepth
file['depth_0'] = annotations[0]
# Created an annotation by depth level, corresponding to folder names
for i in range(1, currDepth + 1):
file['depth_' + str(i)] = annotations[i]
# Apply default file extension filter
for extension in self.inclusionList:
if fileName.endswith(extension):
# FileListExt = file list created with default inclusion criteria (text extensions from inclusionList)
fileListExt.append(file)
fileList.append(file)
# apply inclusion filter
if self.applyInclusion:
fileListIncl = [
file for file in fileList
# match in inclusion list
if self.match(file['fileName'], self.inclusionsUserAsList)
]
else:
fileListIncl = fileListExt
# apply exclusion filter
if self.applyExclusion:
fileListExcl = [
file for file in fileListIncl
# no match in exclusion list
if not self.match(file['fileName'], self.exclusionsUserAsList)
]
else:
fileListExcl = fileListIncl
# output file list
self.fileList = fileListExcl
if self.fileList:
self.maxDepth = max(depthList)
self.fileList = self.sampleFileList()
self.openFileList()
else:
self.maxDepth = 0
progressBarZero.finish()
# Test if file contains one of the patterns in patternList
def match(self, file, patternList):
for pattern in patternList:
if pattern in file:
return True
return False
def openFileList(self):
tempFileList = list()
progressBarOpen = gui.ProgressBar(self, iterations=len(self.fileList))
for file in self.fileList:
fileContent = ""
try:
filePath = file['absoluteFilePath']
except TypeError:
pass
encodings = getPredefinedEncodings()
try:
with open(filePath, 'rb') as openedFile:
fileContent = openedFile.read()
charsetDict = chardet.detect(fileContent)
detectedEncoding = charsetDict['encoding']
detectedConfidence = charsetDict['confidence']
# Chunking functionnality should be added here
try:
encodings.remove(detectedEncoding)
encodings.insert(0, detectedEncoding)
except ValueError:
pass
for encoding in encodings:
try:
self.fileContent = fileContent.decode(encoding)
except:
pass
file['encoding'] = detectedEncoding
file['fileContent'] = self.fileContent
file['encodingConfidence'] = detectedConfidence
progressBarOpen.advance()
tempFileList.append(file)
except IOError:
if len(myFiles) > 1:
message = u"Couldn't open file '%s'." % filePath
else:
message = u"Couldn't open file."
self.infoBox.setText(message, 'error')
self.send('Text data', None, self)
return
self.fileList = tempFileList
self.folder = {
'rootPath': self.rootFolderPath,
'maxDepth': self.maxDepth,
'inclusionsUser': self.inclusionsUser,
'exclusionsUser': self.exclusionsUser,
'samplingRate': self.samplingRate,
'fileList': self.fileList
}
progressBarOpen.finish()
def browse(self):
"""Display a QFileDialog and select a folder"""
rootFolderPath = QFileDialog.getExistingDirectory(
self,
u'Select Folder(s)',
self.lastLocation,
)
if not rootFolderPath:
return
rootFolderPath = os.path.normpath(rootFolderPath)
self.rootFolderPath = rootFolderPath
self.lastLocation = rootFolderPath
if self.displayAdvancedSettings:
pass
else:
self.getFileList()
self.folder = {
'rootPath': self.rootFolderPath,
'maxDepth': self.maxDepth,
'fileList': self.fileList,
}
self.sendButton.settingsChanged()
self.updateGUI()
def moveUp(self):
"""Move folder upward in folders listbox"""
if self.selectedFolderLabels:
index = self.selectedFolderLabels[0]
if index > 0:
temp = self.folders[index - 1]
self.folders[index - 1] = self.folders[index]
self.folders[index] = temp
self.selectedFolderLabels.listBox.item(index -
1).setSelected(1)
self.sendButton.settingsChanged()
def moveDown(self):
"""Move folder downward in folders listbox"""
if self.selectedFolderLabels:
index = self.selectedFolderLabels[0]
if index < len(self.folders) - 1:
temp = self.folders[index + 1]
self.folders[index + 1] = self.folders[index]
self.folders[index] = temp
self.selectedFolderLabels.listBox.item(index +
1).setSelected(1)
self.sendButton.settingsChanged()
def clearAll(self):
"""Remove all folders from folders attr"""
del self.folders[:]
del self.selectedFolderLabels[:]
self.sendButton.settingsChanged()
def remove(self):
"""Remove folder from folders attr"""
if self.selectedFolderLabels:
index = self.selectedFolderLabels[0]
self.folders.pop(index)
del self.selectedFolderLabels[:]
self.sendButton.settingsChanged()
def add(self):
"""Add folders to folders attr"""
# Identify sequences separated by a comma (,) and deletes existing whitespaces
self.inclusionsUserAsList = [
x.strip() for x in self.inclusionsUser.split(",") if x.strip()
]
self.exclusionsUserAsList = [
x.strip() for x in self.exclusionsUser.split(",") if x.strip()
]
# Calling the GetFileList function returns a self.fileList list of all files corresponding to either defaults
# or optional settings
self.getFileList()
self.folders.append(self.folder)
self.sendButton.settingsChanged()
def sampleFileList(self):
myList = list(self.fileList)
# Sampling rate from input allows calculation of the sampling percentage
samplePercentage = self.samplingRate / 100.0
# The initial list is shuffled so that files from all folders can be picked randomly
random.shuffle(myList)
# Files are picked randomly from the previously shuffled list
nOfFiles = int(math.ceil(len(myList) * samplePercentage))
return myList[:nOfFiles]
def updateGUI(self):
"""Update GUI state"""
if self.displayAdvancedSettings:
if self.selectedFolderLabels:
cachedLabel = self.selectedFolderLabels[0]
else:
cachedLabel = None
del self.folderLabels[:]
folderLabels = []
if self.folders:
folderRootPathsList = [f['rootPath'] for f in self.folders]
maxDepthList = ['%s' % f['maxDepth'] for f in self.folders]
inclusionsUserList = [
f['inclusionsUser'] for f in self.folders
]
exclusionsUserList = [
f['exclusionsUser'] for f in self.folders
]
samplingRatesList = [
'%s' % f['samplingRate'] for f in self.folders
]
folderNamesList = [
os.path.basename(p) for p in folderRootPathsList
]
maxFolderNameLen = max([len(n) for n in folderNamesList])
for index in range(len(self.folders)):
format = u'%-' + str(maxFolderNameLen + 2) + u's'
folderLabel = format % folderNamesList[index]
folderLabel += "[d]:{" + maxDepthList[index] + "} "
folderLabel += "[i]:{" + inclusionsUserList[index] + "} "
folderLabel += "[e]:{" + exclusionsUserList[index] + "} "
folderLabel += "[s]:{" + samplingRatesList[index] + "%}"
folderLabels.append(folderLabel)
self.folderLabels = folderLabels
if cachedLabel is not None:
self.sendButton.sendIfPreCallback = None
self.selectedFolderLabels.listBox.item(
cachedLabel).setSelected(1)
self.sendButton.sendIfPreCallback = self.updateGUI
if self.rootFolderPath:
if (self.newAnnotationKey and self.newAnnotationValue) or (
not self.newAnnotationKey
and not self.newAnnotationValue):
self.addButton.setDisabled(False)
else:
self.addButton.setDisabled(True)
else:
self.addButton.setDisabled(True)
if self.autoNumber:
self.autoNumberKeyLineEdit.setDisabled(False)
else:
self.autoNumberKeyLineEdit.setDisabled(True)
self.updatefolderBoxButtons()
self.advancedSettings.setVisible(True)
else:
self.advancedSettings.setVisible(False)
def updatefolderBoxButtons(self):
"""Update state of folder box buttons"""
if self.selectedFolderLabels:
self.removeButton.setDisabled(False)
if self.selectedFolderLabels[0] > 0:
self.moveUpButton.setDisabled(False)
else:
self.moveUpButton.setDisabled(True)
if self.selectedFolderLabels[0] < len(self.folders) - 1:
self.moveDownButton.setDisabled(False)
else:
self.moveDownButton.setDisabled(True)
else:
self.moveUpButton.setDisabled(True)
self.moveDownButton.setDisabled(True)
self.removeButton.setDisabled(True)
if len(self.folders):
self.clearAllButton.setDisabled(False)
self.exportButton.setDisabled(True)
else:
self.clearAllButton.setDisabled(True)
self.exportButton.setDisabled(True)
def setCaption(self, title):
if 'captionTitle' in dir(self):
changed = title != self.captionTitle
super().setCaption(title)
if changed:
self.sendButton.settingsChanged()
else:
super().setCaption(title)
def onDeleteWidget(self):
self.clearCreatedInputs()
class SuperTextFiles(OWTextableBaseWidget):
"""Textable widget to import PDF files and if necessary to do an Optical
Character Recognition (OCR)"""
#----------------------------------------------------------------------
# Widget's metadata...
name = "Super Text Files"
description = "Import data from raw text and PDF files"
icon = "icons/SuperTextFiles.svg"
priority = 1 # TODO
#----------------------------------------------------------------------
# Channel definitions....
inputs = [('Message', JSONMessage, "inputMessage", widget.Single)]
outputs = [('Text data', Segmentation)]
#----------------------------------------------------------------------
# Layout parameters...
want_main_area = False
#----------------------------------------------------------------------
# Settings...
settingsHandler = VersionedSettingsHandler(
version=__version__.rsplit(".", 1)[0])
files = settings.Setting([])
encoding = settings.Setting('(auto-detect)')
autoNumber = settings.Setting(False)
autoNumberKey = settings.Setting(u'num')
importFilenames = settings.Setting(True)
importFilenamesKey = settings.Setting(u'filename')
lastLocation = settings.Setting('.')
displayAdvancedSettings = settings.Setting(False)
file = settings.Setting(u'')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Other attributes...
self.segmentation = None
self.createdInputs = list()
self.fileLabels = list()
self.selectedFileLabels = list()
self.newFiles = u''
self.newAnnotationKey = u''
self.newAnnotationValue = u''
self.pdfPassword = u'' # SuperTextFiles
self.ocrForce = False # SuperTextFiles
self.ocrLanguages = u'eng' # SuperTextFiles
self.infoBox = InfoBox(widget=self.controlArea)
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute='infoBox',
sendIfPreCallback=self.updateGUI,
)
self.advancedSettings = AdvancedSettings(
widget=self.controlArea,
master=self,
callback=self.sendButton.settingsChanged,
)
# GUI...
# Advanced settings checkbox...
self.advancedSettings.draw()
# BASIC GUI...
# Basic file box
basicFileBox = gui.widgetBox(
widget=self.controlArea,
box=u'Source',
orientation='vertical',
addSpace=False,
)
basicFileBoxLine1 = gui.widgetBox(
widget=basicFileBox,
box=False,
orientation='horizontal',
)
gui.lineEdit(
widget=basicFileBoxLine1,
master=self,
value='file',
orientation='horizontal',
label=u'File path:',
labelWidth=101,
callback=self.sendButton.settingsChanged,
tooltip=(u"The path of the file."),
)
gui.separator(widget=basicFileBoxLine1, width=5)
gui.button(
widget=basicFileBoxLine1,
master=self,
label=u'Browse',
callback=self.browse,
tooltip=(u"Open a dialog for selecting file."),
)
gui.separator(widget=basicFileBox, width=3)
advancedEncodingsCombobox = gui.comboBox(
widget=basicFileBox,
master=self,
value='encoding',
items=getPredefinedEncodings(),
sendSelectedValue=True,
orientation='horizontal',
label=u'Encoding:',
labelWidth=101,
callback=self.sendButton.settingsChanged,
tooltip=(u"Select input file(s) encoding."),
)
addSeparatorAfterDefaultEncodings(advancedEncodingsCombobox)
addAutoDetectEncoding(advancedEncodingsCombobox)
gui.separator(widget=basicFileBox, width=3)
self.advancedSettings.basicWidgets.append(basicFileBox)
self.advancedSettings.basicWidgetsAppendSeparator()
# ADVANCED GUI...
defaultLabelWidth = 120 # SuperTextFiles
# File box
fileBox = gui.widgetBox(
widget=self.controlArea,
box=u'Sources',
orientation='vertical',
addSpace=False,
)
fileBoxLine1 = gui.widgetBox(
widget=fileBox,
box=False,
orientation='horizontal',
addSpace=True,
)
self.fileListbox = gui.listBox(
widget=fileBoxLine1,
master=self,
value='selectedFileLabels',
labels='fileLabels',
callback=self.updateFileBoxButtons,
tooltip=(
u"The list of files whose content will be imported.\n"
u"\nIn the output segmentation, the content of each\n"
u"file appears in the same position as in the list.\n"
u"\nColumn 1 shows the file's name.\n"
u"Column 2 shows the file's annotation (if any).\n"
# Start SuperTextFiles
# u"Column 3 shows the file's encoding." # removed
u"Column 3 shows the file's password (if any).\n"
u"Column 4 shows the file's languages (if any).\n"
u"Column 5 shows if OCR is forced.\n"
u"Column 6 shows the file's encoding."
# End SuperTextFiles
),
)
font = QFont()
font.setFamily('Courier')
font.setStyleHint(QFont.Courier)
font.setPixelSize(12)
self.fileListbox.setFont(font)
fileBoxCol2 = gui.widgetBox(
widget=fileBoxLine1,
orientation='vertical',
)
self.moveUpButton = gui.button(
widget=fileBoxCol2,
master=self,
label=u'Move Up',
callback=self.moveUp,
tooltip=(u"Move the selected file upward in the list."),
)
self.moveDownButton = gui.button(
widget=fileBoxCol2,
master=self,
label=u'Move Down',
callback=self.moveDown,
tooltip=(u"Move the selected file downward in the list."),
)
self.removeButton = gui.button(
widget=fileBoxCol2,
master=self,
label=u'Remove',
callback=self.remove,
tooltip=(u"Remove the selected file from the list."),
)
self.clearAllButton = gui.button(
widget=fileBoxCol2,
master=self,
label=u'Clear All',
callback=self.clearAll,
tooltip=(u"Remove all files from the list."),
)
self.exportButton = gui.button(
widget=fileBoxCol2,
master=self,
label=u'Export List',
callback=self.exportList,
tooltip=(u"Open a dialog for selecting a file where the file\n"
u"list can be exported in JSON format."),
)
self.importButton = gui.button(
widget=fileBoxCol2,
master=self,
label=u'Import List',
callback=self.importList,
tooltip=(u"Open a dialog for selecting a file list to\n"
u"import (in JSON format). Files from this list\n"
u"will be added to those already imported."),
)
fileBoxLine2 = gui.widgetBox(
widget=fileBox,
box=False,
orientation='vertical',
)
# Add file box
addFileBox = gui.widgetBox(
widget=fileBoxLine2,
box=True,
orientation='vertical',
)
addFileBoxLine1 = gui.widgetBox(
widget=addFileBox,
orientation='horizontal',
)
gui.lineEdit(
widget=addFileBoxLine1,
master=self,
value='newFiles',
orientation='horizontal',
label=u'File path(s):',
labelWidth=defaultLabelWidth,
callback=self.updateGUI,
tooltip=(u"The paths of the files that will be added to the\n"
u"list when button 'Add' is clicked.\n\n"
u"Successive paths must be separated with ' / ' \n"
u"(whitespace + slash + whitespace). Their order in\n"
u"the list will be the same as in this field."),
)
gui.separator(widget=addFileBoxLine1, width=5)
gui.button(
widget=addFileBoxLine1,
master=self,
label=u'Browse',
callback=self.browse,
tooltip=(u"Open a dialog for selecting files.\n\n"
u"To select multiple files at once, either draw a\n"
u"selection box around them, or use shift and/or\n"
u"ctrl + click.\n\n"
u"Selected file paths will appear in the field to\n"
u"the left of this button afterwards, ready to be\n"
u"added to the list when button 'Add' is clicked."),
)
gui.separator(widget=addFileBox, width=3)
basicEncodingsCombobox = gui.comboBox(
widget=addFileBox,
master=self,
value='encoding',
items=getPredefinedEncodings(),
sendSelectedValue=True,
orientation='horizontal',
label=u'Encoding:',
labelWidth=defaultLabelWidth,
callback=self.updateGUI,
tooltip=(u"Select input file(s) encoding."),
)
addSeparatorAfterDefaultEncodings(basicEncodingsCombobox)
addAutoDetectEncoding(basicEncodingsCombobox)
self.encoding = self.encoding
gui.separator(widget=addFileBox, width=3)
gui.lineEdit(
widget=addFileBox,
master=self,
value='newAnnotationKey',
orientation='horizontal',
label=u'Annotation key:',
labelWidth=defaultLabelWidth,
callback=self.updateGUI,
tooltip=(u"This field lets you specify a custom annotation\n"
u"key associated with each file that is about to be\n"
u"added to the list."),
)
gui.separator(widget=addFileBox, width=3)
gui.lineEdit(
widget=addFileBox,
master=self,
value='newAnnotationValue',
orientation='horizontal',
label=u'Annotation value:',
labelWidth=defaultLabelWidth,
callback=self.updateGUI,
tooltip=(u"This field lets you specify the annotation value\n"
u"associated with the above annotation key."),
)
### Start SuperTextFiles addition
gui.separator(widget=addFileBox, width=3)
# Field for PDF password
gui.lineEdit(
widget=addFileBox,
master=self,
value='pdfPassword',
orientation='horizontal',
label=u'PDF password:'******'ocrLanguages',
orientation='horizontal',
label=u'OCR Language(s):',
labelWidth=defaultLabelWidth,
callback=self.updateGUI,
tooltip=(u"This field lets you specify languages\n"
u"for the OCR process. Ex.: fra+ita"),
)
gui.checkBox(
widget=addFileBox,
master=self,
value='ocrForce',
label=u'Force OCR',
labelWidth=defaultLabelWidth,
callback=self.updateGUI,
tooltip=(u"Force to use an OCR detection on this file"),
)
### End SuperTextFiles addition
gui.separator(widget=addFileBox, width=3)
self.addButton = gui.button(
widget=addFileBox,
master=self,
label=u'Add',
callback=self.add,
tooltip=(u"Add the file(s) currently displayed in the\n"
u"'Files' text field to the list.\n\n"
u"Each of these files will be associated with the\n"
u"specified encoding and annotation (if any).\n\n"
u"Other files may be selected afterwards and\n"
u"assigned a different encoding and annotation."),
)
self.advancedSettings.advancedWidgets.append(fileBox)
self.advancedSettings.advancedWidgetsAppendSeparator()
# Options box...
optionsBox = gui.widgetBox(
widget=self.controlArea,
box=u'Options',
orientation='vertical',
addSpace=False,
)
optionsBoxLine1 = gui.widgetBox(
widget=optionsBox,
box=False,
orientation='horizontal',
)
gui.checkBox(
widget=optionsBoxLine1,
master=self,
value='importFilenames',
label=u'Import file names with key:',
labelWidth=180,
callback=self.sendButton.settingsChanged,
tooltip=(u"Import file names as annotations."),
)
self.importFilenamesKeyLineEdit = gui.lineEdit(
widget=optionsBoxLine1,
master=self,
value='importFilenamesKey',
orientation='horizontal',
callback=self.sendButton.settingsChanged,
tooltip=(u"Annotation key for importing file names."),
)
gui.separator(widget=optionsBox, width=3)
optionsBoxLine2 = gui.widgetBox(
widget=optionsBox,
box=False,
orientation='horizontal',
)
gui.checkBox(
widget=optionsBoxLine2,
master=self,
value='autoNumber',
label=u'Auto-number with key:',
labelWidth=180,
callback=self.sendButton.settingsChanged,
tooltip=(u"Annotate files with increasing numeric indices."),
)
self.autoNumberKeyLineEdit = gui.lineEdit(
widget=optionsBoxLine2,
master=self,
value='autoNumberKey',
orientation='horizontal',
callback=self.sendButton.settingsChanged,
tooltip=(u"Annotation key for file auto-numbering."),
)
gui.separator(widget=optionsBox, width=3)
self.advancedSettings.advancedWidgets.append(optionsBox)
self.advancedSettings.advancedWidgetsAppendSeparator()
gui.rubber(self.controlArea)
# Send button...
self.sendButton.draw()
# Info box...
self.infoBox.draw()
self.adjustSizeWithTimer()
QTimer.singleShot(0, self.sendButton.sendIf)
def inputMessage(self, message):
"""Handle JSON message on input connection"""
if not message:
return
self.displayAdvancedSettings = True
self.advancedSettings.setVisible(True)
self.clearAll()
self.infoBox.inputChanged()
try:
json_data = json.loads(message.content)
temp_files = list()
for entry in json_data:
path = entry.get('path', '')
encoding = entry.get('encoding', '')
annotationKey = entry.get('annotation_key', '')
annotationValue = entry.get('annotation_value', '')
pdfPassword = entry.get('pdf_password', '') # SuperTextFiles
ocrLanguages = entry.get('ocr_languages', '') # SuperTextFiles
ocrForce = entry.get('ocr_force', '') # SuperTextFiles
if path == '' or encoding == '' or ocrForce == '':
self.infoBox.setText(
u"Please verify keys and values of incoming "
u"JSON message.", 'error')
self.send('Text data', None, self)
return
temp_files.append((
path,
encoding,
annotationKey,
annotationValue,
pdfPassword, # SuperTextFiles
ocrLanguages, # SuperTextFiles
ocrForce, # SuperTextFiles
))
self.files.extend(temp_files)
self.sendButton.settingsChanged()
except ValueError:
self.infoBox.setText(
u"Please make sure that incoming message is valid JSON.",
'error')
self.send('Text data', None, self)
return
def sendData(self):
"""Load files, create and send segmentation"""
# Check that there's something on input...
if ((self.displayAdvancedSettings and not self.files)
or not (self.file or self.displayAdvancedSettings)):
self.infoBox.setText(u'Please select input file.', 'warning')
self.send('Text data', None, self)
return
# Check that autoNumberKey is not empty (if necessary)...
if self.displayAdvancedSettings and self.autoNumber:
if self.autoNumberKey:
autoNumberKey = self.autoNumberKey
else:
self.infoBox.setText(
u'Please enter an annotation key for auto-numbering.',
'warning')
self.send('Text data', None, self)
return
else:
autoNumberKey = None
# Clear created Inputs...
self.clearCreatedInputs()
fileContents = list()
annotations = list()
counter = 1
if self.displayAdvancedSettings:
myFiles = self.files
else:
myFiles = [[self.file, self.encoding, "", "", "", "eng", False]]
self.infoBox.setText(u"Processing, please wait...", "warning")
self.controlArea.setDisabled(True)
progressBar = ProgressBar(self, iterations=len(myFiles))
# Open and process each file successively...
for myFile in myFiles:
filePath = myFile[0]
encoding = myFile[1]
encoding = re.sub(r"[ ]\(.+", "", encoding)
annotation_key = myFile[2]
annotation_value = myFile[3]
pdf_password = myFile[4] # SuperTextFiles
ocr_languages = myFile[5] # SuperTextFiles
ocr_force = myFile[6] # SuperTextFiles
myFiletype = filetype.guess(myFile[0]) # SuperTextFiles
# Try to open the file...
self.error()
# Start SuperTextFiles
try:
if myFiletype is None:
fileContent = self.extract_raw_text(filePath, encoding)
elif myFiletype.extension == "pdf":
if ocr_force is True:
fileContent = self.get_pdf_content(
filePath,
ocr_languages,
)
else:
if self.is_textual_pdf_file(filePath) is True:
fileContent = self.extract_text_from_pdf(filePath)
else:
fileContent = self.get_pdf_content(
filePath,
ocr_languages,
)
elif myFiletype.extension in IMG_FILETYPES:
fileContent = self.ocrize(filePath, ocr_languages)
if fileContent == -1:
message = u"Couldn't open file."
self.infoBox.setText(message, 'error')
self.send('Text data', None, self)
self.controlArea.setDisabled(False)
return
# End SuperTextFiles
except IOError as e:
if "tesseract" in str(e):
QMessageBox.warning(None, 'Textable', str(e),
QMessageBox.Ok)
progressBar.finish()
if len(myFiles) > 1:
message = u"Couldn't open file '%s'." % filePath
else:
message = u"Couldn't open file."
self.infoBox.setText(message, 'error')
self.send('Text data', None, self)
self.controlArea.setDisabled(False)
return
# Remove utf-8 BOM if necessary...
if encoding == u'utf-8':
fileContent = fileContent.lstrip(
codecs.BOM_UTF8.decode('utf-8'))
# Normalize text (canonical decomposition then composition)...
fileContent = normalize('NFC', fileContent)
fileContents.append(fileContent)
# Annotations...
annotation = dict()
if self.displayAdvancedSettings:
if annotation_key and annotation_value:
annotation[annotation_key] = annotation_value
if self.importFilenames and self.importFilenamesKey:
filename = os.path.basename(filePath)
annotation[self.importFilenamesKey] = filename
if self.autoNumber and self.autoNumberKey:
annotation[self.autoNumberKey] = counter
counter += 1
annotations.append(annotation)
progressBar.advance()
# Create an LTTL.Input for each file...
if len(fileContents) == 1:
label = self.captionTitle
else:
label = None
for index in range(len(fileContents)):
myInput = Input(fileContents[index], label)
segment = myInput[0]
segment.annotations.update(annotations[index])
myInput[0] = segment
self.createdInputs.append(myInput)
# If there's only one file, the widget's output is the created Input.
if len(fileContents) == 1:
self.segmentation = self.createdInputs[0]
# Otherwise the widget's output is a concatenation...
else:
self.segmentation = Segmenter.concatenate(
segmentations=self.createdInputs,
label=self.captionTitle,
copy_annotations=True,
import_labels_as=None,
sort=False,
auto_number_as=None,
merge_duplicates=False,
progress_callback=None,
)
message = u'%i segment@p sent to output ' % len(self.segmentation)
message = pluralize(message, len(self.segmentation))
numChars = 0
for segment in self.segmentation:
segmentLength = len(Segmentation.get_data(segment.str_index))
numChars += segmentLength
message += u'(%i character@p).' % numChars
message = pluralize(message, numChars)
self.infoBox.setText(message)
progressBar.finish()
self.controlArea.setDisabled(False)
self.send('Text data', self.segmentation, self)
self.sendButton.resetSettingsChangedFlag()
def extract_raw_text(self, filePath, encoding):
"""This function receive a filePath and an encoding value and return a
string with the text of the given file."""
if encoding == "(auto-detect)":
detector = UniversalDetector()
fh = open(filePath, 'rb')
for line in fh:
detector.feed(line)
if detector.done: break
detector.close()
fh.close()
encoding = detector.result['encoding']
fh = open(
filePath,
mode='rU',
encoding=encoding,
)
try:
i = 0
fileContent = ""
chunks = list()
for chunk in iter(lambda: fh.read(CHUNK_LENGTH), ""):
chunks.append('\n'.join(chunk.splitlines()))
i += CHUNK_LENGTH
if i % (CHUNK_NUM * CHUNK_LENGTH) == 0:
fileContent += "".join(chunks)
chunks = list()
if len(chunks):
fileContent += "".join(chunks)
del chunks
return fileContent
except UnicodeError:
progressBar.finish()
if len(myFiles) > 1:
message = u"Please select another encoding " \
+ u"for file %s." % filePath
else:
message = u"Please select another encoding."
self.infoBox.setText(message, 'error')
self.send('Text data', None, self)
self.controlArea.setDisabled(False)
return
finally:
fh.close()
def is_textual_pdf_file(self, filePath):
"""Evaluate the content of the pdf file"""
with pdfplumber.open(filePath, password=self.pdfPassword) as fh:
first_page = fh.pages[0]
text = first_page.extract_text()
if text is None or text.isspace() is True:
return False
else:
return True
def extract_text_from_pdf(self, filePath):
"""Extract all readable text contents"""
fileContent = ""
with pdfplumber.open(filePath, password=self.pdfPassword) as fh:
for page in fh.pages:
fileContent += page.extract_text()
return fileContent
def get_pdf_content(self, filePath, languages):
""" First this function get all texts in the file if exist. Then it
creates a list of pictures to make the OCR method."""
text = ""
with fitz.open(filePath) as doc:
images = []
for page in doc:
text += page.getText("text")
images += doc.getPageImageList(page.number)
for image in images:
xref = image[0]
picture = fitz.Pixmap(doc, xref)
if picture.n > 4: # CMYK colorspace
picture = fitz.Pixmap(fitz.csRGB,
picture) # convert to RGB
bytes_img = BytesIO(picture.getImageData())
page_text = self.ocrize(bytes_img, languages)
if page_text == -1:
text = -1
break
elif page_text:
text += page_text
return text
def ocrize(self, image, languages):
"""Make an OCR on a list of images or an image file"""
languages = languages.strip() # remove trailing spaces
if languages == "":
languages = "eng"
try:
ocrized_text = image_to_string(Image.open(image), lang=languages)
return ocrized_text
except TesseractError as e:
if "load" in str(e):
QMessageBox.warning(
None, 'Textable',
"Please make sure all Tesseract parameter files for "
"language(s) '%s' have been installed." % languages,
QMessageBox.Ok)
return -1
def clearCreatedInputs(self):
for i in self.createdInputs:
Segmentation.set_data(i[0].str_index, None)
del self.createdInputs[:]
def importList(self):
"""Display a FileDialog and import file list"""
filePath, _ = QFileDialog.getOpenFileName(self, u'Import File List',
self.lastLocation,
u'Text files (*)')
if not filePath:
return
self.file = os.path.normpath(filePath)
self.lastLocation = os.path.dirname(filePath)
self.error()
try:
fileHandle = codecs.open(filePath, encoding='utf8')
fileContent = fileHandle.read()
fileHandle.close()
except IOError:
QMessageBox.warning(None, 'Textable', "Couldn't open file.",
QMessageBox.Ok)
return
try:
json_data = json.loads(fileContent)
temp_files = list()
for entry in json_data:
path = entry.get('path', '')
encoding = entry.get('encoding', '')
annotationKey = entry.get('annotation_key', '')
annotationValue = entry.get('annotation_value', '')
pdfPassword = entry.get('pdf_password', '') # SuperTextFiles
ocrLanguages = entry.get('ocr_languages', '') # SuperTextFiles
ocrForce = entry.get('ocr_force', '') # SuperTextFiles
if path == '' or encoding == '' or ocrForce == '':
QMessageBox.warning(
None, 'Textable',
"Selected JSON file doesn't have the right keys "
"and/or values.", QMessageBox.Ok)
return
temp_files.append((
path,
encoding,
annotationKey,
annotationValue,
pdfPassword, # SuperTextFiles
ocrLanguages, # SuperTextFiles
ocrForce, # SuperTextFiles
))
self.files.extend(temp_files)
if temp_files:
self.sendButton.settingsChanged()
except ValueError:
QMessageBox.warning(None, 'Textable', "JSON parsing error.",
QMessageBox.Ok)
return
def exportList(self):
"""Display a FileDialog and export file list"""
toDump = list()
for myfile in self.files:
toDump.append({
'path': myfile[0],
'encoding': myfile[1],
})
if myfile[2] and myfile[3]:
toDump[-1]['annotation_key'] = myfile[2]
toDump[-1]['annotation_value'] = myfile[3]
# Start SuperTextFiles
if myfile[4]:
toDump[-1]['pdf_password'] = myfile[4]
if myfile[5]:
toDump[-1]['ocr_languages'] = myfile[5]
toDump[-1]['ocr_force'] = myfile[6]
# End SuperTextFiles
filePath, _ = QFileDialog.getSaveFileName(
self,
u'Export File List',
self.lastLocation,
)
if filePath:
self.lastLocation = os.path.dirname(filePath)
outputFile = codecs.open(
filePath,
encoding='utf8',
mode='w',
errors='xmlcharrefreplace',
)
outputFile.write(
normalizeCarriageReturns(
json.dumps(toDump, sort_keys=True, indent=4)))
outputFile.close()
QMessageBox.information(None, 'Textable',
'File list correctly exported',
QMessageBox.Ok)
def browse(self):
"""Display a FileDialog and select files"""
if self.displayAdvancedSettings:
filePathList, _ = QFileDialog.getOpenFileNames(
self, u'Select Text File(s)', self.lastLocation,
u'Text files (*)')
if not filePathList:
return
filePathList = [os.path.normpath(f) for f in filePathList]
self.newFiles = u' / '.join(filePathList)
self.lastLocation = os.path.dirname(filePathList[-1])
self.updateGUI()
else:
filePath, _ = QFileDialog.getOpenFileName(self, u'Open Text File',
self.lastLocation,
u'Text files (*)')
if not filePath:
return
self.file = os.path.normpath(filePath)
self.lastLocation = os.path.dirname(filePath)
self.updateGUI()
self.sendButton.settingsChanged()
def moveUp(self):
"""Move file upward in Files listbox"""
if self.selectedFileLabels:
index = self.selectedFileLabels[0]
if index > 0:
temp = self.files[index - 1]
self.files[index - 1] = self.files[index]
self.files[index] = temp
self.selectedFileLabels = [index - 1]
self.sendButton.settingsChanged()
def moveDown(self):
"""Move file downward in Files listbox"""
if self.selectedFileLabels:
index = self.selectedFileLabels[0]
if index < len(self.files) - 1:
temp = self.files[index + 1]
self.files[index + 1] = self.files[index]
self.files[index] = temp
self.selectedFileLabels = [index + 1]
self.sendButton.settingsChanged()
def clearAll(self):
"""Remove all files from files attr"""
del self.files[:]
del self.selectedFileLabels[:]
self.sendButton.settingsChanged()
def remove(self):
"""Remove file from files attr"""
if self.selectedFileLabels:
index = self.selectedFileLabels[0]
self.files.pop(index)
del self.selectedFileLabels[:]
self.sendButton.settingsChanged()
def add(self):
"""Add files to files attr"""
filePathList = re.split(r' +/ +', self.newFiles)
for filePath in filePathList:
encoding = re.sub(r"[ ]\(.+", "", self.encoding)
self.files.append((
filePath,
encoding,
self.newAnnotationKey,
self.newAnnotationValue,
self.pdfPassword, # SuperTextFiles
self.ocrLanguages, # SuperTextFiles
self.ocrForce, # SuperTextFiles
))
self.sendButton.settingsChanged()
def updateGUI(self):
"""Update GUI state"""
if self.displayAdvancedSettings:
if self.selectedFileLabels:
cachedLabel = self.selectedFileLabels[0]
else:
cachedLabel = None
del self.fileLabels[:]
if self.files:
filePaths = [f[0] for f in self.files]
filenames = [os.path.basename(p) for p in filePaths]
encodings = [f[1] for f in self.files]
annotations = ['{%s: %s}' % (f[2], f[3]) for f in self.files]
maxFilenameLen = max([len(n) for n in filenames])
maxAnnoLen = max([len(a) for a in annotations])
# Start SuperTextFiles
pdfPassword = [f[4] for f in self.files]
ocrLanguages = [f[5] for f in self.files]
ocrForce = [str(f[6]) for f in self.files]
maxPdfPasswordLen = max([len(n) for n in pdfPassword])
maxOcrLanguagesLen = max([len(n) for n in ocrLanguages])
# End SuperTextFiles
for index in range(len(self.files)):
format = u'%-' + str(maxFilenameLen + 2) + u's'
fileLabel = format % filenames[index]
if maxAnnoLen > 4:
if len(annotations[index]) > 4:
format = u'%-' + str(maxAnnoLen + 2) + u's'
fileLabel += format % annotations[index]
else:
fileLabel += u' ' * (maxAnnoLen + 2)
# Start SuperTextFiles
format = u'%-' + str(maxPdfPasswordLen + 2) + u's'
fileLabel += format % pdfPassword[index]
format = u'%-' + str(maxOcrLanguagesLen + 2) + u's'
fileLabel += format % ocrLanguages[index]
format = u'%-' + str(5 + 2) + u's'
fileLabel += format % ocrForce[index]
# End SuperTextFiles
fileLabel += encodings[index]
self.fileLabels.append(fileLabel)
self.fileLabels = self.fileLabels
if cachedLabel is not None:
self.sendButton.sendIfPreCallback = None
self.selectedFileLabels = [cachedLabel]
self.sendButton.sendIfPreCallback = self.updateGUI
if self.newFiles:
if ((self.newAnnotationKey and self.newAnnotationValue)
or (not self.newAnnotationKey
and not self.newAnnotationValue)):
self.addButton.setDisabled(False)
else:
self.addButton.setDisabled(True)
else:
self.addButton.setDisabled(True)
if self.autoNumber:
self.autoNumberKeyLineEdit.setDisabled(False)
else:
self.autoNumberKeyLineEdit.setDisabled(True)
if self.importFilenames:
self.importFilenamesKeyLineEdit.setDisabled(False)
else:
self.importFilenamesKeyLineEdit.setDisabled(True)
self.updateFileBoxButtons()
self.advancedSettings.setVisible(True)
else:
self.advancedSettings.setVisible(False)
def updateFileBoxButtons(self):
"""Update state of File box buttons"""
if self.selectedFileLabels:
self.removeButton.setDisabled(False)
if self.selectedFileLabels[0] > 0:
self.moveUpButton.setDisabled(False)
else:
self.moveUpButton.setDisabled(True)
if self.selectedFileLabels[0] < len(self.files) - 1:
self.moveDownButton.setDisabled(False)
else:
self.moveDownButton.setDisabled(True)
else:
self.moveUpButton.setDisabled(True)
self.moveDownButton.setDisabled(True)
self.removeButton.setDisabled(True)
if len(self.files):
self.clearAllButton.setDisabled(False)
self.exportButton.setDisabled(False)
else:
self.clearAllButton.setDisabled(True)
self.exportButton.setDisabled(True)
def setCaption(self, title):
if 'captionTitle' in dir(self):
changed = title != self.captionTitle
super().setCaption(title)
if changed:
self.sendButton.settingsChanged()
else:
super().setCaption(title)
def onDeleteWidget(self):
self.clearCreatedInputs()
class OWTextableTextTree(OWTextableBaseWidget):
"""Orange widget for loading text folders"""
name = "Text Tree"
description = "Import data from raw text trees"
icon = "icons/Textfolders.png"
icon = "icons/textTree.svg"
priority = 2
# Input and output channels...
inputs = [
('Message', JSONMessage, "inputMessage", widget.Single)
]
outputs = [('Text data', Segmentation)]
settingsHandler = VersionedSettingsHandler(
version=__version__.rsplit(".", 1)[0]
)
# Settings...
autoSend = settings.Setting(True)
folders = settings.Setting([])
encoding = settings.Setting('iso-8859-1')
operation = settings.Setting('nothing')
sampling =settings.Setting(100)
autoNumber = settings.Setting(False)
autoNumberKey = settings.Setting(u'num')
importFilenames = settings.Setting(True)
importFolderName = settings.Setting(True)
importFolderNameKey = settings.Setting(u'folderName')
importFileNameKey = settings.Setting(u'filename')
FolderDepth1Key = settings.Setting(u'depth 1')
FolderDepth2Key = settings.Setting(u'depth 2')
FolderDepth2Key = settings.Setting(u'depth 3')
FolderDepth2Key = settings.Setting(u'depth 4')
FolderDepthLvl = settings.Setting(u'depth level')
lastLocation = settings.Setting('.')
displayAdvancedSettings = settings.Setting(False)
folder = settings.Setting(u'')
want_main_area = False
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Other attributes...
self.segmentation = None
self.operation = "no"
self.applyInclusion = False
self.applyExclusion = False
self.applySampling = True
self.samplingRate = 100
self.createdInputs = list()
self.folderLabels = list()
self.selectedfolderLabels = list()
self.rootFolderPath = u''
self.inclusionsUser = u''
self.exclusionsUser = u''
self.newAnnotationKey = u''
self.newAnnotationValue = u''
self.folders = list() # self.folders is a list of dictionaries with each dictionaries being a a folder
self.inclusionList = [".txt",".html",".xml",".csv"] #by default empty list
# self.exclusionList = [".png,",".PNG",".jpg",".JPG",".gif",".GIF",".tiff",".TIFF",".jpeg",".JPEG",".DS_Store"] # by default exclusions : img files, .DS_Store (macOS)
self.exclusionList = [] # by default null
self.infoBox = InfoBox(widget=self.controlArea)
# self.fileList = list() #output file list
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute='infoBox',
sendIfPreCallback=self.updateGUI,
)
self.advancedSettings = AdvancedSettings(
widget=self.controlArea,
master=self,
callback=self.sendButton.settingsChanged,
)
# GUI...
# Advanced settings checkbox...
self.advancedSettings.draw()
# BASIC GUI...
# Basic folder box
basicfolderBox = gui.widgetBox(
widget=self.controlArea,
box=u'Source',
orientation='vertical',
addSpace=False,
)
basicfolderBoxLine1 = gui.widgetBox(
widget=basicfolderBox,
box=False,
orientation='horizontal',
)
gui.lineEdit(
widget=basicfolderBoxLine1,
master=self,
value='rootFolderPath',
orientation='horizontal',
label=u'Folder path:',
labelWidth=101,
callback=self.add,
tooltip=(
u"The path of the folder."
),
)
gui.separator(widget=basicfolderBoxLine1, width=5)
gui.button(
widget=basicfolderBoxLine1,
master=self,
label=u'Browse',
callback=self.browse,
tooltip=(
u"Open a dialog for selecting a top folder."
),
)
#gui.separator(widget=basicfolderBox, width=3)
#gui.comboBox(
# widget=basicfolderBox,
# master=self,
# value='encoding',
# items=getPredefinedEncodings(),
# sendSelectedValue=True,
# orientation='horizontal',
# label=u'Encoding:',
# labelWidth=101,
# callback=self.sendButton.settingsChanged,
# tooltip=(
# u"Select input folder(s) encoding."
# ),
# )
gui.separator(widget=basicfolderBox, width=3)
self.advancedSettings.basicWidgets.append(basicfolderBox)
self.advancedSettings.basicWidgetsAppendSeparator()
# ADVANCED GUI...
# folder box
folderBox = gui.widgetBox(
widget=self.controlArea,
box=u'Sources',
orientation='vertical',
addSpace=False,
)
folderBoxLine1 = gui.widgetBox(
widget=folderBox,
box=False,
orientation='horizontal',
addSpace=True,
)
self.folderListbox = gui.listBox(
widget=folderBoxLine1,
master=self,
value='selectedfolderLabels',
labels='folderLabels',
callback=self.updatefolderBoxButtons,
tooltip=(
u"The list of folders whose content will be imported.\n"
u"\nIn the output segmentation, the content of each\n"
u"folder appears in the same position as in the list.\n"
u"\nColumn 1 shows the folder's name.\n"
u"Column 2 shows the folder's annotation (if any).\n"
u"Column 3 shows the folder's encoding."
),
)
font = QFont()
font.setFamily('Courier')
font.setStyleHint(QFont.Courier)
font.setPixelSize(12)
self.folderListbox.setFont(font)
folderBoxCol2 = gui.widgetBox(
widget=folderBoxLine1,
orientation='vertical',
)
self.moveUpButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Move Up',
callback=self.moveUp,
tooltip=(
u"Move the selected folder upward in the list."
),
)
self.moveDownButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Move Down',
callback=self.moveDown,
tooltip=(
u"Move the selected folder downward in the list."
),
)
self.removeButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Remove',
callback=self.remove,
tooltip=(
u"Remove the selected folder from the list."
),
)
self.clearAllButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Clear All',
callback=self.clearAll,
tooltip=(
u"Remove all folders from the list."
),
)
self.exportButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Export List',
callback=self.exportList,
tooltip=(
u"Open a dialog for selecting a folder where the folder\n"
u"list can be exported in JSON format."
),
)
self.importButton = gui.button(
widget=folderBoxCol2,
master=self,
label=u'Import List',
callback=self.importList,
tooltip=(
u"Open a dialog for selecting a folder list to\n"
u"import (in JSON format). folders from this list\n"
u"will be added to those already imported."
),
)
folderBoxLine2 = gui.widgetBox(
widget=folderBox,
box=False,
orientation='vertical',
)
# Add folder box
addfolderBox = gui.widgetBox(
widget=folderBoxLine2,
box=True,
orientation='vertical',
)
addfolderBoxLine1 = gui.widgetBox(
widget=addfolderBox,
orientation='horizontal',
)
# Folder path input
gui.lineEdit(
widget=addfolderBoxLine1,
master=self,
value='rootFolderPath',
orientation='horizontal',
label=u'Folder path:',
labelWidth=101,
callback=self.updateGUI,
tooltip=(
u"The paths of the folders that will be added to the\n"
u"list when button 'Add' is clicked.\n\n"
u"Successive paths must be separated with ' / ' \n"
u"(whitespace + slash + whitespace). Their order in\n"
u"the list will be the same as in this field."
),
)
gui.separator(widget=addfolderBoxLine1, width=5)
# Button Browse
gui.button(
widget=addfolderBoxLine1,
master=self,
label=u'Browse',
callback=self.browse,
tooltip=(
u"Open a dialog for selecting a top folder.\n\n"
u"Selected folder paths will appear in the field to\n"
u"the left of this button afterwards, ready to be\n"
u"added to the list when button 'Add' is clicked."
),
)
gui.separator(widget=addfolderBox, width=10)
# Filter choice to include only certain files or to exclude files
# ------------
# self.applyInclusion = False à mettre dans le init
# gui.checkbox()
# callback = lambda t=self.applyInclusion : includeLineEdit.setDisabled(not t)
# includeLineEdit = gui.lineEdit()
# ------------
# Filter box to input include only
gui.separator(widget=addfolderBox, width=3)
includeBoxLine1 = gui.widgetBox(
widget=addfolderBox,
box=False,
orientation='horizontal',
)
# Include only box
gui.checkBox(
widget=includeBoxLine1,
master=self,
value='applyInclusion',
label=u'Include only',
labelWidth=100,
callback = lambda: includeLineEdit.setDisabled(not self.applyInclusion),
tooltip=(
u"Choose the inclusion"
),
)
includeLineEdit = gui.lineEdit(
widget=includeBoxLine1,
master=self,
value='inclusionsUser',
orientation='horizontal',
label=u'',
disabled = True,
labelWidth=101,
tooltip=(
u"This field lets you specify a custom filter\n"
u"to select the folders to be\n"
u"added to the list."
),
)
# Filter box to exclude only
gui.separator(widget=addfolderBox, width=3)
excludeBoxLine1 = gui.widgetBox(
widget=addfolderBox,
box=False,
orientation='horizontal',
)
# Exclude only box
gui.checkBox(
widget=excludeBoxLine1,
master=self,
value='applyExclusion',
label=u'Exclude',
labelWidth=100,
disabled = False,
callback = lambda: includeLineEdit2.setDisabled(not self.applyExclusion),
tooltip=(
u"Exclude the inclusion"
),
)
includeLineEdit2=gui.lineEdit(
widget=excludeBoxLine1,
master=self,
value='exclusionsUser',
orientation='horizontal',
label=u'',
disabled = True,
labelWidth=101,
tooltip=(
u"This field lets you specify a custom filter\n"
u"to select the folders to be\n"
u"added to the list."
),
)
# Sampling box to input the level of sampling
gui.separator(widget=addfolderBox, width=3)
samplingBoxLine1 = gui.widgetBox(
widget=addfolderBox,
box=False,
orientation='horizontal',
)
# Check box for sampling
gui.checkBox(
widget=samplingBoxLine1,
master=self,
value='applySampling',
label=u'Sampling',
labelWidth=100,
disabled = False,
callback = lambda: samplingSpin.setDisabled(not self.applySampling),
tooltip=(
u"Choose the sampling level"
),
)
# Box to input the level of samplig, spin minv = 10 and maxv = 100
# self.importFilenamesKeyLineEdit = gui.spin(
samplingSpin = gui.spin(
widget=samplingBoxLine1,
master=self,
value='samplingRate',
minv = 10,
maxv = 100,
labelWidth=50,
orientation='horizontal',
tooltip=(
u"sampling level"
),
)
gui.separator(widget=addfolderBox, width=3)
self.addButton = gui.button(
widget=addfolderBox,
master=self,
label=u'Add',
callback=self.add,
tooltip=(
u"Add the folder(s) currently displayed in the\n"
u"'folders' text field to the list.\n\n"
u"Each of these folders will be associated with the\n"
u"specified encoding and annotation (if any).\n\n"
u"Other folders may be selected afterwards and\n"
u"assigned a different encoding and annotation."
),
)
self.advancedSettings.advancedWidgets.append(folderBox)
self.advancedSettings.advancedWidgetsAppendSeparator()
# Options box...
optionsBox = gui.widgetBox(
widget=self.controlArea,
box=u'Options',
orientation='vertical',
addSpace=False,
)
optionsBoxLine1 = gui.widgetBox(
widget=optionsBox,
box=False,
orientation='horizontal',
)
# gui.checkBox(
# widget=optionsBoxLine1,
# master=self,
# value='importFilenames',
# label=u'Import folder names with key:',
# labelWidth=180,
# callback=self.sendButton.settingsChanged,
# tooltip=(
# u"Import folder names as annotations."
# ),
# )
# self.importFilenamesKeyLineEdit = gui.lineEdit(
# widget=optionsBoxLine1,
# master=self,
# value='importFilenamesKey',
# orientation='horizontal',
# callback=self.sendButton.settingsChanged,
# tooltip=(
# u"Annotation key for importing folder names."
# ),
# )
gui.separator(widget=optionsBox, width=3)
optionsBoxLine2 = gui.widgetBox(
widget=optionsBox,
box=False,
orientation='horizontal',
)
gui.checkBox(
widget=optionsBoxLine2,
master=self,
value='autoNumber',
label=u'Auto-number with key:',
labelWidth=180,
callback=self.sendButton.settingsChanged,
tooltip=(
u"Annotate folders with increasing numeric indices."
),
)
self.autoNumberKeyLineEdit = gui.lineEdit(
widget=optionsBoxLine2,
master=self,
value='autoNumberKey',
orientation='horizontal',
callback=self.sendButton.settingsChanged,
tooltip=(
u"Annotation key for folder auto-numbering."
),
)
gui.separator(widget=optionsBox, width=3)
self.advancedSettings.advancedWidgets.append(optionsBox)
self.advancedSettings.advancedWidgetsAppendSeparator()
gui.rubber(self.controlArea)
# Send button...
self.sendButton.draw()
# Info box...
self.infoBox.draw()
self.adjustSizeWithTimer()
QTimer.singleShot(0, self.sendButton.sendIf)
def inputMessage(self, message):
"""Handle JSON message on input connection"""
if not message:
return
self.displayAdvancedSettings = True
self.advancedSettings.setVisible(True)
self.clearAll()
self.infoBox.inputChanged()
try:
json_data = json.loads(message.content)
temp_folders = list()
for entry in json_data:
path = entry.get('path', '')
encoding = entry.get('encoding', '')
annotationKey = entry.get('annotation_key', '')
annotationValue = entry.get('annotation_value', '')
if path == '' or encoding == '':
self.infoBox.setText(
u"Please verify keys and values of incoming "
u"JSON message.",
'error'
)
self.send('Text data', None, self)
return
depth = "0"
options = "[i]:{unicorn}"
temp_folders.append((
name,
path,
depth,
options,
))
self.folders.extend(temp_folders)
self.sendButton.settingsChanged()
except ValueError:
self.infoBox.setText(
u"Please make sure that incoming message is valid JSON.",
'error'
)
self.send('Text data', None, self)
return
def sendData(self):
"""Load folders, create and send segmentation"""
# Check that there's something on input...
if (
(self.displayAdvancedSettings and not self.folders) or
not (self.rootFolderPath or self.displayAdvancedSettings)
):
self.infoBox.setText(u'Please select input folder.', 'warning')
self.send('Text data', None, self)
return
# Check that autoNumberKey is not empty (if necessary)...
if self.displayAdvancedSettings and self.autoNumber:
if self.autoNumberKey:
autoNumberKey = self.autoNumberKey
else:
self.infoBox.setText(
u'Please enter an annotation key for auto-numbering.',
'warning'
)
self.send('Text data', None, self)
return
else:
autoNumberKey = None
# Clear created Inputs...
self.clearCreatedInputs()
fileContents = list()
annotations = list()
counter = 1
if self.displayAdvancedSettings:
myFolders = self.folders
else:
myFolders = [[self.rootFolderPath]]
progressBar = gui.ProgressBar(
self,
iterations=len(myFolders)
)
# Walk through each folder and open each files successively...
fileContents = self.fileContents
# Annotations...
myFolders = self.folders
for myFolder in myFolders:
myFiles = myFolder['fileList']
for myFile in myFiles:
# print(myFile)
annotation = dict()
if self.importFileNameKey:
annotation[self.importFileNameKey] = myFile['fileName']
if self.importFolderNameKey:
annotation[self.importFolderNameKey] = myFile['folderName']
if self.FolderDepth1Key:
annotation[self.FolderDepth1Key] = myFile['depth1']
if self.FolderDepth2Key:
annotation[self.FolderDepth2Key] = myFile['depth2']
if self.FolderDepthLvl:
annotation[self.FolderDepthLvl] = myFile['depthLvl']
annotations.append(annotation)
# progressBar.advance()
# Create an LTTL.Input for each files...
if len(fileContents) == 1:
label = self.captionTitle
else:
label = None
for index in range(len(fileContents)):
myInput = Input(fileContents[index], label)
segment = myInput[0]
segment.annotations.update(annotations[index])
myInput[0] = segment
self.createdInputs.append(myInput)
# If there's only one file, the widget's output is the created Input.
if len(fileContents) == 1:
self.segmentation = self.createdInputs[0]
# Otherwise the widget's output is a concatenation...
else:
self.segmentation = Segmenter.concatenate(
segmentations=self.createdInputs,
label=self.captionTitle,
copy_annotations=True,
import_labels_as=None,
sort=False,
auto_number_as=None,
merge_duplicates=False,
progress_callback=None,
)
message = u'%i segment@p sent to output ' % len(self.segmentation)
message = pluralize(message, len(self.segmentation))
numChars = 0
for segment in self.segmentation:
segmentLength = len(Segmentation.get_data(segment.str_index))
numChars += segmentLength
message += u'(%i character@p).' % numChars
message = pluralize(message, numChars)
self.infoBox.setText(message)
progressBar.finish()
self.send('Text data', self.segmentation, self)
self.sendButton.resetSettingsChangedFlag()
def clearCreatedInputs(self):
for i in self.createdInputs:
Segmentation.set_data(i[0].str_index, None)
del self.createdInputs[:]
def importList(self):
"""Display a folderDialog and import folder list"""
folderPath = QFileDialog.getOpenFileName(
self,
u'Import folder List',
self.lastLocation,
u'Text folders (*)'
)
if not folderPath:
return
self.rootFolderPath = os.path.normpath(folderPath)
self.lastLocation = os.path.dirname(folderPath)
self.error()
try:
folderHandle = codecs.open(folderPath, encoding='utf8')
folderContent = folderHandle.read()
folderHandle.close()
except IOError:
QMessageBox.warning(
None,
'Textable',
"Couldn't open folder.",
QMessageBox.Ok
)
return
try:
json_data = json.loads(folderContent)
temp_folders = list()
for entry in json_data:
path = entry.get('path', '')
encoding = entry.get('encoding', '')
annotationKey = entry.get('annotation_key', '')
annotationValue = entry.get('annotation_value', '')
if path == '' or encoding == '':
QMessageBox.warning(
None,
'Textable',
"Selected JSON folder doesn't have the right keys "
"and/or values.",
QMessageBox.Ok
)
return
temp_folders.append((
path,
encoding,
annotationKey,
annotationValue,
))
self.folders.extend(temp_folders)
if temp_folders:
self.sendButton.settingsChanged()
except ValueError:
QMessageBox.warning(
None,
'Textable',
"JSON parsing error.",
QMessageBox.Ok
)
return
def exportList(self):
"""Display a folderDialog and export folder list"""
toDump = list()
myFolders = self.folders
for myFolder in myFolders:
toDump.append({
'path': myFolder[0],
'encoding': myFolder[1],
})
if myFolder[2] and myFolder[3]:
toDump[-1]['annotation_key'] = myFolder[2]
toDump[-1]['annotation_value'] = myFolder[3]
folderPath =QFileDialog.getSaveFileName(
self,
u'Export folder List',
self.lastLocation,
)
if folderPath:
self.lastLocation = os.path.dirname(folderPath)
outputfolder = codecs.open(
folderPath,
encoding='utf8',
mode='w',
errors='xmlcharrefreplace',
)
outputfolder.write(
normalizeCarriageReturns(
json.dumps(toDump, sort_keys=True, indent=4)
)
)
outputfolder.close()
QMessageBox.information(
None,
'Textable',
'folder list correctly exported',
QMessageBox.Ok
)
def getFileList(self):
#print("getFileList")
initialRootParentPath, _ = os.path.split(self.rootFolderPath) #initial parent path is selected's folder parent folder
fileListExt = list() # list of files matching default extension
depthList = list()
progressBar = gui.ProgressBar(self, iterations=1)
for curr_path, dirnames, filenames in os.walk(self.rootFolderPath):
#curr_path is a STRING, the path to the directory.
#dirnames is a LIST of the names of subdirectories.
#filenames is a LIST of the names of the files in curr_path
#symlink non traités
curr_rel_path = curr_path[len(initialRootParentPath)+1:] #defines current relative path by similar initial parent path part
curr_rel_path_list = os.path.normpath(curr_rel_path).split(os.sep) #splits current relative path by os separator
for filename in filenames:
file = dict()
# file = {"absoluteFilePath","foldername","filename","depth1","depth2","depth3","depth4","depth5","depth lvl"}
# prev_non_excl_check = False
# curr_non_excl_check = prev_non_excl_check #importing previous state of the "non-exclusion check" (opposite of exclusion check)
annotations = curr_rel_path_list[:] # annotations are different subfolders browsed
# print(annotations)
curr_depth = len(annotations)
depthList.append(curr_depth)
file['absoluteFilePath'] = os.path.join(curr_path,filename)
file['fileName'] = filename
file['depthLvl'] = curr_depth
file['folderName'] = annotations[0]
for i in range(1, curr_depth):
file['depth' + str(i)] = annotations[i]
for i in range(curr_depth, 5):
file['depth' + str(i)] = "0"
# apply default file extension filter
for extension in self.inclusionList:
if filename.endswith(extension):
fileListExt.append(file)
# apply inclusion filter
if self.applyInclusion:
fileListIncl = [file for file in fileListExt
# match in inclusion list
if self.match(file['fileName'], self.inclusionsUserAsList)]
else:
fileListIncl = fileListExt
# apply exclusion filter
if self.applyExclusion:
fileListExcl = [file for file in fileListIncl
# no match in exclusion list
if not self.match(file['fileName'], self.exclusionsUserAsList)]
else:
fileListExcl = fileListIncl
# output file list
self.fileList = fileListExcl
if self.fileList:
self.maxDepth = max(depthList)
self.openFileList()
else:
self.maxDepth = 0
progressBar.advance()
progressBar.finish()
# test if file contains one of the patterns in patternList
def match(self, file, patternList):
for pattern in patternList:
if pattern in file:
return True
return False
def openFileList(self):
self.fileContents = list()
for file in self.fileList:
fileContent = ""
try:
file_path = file['absoluteFilePath']
except TypeError:
pass
encodings = getPredefinedEncodings()
with open(file_path,'rb') as opened_file:
fileContent = opened_file.read()
charset_dict = chardet.detect(fileContent)
detected_encoding = charset_dict['encoding']
# i = 0
# chunks = list()
#
# for chunk in iter(lambda: opened_file.read(CHUNK_LENGTH), ""):
# chunks.append('\n'.join(chunk.splitlines()))
# i += CHUNK_LENGTH
# if i % (CHUNK_NUM * CHUNK_LENGTH) == 0:
# fileContent += "".join(str(chunks)
# chunk = list()
#
# if len(chunks):
# fileContent += "".join(str(chunks))
# del chunks
try:
encodings.remove(detected_encoding)
encodings.insert(0,detected_encoding)
except ValueError:
pass
for encoding in encodings:
try:
self.fileContent = fileContent.decode(encoding)
except:
pass
# fileContent = normalize('NFC', str(fileContent))
# fileContents.append(fileContent)
self.fileContents.append(self.fileContent)
del self.fileContents[-1]
# print(self.fileContents)
def browse(self):
"""Display a QFileDialog and select a folder"""
rootFolderPath = QFileDialog.getExistingDirectory( #Use QFileDialog.getExistingDirectory
self,
u'Select Folder(s)',
self.lastLocation,
)
if not rootFolderPath:
return
rootFolderPath = os.path.normpath(rootFolderPath)
self.rootFolderPath = rootFolderPath
self.lastLocation = rootFolderPath
if self.displayAdvancedSettings:
pass
else:
self.add()
self.updateGUI()
def moveUp(self):
"""Move folder upward in folders listbox"""
if self.selectedfolderLabels:
index = self.selectedfolderLabels[0]
if index > 0:
temp = self.folders[index - 1]
self.folders[index - 1] = self.folders[index]
self.folders[index] = temp
self.selectedfolderLabels.listBox.item(index - 1).setSelected(1)
self.sendButton.settingsChanged()
def moveDown(self):
"""Move folder downward in folders listbox"""
if self.selectedfolderLabels:
index = self.selectedfolderLabels[0]
if index < len(self.folders) - 1:
temp = self.folders[index + 1]
self.folders[index + 1] = self.folders[index]
self.folders[index] = temp
self.selectedfolderLabels.listBox.item(index + 1).setSelected(1)
self.sendButton.settingsChanged()
def clearAll(self):
"""Remove all folders from folders attr"""
del self.folders[:]
del self.selectedfolderLabels[:]
self.sendButton.settingsChanged()
def remove(self):
"""Remove folder from folders attr"""
if self.selectedfolderLabels:
index = self.selectedfolderLabels[0]
self.folders.pop(index)
del self.selectedfolderLabels[:]
self.sendButton.settingsChanged()
def add(self):
"""Add folders to folders attr"""
#rootFolderPathList = re.split(r' +/ +', self.rootFolderPath) #self.rootFolderPath = name
# identify sequences separated by a "," and suppress the white spaces
self.inclusionsUserAsList = [x.strip() for x in self.inclusionsUser.split(",") if x.strip()]
self.exclusionsUserAsList = [x.strip() for x in self.exclusionsUser.split(",") if x.strip()]
self.getFileList()
# display the list of files
print("Files: ", list(map(lambda f: f['fileName'], self.fileList)))
sampleFileList = self.sampleFileList()
# display the list of sampled files
print("Files after sampling: ", list(map(lambda f: f['fileName'], sampleFileList)))
self.folders.append(
{
'rootPath' : self.rootFolderPath,
'maxDepth' : self.maxDepth,
'inclusionsUser' : self.inclusionsUser,
'exclusionsUser' : self.exclusionsUser,
'samplingRate' : self.samplingRate,
'fileList' : sampleFileList,
}
)
# print(self.folders)
self.sendButton.settingsChanged()
# for folderDict in self.folders:
# fileList = folderDict['fileList']
def sampleFileList(self):
# Utilisation de la variable fileList
# On fait une copie pour eviter de modifier self.fileList avec shuffle plus bas
myList = list(self.fileList)
# Initialisation d'un parametre qui decidera de l'echantillonage
samplePercentage = self.samplingRate / 100.0
# print(samplePercentage)
# On melange la liste pour prendre ensuite les "samplePercentage" premiers
random.shuffle(myList)
# On definit le nombre de fichiers voulus selon le parametre d'echantillonage "samplePercentage", arrondi au superieur
nOfFiles = int(math.ceil(len(myList) * samplePercentage))
# On prend les "nOfFiles" premiers fichiers de la liste melangee
return myList[:nOfFiles]
def updateGUI(self):
"""Update GUI state"""
if self.displayAdvancedSettings:
if self.selectedfolderLabels:
cachedLabel = self.selectedfolderLabels[0]
else:
cachedLabel = None
del self.folderLabels[:]
folderLabels = []
if self.folders:
folderRootPathsList = [f['rootPath'] for f in self.folders]
maxDepthList = ['%s' % f['maxDepth'] for f in self.folders]
inclusionsUserList = [f['inclusionsUser'] for f in self.folders]
# print(inclusionsUserList)
exclusionsUserList = [f['exclusionsUser'] for f in self.folders]
samplingRatesList = ['%s' % f['samplingRate'] for f in self.folders]
folderNamesList = [os.path.basename(p) for p in folderRootPathsList]
maxFolderNameLen = max([len(n) for n in folderNamesList])
for index in range(len(self.folders)):
format = u'%-' + str(maxFolderNameLen + 2) + u's'
# folderLabel = format % folderNamesList[index],
folderLabel = format % folderNamesList[index]
# print(inclusionsUserList[index])
folderLabel += "[d]:{"+maxDepthList[index]+"} "
folderLabel += "[i]:{"+inclusionsUserList[index]+"} "
folderLabel += "[e]:{"+exclusionsUserList[index]+"} "
folderLabel += "[s]:{"+samplingRatesList[index]+"%}"
folderLabels.append(folderLabel)
self.folderLabels = folderLabels
if cachedLabel is not None:
self.sendButton.sendIfPreCallback = None
self.selectedfolderLabels.listBox.item(
cachedLabel
).setSelected(1)
self.sendButton.sendIfPreCallback = self.updateGUI
if self.rootFolderPath:
if (
(self.newAnnotationKey and self.newAnnotationValue) or
(not self.newAnnotationKey and not self.newAnnotationValue)
):
self.addButton.setDisabled(False)
else:
self.addButton.setDisabled(True)
else:
self.addButton.setDisabled(True)
if self.autoNumber:
self.autoNumberKeyLineEdit.setDisabled(False)
else:
self.autoNumberKeyLineEdit.setDisabled(True)
# if self.importFilenames:
# self.importFilenamesKeyLineEdit.setDisabled(False)
# else:
# self.importFilenamesKeyLineEdit.setDisabled(True)
self.updatefolderBoxButtons()
self.advancedSettings.setVisible(True)
else:
self.advancedSettings.setVisible(False)
def updatefolderBoxButtons(self):
"""Update state of folder box buttons"""
if self.selectedfolderLabels:
self.removeButton.setDisabled(False)
if self.selectedfolderLabels[0] > 0:
self.moveUpButton.setDisabled(False)
else:
self.moveUpButton.setDisabled(True)
if self.selectedfolderLabels[0] < len(self.folders) - 1:
self.moveDownButton.setDisabled(False)
else:
self.moveDownButton.setDisabled(True)
else:
self.moveUpButton.setDisabled(True)
self.moveDownButton.setDisabled(True)
self.removeButton.setDisabled(True)
if len(self.folders):
self.clearAllButton.setDisabled(False)
self.exportButton.setDisabled(False)
else:
self.clearAllButton.setDisabled(True)
self.exportButton.setDisabled(True)
def setCaption(self, title):
if 'captionTitle' in dir(self):
changed = title != self.captionTitle
super().setCaption(title)
if changed:
self.sendButton.settingsChanged()
else:
super().setCaption(title)
def onDeleteWidget(self):
self.clearCreatedInputs()