def testRemoveControlChar(self):
text = u''.join([chr(i) for i in range(32)])
text += u'\uFFEF\uFFFE\uFFFF'
text += u'\uDA00toto\U00011234'
norm = removeControlChar(text)
self.assertEquals(u"\t\n\r\uFFEFtoto\U00011234", norm)
self.assertEquals(unicode, type(norm))
def testRemoveControlChar(self):
text = u''.join([chr(i) for i in range(32)])
text += u'\uFFEF\uFFFE\uFFFF'
text += u'\uDA00toto\U00011234'
norm = removeControlChar(text)
self.assertEquals(u"\t\n\r\uFFEFtoto\U00011234", norm)
self.assertEquals(unicode, type(norm))
def indexDocument(self, futureDoc):
futureDoc.file_state=FileInfo.CREATED_FILE_STATE
if self.verbose:
print "Requesting indexation of %s" % futureDoc.filename,
try:
futureDoc.title = removeControlChar(futureDoc.title)
futureDoc.text = removeControlChar(futureDoc.text)
if self.verbose:
print '('+futureDoc.title.encode('utf-8')+')'
self.serverProxy.indexDocument(self.cnxId, futureDoc)
except (Fault, ProtocolError), exc:
if self.verbose:
print "An error occured on the server while indexing %s" % \
futureDoc.filename.encode('iso-8859-1')
print exc
print "See server log for details"
else:
print "Error indexing %s: %s" % (futureDoc.filename.encode('iso-8859-1'), exc)
def indexDocument(self, futureDoc):
futureDoc.file_state=FileInfo.CREATED_FILE_STATE
if self.verbose:
print "Requesting indexation of %s" % \
safe_encode(futureDoc.filename),
try:
futureDoc.title = removeControlChar(futureDoc.title)
futureDoc.text = removeControlChar(futureDoc.text)
if self.verbose:
print '('+safe_encode(futureDoc.title)+')'
self.serverProxy.indexDocument(self.cnxId, futureDoc)
except (Fault, ProtocolError), exc:
if self.verbose:
print "An error occured on the Node while indexing %s" % \
safe_encode(futureDoc.filename)
print exc
print "See Node log for details"
else:
print "Error indexing %s: %s" % \
(safe_encode(futureDoc.filename), exc)
def indexDocument(self, futureDoc):
futureDoc.file_state = FileInfo.CREATED_FILE_STATE
if self.verbose:
print "Requesting indexation of %s" % futureDoc.filename,
try:
futureDoc.title = removeControlChar(futureDoc.title)
futureDoc.text = removeControlChar(futureDoc.text)
if self.verbose:
print '(' + futureDoc.title.encode('utf-8') + ')'
self.serverProxy.indexDocument(self.cnxId, futureDoc)
except (Fault, ProtocolError), exc:
if self.verbose:
print "An error occured on the server while indexing %s" % \
futureDoc.filename.encode('iso-8859-1')
print exc
print "See server log for details"
else:
print "Error indexing %s: %s" % (
futureDoc.filename.encode('iso-8859-1'), exc)
def indexDocument(self, futureDoc):
futureDoc.file_state=FileInfo.CREATED_FILE_STATE
if self.verbose:
print "[local] Requesting indexation of %s" % \
safe_encode(futureDoc.filename),
try:
futureDoc.title = removeControlChar(futureDoc.title)
futureDoc.text = removeControlChar(futureDoc.text)
if self.verbose:
print '[local] ('+safe_encode(futureDoc.title)+')'
# first argument of indexDocument set to None means we're
# working locally
self.querier.indexDocument(None, futureDoc)
except Exception, exc:
if self.verbose:
print "[local] An error occured on the Node while indexing %s" % \
safe_encode(futureDoc.filename)
import traceback
traceback.print_exc()
print "[local] See Node log for details"
else:
print "[local] Error indexing %s: %s" % \
(safe_encode(futureDoc.filename), exc)