def createLogMessage(cls,
logValue,
traceStack,
shaveStackTrace,
htmlEscape,
prefix=None,
**kwargs):
""" Formats log message data into a string for output """
doIndent = kwargs.get('indent', True)
if doIndent:
logValue = cls.formatAsString(logValue)
elif isinstance(logValue, (list, tuple)):
logValue = '\n'.join(logValue)
if htmlEscape:
logValue = StringUtils.htmlEscape(logValue)
if doIndent:
logValue = logValue.replace('\n', '\n ')
logValue = StringUtils.strToUnicode(logValue)
if not StringUtils.isStringType(logValue):
logValue = 'FAILED TO LOG RESPONSE'
out = {'log': logValue}
if prefix:
logPrefix = StringUtils.strToUnicode(prefix)
if not StringUtils.isStringType(logPrefix):
logPrefix = 'FAILED TO CREATE PREFIX'
out['prefix'] = logPrefix
if traceStack:
logStack = StringUtils.strToUnicode(
'Stack Trace:\n' + cls.getFormattedStackTrace(shaveStackTrace))
if not StringUtils.isStringType(logStack):
logStack = 'FAILED TO CREATE STACK'
out['stack'] = logStack
return out
def append(self, text):
#self.statusText.moveCursor(QtGui.QTextCursor.End)
#self.statusText.insertHtml(StringUtils.strToUnicode(text))
self.statusText.append(StringUtils.strToUnicode(text))
self.statusText.moveCursor(QtGui.QTextCursor.End)
self.statusText.moveCursor(QtGui.QTextCursor.StartOfLine)
print(u'\n' + 60*u'-')
print(text)
print(u'--- RESULT ---')
print(self.statusText.toHtml())
def found_terminator(self):
if self.handling:
return
self.set_terminator(None) # connections sometimes over-send
self.handling = True
# noinspection PyTypeChecker
self._chunk.writeString(''.join(self._data))
self._chunk.position = 0
self._requestFlags = self._chunk.readUint32()
self._message = StringUtils.strToUnicode(str(self._chunk.read(-1)))
self.handle_request()
def found_terminator(self):
if self.handling:
return
self.set_terminator(None) # connections sometimes over-send
self.handling = True
# noinspection PyTypeChecker
self._chunk.writeString(''.join(self._data))
self._chunk.position = 0
self._requestFlags = self._chunk.readUint32()
self._message = StringUtils.strToUnicode(str(self._chunk.read(-1)))
self.handle_request()
def createLogMessage(
cls, logValue, traceStack, shaveStackTrace, htmlEscape, prefix =None, **kwargs
):
""" Formats log message data into a string for output """
doIndent = kwargs.get('indent', True)
if doIndent:
logValue = cls.formatAsString(logValue)
elif isinstance(logValue, (list, tuple)):
logValue = '\n'.join(logValue)
if htmlEscape:
logValue = StringUtils.htmlEscape(logValue)
if doIndent:
logValue = logValue.replace('\n', '\n ')
logValue = StringUtils.strToUnicode(logValue)
if not StringUtils.isStringType(logValue):
logValue = 'FAILED TO LOG RESPONSE'
out = {'log':logValue}
if prefix:
logPrefix = StringUtils.strToUnicode(prefix)
if not StringUtils.isStringType(logPrefix):
logPrefix = 'FAILED TO CREATE PREFIX'
out['prefix'] = logPrefix
if traceStack:
logStack = StringUtils.strToUnicode(
'Stack Trace:\n' + cls.getFormattedStackTrace(shaveStackTrace))
if not StringUtils.isStringType(logStack):
logStack = 'FAILED TO CREATE STACK'
out['stack'] = logStack
return out
def cleanBytesToText(cls, source, inPlace =True):
"""cleanBytesToText doc..."""
out = source if inPlace else dict()
from pyaid.list.ListUtils import ListUtils
for n, v in cls.iter(source):
if isinstance(v, (tuple, list)):
v = ListUtils.cleanBytesToText(v, inPlace=inPlace)
elif isinstance(v, dict):
v = cls.cleanBytesToText(v)
else:
v = StringUtils.strToUnicode(v, force=False)
out[StringUtils.toStrStr(n)] = v
return out
def itemsToUnicode(cls, target, inPlace =False):
""" Iterates through the elements of the target list and converts each of them to unicode
strings, including decoding byte strings to unicode strings."""
from pyaid.string.StringUtils import StringUtils
output = target if inPlace else []
index = 0
while index < len(target):
source = target[index]
if StringUtils.isStringType(source):
output[index] = StringUtils.strToUnicode(source)
else:
output[index] = StringUtils.toUnicode(source)
return output
def itemsToUnicode(cls, target, inPlace=False):
""" Iterates through the elements of the target list and converts each of them to unicode
strings, including decoding byte strings to unicode strings."""
from pyaid.string.StringUtils import StringUtils
output = target if inPlace else []
index = 0
while index < len(target):
source = target[index]
if StringUtils.isStringType(source):
output[index] = StringUtils.strToUnicode(source)
else:
output[index] = StringUtils.toUnicode(source)
return output
def prettyPrint(cls, source, separator=', '):
"""prettyPrint doc..."""
out = []
from pyaid.dict.DictUtils import DictUtils
for v in source:
if isinstance(v, (list, tuple)):
v = cls.prettyPrint(v, separator=',')
if isinstance(v, dict):
v = DictUtils.prettyPrint(v)
elif isinstance(v, StringUtils.BINARY_TYPE):
v = StringUtils.strToUnicode(v)
else:
v = StringUtils.toUnicode(v)
out.append(v)
return '[%s]' % separator.join(out)
def prettyPrint(cls, source, separator = ', '):
"""prettyPrint doc..."""
out = []
from pyaid.dict.DictUtils import DictUtils
for v in source:
if isinstance(v, (list, tuple)):
v = cls.prettyPrint(v, separator=',')
if isinstance(v, dict):
v = DictUtils.prettyPrint(v)
elif isinstance(v, StringUtils.BINARY_TYPE):
v = StringUtils.strToUnicode(v)
else:
v = StringUtils.toUnicode(v)
out.append(v)
return '[%s]' % separator.join(out)
def cleanBytesToText(cls, source, inPlace =True):
"""cleanBytesToText doc..."""
out = source if inPlace else []
from pyaid.dict.DictUtils import DictUtils
for i in range(len(source)):
v = source[i]
if isinstance(v, (tuple, list)):
v = cls.cleanBytesToText(v, inPlace=inPlace)
elif isinstance(v, dict):
v = DictUtils.cleanBytesToText(v, inPlace=inPlace)
else:
v = StringUtils.strToUnicode(v, force=False)
if inPlace:
out[i] = v
else:
out.append(v)
return out
def cleanBytesToText(cls, source, inPlace=True):
"""cleanBytesToText doc..."""
out = source if inPlace else []
from pyaid.dict.DictUtils import DictUtils
for i in range(len(source)):
v = source[i]
if isinstance(v, (tuple, list)):
v = cls.cleanBytesToText(v, inPlace=inPlace)
elif isinstance(v, dict):
v = DictUtils.cleanBytesToText(v, inPlace=inPlace)
else:
v = StringUtils.strToUnicode(v, force=False)
if inPlace:
out[i] = v
else:
out.append(v)
return out
def itemsToString(cls, target, inPlace =False, toUnicode =True):
""" Iterates through the elements of the target list and converts each of them to binary
strings, including decoding unicode strings to byte strings."""
from pyaid.string.StringUtils import StringUtils
output = target if inPlace else (target + [])
index = 0
while index < len(target):
source = target[index]
if StringUtils.isStringType(source):
if toUnicode:
output[index] = StringUtils.strToUnicode(source)
else:
output[index] = StringUtils.unicodeToStr(source, force=True)
else:
output[index] = str(source)
index += 1
return output
def itemsToString(cls, target, inPlace=False, toUnicode=True):
""" Iterates through the elements of the target list and converts each of them to binary
strings, including decoding unicode strings to byte strings."""
from pyaid.string.StringUtils import StringUtils
output = target if inPlace else (target + [])
index = 0
while index < len(target):
source = target[index]
if StringUtils.isStringType(source):
if toUnicode:
output[index] = StringUtils.strToUnicode(source)
else:
output[index] = StringUtils.unicodeToStr(source,
force=True)
else:
output[index] = str(source)
index += 1
return output
def prettyPrint(cls, source, delimiter = ' | ', separator = ': '):
if not source:
return '[EMPTY]'
from pyaid.list.ListUtils import ListUtils
out = []
for n,v in cls.iter(source):
n = StringUtils.toUnicode(n)
if isinstance(v, dict):
v = '{ ' + cls.prettyPrint(v, delimiter=delimiter, separator=separator) + ' }'
elif isinstance(v, StringUtils.BINARY_TYPE):
v = StringUtils.strToUnicode(v)
elif isinstance(v, (list, tuple)):
v = ListUtils.prettyPrint(v)
else:
v = StringUtils.toUnicode(v)
out.append(n + separator + v)
out.sort(key=StringUtils.TEXT_TYPE.lower)
return delimiter.join(out)
def _runImpl(self):
model = Tracks_SiteMap.MASTER
session = self._session if self._session else model.createSession()
try:
self._log.write(u'<h1>Beginning Sitemap Import...</h1>')
if self._path is None or not os.path.exists(self._path):
self._log.write(u'<h2>Invalid or missing path</h2>')
return 1
# Delete all existing rows
rowCount = session.query(model).count()
if rowCount > 0:
session.query(model).delete()
with open(self._path, 'rU') as f:
try:
reader = csv.reader(
f,
delimiter=StringUtils.toStr2(','),
quotechar=StringUtils.toStr2('"'))
except Exception as err:
self._writeError({
'message':u'ERROR: Unable to read CSV file "%s"' % self._path,
'error':err })
return
if reader is None:
self._writeError({
'message':u'ERROR: Failed to create CSV reader for file "%s"' % self._path })
return
for row in reader:
# Skip any rows that don't start with the proper numeric index value, which
# includes the header row (if it exists) with the column names
try:
index = int(row[0])
except Exception as err:
continue
rowDict = dict()
for column in Reflection.getReflectionList(SitemapCsvColumnEnum):
value = row[column.index]
value = StringUtils.strToUnicode(value)
if value != u'' or value is None:
rowDict[column.name] = value
self._fromSpreadsheetEntry(rowDict, session)
except Exception as err:
if not self._session:
session.rollback()
session.close()
self._log.writeError(u'ERROR: Sitemap Importing Error', err)
return 1
if self._session is None:
session.commit()
session.close()
self._log.write(u'<h1>Sitemap Import Complete</h1>')
return 0
def _sendRemote(self, nimbleData):
responseFlags = 0
message = u''
retry = NimbleEnvironment.REMOTE_RETRY_COUNT
while retry > 0:
try:
self.open()
except Exception as err:
failure = [
'[ERROR | NIMBLE COMMUNICATION] Unable to open connection',
err ]
retry -= 1
if retry == 0:
if not nimble.quietFailure:
NimbleEnvironment.logError(failure[0], failure[1])
return None
continue
try:
serialData = nimbleData.serialize()
except Exception as err:
failure = [
'[ERROR | NIMBLE COMMUNICATION] Unable to serialize data for transmission',
err ]
if not nimble.quietFailure:
NimbleEnvironment.logError(failure[0], failure[1])
return None
try:
self._chunk.clear()
self._chunk.writeUint32(NimbleEnvironment.CONNECTION_FLAGS)
self._chunk.writeString(serialData + NimbleEnvironment.TERMINATION_IDENTIFIER)
self._socket.sendall(self._chunk.byteArray)
except Exception as err:
failure = [
'[ERROR | NIMBLE COMMUNICATION] Unable to send data',
err ]
self.close()
retry -= 1
if retry == 0:
if not nimble.quietFailure:
NimbleEnvironment.logError(failure[0], failure[1])
return None
continue
try:
self._chunk.clear()
b = SocketUtils.receiveInChunks(
self._socket,
chunkSize=NimbleEnvironment.SOCKET_RESPONSE_CHUNK_SIZE)
self._chunk.writeString(b)
self._chunk.position = 0
responseFlags = self._chunk.readUint32()
message = StringUtils.strToUnicode(self._chunk.read(-1))
# Break while loop on successful reading of the result
if message is not None:
break
except Exception as err:
if not nimble.quietFailure:
NimbleEnvironment.logError(
'[ERROR | NIMBLE COMMUNICATION] Unable to read response', err)
self.close()
return None
try:
if not (responseFlags & ConnectionFlags.KEEP_ALIVE):
self.close()
except Exception as err:
if not nimble.quietFailure:
NimbleEnvironment.logError(
'[ERROR | NIMBLE COMMUNICATION] Unable to close connection', err)
try:
return NimbleData.fromMessage(message)
except Exception as err:
if not nimble.quietFailure:
NimbleEnvironment.logError(
'[ERROR | NIMBLE COMMUNICATION] Response data parsing failure', err)
return None
def _sendRemote(self, nimbleData):
responseFlags = 0
message = u''
retry = NimbleEnvironment.REMOTE_RETRY_COUNT
while retry > 0:
try:
self.open()
except Exception as err:
failure = [
'[ERROR | NIMBLE COMMUNICATION] Unable to open connection',
err
]
retry -= 1
if retry == 0:
if not nimble.quietFailure:
NimbleEnvironment.logError(failure[0], failure[1])
return None
continue
try:
serialData = nimbleData.serialize()
except Exception as err:
failure = [
'[ERROR | NIMBLE COMMUNICATION] Unable to serialize data for transmission',
err
]
if not nimble.quietFailure:
NimbleEnvironment.logError(failure[0], failure[1])
return None
try:
self._chunk.clear()
self._chunk.writeUint32(NimbleEnvironment.CONNECTION_FLAGS)
self._chunk.writeString(
serialData + NimbleEnvironment.TERMINATION_IDENTIFIER)
self._socket.sendall(self._chunk.byteArray)
except Exception as err:
failure = [
'[ERROR | NIMBLE COMMUNICATION] Unable to send data', err
]
self.close()
retry -= 1
if retry == 0:
if not nimble.quietFailure:
NimbleEnvironment.logError(failure[0], failure[1])
return None
continue
try:
self._chunk.clear()
b = SocketUtils.receiveInChunks(
self._socket,
chunkSize=NimbleEnvironment.SOCKET_RESPONSE_CHUNK_SIZE)
self._chunk.writeString(b)
self._chunk.position = 0
responseFlags = self._chunk.readUint32()
message = StringUtils.strToUnicode(self._chunk.read(-1))
# Break while loop on successful reading of the result
if message is not None:
break
except Exception as err:
if not nimble.quietFailure:
NimbleEnvironment.logError(
'[ERROR | NIMBLE COMMUNICATION] Unable to read response',
err)
self.close()
return None
try:
if not (responseFlags & ConnectionFlags.KEEP_ALIVE):
self.close()
except Exception as err:
if not nimble.quietFailure:
NimbleEnvironment.logError(
'[ERROR | NIMBLE COMMUNICATION] Unable to close connection',
err)
try:
return NimbleData.fromMessage(message)
except Exception as err:
if not nimble.quietFailure:
NimbleEnvironment.logError(
'[ERROR | NIMBLE COMMUNICATION] Response data parsing failure',
err)
return None
def _reformat(cls, src):
out = dict()
for n,v in DictUtils.iter(src):
n = StringUtils.strToUnicode(n)
out[n] = cls._reformatValue(v)
return out
def _reformat(cls, src):
out = dict()
for n, v in DictUtils.iter(src):
n = StringUtils.strToUnicode(n)
out[n] = cls._reformatValue(v)
return out
