def decodeMessage(wrapper, message):
result = ""
decoded = Base64.decodeBase64(message)
try:
Z = String(decoded, "UTF-8")
result = Z.split(wrapper)[1]
except Exception as ex:
# Error decoding a Tango message.
pass
return result
def decodeMessage(wrapper, message):
result = ""
decoded = Base64.decodeBase64(message)
try:
Z = String(decoded, "UTF-8")
result = Z.split(wrapper)[1]
except Exception as ex:
self._logger.log(Level.SEVERE, "Error decoding a Tango message", ex)
self._logger.log(Level.SEVERE, traceback.format_exc())
return result
def decodeMessage(wrapper, message):
result = ""
decoded = Base64.decodeBase64(message)
try:
Z = String(decoded, "UTF-8")
result = Z.split(wrapper)[1]
except Exception as ex:
# Error decoding a Tango message.
pass
return result
def parseCommandLine(argv):
fullArgPairPattern = Pattern.compile("--\\w+=\\S*")
justArgNamePattern = Pattern.compile("--\\w+")
cmdParamProps = {}
if (len(argv) > 0):
for param in argv:
cmdParam = String(param)
fullMatcher = fullArgPairPattern.matcher(cmdParam)
if (fullMatcher.matches()):
(paramName, paramValue) = cmdParam.split("=")
cmdParamProps[paramName] = paramValue
else:
nameMatcher = justArgNamePattern.matcher(cmdParam)
if (nameMatcher.matches()):
cmdParamProps[param] = None
else:
print("This " + param + " is not a Command Line parameter")
return cmdParamProps
def splitIntoWords(str):
"""Returns a list of non-word-character-separated words in str."""
nonempty = lambda s: len(s) > 0
return filter(nonempty, list(String.split(str, "\\W+")))
def splitIntoWords(s):
nonempty = lambda s: len(s) > 0
return filter(nonempty, list(String.split(s, "\\W+")))
def splitIntoWords(s):
nonempty = lambda s: len(s) > 0
return filter(nonempty, list(String.split(s, "\\W+")))
def __reportSearch(self):
self.reportId = self.request.getParameter("id")
self.format = self.request.getParameter("format")
self.report = self.reportManager.getReports().get(self.reportId)
self.reportQuery = self.report.getQueryAsString()
self.log.debug("Report query: " + self.reportQuery)
#Get a total number of records
try:
out = ByteArrayOutputStream()
recnumreq = SearchRequest(self.reportQuery)
recnumreq.setParam("rows", "0")
self.indexer.search(recnumreq, out)
recnumres = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__rowsFoundSolr = "%s" % recnumres.getNumFound()
except:
self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % (
sys.exc_info()[0], sys.exc_info()[1])
self.log.error(
"Reporting threw an exception (report was %s): %s - %s" %
(self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1]))
return
#Setup the main query
req = SearchRequest(self.reportQuery)
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", self.__rowsFoundSolr)
try:
#Now do the master search
out = ByteArrayOutputStream()
self.indexer.search(req, out)
self.__reportResult = SolrResult(
ByteArrayInputStream(out.toByteArray()))
self.__checkResults()
except:
self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % (
sys.exc_info()[0], sys.exc_info()[1])
self.log.error(
"Reporting threw an exception (report was %s): %s - %s" %
(self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1]))
return
#At this point the display template has enough to go with.
#We just need to handle the CSV now
if (self.format == "csv"):
#Setup the main query - we need to requery to make sure we return
#only the required fields. We'll use the specific IDs that met the
#__checkResults check
req = SearchRequest(self.reportQuery)
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", self.__rowsFoundSolr)
req.setParam("csv.mv.separator", ";")
#we need to get a list of the matching IDs from Solr
#this doesn't work for long queries so it's abandoned
#but left here commented to make sure we don't try it again
#idQry = ""
#for item in self.getProcessedResultsList():
# idQry += item.get("id") + " OR "
#req.setParam("fq", 'id:(%s)' % idQry[:len(idQry)-4])
#Create a list of IDs for reference when preparing the CSV
idQryList = []
for item in self.getProcessedResultsList():
idQryList.append(item.get("id"))
#Setup SOLR query with the required fields
self.fields = self.systemConfig.getArray("redbox-reports",
"csv-output-fields")
#We must have an ID field and it must be the first field
fieldString = "id,"
if self.fields is not None:
for field in self.fields:
fieldString = fieldString + field.get("field-name") + ","
fieldString = fieldString[:-1]
req.setParam("fl", fieldString)
out = ByteArrayOutputStream()
try:
self.indexer.search(req, out, self.format)
except:
#We can't get the result back from SOLR so fail back to the template display
self.errorMsg = "Query failure. Failed to load the data - this issue has been logged (%s - %s)." % (
sys.exc_info()[0], sys.exc_info()[1])
self.log.error(
"Reporting threw an exception (report was %s); Error: %s - %s"
% (self.report.getLabel(), sys.exc_info()[0],
sys.exc_info()[1]))
return
try:
csvResponseString = String(out.toByteArray(), "utf-8")
csvResponseLines = csvResponseString.split("\n")
except:
#We can't get the result back from SOLR so fail back to the template display
self.errorMsg = "Query failure. Failed to prepare the CSV - this issue has been logged (%s - %s)." % (
sys.exc_info()[0], sys.exc_info()[1])
self.log.error(
"Reporting threw an exception (report was %s); Error: %s - %s"
% (self.report.getLabel(), sys.exc_info()[0],
sys.exc_info()[1]))
return
fileName = self.urlEncode(self.report.getLabel())
self.log.debug("Generating CSV report with file name: " + fileName)
self.response.setHeader("Content-Disposition",
"attachment; filename=%s.csv" % fileName)
sw = StringWriter()
parser = CSVParser()
writer = CSVWriter(sw)
count = 0
prevLine = ""
badRowFlag = False
for line in csvResponseLines:
if badRowFlag:
#In this section of code we'll handle errors by either trying to fix the problem
#or by adding an error line in the CSV. We'll then move to the next row and keep going
try:
self.log.debug(
"Reporting - trying to append the previous line with the previous faulty one. Line appears as: %s"
% prevLine + line)
csvLine = parser.parseLine(prevLine + line)
badRowFlag = False
prevLine = ""
self.log.debug(
"Reporting - remedy appears to have worked. Line appears as: %s"
% prevLine + line)
except:
#We tried to rescue the file but failed on the second run so give up
writer.writeNext(
["Failed to transfer record to CSV - check logs"])
self.log.error(
"Reporting threw an exception (report was %s); Error: %s - %s; Result line: %s"
% (self.report.getLabel(), sys.exc_info()[0],
sys.exc_info()[1], prevLine + line))
else:
try:
csvLine = parser.parseLine(line)
badRowFlag = False
prevLine = ""
except:
#This can happen if there's a newline in the index data
#so we raise the badRowFlag and see if we can join this
#row to the next one to fix it
self.log.debug(
"Reporting threw an exception but I'll see if it's just a formatting issue (report was %s); Error: %s - %s; Result line: %s"
% (self.report.getLabel(), sys.exc_info()[0],
sys.exc_info()[1], line))
badRowFlag = True
prevLine = line
continue
if count == 0:
#Header row
count += 1
for idx, csvValue in enumerate(csvLine):
csvLine[idx] = self.findDisplayLabel(csvValue)
elif csvLine[0] not in idQryList:
#ignore
continue
writer.writeNext(csvLine)
#Now send off the CSV
self.out = self.response.getOutputStream("text/csv")
self.out.print(sw.toString())
self.out.close()
def splitIntoWords(str):
"""Returns a list of non-word-character-separated words in str."""
nonempty = lambda s: len(s) > 0
return filter(nonempty, list(String.split(str, "\\W+")))
def __reportSearch(self):
self.reportId = self.request.getParameter("id")
self.format = self.request.getParameter("format")
self.report = self.reportManager.getReports().get(self.reportId)
self.reportQuery = self.report.getQueryAsString()
self.log.debug("Report query: " + self.reportQuery)
#Get a total number of records
try:
out = ByteArrayOutputStream()
recnumreq = SearchRequest(self.reportQuery)
recnumreq.setParam("rows", "0")
self.indexer.search(recnumreq, out)
recnumres = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__rowsFoundSolr = "%s" % recnumres.getNumFound()
except:
self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1])
self.log.error("Reporting threw an exception (report was %s): %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1]))
return
#Setup the main query
req = SearchRequest(self.reportQuery)
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", self.__rowsFoundSolr)
try:
#Now do the master search
out = ByteArrayOutputStream()
self.indexer.search(req, out)
self.__reportResult = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__checkResults()
except:
self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1])
self.log.error("Reporting threw an exception (report was %s): %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1]))
return
#At this point the display template has enough to go with.
#We just need to handle the CSV now
if (self.format == "csv"):
#Setup the main query - we need to requery to make sure we return
#only the required fields. We'll use the specific IDs that met the
#__checkResults check
req = SearchRequest(self.reportQuery)
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", self.__rowsFoundSolr)
req.setParam("csv.mv.separator",";")
#we need to get a list of the matching IDs from Solr
#this doesn't work for long queries so it's abandoned
#but left here commented to make sure we don't try it again
#idQry = ""
#for item in self.getProcessedResultsList():
# idQry += item.get("id") + " OR "
#req.setParam("fq", 'id:(%s)' % idQry[:len(idQry)-4])
#Create a list of IDs for reference when preparing the CSV
idQryList = []
for item in self.getProcessedResultsList():
idQryList.append(item.get("id"))
#Setup SOLR query with the required fields
self.fields = self.systemConfig.getArray("redbox-reports","csv-output-fields")
#We must have an ID field and it must be the first field
fieldString = "id,"
if self.fields is not None:
for field in self.fields:
fieldString = fieldString+ field.get("field-name")+","
fieldString = fieldString[:-1]
req.setParam("fl",fieldString)
out = ByteArrayOutputStream()
try:
self.indexer.search(req, out, self.format)
except:
#We can't get the result back from SOLR so fail back to the template display
self.errorMsg = "Query failure. Failed to load the data - this issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1])
self.log.error("Reporting threw an exception (report was %s); Error: %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1]))
return
try:
csvResponseString = String(out.toByteArray(),"utf-8")
csvResponseLines = csvResponseString.split("\n")
except:
#We can't get the result back from SOLR so fail back to the template display
self.errorMsg = "Query failure. Failed to prepare the CSV - this issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1])
self.log.error("Reporting threw an exception (report was %s); Error: %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1]))
return
fileName = self.urlEncode(self.report.getLabel())
self.log.debug("Generating CSV report with file name: " + fileName)
self.response.setHeader("Content-Disposition", "attachment; filename=%s.csv" % fileName)
sw = StringWriter()
parser = CSVParser()
writer = CSVWriter(sw)
count = 0
prevLine = ""
badRowFlag = False
for line in csvResponseLines:
if badRowFlag:
#In this section of code we'll handle errors by either trying to fix the problem
#or by adding an error line in the CSV. We'll then move to the next row and keep going
try:
self.log.debug("Reporting - trying to append the previous line with the previous faulty one. Line appears as: %s" % prevLine + line)
csvLine = parser.parseLine(prevLine + line)
badRowFlag = False
prevLine = ""
self.log.debug("Reporting - remedy appears to have worked. Line appears as: %s" % prevLine + line)
except:
#We tried to rescue the file but failed on the second run so give up
writer.writeNext(["Failed to transfer record to CSV - check logs"])
self.log.error("Reporting threw an exception (report was %s); Error: %s - %s; Result line: %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1], prevLine + line))
else:
try:
csvLine = parser.parseLine(line)
badRowFlag = False
prevLine = ""
except:
#This can happen if there's a newline in the index data
#so we raise the badRowFlag and see if we can join this
#row to the next one to fix it
self.log.debug("Reporting threw an exception but I'll see if it's just a formatting issue (report was %s); Error: %s - %s; Result line: %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1], line))
badRowFlag = True
prevLine = line
continue
if count == 0 :
#Header row
count += 1
for idx, csvValue in enumerate(csvLine):
csvLine[idx] = self.findDisplayLabel(csvValue)
elif csvLine[0] not in idQryList:
#ignore
continue
writer.writeNext(csvLine)
#Now send off the CSV
self.out = self.response.getOutputStream("text/csv")
self.out.print(sw.toString())
self.out.close()
