def decodeMessage(wrapper, message): result = "" decoded = Base64.decodeBase64(message) try: Z = String(decoded, "UTF-8") result = Z.split(wrapper)[1] except Exception as ex: # Error decoding a Tango message. pass return result
def decodeMessage(wrapper, message): result = "" decoded = Base64.decodeBase64(message) try: Z = String(decoded, "UTF-8") result = Z.split(wrapper)[1] except Exception as ex: self._logger.log(Level.SEVERE, "Error decoding a Tango message", ex) self._logger.log(Level.SEVERE, traceback.format_exc()) return result
def parseCommandLine(argv): fullArgPairPattern = Pattern.compile("--\\w+=\\S*") justArgNamePattern = Pattern.compile("--\\w+") cmdParamProps = {} if (len(argv) > 0): for param in argv: cmdParam = String(param) fullMatcher = fullArgPairPattern.matcher(cmdParam) if (fullMatcher.matches()): (paramName, paramValue) = cmdParam.split("=") cmdParamProps[paramName] = paramValue else: nameMatcher = justArgNamePattern.matcher(cmdParam) if (nameMatcher.matches()): cmdParamProps[param] = None else: print("This " + param + " is not a Command Line parameter") return cmdParamProps
def splitIntoWords(str): """Returns a list of non-word-character-separated words in str.""" nonempty = lambda s: len(s) > 0 return filter(nonempty, list(String.split(str, "\\W+")))
def splitIntoWords(s): nonempty = lambda s: len(s) > 0 return filter(nonempty, list(String.split(s, "\\W+")))
def __reportSearch(self): self.reportId = self.request.getParameter("id") self.format = self.request.getParameter("format") self.report = self.reportManager.getReports().get(self.reportId) self.reportQuery = self.report.getQueryAsString() self.log.debug("Report query: " + self.reportQuery) #Get a total number of records try: out = ByteArrayOutputStream() recnumreq = SearchRequest(self.reportQuery) recnumreq.setParam("rows", "0") self.indexer.search(recnumreq, out) recnumres = SolrResult(ByteArrayInputStream(out.toByteArray())) self.__rowsFoundSolr = "%s" % recnumres.getNumFound() except: self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % ( sys.exc_info()[0], sys.exc_info()[1]) self.log.error( "Reporting threw an exception (report was %s): %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return #Setup the main query req = SearchRequest(self.reportQuery) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", self.__rowsFoundSolr) try: #Now do the master search out = ByteArrayOutputStream() self.indexer.search(req, out) self.__reportResult = SolrResult( ByteArrayInputStream(out.toByteArray())) self.__checkResults() except: self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % ( sys.exc_info()[0], sys.exc_info()[1]) self.log.error( "Reporting threw an exception (report was %s): %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return #At this point the display template has enough to go with. #We just need to handle the CSV now if (self.format == "csv"): #Setup the main query - we need to requery to make sure we return #only the required fields. We'll use the specific IDs that met the #__checkResults check req = SearchRequest(self.reportQuery) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", self.__rowsFoundSolr) req.setParam("csv.mv.separator", ";") #we need to get a list of the matching IDs from Solr #this doesn't work for long queries so it's abandoned #but left here commented to make sure we don't try it again #idQry = "" #for item in self.getProcessedResultsList(): # idQry += item.get("id") + " OR " #req.setParam("fq", 'id:(%s)' % idQry[:len(idQry)-4]) #Create a list of IDs for reference when preparing the CSV idQryList = [] for item in self.getProcessedResultsList(): idQryList.append(item.get("id")) #Setup SOLR query with the required fields self.fields = self.systemConfig.getArray("redbox-reports", "csv-output-fields") #We must have an ID field and it must be the first field fieldString = "id," if self.fields is not None: for field in self.fields: fieldString = fieldString + field.get("field-name") + "," fieldString = fieldString[:-1] req.setParam("fl", fieldString) out = ByteArrayOutputStream() try: self.indexer.search(req, out, self.format) except: #We can't get the result back from SOLR so fail back to the template display self.errorMsg = "Query failure. Failed to load the data - this issue has been logged (%s - %s)." % ( sys.exc_info()[0], sys.exc_info()[1]) self.log.error( "Reporting threw an exception (report was %s); Error: %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return try: csvResponseString = String(out.toByteArray(), "utf-8") csvResponseLines = csvResponseString.split("\n") except: #We can't get the result back from SOLR so fail back to the template display self.errorMsg = "Query failure. Failed to prepare the CSV - this issue has been logged (%s - %s)." % ( sys.exc_info()[0], sys.exc_info()[1]) self.log.error( "Reporting threw an exception (report was %s); Error: %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return fileName = self.urlEncode(self.report.getLabel()) self.log.debug("Generating CSV report with file name: " + fileName) self.response.setHeader("Content-Disposition", "attachment; filename=%s.csv" % fileName) sw = StringWriter() parser = CSVParser() writer = CSVWriter(sw) count = 0 prevLine = "" badRowFlag = False for line in csvResponseLines: if badRowFlag: #In this section of code we'll handle errors by either trying to fix the problem #or by adding an error line in the CSV. We'll then move to the next row and keep going try: self.log.debug( "Reporting - trying to append the previous line with the previous faulty one. Line appears as: %s" % prevLine + line) csvLine = parser.parseLine(prevLine + line) badRowFlag = False prevLine = "" self.log.debug( "Reporting - remedy appears to have worked. Line appears as: %s" % prevLine + line) except: #We tried to rescue the file but failed on the second run so give up writer.writeNext( ["Failed to transfer record to CSV - check logs"]) self.log.error( "Reporting threw an exception (report was %s); Error: %s - %s; Result line: %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1], prevLine + line)) else: try: csvLine = parser.parseLine(line) badRowFlag = False prevLine = "" except: #This can happen if there's a newline in the index data #so we raise the badRowFlag and see if we can join this #row to the next one to fix it self.log.debug( "Reporting threw an exception but I'll see if it's just a formatting issue (report was %s); Error: %s - %s; Result line: %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1], line)) badRowFlag = True prevLine = line continue if count == 0: #Header row count += 1 for idx, csvValue in enumerate(csvLine): csvLine[idx] = self.findDisplayLabel(csvValue) elif csvLine[0] not in idQryList: #ignore continue writer.writeNext(csvLine) #Now send off the CSV self.out = self.response.getOutputStream("text/csv") self.out.print(sw.toString()) self.out.close()
def __reportSearch(self): self.reportId = self.request.getParameter("id") self.format = self.request.getParameter("format") self.report = self.reportManager.getReports().get(self.reportId) self.reportQuery = self.report.getQueryAsString() self.log.debug("Report query: " + self.reportQuery) #Get a total number of records try: out = ByteArrayOutputStream() recnumreq = SearchRequest(self.reportQuery) recnumreq.setParam("rows", "0") self.indexer.search(recnumreq, out) recnumres = SolrResult(ByteArrayInputStream(out.toByteArray())) self.__rowsFoundSolr = "%s" % recnumres.getNumFound() except: self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Reporting threw an exception (report was %s): %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return #Setup the main query req = SearchRequest(self.reportQuery) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", self.__rowsFoundSolr) try: #Now do the master search out = ByteArrayOutputStream() self.indexer.search(req, out) self.__reportResult = SolrResult(ByteArrayInputStream(out.toByteArray())) self.__checkResults() except: self.errorMsg = "Query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Reporting threw an exception (report was %s): %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return #At this point the display template has enough to go with. #We just need to handle the CSV now if (self.format == "csv"): #Setup the main query - we need to requery to make sure we return #only the required fields. We'll use the specific IDs that met the #__checkResults check req = SearchRequest(self.reportQuery) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", self.__rowsFoundSolr) req.setParam("csv.mv.separator",";") #we need to get a list of the matching IDs from Solr #this doesn't work for long queries so it's abandoned #but left here commented to make sure we don't try it again #idQry = "" #for item in self.getProcessedResultsList(): # idQry += item.get("id") + " OR " #req.setParam("fq", 'id:(%s)' % idQry[:len(idQry)-4]) #Create a list of IDs for reference when preparing the CSV idQryList = [] for item in self.getProcessedResultsList(): idQryList.append(item.get("id")) #Setup SOLR query with the required fields self.fields = self.systemConfig.getArray("redbox-reports","csv-output-fields") #We must have an ID field and it must be the first field fieldString = "id," if self.fields is not None: for field in self.fields: fieldString = fieldString+ field.get("field-name")+"," fieldString = fieldString[:-1] req.setParam("fl",fieldString) out = ByteArrayOutputStream() try: self.indexer.search(req, out, self.format) except: #We can't get the result back from SOLR so fail back to the template display self.errorMsg = "Query failure. Failed to load the data - this issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Reporting threw an exception (report was %s); Error: %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return try: csvResponseString = String(out.toByteArray(),"utf-8") csvResponseLines = csvResponseString.split("\n") except: #We can't get the result back from SOLR so fail back to the template display self.errorMsg = "Query failure. Failed to prepare the CSV - this issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Reporting threw an exception (report was %s); Error: %s - %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1])) return fileName = self.urlEncode(self.report.getLabel()) self.log.debug("Generating CSV report with file name: " + fileName) self.response.setHeader("Content-Disposition", "attachment; filename=%s.csv" % fileName) sw = StringWriter() parser = CSVParser() writer = CSVWriter(sw) count = 0 prevLine = "" badRowFlag = False for line in csvResponseLines: if badRowFlag: #In this section of code we'll handle errors by either trying to fix the problem #or by adding an error line in the CSV. We'll then move to the next row and keep going try: self.log.debug("Reporting - trying to append the previous line with the previous faulty one. Line appears as: %s" % prevLine + line) csvLine = parser.parseLine(prevLine + line) badRowFlag = False prevLine = "" self.log.debug("Reporting - remedy appears to have worked. Line appears as: %s" % prevLine + line) except: #We tried to rescue the file but failed on the second run so give up writer.writeNext(["Failed to transfer record to CSV - check logs"]) self.log.error("Reporting threw an exception (report was %s); Error: %s - %s; Result line: %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1], prevLine + line)) else: try: csvLine = parser.parseLine(line) badRowFlag = False prevLine = "" except: #This can happen if there's a newline in the index data #so we raise the badRowFlag and see if we can join this #row to the next one to fix it self.log.debug("Reporting threw an exception but I'll see if it's just a formatting issue (report was %s); Error: %s - %s; Result line: %s" % (self.report.getLabel(), sys.exc_info()[0], sys.exc_info()[1], line)) badRowFlag = True prevLine = line continue if count == 0 : #Header row count += 1 for idx, csvValue in enumerate(csvLine): csvLine[idx] = self.findDisplayLabel(csvValue) elif csvLine[0] not in idQryList: #ignore continue writer.writeNext(csvLine) #Now send off the CSV self.out = self.response.getOutputStream("text/csv") self.out.print(sw.toString()) self.out.close()