def numberOfModifiedRecord(self): indexer = self.services.getIndexer() portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery() portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery() # Security prep work current_user = self.page.authentication.get_username() security_roles = self.page.authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" req = SearchRequest("modified:true") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.addParam("fq", "") req.setParam("rows", "0") if not self.page.authentication.is_admin(): req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray())) return self.__result.getString(None, "response", "numFound")
def handleQuery(self, query, fieldName, formatStr): out = ByteArrayOutputStream() req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): results = res.getResults() for searchRes in results: searchResList = searchRes.getList(fieldName) if (searchResList.isEmpty() == False): for hit in searchResList: if self.term is not None: if hit.find(self.term) != -1: hits.add(hit) else: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print("," + formatStr % {"hit": hit}) else: self.writer.print(formatStr % {"hit": hit}) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def handleWorkflowStep(self): out = ByteArrayOutputStream() req = SearchRequest("workflow_step_label:[* TO *]" ) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): recordTypeResults = res.getResults() for recordTypeResult in recordTypeResults: recordTypeList = recordTypeResult.getList("workflow_step_label") if (recordTypeList.isEmpty()==False): for hit in recordTypeList: hits.add(hit) self.writer.println("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.println(",{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit)) else: self.writer.println("{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit)) hitnum += 1 self.writer.println("]") else: self.writer.println("[\"\"]") self.writer.close()
def __getUsers(self, oid): indexer = Services.getIndexer() req = SearchRequest("id:" + oid) req.setParam("fl", "security_exception,owner") out = ByteArrayOutputStream() indexer.search(req, out) rtJson = "" try: qresult = SolrResult(ByteArrayInputStream( out.toByteArray())).getResults().get(0) owner = qresult.getString(None, 'owner') secException = qresult.getArray('security_exception') if secException is None: secException = JSONArray() self.log.debug("Owner of object: " + owner) self.log.debug("Viewer(s) of object: " + secException.toString()) if secException.contains(owner): secException.remove(owner) return '{"owner":"' + owner + '", "viewers": ' + secException.toString( ) + '}' except Exception, e: self.log.error("Error during query/package ownership data" + str(e))
def __saveManifest(self, oid): object = self.services.getStorage().getObject(oid) sourceId = object.getSourceId() manifestStr = String(self.__manifest.toString()) object.updatePayload( sourceId, ByteArrayInputStream(manifestStr.getBytes("UTF-8"))) object.close()
def __search(self): query = self.formData.get("query") searchQuery = self.sessionState.get("searchQuery") if query is None or query == "": query = "*:*" if searchQuery and query == "*:*": query = searchQuery elif searchQuery: query += " AND " + searchQuery facetField = self.formData.get("facet.field") req = SearchRequest(query) req.setParam("facet", "true") req.setParam("fl", "id") req.setParam("rows", "0") req.setParam("facet.limit", "-1") req.setParam("facet.field", facetField) fq = self.sessionState.get("fq") if fq is not None: req.setParam("fq", fq) req.addParam("fq", 'item_type:"object"') # Make sure 'fq' has already been set in the session security_roles = self.auth.get_roles_list() security_query = 'security_filter:("' + '" OR "'.join( security_roles) + '")' req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer = self.services.indexer indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) return FacetList(facetField, result)
def findPublishedRecords(self): #req = SearchRequest("published:\"true\"") req = SearchRequest("storage_id:\"c6a214670dc644e5ebdaede4a2243f67\"") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def Parse(self, data, isfinal=False): # The 'data' argument should be an encoded text: a str instance that # represents an array of bytes. If instead it is a unicode string, # only the us-ascii range is considered safe enough to be silently # converted. if isinstance(data, unicode): data = data.encode(sys.getdefaultencoding()) self._data.append(data) if isfinal: bytes = StringUtil.toBytes(self._data.toString()) byte_stream = ByteArrayInputStream(bytes) source = InputSource(byte_stream) if self.encoding is not None: source.setEncoding(self.encoding) try: self._reader.parse(source) except SAXParseException, sax_error: # Experiments tend to show that the '_Next*' parser locations # match more closely expat behavior than the 'Current*' or sax # error locations. self.ErrorLineNumber = self._NextLineNumber self.ErrorColumnNumber = self._NextColumnNumber self.ErrorCode = None raise self._expat_error(sax_error) return 1
def handleGrantNumber(self): out = ByteArrayOutputStream() req = SearchRequest("grant_numbers:%s*" % self.term) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): creatorResults = res.getResults() for creatorRes in creatorResults: creatorList = creatorRes.getList("grant_numbers") if (creatorList.isEmpty()==False): for hit in creatorList: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print(",\"%s\"" % hit) else: self.writer.print("\"%s\"" % hit) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def export(self, exportType): exportQuery = "%s:%s" % (self.facetField, self.facetFieldValue) outputType = "text/%s; charset=UTF-8" % type responseHeader = "attachment; filename=%s.%s" % (self.facetFieldValue, exportType) try: out = ByteArrayOutputStream() recnumreq = SearchRequest(exportQuery) recnumreq.setParam("fl","create_timestamp") recnumreq.setParam("rows", "0") self.indexer.search(recnumreq, out) recnumres = SolrResult(ByteArrayInputStream(out.toByteArray())) self.__rowsFoundSolr = "%s" % recnumres.getNumFound() except: self.errorMsg = "Export query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Export query threw an exception (package type was %s): %s - %s" % (self.facetFieldValue, sys.exc_info()[0], sys.exc_info()[1])) return out = ByteArrayOutputStream() req = SearchRequest(exportQuery) req.setParam("wt", exportType) req.setParam("rows", self.__rowsFoundSolr) self.indexer.search(req, out) self.response.setHeader("Content-Disposition", responseHeader) writer = self.response.getPrintWriter(outputType) writer.println(out.toString("UTF-8")) writer.close()
def mainproc(attributes): conn = ConnectionFactory.getInstance().acquire() try: stmt = conn.prepareCall( "{call [dbo].[grid_download2] (?, ?, ?, ?, ?, ?, ?, ?, ?)}") try: stmt.setString(1, attributes.getMainContext()) stmt.setString(2, attributes.getAddContext()) setSQLXMLParam(stmt, 3, attributes.getFilterinfo()) setSQLXMLParam(stmt, 4, attributes.getSessionContext()) stmt.setString(5, attributes.getElementId()) stmt.setString(6, attributes.getRecordId()) stmt.registerOutParameter(7, Types.VARCHAR) stmt.registerOutParameter(8, Types.BLOB) stmt.registerOutParameter(9, Types.VARCHAR) stmt.execute() fileName = stmt.getString(7) bt = stmt.getBytes(8) if bt != None: return JythonDownloadResult(ByteArrayInputStream(bt), fileName) else: return JythonDownloadResult(None, fileName) finally: stmt.close() finally: ConnectionFactory.getInstance().release(conn) res = JythonDTO(None, None) return res
def parseDom(xmlText): baos = ByteArrayInputStream(xmlText.getBytes("UTF-8")) dfactory = DocumentBuilderFactory.newInstance() dfactory.setNamespaceAware(true) dfactory.setValidating(false) documentBuilder = dfactory.newDocumentBuilder() return documentBuilder.parse(baos)
def checkEventLogForEdits(self, oid): req = SearchRequest("oid:" + oid + " AND context:Workflow AND eventType:Save") out = ByteArrayOutputStream() self.indexer.searchByIndex(req, out, "eventLog") solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getRows() > 0
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper") externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder") oid = self.request.getParameter("oid") if oid is None : identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) relationshipType = self.request.getParameter("relationship") curatedPid = self.request.getParameter("curatedPid") sourceId = self.request.getParameter("sourceIdentifier") digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJson = self.getTfPackage(digitalObject) relationships = metadataJson.getArray("relationships") found = False for relationship in relationships: if relationship.get("identifier") == sourceId: relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) found = True if not found: relationship = JsonObject() relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) relationship.put("relationship",relationshipType) relationship.put("identifier",sourceId) relationships.add(relationship) self.log.info(metadataJson.toString(True)) out.println(metadataJson.toString(True)) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) for pid in digitalObject.getPayloadIdList(): if pid.endswith(".tfpackage"): StorageUtils.createOrUpdatePayload(digitalObject,pid,istream) out.close() finally: self.sessionState.remove("username")
def _extract_cert_from_data(f, password=None, key_converter=None, cert_converter=None): certs = [] private_key = None if isinstance(f, unicode): f = StringIO(str(f)) elif isinstance(f, str): f = StringIO(f) if not hasattr(f, 'seek'): raise TypeError( "PEM lib (data must be a file like object, string or bytes") if _is_cert_pem(f): certs, private_key = _read_pem_cert_from_data(f, password, key_converter, cert_converter) else: cf = CertificateFactory.getInstance("X.509") certs = list(cf.generateCertificates(ByteArrayInputStream(f.read()))) return certs, private_key
def search_solr(self): query = "(rootUri:" if self.rootUriList: query += "(" + " OR ".join(self.rootUriList) + ")" else: query += "\"" + self.rootUri + "\"" if self.type: query += " AND type:\"" + self.type + "\"" query += ")" #print "**********", query req = SearchRequest(query) req.setParam("facet", "false") req.setParam("rows", str(99999)) req.setParam("sort", "dateCreated asc") req.setParam("start", str(0)) #security_roles = page.authentication.get_roles_list(); #security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' #req.addParam("fq", security_query) out = ByteArrayOutputStream() Services.indexer.annotateSearch(req, out) result = SolrResult(ByteArrayInputStream( out.toByteArray())).getResults() # Every annotation for this URI if self.type == "http://www.purl.org/anotar/ns/type/0.1#Tag": return self.process_tags(result) else: return self.process_response(result)
def __feed(self): self.portal = self.services.getPortalManager().get(self.vc("portalId")) recordsPerPage = self.portal.recordsPerPage pageNum = self.vc("sessionState").get("pageNum", 1) query = "*:*" if self.vc("formData").get("query"): query = self.vc("formData").get("query") query = self.__escapeQuery(query) req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", self.portal.facetFieldList) req.setParam("facet.sort", "true") req.setParam("facet.limit", str(self.portal.facetCount)) req.setParam("sort", "f_dc_title asc") portalQuery = self.portal.query if portalQuery: req.addParam("fq", portalQuery) else: fq = self.vc("sessionState").get("fq") if fq is not None: req.setParam("fq", fq) req.setParam("start", str((pageNum - 1) * recordsPerPage)) self.log.debug(" * Query: '{}'", query) self.log.debug(" * portalQuery: '{}'", portalQuery) self.log.debug(" * feed.py: '{}'", req) out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __call__(self, _input, profile_run=False, **kw): nsmap = dict(xsl='http://www.w3.org/1999/XSL/Transform') output_method = 'xml' output_encoding = 'utf-8' nodes = self.xsl_tree.xpath('xsl:output/@method', namespaces=nsmap) if len(nodes) > 0: output_method = nodes[0] nodes = self.xsl_tree.xpath('xsl:output/@encoding', namespaces=nsmap) if len(nodes) > 0: output_encoding = nodes[0] #print tostring(_input) if isinstance(_input, _ElementTree): doc_source = DOMSource(_input._dom_doc) elif isinstance(_input, _Element): # Xalan-J 2.7.1 does not support a DOMSource from an Element # so we build new document dom_doc = builder.newDocument() dom_root = dom_doc.importNode(_input._dom_element, True) dom_doc.appendChild(dom_root) doc_source = DOMSource(dom_doc) else: raise NotImplementedError() if output_method in ('xml', 'html'): # TODO: for testing outputstream = ByteArrayOutputStream() result = StreamResult(outputstream) self.transformer.transform(doc_source, result) bytes = outputstream.toByteArray() inputstream = ByteArrayInputStream(bytes) try: dom_doc = builder.parse(inputstream) except: import sys sys.stderr.write(bytes.tostring()) raise result_tree = _ElementTree(dom_doc) return result_tree result = DOMResult() self.transformer.transform(doc_source, result) dom_doc = result.getNode() result_tree = _ElementTree(dom_doc) #print tostring(result_tree) return result_tree else: outputstream = ByteArrayOutputStream() result = StreamResult(outputstream) self.transformer.transform(doc_source, result) resultdoc = builder.newDocument() resulttree = _XSLTResultTree(resultdoc) resulttree._text = outputstream.toString(output_encoding) return resulttree
def _read_image(self, fp): if sys.platform[0:4] == 'java': from javax.imageio import ImageIO from java.io import ByteArrayInputStream input_stream = ByteArrayInputStream(fp.read()) return ImageIO.read(input_stream) elif PILImage: return PILImage.open(fp)
def _addPropertyValueToTFMeta(self, object, tfMetaPropertyValue): objectMetadata = object.getMetadata() objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue) output = ByteArrayOutputStream(); objectMetadata.store(output, None); input = ByteArrayInputStream(output.toByteArray()); StorageUtils.createOrUpdatePayload(object,"TF-OBJ-META",input);
def testEscapeBackslash(self): # JSON "xy\\.z" should become String "xy\.z" text = "{ \"abc\": \"xy\\\\.z\" }" stream = ByteArrayInputStream(text.encode('utf-8')) json_translator = JsonStreamTranslator("String", stream) result = json_translator.parse() abc = result['abc'] self.assertEquals("xy\\.z", abc, "Should be single slash")
def write(self, name, file): fname = self.getName(name) entry = cab.CabFileEntry(name=fname, date=Date()) if isinstance(file, ByteArrayOutputStream): file = ByteArrayInputStream(file.toByteArray()) elif isinstance(file, type('')): file = FileInputStream(file) self.cabfile.addStream(file, entry)
def parseFileContent(self, bytes): byteInputStream = ByteArrayInputStream(bytes) reader = InputStreamReader(byteInputStream, self.encoding) parser = csvParser.Parser(reader, self.delimiter) parser.setQuoteSymbol(self.quoteSymbol) parser.setRowToStartIndex(self.rowToStartIndex) return csvParser.Processor().processByParser(parser).getRows()
def _getXmlRootFromString(self, xmlString): """ Parses string xml representation and returns root element str->Element @raise JavaException: XML parsing failed """ xmlString = ''.join([line.strip() for line in xmlString.split('\n') if line]) strContent = String(xmlString) return SAXBuilder().build(ByteArrayInputStream(strContent.getBytes('utf-8'))).getRootElement()
def __searchExecute(self, search, count): try: search.setParam("start", str(count)) out = ByteArrayOutputStream() self.services.indexer.search(search, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Error during search: ", e) return None
def serialize(o, special=False): b = ByteArrayOutputStream() objs = ObjectOutputStream(b) objs.writeObject(o) if not special: OIS = ObjectInputStream else: OIS = PythonObjectInputStream objs = OIS(ByteArrayInputStream(b.toByteArray())) return objs.readObject()
def test_serialization(self): s = set(range(5, 10)) output = ByteArrayOutputStream() serializer = ObjectOutputStream(output) serializer.writeObject(s) serializer.close() input = ByteArrayInputStream(output.toByteArray()) unserializer = ObjectInputStream(input) self.assertEqual(s, unserializer.readObject())
def from_json(json_object): if not ITEMSTACK_JSON_NAME in json_object: return json_object raw_bytes = binascii.unhexlify(json_object[ITEMSTACK_JSON_NAME]) byte_array_in = ByteArrayInputStream(raw_bytes) bukkit_in = BukkitObjectInputStream(byte_array_in) bukkit_item = bukkit_in.readObject() bukkit_in.close() return bukkit_item
def getAttachments(self): attachmentType = "review-attachments" req = SearchRequest("attached_to:%s AND attachment_type:%s" % (self.oid, attachmentType)) req.setParam("rows", "1000") out = ByteArrayOutputStream() self.Services.indexer.search(req, out) response = SolrResult(ByteArrayInputStream(out.toByteArray())) return response.getResults()
def findPackagesToTransition(self, fromWorkflowId, fromWorkflowStage): req = SearchRequest("workflow_id:" + fromWorkflowId + " AND _query_:\"workflow_step:" + fromWorkflowStage + "\"") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()