def numberOfModifiedRecord(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
req = SearchRequest("modified:true")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
return self.__result.getString(None, "response", "numFound")
def handleQuery(self, query, fieldName, formatStr):
out = ByteArrayOutputStream()
req = SearchRequest(query)
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", "1000")
self.indexer.search(req, out)
res = SolrResult(ByteArrayInputStream(out.toByteArray()))
hits = HashSet()
if (res.getNumFound() > 0):
results = res.getResults()
for searchRes in results:
searchResList = searchRes.getList(fieldName)
if (searchResList.isEmpty() == False):
for hit in searchResList:
if self.term is not None:
if hit.find(self.term) != -1:
hits.add(hit)
else:
hits.add(hit)
self.writer.print("[")
hitnum = 0
for hit in hits:
if (hitnum > 0):
self.writer.print("," + formatStr % {"hit": hit})
else:
self.writer.print(formatStr % {"hit": hit})
hitnum += 1
self.writer.print("]")
else:
self.writer.println("[\"\"]")
self.writer.close()
def handleWorkflowStep(self):
out = ByteArrayOutputStream()
req = SearchRequest("workflow_step_label:[* TO *]" )
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", "1000")
self.indexer.search(req, out)
res = SolrResult(ByteArrayInputStream(out.toByteArray()))
hits = HashSet()
if (res.getNumFound() > 0):
recordTypeResults = res.getResults()
for recordTypeResult in recordTypeResults:
recordTypeList = recordTypeResult.getList("workflow_step_label")
if (recordTypeList.isEmpty()==False):
for hit in recordTypeList:
hits.add(hit)
self.writer.println("[")
hitnum = 0
for hit in hits:
if (hitnum > 0):
self.writer.println(",{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit))
else:
self.writer.println("{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit))
hitnum += 1
self.writer.println("]")
else:
self.writer.println("[\"\"]")
self.writer.close()
def __getUsers(self, oid):
indexer = Services.getIndexer()
req = SearchRequest("id:" + oid)
req.setParam("fl", "security_exception,owner")
out = ByteArrayOutputStream()
indexer.search(req, out)
rtJson = ""
try:
qresult = SolrResult(ByteArrayInputStream(
out.toByteArray())).getResults().get(0)
owner = qresult.getString(None, 'owner')
secException = qresult.getArray('security_exception')
if secException is None:
secException = JSONArray()
self.log.debug("Owner of object: " + owner)
self.log.debug("Viewer(s) of object: " + secException.toString())
if secException.contains(owner):
secException.remove(owner)
return '{"owner":"' + owner + '", "viewers": ' + secException.toString(
) + '}'
except Exception, e:
self.log.error("Error during query/package ownership data" +
str(e))
def __saveManifest(self, oid):
object = self.services.getStorage().getObject(oid)
sourceId = object.getSourceId()
manifestStr = String(self.__manifest.toString())
object.updatePayload(
sourceId, ByteArrayInputStream(manifestStr.getBytes("UTF-8")))
object.close()
def __search(self):
query = self.formData.get("query")
searchQuery = self.sessionState.get("searchQuery")
if query is None or query == "":
query = "*:*"
if searchQuery and query == "*:*":
query = searchQuery
elif searchQuery:
query += " AND " + searchQuery
facetField = self.formData.get("facet.field")
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("fl", "id")
req.setParam("rows", "0")
req.setParam("facet.limit", "-1")
req.setParam("facet.field", facetField)
fq = self.sessionState.get("fq")
if fq is not None:
req.setParam("fq", fq)
req.addParam("fq", 'item_type:"object"')
# Make sure 'fq' has already been set in the session
security_roles = self.auth.get_roles_list()
security_query = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.indexer
indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
return FacetList(facetField, result)
def findPublishedRecords(self):
#req = SearchRequest("published:\"true\"")
req = SearchRequest("storage_id:\"c6a214670dc644e5ebdaede4a2243f67\"")
out = ByteArrayOutputStream()
self.indexer.search(req, out)
solrResult = SolrResult(ByteArrayInputStream(out.toByteArray()))
return solrResult.getResults()
def Parse(self, data, isfinal=False):
# The 'data' argument should be an encoded text: a str instance that
# represents an array of bytes. If instead it is a unicode string,
# only the us-ascii range is considered safe enough to be silently
# converted.
if isinstance(data, unicode):
data = data.encode(sys.getdefaultencoding())
self._data.append(data)
if isfinal:
bytes = StringUtil.toBytes(self._data.toString())
byte_stream = ByteArrayInputStream(bytes)
source = InputSource(byte_stream)
if self.encoding is not None:
source.setEncoding(self.encoding)
try:
self._reader.parse(source)
except SAXParseException, sax_error:
# Experiments tend to show that the '_Next*' parser locations
# match more closely expat behavior than the 'Current*' or sax
# error locations.
self.ErrorLineNumber = self._NextLineNumber
self.ErrorColumnNumber = self._NextColumnNumber
self.ErrorCode = None
raise self._expat_error(sax_error)
return 1
def handleGrantNumber(self):
out = ByteArrayOutputStream()
req = SearchRequest("grant_numbers:%s*" % self.term)
req.setParam("fq", 'item_type:"object"')
req.setParam("fq", 'workflow_id:"dataset"')
req.setParam("rows", "1000")
self.indexer.search(req, out)
res = SolrResult(ByteArrayInputStream(out.toByteArray()))
hits = HashSet()
if (res.getNumFound() > 0):
creatorResults = res.getResults()
for creatorRes in creatorResults:
creatorList = creatorRes.getList("grant_numbers")
if (creatorList.isEmpty()==False):
for hit in creatorList:
hits.add(hit)
self.writer.print("[")
hitnum = 0
for hit in hits:
if (hitnum > 0):
self.writer.print(",\"%s\"" % hit)
else:
self.writer.print("\"%s\"" % hit)
hitnum += 1
self.writer.print("]")
else:
self.writer.println("[\"\"]")
self.writer.close()
def export(self, exportType):
exportQuery = "%s:%s" % (self.facetField, self.facetFieldValue)
outputType = "text/%s; charset=UTF-8" % type
responseHeader = "attachment; filename=%s.%s" % (self.facetFieldValue, exportType)
try:
out = ByteArrayOutputStream()
recnumreq = SearchRequest(exportQuery)
recnumreq.setParam("fl","create_timestamp")
recnumreq.setParam("rows", "0")
self.indexer.search(recnumreq, out)
recnumres = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__rowsFoundSolr = "%s" % recnumres.getNumFound()
except:
self.errorMsg = "Export query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1])
self.log.error("Export query threw an exception (package type was %s): %s - %s" % (self.facetFieldValue, sys.exc_info()[0], sys.exc_info()[1]))
return
out = ByteArrayOutputStream()
req = SearchRequest(exportQuery)
req.setParam("wt", exportType)
req.setParam("rows", self.__rowsFoundSolr)
self.indexer.search(req, out)
self.response.setHeader("Content-Disposition", responseHeader)
writer = self.response.getPrintWriter(outputType)
writer.println(out.toString("UTF-8"))
writer.close()
def mainproc(attributes):
conn = ConnectionFactory.getInstance().acquire()
try:
stmt = conn.prepareCall(
"{call [dbo].[grid_download2] (?, ?, ?, ?, ?, ?, ?, ?, ?)}")
try:
stmt.setString(1, attributes.getMainContext())
stmt.setString(2, attributes.getAddContext())
setSQLXMLParam(stmt, 3, attributes.getFilterinfo())
setSQLXMLParam(stmt, 4, attributes.getSessionContext())
stmt.setString(5, attributes.getElementId())
stmt.setString(6, attributes.getRecordId())
stmt.registerOutParameter(7, Types.VARCHAR)
stmt.registerOutParameter(8, Types.BLOB)
stmt.registerOutParameter(9, Types.VARCHAR)
stmt.execute()
fileName = stmt.getString(7)
bt = stmt.getBytes(8)
if bt != None:
return JythonDownloadResult(ByteArrayInputStream(bt), fileName)
else:
return JythonDownloadResult(None, fileName)
finally:
stmt.close()
finally:
ConnectionFactory.getInstance().release(conn)
res = JythonDTO(None, None)
return res
def parseDom(xmlText):
baos = ByteArrayInputStream(xmlText.getBytes("UTF-8"))
dfactory = DocumentBuilderFactory.newInstance()
dfactory.setNamespaceAware(true)
dfactory.setValidating(false)
documentBuilder = dfactory.newDocumentBuilder()
return documentBuilder.parse(baos)
def checkEventLogForEdits(self, oid):
req = SearchRequest("oid:" + oid +
" AND context:Workflow AND eventType:Save")
out = ByteArrayOutputStream()
self.indexer.searchByIndex(req, out, "eventLog")
solrResult = SolrResult(ByteArrayInputStream(out.toByteArray()))
return solrResult.getRows() > 0
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper")
externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder")
oid = self.request.getParameter("oid")
if oid is None :
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
relationshipType = self.request.getParameter("relationship")
curatedPid = self.request.getParameter("curatedPid")
sourceId = self.request.getParameter("sourceIdentifier")
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJson = self.getTfPackage(digitalObject)
relationships = metadataJson.getArray("relationships")
found = False
for relationship in relationships:
if relationship.get("identifier") == sourceId:
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
found = True
if not found:
relationship = JsonObject()
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
relationship.put("relationship",relationshipType)
relationship.put("identifier",sourceId)
relationships.add(relationship)
self.log.info(metadataJson.toString(True))
out.println(metadataJson.toString(True))
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
for pid in digitalObject.getPayloadIdList():
if pid.endswith(".tfpackage"):
StorageUtils.createOrUpdatePayload(digitalObject,pid,istream)
out.close()
finally:
self.sessionState.remove("username")
def _extract_cert_from_data(f,
password=None,
key_converter=None,
cert_converter=None):
certs = []
private_key = None
if isinstance(f, unicode):
f = StringIO(str(f))
elif isinstance(f, str):
f = StringIO(f)
if not hasattr(f, 'seek'):
raise TypeError(
"PEM lib (data must be a file like object, string or bytes")
if _is_cert_pem(f):
certs, private_key = _read_pem_cert_from_data(f, password,
key_converter,
cert_converter)
else:
cf = CertificateFactory.getInstance("X.509")
certs = list(cf.generateCertificates(ByteArrayInputStream(f.read())))
return certs, private_key
def search_solr(self):
query = "(rootUri:"
if self.rootUriList:
query += "(" + " OR ".join(self.rootUriList) + ")"
else:
query += "\"" + self.rootUri + "\""
if self.type:
query += " AND type:\"" + self.type + "\""
query += ")"
#print "**********", query
req = SearchRequest(query)
req.setParam("facet", "false")
req.setParam("rows", str(99999))
req.setParam("sort", "dateCreated asc")
req.setParam("start", str(0))
#security_roles = page.authentication.get_roles_list();
#security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
#req.addParam("fq", security_query)
out = ByteArrayOutputStream()
Services.indexer.annotateSearch(req, out)
result = SolrResult(ByteArrayInputStream(
out.toByteArray())).getResults()
# Every annotation for this URI
if self.type == "http://www.purl.org/anotar/ns/type/0.1#Tag":
return self.process_tags(result)
else:
return self.process_response(result)
def __feed(self):
self.portal = self.services.getPortalManager().get(self.vc("portalId"))
recordsPerPage = self.portal.recordsPerPage
pageNum = self.vc("sessionState").get("pageNum", 1)
query = "*:*"
if self.vc("formData").get("query"):
query = self.vc("formData").get("query")
query = self.__escapeQuery(query)
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", self.portal.facetFieldList)
req.setParam("facet.sort", "true")
req.setParam("facet.limit", str(self.portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = self.portal.query
if portalQuery:
req.addParam("fq", portalQuery)
else:
fq = self.vc("sessionState").get("fq")
if fq is not None:
req.setParam("fq", fq)
req.setParam("start", str((pageNum - 1) * recordsPerPage))
self.log.debug(" * Query: '{}'", query)
self.log.debug(" * portalQuery: '{}'", portalQuery)
self.log.debug(" * feed.py: '{}'", req)
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __call__(self, _input, profile_run=False, **kw):
nsmap = dict(xsl='http://www.w3.org/1999/XSL/Transform')
output_method = 'xml'
output_encoding = 'utf-8'
nodes = self.xsl_tree.xpath('xsl:output/@method', namespaces=nsmap)
if len(nodes) > 0:
output_method = nodes[0]
nodes = self.xsl_tree.xpath('xsl:output/@encoding', namespaces=nsmap)
if len(nodes) > 0:
output_encoding = nodes[0]
#print tostring(_input)
if isinstance(_input, _ElementTree):
doc_source = DOMSource(_input._dom_doc)
elif isinstance(_input, _Element):
# Xalan-J 2.7.1 does not support a DOMSource from an Element
# so we build new document
dom_doc = builder.newDocument()
dom_root = dom_doc.importNode(_input._dom_element, True)
dom_doc.appendChild(dom_root)
doc_source = DOMSource(dom_doc)
else:
raise NotImplementedError()
if output_method in ('xml', 'html'):
# TODO: for testing
outputstream = ByteArrayOutputStream()
result = StreamResult(outputstream)
self.transformer.transform(doc_source, result)
bytes = outputstream.toByteArray()
inputstream = ByteArrayInputStream(bytes)
try:
dom_doc = builder.parse(inputstream)
except:
import sys
sys.stderr.write(bytes.tostring())
raise
result_tree = _ElementTree(dom_doc)
return result_tree
result = DOMResult()
self.transformer.transform(doc_source, result)
dom_doc = result.getNode()
result_tree = _ElementTree(dom_doc)
#print tostring(result_tree)
return result_tree
else:
outputstream = ByteArrayOutputStream()
result = StreamResult(outputstream)
self.transformer.transform(doc_source, result)
resultdoc = builder.newDocument()
resulttree = _XSLTResultTree(resultdoc)
resulttree._text = outputstream.toString(output_encoding)
return resulttree
def _read_image(self, fp):
if sys.platform[0:4] == 'java':
from javax.imageio import ImageIO
from java.io import ByteArrayInputStream
input_stream = ByteArrayInputStream(fp.read())
return ImageIO.read(input_stream)
elif PILImage:
return PILImage.open(fp)
def _addPropertyValueToTFMeta(self, object, tfMetaPropertyValue):
objectMetadata = object.getMetadata()
objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue)
output = ByteArrayOutputStream();
objectMetadata.store(output, None);
input = ByteArrayInputStream(output.toByteArray());
StorageUtils.createOrUpdatePayload(object,"TF-OBJ-META",input);
def testEscapeBackslash(self):
# JSON "xy\\.z" should become String "xy\.z"
text = "{ \"abc\": \"xy\\\\.z\" }"
stream = ByteArrayInputStream(text.encode('utf-8'))
json_translator = JsonStreamTranslator("String", stream)
result = json_translator.parse()
abc = result['abc']
self.assertEquals("xy\\.z", abc, "Should be single slash")
def write(self, name, file):
fname = self.getName(name)
entry = cab.CabFileEntry(name=fname, date=Date())
if isinstance(file, ByteArrayOutputStream):
file = ByteArrayInputStream(file.toByteArray())
elif isinstance(file, type('')):
file = FileInputStream(file)
self.cabfile.addStream(file, entry)
def parseFileContent(self, bytes):
byteInputStream = ByteArrayInputStream(bytes)
reader = InputStreamReader(byteInputStream, self.encoding)
parser = csvParser.Parser(reader, self.delimiter)
parser.setQuoteSymbol(self.quoteSymbol)
parser.setRowToStartIndex(self.rowToStartIndex)
return csvParser.Processor().processByParser(parser).getRows()
def _getXmlRootFromString(self, xmlString):
"""
Parses string xml representation and returns root element
str->Element
@raise JavaException: XML parsing failed
"""
xmlString = ''.join([line.strip() for line in xmlString.split('\n') if line])
strContent = String(xmlString)
return SAXBuilder().build(ByteArrayInputStream(strContent.getBytes('utf-8'))).getRootElement()
def __searchExecute(self, search, count):
try:
search.setParam("start", str(count))
out = ByteArrayOutputStream()
self.services.indexer.search(search, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except Exception, e:
self.log.error("Error during search: ", e)
return None
def serialize(o, special=False):
b = ByteArrayOutputStream()
objs = ObjectOutputStream(b)
objs.writeObject(o)
if not special:
OIS = ObjectInputStream
else:
OIS = PythonObjectInputStream
objs = OIS(ByteArrayInputStream(b.toByteArray()))
return objs.readObject()
def test_serialization(self):
s = set(range(5, 10))
output = ByteArrayOutputStream()
serializer = ObjectOutputStream(output)
serializer.writeObject(s)
serializer.close()
input = ByteArrayInputStream(output.toByteArray())
unserializer = ObjectInputStream(input)
self.assertEqual(s, unserializer.readObject())
def from_json(json_object):
if not ITEMSTACK_JSON_NAME in json_object:
return json_object
raw_bytes = binascii.unhexlify(json_object[ITEMSTACK_JSON_NAME])
byte_array_in = ByteArrayInputStream(raw_bytes)
bukkit_in = BukkitObjectInputStream(byte_array_in)
bukkit_item = bukkit_in.readObject()
bukkit_in.close()
return bukkit_item
def getAttachments(self):
attachmentType = "review-attachments"
req = SearchRequest("attached_to:%s AND attachment_type:%s" %
(self.oid, attachmentType))
req.setParam("rows", "1000")
out = ByteArrayOutputStream()
self.Services.indexer.search(req, out)
response = SolrResult(ByteArrayInputStream(out.toByteArray()))
return response.getResults()
def findPackagesToTransition(self, fromWorkflowId, fromWorkflowStage):
req = SearchRequest("workflow_id:" + fromWorkflowId +
" AND _query_:\"workflow_step:" +
fromWorkflowStage + "\"")
req.setParam("fq", "owner:[* TO *]")
req.setParam("fq", "security_filter:[* TO *]")
out = ByteArrayOutputStream()
self.indexer.search(req, out)
solrResult = SolrResult(ByteArrayInputStream(out.toByteArray()))
return solrResult.getResults()
