def numberOfModifiedRecord(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
req = SearchRequest("modified:true")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
return self.__result.getString(None, "response", "numFound")
def __search(self, searchField):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(
self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(
self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
startRow = 0
numPerPage = 25
numFound = 0
req = SearchRequest(searchField)
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
objectIdList = []
while True:
req.addParam("fq", 'item_type:"object"')
req.addParam("rows", str(numPerPage))
req.addParam("start", str(startRow))
out = ByteArrayOutputStream()
indexer.search(req, out)
result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response", "docs")
docIds = []
for doc in docs:
docId = doc.getString(None, "storage_id")
if docId is not None:
docIds.append(docId)
objectIdList.extend(docs)
startRow += numPerPage
numFound = int(result.getString(None, "response", "numFound"))
if (startRow > numFound):
break
return objectIdList
def __getPackageTypeAndJsonConfigFile(self, packageType):
try:
types = JsonSimpleConfig().getJsonSimpleMap(["portal", "packageTypes"])
pt = None
if types is not None and not types.isEmpty():
pt = types.get(packageType)
if pt is None:
configFile = "packaging-config.json"
else:
configFile = pt.getString("packaging-config.json", ["jsonconfig"])
except Exception, e:
configFile = "packaging-config.json"
def __getPackageTypeAndJsonConfigFile(self, packageType):
try:
types = JsonSimpleConfig().getJsonSimpleMap(
["portal", "packageTypes"])
pt = None
if types is not None and not types.isEmpty():
pt = types.get(packageType)
if pt is None:
configFile = "packaging-config.json"
else:
configFile = pt.getString("packaging-config.json",
["jsonconfig"])
except Exception, e:
configFile = "packaging-config.json"
def __search(self, searchField):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
startRow = 0
numPerPage = 25
numFound = 0
req = SearchRequest(searchField)
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
objectIdList = []
while True:
req.addParam("fq", 'item_type:"object"')
req.addParam("rows", str(numPerPage))
req.addParam("start", str(startRow))
out = ByteArrayOutputStream()
indexer.search(req, out)
result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response", "docs")
docIds = []
for doc in docs:
docId = doc.getString(None, "storage_id")
if docId is not None:
docIds.append(docId)
objectIdList.extend(docs)
startRow += numPerPage
numFound = int(result.getString(None, "response", "numFound"))
if (startRow > numFound):
break
return objectIdList
def __getPackageTypeAndJsonConfigFile(self):
try:
packageType = self.vc("formData").get("packageType", "default")
if packageType == "":
packageType = "default"
types = JsonSimpleConfig().getJsonSimpleMap(["portal", "packageTypes"])
pt = None
if types is not None and not types.isEmpty():
pt = types.get(packageType)
if pt is None:
configFile = "packaging-config.json"
else:
configFile = pt.getString("packaging-config.json", ["jsonconfig"])
except Exception, e:
configFile = "packaging-config.json"
def getWatcherConfig(self):
watcherPath = self.sysConfig.getString("${fascinator.home}/watcher)",
["watcher", "path"])
configFile = File("%s/app/config.json" % watcherPath)
if configFile.exists():
return JsonSimpleConfig(configFile)
return None
class QueuesData:
def __init__(self):
pass
def __activate__(self, context):
self.request = context["request"]
self.response = context["response"]
self.formData = context["formData"]
if self.request.isXHR():
print " **** formData: %s" % self.formData
queue = self.formData.get("queueName")
msg = self.formData.get("queueMessage")
self.queueMessage(queue, msg)
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
out.println(self.formData)
out.close()
self.config = JsonSimpleConfig()
self.threads = self.config.getJsonSimpleList(["messaging", "threads"])
def getDescription(self, queue):
for thread in self.threads:
name = thread.getString(None, ["config", "name"])
if name == queue:
return thread.getString(None, ["description"])
def queueMessage(self, queue, msg):
ms = MessagingServices.getInstance()
ms.queueMessage(queue, msg)
ms.release()
def __activate__(self, context):
if self.tokensDB is None:
self.tokensDB = TokensDatabase(context)
# Set up configuration
self.systemConfig = JsonSimpleConfig()
self.oaiConfig = None
self.getMetadataFormats()
self.velocityContext = context
self.services = context["Services"]
self.log = context["log"]
self.sessionState = context["sessionState"]
self.portalDir = context["portalDir"]
self.__result = None
self.lastPage = False
# Check if the OAI request has an overriding portal ('set') to the URL
paramSet = self.vc("formData").get("set")
self.__portalName = context["page"].getPortal().getName()
illegalSet = False
if paramSet is not None:
portals = self.vc("page").getPortals().keySet()
if portals.contains(paramSet):
self.__portalName = paramSet
else:
illegalSet = True
self.__metadataPrefix = ""
self.__sessionExpiry = self.systemConfig.getInteger(None, ["portal", "oai-pmh", "sessionExpiry"])
# Check if there's a resumption token in the formData
self.__currentToken = None
resumptionToken = self.vc("formData").get("resumptionToken")
if resumptionToken is not None:
# This could still be be null
self.__currentToken = self.tokensDB.getToken(resumptionToken)
# Process/parse the request we've received for validity
self.vc("request").setAttribute("Content-Type", "text/xml")
self.__request = OaiPmhVerb(context, self.tokensDB, self.__currentToken)
if self.getError() is None and illegalSet:
self.__request.setError("badArgument", "Set '%s' is not valid!" % paramSet)
# If there are no errors... and the request requires some additional
# data (like a search result) do so now. Everything else can be
# handled in the templates.
if self.getError() is None and \
self.getVerb() in ["GetRecord", "ListIdentifiers", "ListRecords"]:
# Find the metadata prefix requested
self.__metadataPrefix = self.vc("formData").get("metadataPrefix")
if self.__metadataPrefix is None:
self.__metadataPrefix = self.__currentToken.getMetadataPrefix()
# Only list records if the metadata format is enabled in this view
if self.isInView(self.__metadataPrefix):
self.__search()
class ApiAdminData:
def __init__(self):
pass
def __activate__(self, context):
self.velocityContext = context
self.formData = self.velocityContext["formData"]
self.request = self.velocityContext["request"]
self.apiKeyService = ApplicationContextProvider.getApplicationContext().getBean("apiKeyTokenService")
self.systemConfig = JsonSimpleConfig()
self.log = self.velocityContext["log"]
if self.request.getMethod() == "POST":
if self.formData.get("action") == "Add":
self.add_key()
if self.formData.get("action") == "Remove":
self.remove_key()
if self.formData.get("action") == "Regenerate":
self.regenerate_key()
self.json = JsonSimple()
def parse_json(self, json_string):
self.json = JsonSimple(json_string)
def regenerate_key(self):
clientArray = self.getKeysArray()
name = self.formData.get("name")
clientToBeReplaced = None
index = 0
for client in clientArray:
if client.get("name") == name:
clientToBeReplaced = client
break
index = index + 1
if clientToBeReplaced is not None:
clientObject = JsonObject()
clientObject.put("name", self.formData.get("name"))
clientObject.put("key", self.get_random_key())
clientArray.set(index,clientObject)
self.apiKeyService.updateAndSaveKeys(clientArray)
def getKeysArray(self):
keysFile = FileUtils.getFile(self.systemConfig.getString("", "api", "apiKeyFile"))
keysFile.createNewFile()
try:
keysJsonSimple = JsonSimple(keysFile)
except IOException, ParseException:
self.log.warn("File may be blank. Creating empty json api keys file...")
FileUtils.writeStringToFile(keysFile, '{"api": {"clients": []}}')
keysJsonSimple = JsonSimple(keysFile)
clientArray = keysJsonSimple.getArray("api", "clients")
self.log.debug("client array is: {}", clientArray)
return clientArray
def __getPackageTypeAndJsonConfigFile(self):
try:
packageType = self.vc("formData").get("packageType", "default")
if packageType == "":
packageType = "default"
types = JsonSimpleConfig().getJsonSimpleMap(
["portal", "packageTypes"])
pt = None
if types is not None and not types.isEmpty():
pt = types.get(packageType)
if pt is None:
configFile = "packaging-config.json"
else:
self.vc("log").error("Getting pt: %s" % pt)
configFile = pt.getString("packaging-config.json",
["jsonconfig"])
self.vc("log").error("Getting configFile: %s" % configFile)
except Exception, e:
configFile = "packaging-config.json"
def __activate__(self, context):
self.velocityContext = context
self.formData = self.velocityContext["formData"]
self.request = self.velocityContext["request"]
self.apiKeyService = ApplicationContextProvider.getApplicationContext().getBean("apiKeyTokenService")
self.systemConfig = JsonSimpleConfig()
self.log = self.velocityContext["log"]
if self.request.getMethod() == "POST":
if self.formData.get("action") == "Add":
self.add_key()
if self.formData.get("action") == "Remove":
self.remove_key()
if self.formData.get("action") == "Regenerate":
self.regenerate_key()
self.json = JsonSimple()
def __activate__(self, context):
self.request = context["request"]
self.response = context["response"]
self.formData = context["formData"]
if self.request.isXHR():
print " **** formData: %s" % self.formData
queue = self.formData.get("queueName")
msg = self.formData.get("queueMessage")
self.queueMessage(queue, msg)
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
out.println(self.formData)
out.close()
self.config = JsonSimpleConfig()
self.threads = self.config.getJsonSimpleList(["messaging", "threads"])
def __activate__(self, context):
self.sysConfig = JsonSimpleConfig()
self.velocityContext = context
self.log = context["log"]
self.__meta = {}
formData = self.vc("formData")
self.isAjax = formData.get("ajax") != None
if self.isAjax:
ok = JsonObject()
ok.put("ok", "OK")
self.json = ok.toString()
else:
self.json = ""
self.__selectedPackageType = formData.get("packageType", "default")
self.log.debug("formData = %s" % self.vc("formData"))
self.log.debug("selectedPackageType = '%s'" %
self.__selectedPackageType)
self.__meta["packageType"] = formData.get("packageType", "default")
self.__meta["description"] = formData.get("description", "")
def getWatcherFile(self):
configFile = FascinatorHome.getPathFile("watcher/config.json")
if configFile.exists():
return JsonSimpleConfig(configFile)
return None
class OaiData:
def __init__(self):
self.tokensDB = None
def __activate__(self, context):
if self.tokensDB is None:
self.tokensDB = TokensDatabase(context)
# Set up configuration
self.systemConfig = JsonSimpleConfig()
self.oaiConfig = None
self.getMetadataFormats()
self.velocityContext = context
self.services = context["Services"]
self.log = context["log"]
self.sessionState = context["sessionState"]
self.portalDir = context["portalDir"]
self.__result = None
self.lastPage = False
# Check if the OAI request has an overriding portal ('set') to the URL
paramSet = self.vc("formData").get("set")
self.__portalName = context["page"].getPortal().getName()
illegalSet = False
if paramSet is not None:
portals = self.vc("page").getPortals().keySet()
if portals.contains(paramSet):
self.__portalName = paramSet
else:
illegalSet = True
self.__metadataPrefix = ""
self.__sessionExpiry = self.systemConfig.getInteger(None, ["portal", "oai-pmh", "sessionExpiry"])
# Check if there's a resumption token in the formData
self.__currentToken = None
resumptionToken = self.vc("formData").get("resumptionToken")
if resumptionToken is not None:
# Split out the start component from the actual resumption token
(resumptionTokenPart, start) = resumptionToken.strip().split(":")
# This could still be be null
self.__currentToken = self.tokensDB.getToken(resumptionTokenPart)
# Code to handle null token is handled later on
if self.__currentToken is not None:
self.__currentToken.setStart(start)
# Process/parse the request we've received for validity
self.vc("request").setAttribute("Content-Type", "text/xml")
self.__request = OaiPmhVerb(context, self.tokensDB, self.__currentToken)
if self.getError() is None and illegalSet:
self.__request.setError("badArgument", "Set '%s' is not valid!" % paramSet)
# If there are no errors... and the request requires some additional
# data (like a search result) do so now. Everything else can be
# handled in the templates.
if self.getError() is None and self.getVerb() in ["GetRecord", "ListIdentifiers", "ListRecords"]:
# Find the metadata prefix requested
self.__metadataPrefix = self.vc("formData").get("metadataPrefix")
if self.__metadataPrefix is None:
self.__metadataPrefix = self.__currentToken.getMetadataPrefix()
# Only list records if the metadata format is enabled in this view
if self.isInView(self.__metadataPrefix):
self.__search()
# Get from velocity context
def vc(self, index):
if self.velocityContext[index] is not None:
return self.velocityContext[index]
else:
self.log.error("ERROR: Requested context entry '" + index + "' doesn't exist")
return None
def isInView(self, format, view=None):
# Sanity check
if format is None or format == "":
return False
# Default to current poral
if view is None:
view = self.__portalName
# Make sure there is some config for this format
formatConfig = self.getMetadataFormats().get(format)
if formatConfig is None:
return False
# Is it visible everywhere?
allViews = formatConfig.getBoolean(False, ["enabledInAllViews"])
if allViews:
return True
# Check if it is visible in this view
else:
allowedViews = formatConfig.getStringList(["enabledViews"])
if view in allowedViews:
return True
# Rejection
return False
def getID(self, item):
identifier = item.getFirst("oai_identifier")
# Fallback to the default
if identifier is None or identifier == "":
return "oai:fascinator.usq.edu.au:" + item.getFirst("id")
# Use the indexed value
return identifier
def isDeleted(self, item):
return bool(item.getFirst("oai_deleted"))
def getSet(self, item):
set = item.getFirst("oai_set")
# Fallback to the portal name
if set is None or set == "":
return self.__portalName
# Use the required set
return set
def getVerb(self):
return self.getRequest().getVerb()
def getError(self):
return self.getRequest().getError()
def getResponseDate(self):
return time.strftime("%Y-%m-%dT%H:%M:%SZ")
def getRequest(self):
return self.__request
def getResult(self):
return self.__result
def getElement(self, elementName, values):
elementStr = ""
if values:
for value in values:
elementStr += "<%s>%s</%s>" % (elementName, value, elementName)
return elementStr
def __search(self):
self.__result = SolrResult(None)
portal = self.services.getPortalManager().get(self.__portalName)
recordsPerPage = portal.recordsPerPage
# Resolve our identifier
id = self.vc("formData").get("identifier")
query = "*:*"
if id is not None and id != "":
# A default TF2 OID
if id.startswith("oai:fascinator.usq.edu.au:"):
idString = id.replace("oai:fascinator.usq.edu.au:", "")
idString = self.__escapeQuery(idString)
query = "id:" + idString
# Or a custom OAI ID
else:
idString = self.__escapeQuery(id)
query = "oai_identifier:" + idString
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", "item_type:object")
# Date data... is supplied
fromDate = self.__request.getFromDate()
untilDate = self.__request.getUntilDate()
if fromDate is not None:
fromStr = fromDate.isoformat() + "Z"
self.log.debug("From Date: '{}'", fromStr)
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilStr)
queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr)
else:
queryStr = "last_modified:[%s TO *]" % (fromStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
else:
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilDate.isoformat())
queryStr = "last_modified:[* TO %s]" % (untilStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
# Check if there's resumption token exist in the formData
newToken = None
if self.__currentToken is not None:
start = int(self.__currentToken.getStart())
totalFound = int(self.__currentToken.getTotalFound())
nextTokenStart = start + recordsPerPage
if nextTokenStart < totalFound:
newToken = self.__currentToken
newToken.resetExpiry(self.__sessionExpiry)
newToken.setStart(nextTokenStart)
# or start a new resumption token
else:
start = 0
newToken = ResumptionToken(None, recordsPerPage, self.__metadataPrefix, self.__sessionExpiry)
req.setParam("start", str(start))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
totalFound = self.__result.getNumFound()
if totalFound == 0:
newToken = None
# If an ID was requested, and not found, this is an error
if id is not None and id != "":
self.__request.setError("idDoesNotExist", "ID: '%s' not found" % id)
else:
self.__request.setError("noRecordsMatch", "No records match this request")
# We need to store this for NEW tokens
elif self.__currentToken is None:
# Assuming there are enough results to even keep the token
if newToken.getStart() < totalFound:
newToken.setTotalFound(totalFound)
else:
newToken = None
# Check if we need to remove the resumption token
else:
if (start + recordsPerPage) >= totalFound:
self.tokensDB.removeToken(self.__currentToken)
self.lastPage = True
# Store/update the resumption token
if newToken is not None:
# Brand new token
if self.__currentToken is None:
self.tokensDB.storeToken(newToken)
# Or update an old token
else:
self.tokensDB.updateToken(newToken)
self.__currentToken = newToken
def getToken(self):
if self.isInView(self.__metadataPrefix) and not self.lastPage:
return self.__currentToken
return None
def getMetadataFormats(self):
if self.oaiConfig is None:
self.oaiConfig = self.systemConfig.getJsonSimpleMap(["portal", "oai-pmh", "metadataFormats"])
return self.oaiConfig
def encodeXml(self, string):
return StringEscapeUtils.escapeXml(string)
def getPayload(self, oid, metadataFileName):
# First get the Object from storage
object = None
try:
object = self.services.getStorage().getObject(oid)
except StorageException, e:
return None
# Check whether the payload exists
try:
return object.getPayload(metadataFileName)
except StorageException, e:
return None
def __init__(self, context, tokenDB, currentToken):
self.log = context["log"]
self.config = JsonSimpleConfig()
formData = context["formData"]
self.__error = None
self.__verb = formData.get("verb")
self.__metadataFormats = self.__metadataFormatList()
self.__fromDate = None
self.__untilDate = None
self.log.debug(" * OAI Verb = '{}'", self.__verb)
# No verb provided
if self.__verb is None:
self.__error = OaiPmhError("badVerb", "No verb was specified")
# Some verbs require additional data
elif self.__verb in ["GetRecord", "ListIdentifiers", "ListRecords"]:
# What format metadata is requested?
self.__metadataPrefix = formData.get("metadataPrefix")
if self.__metadataPrefix is None:
# No metadata supplied, error...
# unless the are resuming an earlier request
if currentToken is not None:
# Make sure the resumption token hasn't expired
if currentToken.getExpiry() > System.currentTimeMillis():
# And retrieve the metadata prefix from the last request
self.__metadataPrefix = currentToken.getMetadataPrefix()
# Expired token, make sure it's not in the database anymore
else:
self.log.error("Using an expired token")
self.__error = OaiPmhError("badResumptionToken", "Token has expired")
success = tokenDB.removeToken(currentToken)
if not success:
self.log.error("Error removing expired token!")
# No prefix and no token. We're done
else:
attemptedToken = context["formData"].get("resumptionToken")
# Either they used an invalid token
if attemptedToken is not None:
self.log.error("Illegal resumption token: '{}'", attemptedToken)
self.__error = OaiPmhError("badResumptionToken", "Illegal resumption token")
# Or were missing their metadata prefix
else:
self.log.error("No metadata prefix supplied, and no token")
self.__error = OaiPmhError("badArgument", "Metadata prefix required")
# These verbs require a metadata format... and we must be able to support it
elif self.__metadataPrefix not in self.__metadataFormats:
self.log.error("Metadata prefix is not valid for this view")
self.__error = OaiPmhError(
"cannotDisseminateFormat", "Record not available as metadata type: %s" % self.__metadataPrefix
)
# These verbs allow for date limits, validate them if found
if self.__verb in ["ListIdentifiers", "ListRecords"]:
# From date
fromStr = context["formData"].get("from")
fromTIndex = None
if fromStr is not None:
fromTIndex = fromStr.find("T")
# Basic dates
try:
if fromTIndex == -1:
self.__fromDate = datetime.strptime(fromStr, "%Y-%m-%d")
# Or datetimes
else:
self.__fromDate = datetime.strptime(fromStr, "%Y-%m-%dT%H:%M:%SZ")
except:
self.log.error("Invalid FROM date: '{}'", fromStr)
self.__error = OaiPmhError("badArgument", "From date not in valid format!")
return
# Until Date
untilStr = context["formData"].get("until")
if untilStr is not None:
untilTIndex = untilStr.find("T")
# Granularity mismatches
if (fromTIndex is not None) and (
(fromTIndex == -1 and untilTIndex != -1) or (fromTIndex != -1 and untilTIndex == -1)
):
self.log.error("Date granularity mismatch: '{}' vs '{}'", fromStr, untilStr)
self.__error = OaiPmhError("badArgument", "Date granularity mismatch")
return
# Basic dates
try:
if untilTIndex == -1:
self.__untilDate = datetime.strptime(untilStr, "%Y-%m-%d")
# Or datetimes
else:
self.__untilDate = datetime.strptime(untilStr, "%Y-%m-%dT%H:%M:%SZ")
except:
self.log.error("Invalid UNTIL date: '{}'", untilStr)
self.__error = OaiPmhError("badArgument", "Until date not in valid format!")
return
# Sanity check
if self.__fromDate is not None and self.__untilDate is not None:
if self.__fromDate > self.__untilDate:
self.log.error("FROM date > UNTIL date: '{}' > '{}'", fromStr, untilStr)
self.__error = OaiPmhError("badArgument", "From date cannot be later then Until date!")
return
# Check for a valid identifier
if self.__verb == "GetRecord":
id = context["formData"].get("identifier")
if id is None or id == "":
self.log.error("GetRecord missing an identifier")
self.__error = OaiPmhError("badArgument", "Identifier required")
return
# Basic verbs we will respond to easily
elif self.__verb in ["Identify", "ListMetadataFormats", "ListSets"]:
pass
# Invalid verb
else:
self.log.error("Invalid verb provided: '{}'", self.__verb)
self.__error = OaiPmhError("badVerb", "Unknown verb: '%s'" % self.__verb)
def __init__(self):
self.sysConfig = JsonSimpleConfig()
class SettingsData:
def __init__(self):
self.sysConfig = JsonSimpleConfig()
def __activate__(self, context):
self.velocityContext = context
self.__portal = None
action = self.vc("formData").get("verb")
if action == "create_view":
fq = [
q for q in self.vc("sessionState").get("fq")
if q != 'item_type:"object"'
]
if fq == []:
name = "new"
desc = "New View"
query = ""
else:
name = ""
desc = ""
query = str(" ".join(fq))
newPortal = Portal(name, desc, query)
newPortal.setFacetFields(
Services.portalManager.default.facetFields)
newPortal.setQuery(query)
self.__portal = newPortal
else:
portalName = self.vc("formData").get("portalName")
print " * settings.py: portalName=%s, portalId=%s" % (
portalName, self.vc("portalId"))
if portalName is None or (self.vc("formData").get("portalAction")
== "Cancel"):
self.__portal = Services.portalManager.get(self.vc("portalId"))
else:
self.__portal = Portal()
self.__portal.name = portalName
Services.portalManager.add(self.__portal)
if self.vc("formData").get("portalAction") == "Update":
self.__updatePortal()
if self.vc("formData").get("emailAction") == "Update":
self.__updateEmail()
# Get from velocity context
def vc(self, index):
if self.velocityContext[index] is not None:
return self.velocityContext[index]
else:
log.error("ERROR: Requested context entry '" + index +
"' doesn't exist")
return None
def isSelected(self, category):
selected = self.vc("sessionState").get("settingsCategory")
if category == selected:
return "selected"
return ""
def __updatePortal(self):
self.__portal.name = self.vc("formData").get("portalName")
self.__portal.description = self.vc("formData").get(
"portalDescription")
self.__portal.query = self.vc("formData").get("portalQuery")
self.__portal.recordsPerPage = int(
self.vc("formData").get("portalRecordsPerPage"))
self.__portal.facetCount = int(
self.vc("formData").get("portalFacetLimit"))
self.__portal.facetSort = self.vc("formData").get(
"portalFacetSort") is not None
facetFields = self.__portal.facetFields
facetFields.clear()
size = int(self.vc("formData").get("portalFacetSize"))
for i in range(1, size + 2):
nameKey = "portalFacet_%s_name" % i
labelKey = "portalFacet_%s_label" % i
name = self.vc("formData").get(nameKey)
label = self.vc("formData").get(labelKey)
print "facet: key: %s, label: %s" % (name, label)
if name is not None and label is not None:
facetFields.put(name, label)
self.__portal.setFacetFields(facetFields)
sortFields = self.__portal.sortFields
sortFields.clear()
size = int(self.vc("formData").get("portalSortSize"))
for i in range(1, size + 2):
nameKey = "portalSort_%s_name" % i
labelKey = "portalSort_%s_label" % i
name = self.vc("formData").get(nameKey)
label = self.vc("formData").get(labelKey)
print "sort: key: %s, label: %s" % (name, label)
if name is not None and label is not None:
sortFields.put(name, label)
self.__portal.setSortFields(sortFields)
Services.portalManager.save(self.__portal)
def getPortal(self):
return self.__portal
def getIndexerPlugins(self):
return PluginManager.getIndexerPlugins()
def getStoragePlugins(self):
return PluginManager.getStoragePlugins()
def getHarvesterPlugins(self):
return PluginManager.getHarvesterPlugins()
def getTransformerPlugins(self):
return PluginManager.getTransformerPlugins()
def getWatcherConfig(self):
watcherPath = self.sysConfig.getString("${fascinator.home}/watcher)",
["watcher", "path"])
configFile = File("%s/app/config.json" % watcherPath)
if configFile.exists():
return JsonSimpleConfig(configFile)
return None
def getEmail(self):
return self.sysConfig.getString(None, ["email"])
def getTimeout(self):
return self.sysConfig.getString(
None, ["portal", "houseKeeping", "config", "frequency"])
def getFacetDisplays(self):
facetDisplays = self.__portal.getObject(["portal", "facet-displays"])
if facetDisplays is None or facetDisplays.isEmpty():
facetDisplays = HashMap()
facetDisplays.put("list", "List menu")
facetDisplays.put("tree", "Dynamic tree")
return facetDisplays
def __activate__(self, context):
if self.tokensDB is None:
self.tokensDB = TokensDatabase(context)
# Set up configuration
self.systemConfig = JsonSimpleConfig()
self.oaiConfig = None
self.getMetadataFormats()
self.velocityContext = context
self.services = context["Services"]
self.log = context["log"]
self.sessionState = context["sessionState"]
self.portalDir = context["portalDir"]
self.__result = None
self.lastPage = False
# Check if the OAI request has an overriding portal ('set') to the URL
paramSet = self.vc("formData").get("set")
self.__portalName = context["page"].getPortal().getName()
illegalSet = False
if paramSet is not None:
portals = self.vc("page").getPortals().keySet()
if portals.contains(paramSet):
self.__portalName = paramSet
else:
illegalSet = True
self.__metadataPrefix = ""
self.__sessionExpiry = self.systemConfig.getInteger(
None, ["portal", "oai-pmh", "sessionExpiry"])
# Check if there's a resumption token in the formData
self.__currentToken = None
resumptionToken = self.vc("formData").get("resumptionToken")
if resumptionToken is not None:
token = self.tokensDB.getToken(resumptionToken)
self.__currentToken = token
# Process/parse the request we've received for validity
self.vc("request").setAttribute("Content-Type", "text/xml")
self.__request = OaiPmhVerb(context, self.tokensDB,
self.__currentToken)
if self.getError() is None and illegalSet:
self.__request.setError("badArgument",
"Set '%s' is not valid!" % paramSet)
# If there are no errors... and the request requires some additional
# data (like a search result) do so now. Everything else can be
# handled in the templates.
if self.getError() is None and \
self.getVerb() in ["GetRecord", "ListIdentifiers", "ListRecords"]:
# Find the metadata prefix requested
self.__metadataPrefix = self.vc("formData").get("metadataPrefix")
if self.__metadataPrefix is None:
self.__metadataPrefix = self.__currentToken.getMetadataPrefix()
if resumptionToken is None:
self.__buildResumptionTokenSets()
else:
self.__result = SolrResult(self.__currentToken.getResultJson())
# Only list records if the metadata format is enabled in this view
if self.isInView(self.__metadataPrefix) == False:
self.__result = None
class OaiData:
def __init__(self):
self.tokensDB = None
def __activate__(self, context):
if self.tokensDB is None:
self.tokensDB = TokensDatabase(context)
# Set up configuration
self.systemConfig = JsonSimpleConfig()
self.oaiConfig = None
self.getMetadataFormats()
self.velocityContext = context
self.services = context["Services"]
self.log = context["log"]
self.sessionState = context["sessionState"]
self.portalDir = context["portalDir"]
self.__result = None
self.lastPage = False
# Check if the OAI request has an overriding portal ('set') to the URL
paramSet = self.vc("formData").get("set")
self.__portalName = context["page"].getPortal().getName()
illegalSet = False
if paramSet is not None:
portals = self.vc("page").getPortals().keySet()
if portals.contains(paramSet):
self.__portalName = paramSet
else:
illegalSet = True
self.__metadataPrefix = ""
self.__sessionExpiry = self.systemConfig.getInteger(
None, ["portal", "oai-pmh", "sessionExpiry"])
# Check if there's a resumption token in the formData
self.__currentToken = None
resumptionToken = self.vc("formData").get("resumptionToken")
if resumptionToken is not None:
token = self.tokensDB.getToken(resumptionToken)
self.__currentToken = token
# Process/parse the request we've received for validity
self.vc("request").setAttribute("Content-Type", "text/xml")
self.__request = OaiPmhVerb(context, self.tokensDB,
self.__currentToken)
if self.getError() is None and illegalSet:
self.__request.setError("badArgument",
"Set '%s' is not valid!" % paramSet)
# If there are no errors... and the request requires some additional
# data (like a search result) do so now. Everything else can be
# handled in the templates.
if self.getError() is None and \
self.getVerb() in ["GetRecord", "ListIdentifiers", "ListRecords"]:
# Find the metadata prefix requested
self.__metadataPrefix = self.vc("formData").get("metadataPrefix")
if self.__metadataPrefix is None:
self.__metadataPrefix = self.__currentToken.getMetadataPrefix()
if resumptionToken is None:
self.__buildResumptionTokenSets()
else:
self.__result = SolrResult(self.__currentToken.getResultJson())
# Only list records if the metadata format is enabled in this view
if self.isInView(self.__metadataPrefix) == False:
self.__result = None
# Get from velocity context
def vc(self, index):
if self.velocityContext[index] is not None:
return self.velocityContext[index]
else:
self.log.error("ERROR: Requested context entry '" + index +
"' doesn't exist")
return None
def isInView(self, format, view=None):
# Sanity check
if format is None or format == "":
return False
# Default to current poral
if view is None:
view = self.__portalName
# Make sure there is some config for this format
formatConfig = self.getMetadataFormats().get(format)
if formatConfig is None:
return False
# Is it visible everywhere?
allViews = formatConfig.getBoolean(False, ["enabledInAllViews"])
if allViews:
return True
# Check if it is visible in this view
else:
allowedViews = formatConfig.getStringList(["enabledViews"])
if view in allowedViews:
return True
# Rejection
return False
def getID(self, item):
identifier = item.getFirst("oai_identifier")
# Fallback to the default
if identifier is None or identifier == "":
return "oai:fascinator.usq.edu.au:" + item.getFirst("id")
# Use the indexed value
return identifier
def isDeleted(self, item):
return bool(item.getFirst("oai_deleted"))
def getSet(self, item):
set = item.getFirst("oai_set")
# Fallback to the portal name
if set is None or set == "":
return self.__portalName
# Use the required set
return set
def getVerb(self):
return self.getRequest().getVerb()
def getError(self):
return self.getRequest().getError()
def getResponseDate(self):
return time.strftime("%Y-%m-%dT%H:%M:%SZ")
def getRequest(self):
return self.__request
def getResult(self):
return self.__result
def getElement(self, elementName, values):
elementStr = ""
if values:
for value in values:
elementStr += "<%s>%s</%s>" % (elementName, value, elementName)
return elementStr
def __buildResumptionTokenSets(self):
self.__result = SolrResult(None)
portal = self.services.getPortalManager().get(self.__portalName)
recordsPerPage = portal.recordsPerPage
# Resolve our identifier
id = self.vc("formData").get("identifier")
query = "*:*"
if id is not None and id != "":
# A default TF2 OID
if id.startswith("oai:fascinator.usq.edu.au:"):
idString = id.replace("oai:fascinator.usq.edu.au:", "")
idString = self.__escapeQuery(idString)
query = "id:" + idString
# Or a custom OAI ID
else:
idString = self.__escapeQuery(id)
query = "oai_identifier:" + idString
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", "item_type:object")
# Date data... is supplied
fromDate = self.__request.getFromDate()
untilDate = self.__request.getUntilDate()
if fromDate is not None:
fromStr = fromDate.isoformat() + "Z"
self.log.debug("From Date: '{}'", fromStr)
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilStr)
queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr)
else:
queryStr = "last_modified:[%s TO *]" % (fromStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
else:
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilDate.isoformat())
queryStr = "last_modified:[* TO %s]" % (untilStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
# Check if there's resumption token exist in the formData
start = 0
req.setParam("start", str(start))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
totalFound = self.__result.getNumFound()
self.log.debug("Total found:" + str(totalFound))
if totalFound > recordsPerPage:
startRow = 0
random.seed()
resumptionToken = "%016x" % random.getrandbits(128)
nextResumptionToken = resumptionToken
firstLoop = True
while True:
self.log.debug("Current Resumption Token: " + resumptionToken)
req.setParam("start", str(startRow))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
tokenObject = ResumptionToken(resumptionToken,
self.__metadataPrefix,
nextResumptionToken,
result.toString())
if firstLoop:
self.__currentToken = ResumptionToken(
None, self.__metadataPrefix, resumptionToken, None)
tokenObject = None
firstLoop = False
startRow = startRow + recordsPerPage
if startRow > totalFound:
tokenObject = ResumptionToken(resumptionToken,
self.__metadataPrefix, "",
result.toString())
self.tokensDB.storeToken(tokenObject)
break
if tokenObject is not None:
self.tokensDB.storeToken(tokenObject)
resumptionToken = nextResumptionToken
nextResumptionToken = "%016x" % random.getrandbits(128)
self.log.debug("Resumption Token: " + resumptionToken +
" next resumption token:" + nextResumptionToken)
def getToken(self):
if self.isInView(self.__metadataPrefix) and not self.lastPage:
return self.__currentToken
return None
def getMetadataFormats(self):
if self.oaiConfig is None:
self.oaiConfig = self.systemConfig.getJsonSimpleMap(
["portal", "oai-pmh", "metadataFormats"])
return self.oaiConfig
def encodeXml(self, string):
return StringEscapeUtils.escapeXml(string)
def getPayload(self, oid, metadataFileName):
# First get the Object from storage
object = None
try:
object = self.services.getStorage().getObject(oid)
except StorageException, e:
return None
# Check whether the payload exists
try:
return object.getPayload(metadataFileName)
except StorageException, e:
return None
def __init__(self, context, tokenDB, currentToken):
self.log = context["log"]
self.config = JsonSimpleConfig()
formData = context["formData"]
self.__error = None
self.__verb = formData.get("verb")
self.__metadataFormats = self.__metadataFormatList()
self.__fromDate = None
self.__untilDate = None
self.log.debug(" * OAI Verb = '{}'", self.__verb)
# No verb provided
if self.__verb is None:
self.__error = OaiPmhError("badVerb", "No verb was specified")
# Some verbs require additional data
elif self.__verb in ["GetRecord", "ListIdentifiers", "ListRecords"]:
# What format metadata is requested?
self.__metadataPrefix = formData.get("metadataPrefix")
if self.__metadataPrefix is None:
# No metadata supplied, error...
# unless the are resuming an earlier request
if currentToken is not None:
# Make sure the resumption token hasn't expired
if currentToken.getExpiry() > System.currentTimeMillis():
# And retrieve the metadata prefix from the last request
self.__metadataPrefix = currentToken.getMetadataPrefix(
)
# Expired token, make sure it's not in the database anymore
else:
self.log.error("Using an expired token")
self.__error = OaiPmhError("badResumptionToken",
"Token has expired")
success = tokenDB.removeToken(currentToken)
if not success:
self.log.error("Error removing expired token!")
# No prefix and no token. We're done
else:
attemptedToken = context["formData"].get("resumptionToken")
# Either they used an invalid token
if attemptedToken is not None:
self.log.error("Illegal resumption token: '{}'",
attemptedToken)
self.__error = OaiPmhError("badResumptionToken",
"Illegal resumption token")
# Or were missing their metadata prefix
else:
self.log.error(
"No metadata prefix supplied, and no token")
self.__error = OaiPmhError("badArgument",
"Metadata prefix required")
# These verbs require a metadata format... and we must be able to support it
elif self.__metadataPrefix not in self.__metadataFormats:
self.log.error("Metadata prefix is not valid for this view")
self.__error = OaiPmhError(
"cannotDisseminateFormat",
"Record not available as metadata type: %s" %
self.__metadataPrefix)
# These verbs allow for date limits, validate them if found
if self.__verb in ["ListIdentifiers", "ListRecords"]:
# From date
fromStr = context["formData"].get("from")
fromTIndex = None
if fromStr is not None:
fromTIndex = fromStr.find("T")
# Basic dates
try:
if fromTIndex == -1:
self.__fromDate = datetime.strptime(
fromStr, "%Y-%m-%d")
# Or datetimes
else:
self.__fromDate = datetime.strptime(
fromStr, "%Y-%m-%dT%H:%M:%SZ")
except:
self.log.error("Invalid FROM date: '{}'", fromStr)
self.__error = OaiPmhError(
"badArgument", "From date not in valid format!")
return
# Until Date
untilStr = context["formData"].get("until")
if untilStr is not None:
untilTIndex = untilStr.find("T")
# Granularity mismatches
if (fromTIndex is not None) and \
((fromTIndex == -1 and untilTIndex != -1) or \
(fromTIndex != -1 and untilTIndex == -1)):
self.log.error(
"Date granularity mismatch: '{}' vs '{}'", fromStr,
untilStr)
self.__error = OaiPmhError(
"badArgument", "Date granularity mismatch")
return
# Basic dates
try:
if untilTIndex == -1:
self.__untilDate = datetime.strptime(
untilStr, "%Y-%m-%d")
# Or datetimes
else:
self.__untilDate = datetime.strptime(
untilStr, "%Y-%m-%dT%H:%M:%SZ")
except:
self.log.error("Invalid UNTIL date: '{}'", untilStr)
self.__error = OaiPmhError(
"badArgument", "Until date not in valid format!")
return
# Sanity check
if self.__fromDate is not None and self.__untilDate is not None:
if self.__fromDate > self.__untilDate:
self.log.error("FROM date > UNTIL date: '{}' > '{}'",
fromStr, untilStr)
self.__error = OaiPmhError(
"badArgument",
"From date cannot be later then Until date!")
return
# Check for a valid identifier
if self.__verb == "GetRecord":
id = context["formData"].get("identifier")
if id is None or id == "":
self.log.error("GetRecord missing an identifier")
self.__error = OaiPmhError("badArgument",
"Identifier required")
return
# Basic verbs we will respond to easily
elif self.__verb in ["Identify", "ListMetadataFormats", "ListSets"]:
pass
# Invalid verb
else:
self.log.error("Invalid verb provided: '{}'", self.__verb)
self.__error = OaiPmhError("badVerb",
"Unknown verb: '%s'" % self.__verb)
def process(self):
self.log.debug(" * settings.py: formData={}", self.vc("formData"))
valid = self.vc("page").csrfSecurePage()
if not valid:
self.throw_error("Invalid request")
return
result = "{}"
portalManager = self.vc("Services").getPortalManager()
portal = portalManager.get(self.vc("portalId"))
func = self.vc("formData").get("func")
if func == "view-update":
portal.setDescription(self.vc("formData").get("view-description"))
portal.setQuery(self.vc("formData").get("view-query"))
portal.setSearchQuery(self.vc("formData").get("view-search-query"))
portal.setRecordsPerPage(int(self.vc("formData").get("view-records-per-page")))
portal.setFacetCount(int(self.vc("formData").get("view-facet-count")))
portal.setFacetDisplay(int(self.vc("formData").get("view-facet-display")))
portal.setFacetSort(self.vc("formData").get("view-facet-sort") is not None)
portalManager.save(portal)
elif func == "general-update":
config = JsonSimpleConfig()
email = StringUtils.trimToEmpty(self.vc("formData").get("general-email"))
systemEmail = StringUtils.trimToEmpty(config.getString(None, ["email"]))
if systemEmail != email:
obj = config.writableSystemConfig()
obj.put("email", self.vc("formData").get("general-email"))
obj.put("configured", "true")
config.storeSystemConfig()
# mark restart
self.vc("Services").getHouseKeepingManager().requestUrgentRestart()
else:
self.log.debug(" * settings.py: email not updated: did not change")
self.throw_error("Email address is the same! No change saved.")
elif func == "facets-update":
portal.getObject(["portal"]).remove("facet-fields")
fields = self.vc("formData").getValues("field")
labels = self.vc("formData").getValues("label")
displays = self.vc("formData").getValues("display")
deletes = self.vc("formData").getValues("delete")
for i in range(0, len(fields)):
field = fields[i]
if deletes[i] == "false":
node = portal.writeObject(["portal", "facet-fields", field])
node.put("label", labels[i])
node.put("display", displays[i])
portalManager.save(portal)
elif func == "sort-update":
portal.getObject(["portal"]).remove("sort-fields")
default = self.vc("formData").get("default")
if default:
portal.setSortFieldDefault(default)
order = self.vc("formData").get("order")
if order:
portal.setSortFieldDefaultOrder(order)
fields = self.vc("formData").getValues("field")
labels = self.vc("formData").getValues("label")
deletes = self.vc("formData").getValues("delete")
for i in range(0, len(fields)):
field = fields[i]
if deletes[i] == "false":
node = portal.writeObject(["portal", "sort-fields"])
node.put(field, labels[i])
portalManager.save(portal)
elif func == "watcher-update":
configFile = self.getWatcherFile()
if configFile is not None:
json = JsonSimpleConfig(configFile)
pathIds = self.vc("formData").get("pathIds").split(",")
actives = self.vc("formData").getValues("watcher-active")
if actives is None:
actives = []
deletes = self.vc("formData").getValues("watcher-delete")
if deletes is None:
deletes = []
for pathId in pathIds:
if pathId not in deletes:
path = self.vc("formData").get("%s-path" % pathId)
stopped = str(pathId not in actives).lower()
watchDir = json.writeObject(["watcher", "watchDirs", path])
watchDir.put("ignoreFileFilter", self.vc("formData").get("%s-file" % pathId))
watchDir.put("ignoreDirectories", self.vc("formData").get("%s-dir" % pathId))
watchDir.put("stopped", stopped)
json.writeArray(["watcher", "watchDirs", path, "cxtTags"])
writer = FileWriter(configFile)
writer.write(json.toString(True))
writer.close()
else:
result = "The Watcher is not installed properly."
elif func == "restore-default-config":
# backup the file
JsonSimpleConfig.backupSystemFile()
# delete the file
JsonSimpleConfig.getSystemFile().delete()
# restore default
JsonSimpleConfig.getSystemFile()
# mark restart
self.vc("Services").getHouseKeepingManager().requestUrgentRestart()
elif func == "housekeeping-update":
config = JsonSimpleConfig()
freq = StringUtils.trimToEmpty(self.vc("formData").get("housekeeping-timeout"))
systemFreq = StringUtils.trimToEmpty(config.getString(None, ["portal", "houseKeeping", "config", "frequency"]))
result = "House Keeper refreshed"
if systemFreq != freq:
# Get writeable access to underlying system
sysConfig = JsonSimple(config.writableSystemConfig())
# Modify the house keeping node
hkConfig = sysConfig.writeObject(["portal", "houseKeeping", "config"])
hkConfig.put("frequency", freq)
# Write the underlying config back to disk
config.storeSystemConfig()
result = "Frequency updated, refreshing House Keeper"
# Refresh the HouseKeeper
message = JsonObject()
message.put("type", "refresh")
self.vc("Services").getHouseKeepingManager().sendMessage(message.toString())
self.writer.println(result)
self.writer.close()
class BatchprocessData:
def __init__(self):
pass
def __activate__(self, context):
self.formData = context["formData"]
self.response = context["response"]
self.services = context["Services"]
self.page = context["page"]
self.portal = self.page.getPortal()
self.vc = context["toolkit"]
func = self.formData.get("func")
if func == "num-modified":
writer = self.response.getPrintWriter("text/plain; charset=UTF-8")
writer.println(self.numberOfModifiedRecord())
writer.close()
def renderUpdateForm(self):
return self.__createBatchForm("update")
def renderExportForm(self):
return self.__createBatchForm("export")
def __createBatchForm(self, processName):
self.formRenderer = self.vc.getFormRenderer()
form = "<form id='%s-form' method='post'>\n" \
"<fieldset class='login'>\n" \
"<legend>Batch %s script file</legend>\n" % (processName, processName)
form += self.formRenderer.ajaxFluidErrorHolder("%s-script-file" % processName) + "\n"
if self.__scriptList(processName) != {}:
form += self.formRenderer.renderFormSelect("%s-script-file" % processName, \
"Batch %s script:" % processName, self.__scriptList(processName))
form += "<div><br/>"
form += self.formRenderer.renderFormElement("%s-upload" % processName, "button", "", "Batch %s" % processName)
form += self.formRenderer.renderFormElement("%s-cancel" % processName, "button", "", "Cancel")
form += self.formRenderer.ajaxProgressLoader("%s-script-file" % processName)
form += "</div>"
else:
form += "<div>There is no script available to do batch processing.</div>"
form += "</fieldset></form>\n"
return form
def __scriptList(self, processName):
scriptDir = "%s/batch-process/%s" % (os.environ.get("TF_HOME"), processName)
scriptDic = {}
if os.path.isdir(scriptDir):
scriptFiles = os.listdir(scriptDir)
if scriptFiles:
for script in scriptFiles:
scriptFilePath = "%s/%s" % (scriptDir, script)
scriptDic[scriptFilePath] = script
return scriptDic
def numberOfModifiedRecord(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
req = SearchRequest("modified:true")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
return self.__result.getString(None, "response", "numFound")
class SettingsData:
def __init__(self):
self.sysConfig = JsonSimpleConfig()
def __activate__(self, context):
self.velocityContext = context
self.__portal = None
action = self.vc("formData").get("verb")
if action == "create_view":
fq = [q for q in self.vc("sessionState").get("fq") if q != 'item_type:"object"']
if fq == []:
name = "new"
desc = "New View"
query = ""
else:
name = ""
desc = ""
query = str(" ".join(fq))
newPortal = Portal(name, desc, query)
newPortal.setFacetFields(Services.portalManager.default.facetFields)
newPortal.setQuery(query)
self.__portal = newPortal
else:
portalName = self.vc("formData").get("portalName")
print " * settings.py: portalName=%s, portalId=%s" % (portalName, self.vc("portalId"))
if portalName is None or (self.vc("formData").get("portalAction") == "Cancel"):
self.__portal = Services.portalManager.get(self.vc("portalId"))
else:
self.__portal = Portal()
self.__portal.name = portalName
Services.portalManager.add(self.__portal)
if self.vc("formData").get("portalAction") == "Update":
self.__updatePortal()
if self.vc("formData").get("emailAction") == "Update":
self.__updateEmail()
# Get from velocity context
def vc(self, index):
if self.velocityContext[index] is not None:
return self.velocityContext[index]
else:
log.error("ERROR: Requested context entry '" + index + "' doesn't exist")
return None
def isSelected(self, category):
selected = self.vc("sessionState").get("settingsCategory")
if category == selected:
return "selected"
return ""
def __updatePortal(self):
self.__portal.name = self.vc("formData").get("portalName")
self.__portal.description = self.vc("formData").get("portalDescription")
self.__portal.query = self.vc("formData").get("portalQuery")
self.__portal.recordsPerPage = int(self.vc("formData").get("portalRecordsPerPage"))
self.__portal.facetCount = int(self.vc("formData").get("portalFacetLimit"))
self.__portal.facetSort = self.vc("formData").get("portalFacetSort") is not None
facetFields = self.__portal.facetFields
facetFields.clear()
size = int(self.vc("formData").get("portalFacetSize"))
for i in range(1,size+2):
nameKey = "portalFacet_%s_name" % i
labelKey = "portalFacet_%s_label" % i
name = self.vc("formData").get(nameKey)
label = self.vc("formData").get(labelKey)
print "facet: key: %s, label: %s" % (name, label)
if name is not None and label is not None:
facetFields.put(name, label)
self.__portal.setFacetFields(facetFields)
sortFields = self.__portal.sortFields
sortFields.clear()
size = int(self.vc("formData").get("portalSortSize"))
for i in range(1,size+2):
nameKey = "portalSort_%s_name" % i
labelKey = "portalSort_%s_label" % i
name = self.vc("formData").get(nameKey)
label = self.vc("formData").get(labelKey)
print "sort: key: %s, label: %s" % (name, label)
if name is not None and label is not None:
sortFields.put(name, label)
self.__portal.setSortFields(sortFields)
Services.portalManager.save(self.__portal)
def getPortal(self):
return self.__portal
def getIndexerPlugins(self):
return PluginManager.getIndexerPlugins()
def getStoragePlugins(self):
return PluginManager.getStoragePlugins()
def getHarvesterPlugins(self):
return PluginManager.getHarvesterPlugins()
def getTransformerPlugins(self):
return PluginManager.getTransformerPlugins()
def getWatcherConfig(self):
watcherPath = self.sysConfig.getString("${fascinator.home}/watcher)", ["watcher", "path"])
configFile = File("%s/app/config.json" % watcherPath)
if configFile.exists():
return JsonSimpleConfig(configFile)
return None
def getEmail(self):
return self.sysConfig.getString(None, ["email"])
def getTimeout(self):
return self.sysConfig.getString(None, ["portal", "houseKeeping", "config", "frequency"])
def getFacetDisplays(self):
facetDisplays = self.__portal.getObject(["portal", "facet-displays"])
if facetDisplays is None or facetDisplays.isEmpty():
facetDisplays = HashMap()
facetDisplays.put("list", "List menu")
facetDisplays.put("tree", "Dynamic tree")
return facetDisplays
def __init__(self):
self.sysConfig = JsonSimpleConfig()
def process(self):
self.log.debug(" * settings.py: formData={}", self.vc("formData"))
valid = self.vc("page").csrfSecurePage()
if not valid:
self.throw_error("Invalid request")
return
result = "{}"
portalManager = self.vc("Services").getPortalManager()
portal = portalManager.get(self.vc("portalId"))
func = self.vc("formData").get("func")
if func == "view-update":
portal.setDescription(self.vc("formData").get("view-description"))
portal.setQuery(self.vc("formData").get("view-query"))
portal.setSearchQuery(self.vc("formData").get("view-search-query"))
portal.setRecordsPerPage(
int(self.vc("formData").get("view-records-per-page")))
portal.setFacetCount(
int(self.vc("formData").get("view-facet-count")))
portal.setFacetDisplay(
int(self.vc("formData").get("view-facet-display")))
portal.setFacetSort(
self.vc("formData").get("view-facet-sort") is not None)
portalManager.save(portal)
elif func == "general-update":
config = JsonSimpleConfig()
email = StringUtils.trimToEmpty(
self.vc("formData").get("general-email"))
systemEmail = StringUtils.trimToEmpty(
config.getString(None, ["email"]))
if systemEmail != email:
obj = config.writableSystemConfig()
obj.put("email", self.vc("formData").get("general-email"))
obj.put("configured", "true")
config.storeSystemConfig()
# mark restart
self.vc("Services").getHouseKeepingManager(
).requestUrgentRestart()
else:
self.log.debug(
" * settings.py: email not updated: did not change")
self.throw_error("Email address is the same! No change saved.")
elif func == "facets-update":
portal.getObject(["portal"]).remove("facet-fields")
fields = self.vc("formData").getValues("field")
labels = self.vc("formData").getValues("label")
displays = self.vc("formData").getValues("display")
deletes = self.vc("formData").getValues("delete")
for i in range(0, len(fields)):
field = fields[i]
if deletes[i] == "false":
node = portal.writeObject(
["portal", "facet-fields", field])
node.put("label", labels[i])
node.put("display", displays[i])
portalManager.save(portal)
elif func == "sort-update":
portal.getObject(["portal"]).remove("sort-fields")
default = self.vc("formData").get("default")
if default:
portal.setSortFieldDefault(default)
order = self.vc("formData").get("order")
if order:
portal.setSortFieldDefaultOrder(order)
fields = self.vc("formData").getValues("field")
labels = self.vc("formData").getValues("label")
deletes = self.vc("formData").getValues("delete")
for i in range(0, len(fields)):
field = fields[i]
if deletes[i] == "false":
node = portal.writeObject(["portal", "sort-fields"])
node.put(field, labels[i])
portalManager.save(portal)
elif func == "watcher-update":
configFile = self.getWatcherFile()
if configFile is not None:
json = JsonSimpleConfig(configFile)
pathIds = self.vc("formData").get("pathIds").split(",")
actives = self.vc("formData").getValues("watcher-active")
if actives is None:
actives = []
deletes = self.vc("formData").getValues("watcher-delete")
if deletes is None:
deletes = []
for pathId in pathIds:
if pathId not in deletes:
path = self.vc("formData").get("%s-path" % pathId)
stopped = str(pathId not in actives).lower()
watchDir = json.writeObject(
["watcher", "watchDirs", path])
watchDir.put(
"ignoreFileFilter",
self.vc("formData").get("%s-file" % pathId))
watchDir.put(
"ignoreDirectories",
self.vc("formData").get("%s-dir" % pathId))
watchDir.put("stopped", stopped)
json.writeArray(
["watcher", "watchDirs", path, "cxtTags"])
writer = FileWriter(configFile)
writer.write(json.toString(True))
writer.close()
else:
result = "The Watcher is not installed properly."
elif func == "restore-default-config":
# backup the file
JsonSimpleConfig.backupSystemFile()
# delete the file
JsonSimpleConfig.getSystemFile().delete()
# restore default
JsonSimpleConfig.getSystemFile()
# mark restart
self.vc("Services").getHouseKeepingManager().requestUrgentRestart()
elif func == "housekeeping-update":
config = JsonSimpleConfig()
freq = StringUtils.trimToEmpty(
self.vc("formData").get("housekeeping-timeout"))
systemFreq = StringUtils.trimToEmpty(
config.getString(
None, ["portal", "houseKeeping", "config", "frequency"]))
result = "House Keeper refreshed"
if systemFreq != freq:
# Get writeable access to underlying system
sysConfig = JsonSimple(config.writableSystemConfig())
# Modify the house keeping node
hkConfig = sysConfig.writeObject(
["portal", "houseKeeping", "config"])
hkConfig.put("frequency", freq)
# Write the underlying config back to disk
config.storeSystemConfig()
result = "Frequency updated, refreshing House Keeper"
# Refresh the HouseKeeper
message = JsonObject()
message.put("type", "refresh")
self.vc("Services").getHouseKeepingManager().sendMessage(
message.toString())
self.writer.println(result)
self.writer.close()