def __init__(self, file, config, baseline):
AlertHandler.__init__(self, file, config, baseline)
docFactory = DocumentFactory()
self.saxReader = SAXReader(docFactory)
self.xmlMapFile = StrSubstitutor.replaceSystemProperties(config['xmlMap'])
if not os.path.exists(self.xmlMapFile):
raise AlertException("Requested xmlMap file %s does not exist." % self.xmlMapFile)
## Make sure we can see our mappings
inStream = FileInputStream(File(self.xmlMapFile))
xmlMappings = JsonSimple(inStream)
self.map = xmlMappings.getObject(["mappings"])
self.exceptions = xmlMappings.getObject(["exceptions"])
self.defaultNamespace = xmlMappings.getObject(["defaultNamespace"])
self.mappedExceptionCount = 0
def __init__(self, file, config, baseline):
AlertHandler.__init__(self, file, config, baseline)
docFactory = DocumentFactory()
self.saxReader = SAXReader(docFactory)
self.xmlMapFile = StrSubstitutor.replaceSystemProperties(
config['xmlMap'])
if not os.path.exists(self.xmlMapFile):
raise AlertException("Requested xmlMap file %s does not exist." %
self.xmlMapFile)
## Make sure we can see our mappings
inStream = FileInputStream(File(self.xmlMapFile))
xmlMappings = JsonSimple(inStream)
self.map = xmlMappings.getObject(["mappings"])
self.exceptions = xmlMappings.getObject(["exceptions"])
self.defaultNamespace = xmlMappings.getObject(["defaultNamespace"])
self.mappedExceptionCount = 0
def __workflow(self):
# Workflow data
WORKFLOW_ID = "servicesUI2"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
pageTitle = "Services Record"
displayType = "package-service"
initialStep = 0
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime)
self.utils.add(self.index, "date_created", solrTime)
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
self.utils.add(self.index, "display_type", displayType)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
if self.owner == "guest":
pageTitle = "Submission Request"
displayType = "submission-request"
initialStep = 0
else:
pageTitle = "Metadata Record"
displayType = "package-dataset"
initialStep = 1
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
#Used to make sure we have a created date
createdDateFlag = False
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime)
self.utils.add(self.index, "date_created", solrTime)
self.log.debug("Set created date to :%s" % solrTime)
createdDateFlag = True
elif field == "redbox:embargo.dc:date":
self.embargoedDate = value
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
# check if this is an array field
fnameparts = field.split(":")
if fnameparts is not None and len(fnameparts) >= 3:
if field.startswith("bibo") or field.startswith("skos"):
arrParts = fnameparts[1].split(".")
else:
arrParts = fnameparts[2].split(".")
# we're not interested in: Relationship, Type and some redbox:origin
if arrParts is not None and len(arrParts) >= 2 and field.find(":Relationship.") == -1 and field.find("dc:type") == -1 and field.find("redbox:origin") == -1 and arrParts[1].isdigit():
# we've got an array field
fldPart = ":%s" % arrParts[0]
prefixEndIdx = field.find(fldPart) + len(fldPart)
suffixStartIdx = prefixEndIdx+len(arrParts[1])+1
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[suffixStartIdx:]
if field.endswith("Name"):
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx]
self.log.debug("Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value))
if field.endswith("Name"):
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
idx = arrParts[1]
fullField = fullFieldMap.get(idx)
if (fullField is None):
fullField = ""
if (field.endswith("givenName")):
fullField = "%s, %s" % (fullField, value)
if (field.endswith("familyName")):
fullField = "%s%s" % (value, fullField)
self.log.debug("fullname now is :%s" % fullField)
fullFieldMap.put(idx, fullField)
else:
fieldlist = self.arrayBucket.get(arrFldName)
if fieldlist is None:
fieldlist = []
self.arrayBucket.put(arrFldName, fieldlist)
fieldlist.append(value)
for compfield in self.compFields:
if field.startswith(compfield):
arrFldName = self.reportingFieldPrefix +compfield
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
fullField = fullFieldMap.get("1")
if fullField is None:
fullField = ""
if field.endswith(self.compFieldsConfig[compfield]["end"]):
fullField = "%s%s%s" % (fullField, self.compFieldsConfig[compfield]["delim"] ,value)
if field.endswith(self.compFieldsConfig[compfield]["start"]):
fullField = "%s%s" % (value, fullField)
self.log.debug("full field now is :%s" % fullField)
fullFieldMap.put("1", fullField)
self.utils.add(self.index, "display_type", displayType)
# Make sure we have a creation date
if not createdDateFlag:
self.utils.add(self.index, "date_created", self.last_modified)
self.log.debug("Forced creation date to %s because it was not explicitly set." % self.last_modified)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self):
# Workflow data
WORKFLOW_ID = "Parties_People"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
initialStep = 0
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
#if self.owner == "guest":
# pageTitle = "Submission Request"
# displayType = "submission-request"
# initialStep = 0
#else:
# pageTitle = "Metadata Record"
# displayType = "package-dataset"
# initialStep = 1
## Harvesting straight into the 'Published' stage
pageTitle = "Metadata Record"
displayType = "package-dataset"
#initialStep = 4
initialStep = 3
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(
None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put(
"label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException:
print(" ERROR updating dataset payload")
# Form processing
coreFields = [
"title", "description", "manifest", "metaList", "relationships",
"responses"
]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ",
parsedTime)
self.utils.add(self.index, "date_created", solrTime)
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
self.utils.add(self.index, "display_type", displayType)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
if self.owner == "guest":
pageTitle = "Submission Request"
displayType = "submission-request"
initialStep = 0
else:
pageTitle = "Metadata Record"
displayType = "package-dataset"
initialStep = 1
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(
None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put(
"label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = [
"title", "description", "manifest", "metaList", "relationships",
"responses"
]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
#Used to make sure we have a created date
createdDateFlag = False
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ",
parsedTime)
self.utils.add(self.index, "date_created", solrTime)
self.log.debug("Set created date to :%s" % solrTime)
createdDateFlag = True
elif field == "redbox:embargo.dc:date":
self.embargoedDate = value
elif field == "create_timestamp":
self.createTimeStamp = value
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
# check if this is an array field
fnameparts = field.split(":")
if fnameparts is not None and len(fnameparts) >= 3:
if field.startswith("bibo") or field.startswith(
"skos"):
arrParts = fnameparts[1].split(".")
else:
arrParts = fnameparts[2].split(".")
# we're not interested in: Relationship, Type and some redbox:origin
if arrParts is not None and len(
arrParts) >= 2 and field.find(
":Relationship.") == -1 and field.find(
"dc:type") == -1 and field.find(
"redbox:origin"
) == -1 and arrParts[1].isdigit():
# we've got an array field
fldPart = ":%s" % arrParts[0]
prefixEndIdx = field.find(fldPart) + len(fldPart)
suffixStartIdx = prefixEndIdx + len(
arrParts[1]) + 1
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[
suffixStartIdx:]
if field.endswith("Name"):
arrFldName = self.reportingFieldPrefix + field[:
prefixEndIdx]
self.log.debug(
"Array Field name is:%s from: %s, with value:%s"
% (arrFldName, field, value))
if field.endswith("Name"):
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(
arrFldName, fullFieldMap)
idx = arrParts[1]
fullField = fullFieldMap.get(idx)
if (fullField is None):
fullField = ""
if (field.endswith("givenName")):
fullField = "%s, %s" % (fullField, value)
if (field.endswith("familyName")):
fullField = "%s%s" % (value, fullField)
self.log.debug("fullname now is :%s" %
fullField)
fullFieldMap.put(idx, fullField)
else:
fieldlist = self.arrayBucket.get(arrFldName)
if fieldlist is None:
fieldlist = []
self.arrayBucket.put(arrFldName, fieldlist)
fieldlist.append(value)
for compfield in self.compFields:
if field.startswith(compfield):
arrFldName = self.reportingFieldPrefix + compfield
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
fullField = fullFieldMap.get("1")
if fullField is None:
fullField = ""
if field.endswith(
self.compFieldsConfig[compfield]["end"]):
fullField = "%s%s%s" % (
fullField,
self.compFieldsConfig[compfield]["delim"],
value)
if field.endswith(
self.compFieldsConfig[compfield]["start"]):
fullField = "%s%s" % (value, fullField)
self.log.debug("full field now is :%s" % fullField)
fullFieldMap.put("1", fullField)
self.utils.add(self.index, "display_type", displayType)
# Make sure we have a creation date
if not createdDateFlag:
self.utils.add(self.index, "date_created", self.last_modified)
self.log.debug(
"Forced creation date to %s because it was not explicitly set."
% self.last_modified)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
