def __ingestJson(self, fileName, jsonObject, move):
if self.configFile is None:
self.configFile = FascinatorHome.getPathFile(
"harvest/workflows/dataset.json")
harvester = None
try:
## Cache the file out to disk... although requires it
## .tfpackage extension due to jsonVelocity transformer
jsonPath = self.pTemp(fileName)
jsonFile = open(jsonPath, "wb")
jsonFile.write(jsonObject.toString(True))
jsonFile.close()
## Now instantiate a HarvestClient just for this File.
harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
harvester.start()
## And cleanup afterwards
oid = harvester.getUploadOid()
self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
if move:
shutil.move(self.pBase(fileName), self.pDone(fileName))
return True
except Exception, e:
## TODO: This block looks to just be a copy of the
## top-level one, yet it runs per ROW, not for the
## whole File. Just the JSON data should be stored
## Move the CSV to the 'failed' directory
shutil.move(self.pBase(fileName), self.pFail(fileName))
## And write our error data to disk beside it
self.writeError(fileName, e)
return False
def __ingestJson(self, fileName, jsonObject, move):
if self.configFile is None:
self.configFile = FascinatorHome.getPathFile("harvest/workflows/dataset.json")
harvester = None
try:
## Cache the file out to disk... although requires it
## .tfpackage extension due to jsonVelocity transformer
jsonPath = self.pTemp(fileName)
jsonFile = open(jsonPath, "wb")
jsonFile.write(jsonObject.toString(True))
jsonFile.close()
## Now instantiate a HarvestClient just for this File.
harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
harvester.start()
## And cleanup afterwards
oid = harvester.getUploadOid()
self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
if move:
shutil.move(self.pBase(fileName), self.pDone(fileName))
return True
except Exception, e:
## TODO: This block looks to just be a copy of the
## top-level one, yet it runs per ROW, not for the
## whole File. Just the JSON data should be stored
## Move the CSV to the 'failed' directory
shutil.move(self.pBase(fileName), self.pFail(fileName))
## And write our error data to disk beside it
self.writeError(fileName, e)
return False
def __createFromSelected(self):
self.vc("log").debug("Creating package from selected...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
# if modifying existing manifest, we already have an identifier,
# otherwise create a new one
manifestId = self.__getActiveManifestId()
if manifestId is None:
manifestHash = "%s.tfpackage" % uuid.uuid4()
else:
manifestHash = self.__getActiveManifestPid()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
manifest = self.__getActiveManifest()
oldType = manifest.getType()
if oldType is None:
manifest.setType(packageType)
else:
manifest.setType(oldType)
#self.vc("log").debug("Manifest: {}", manifest)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
if manifestId is None:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, and make sure they are both deployed
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
else:
# update existing object
object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId)
manifestStream = FileUtils.openInputStream(manifestFile)
StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream)
manifestStream.close()
object.close()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
self.vc("log").debug("packageType = '{}'", packageType)
self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
if self.vc("formData").getValues("sequencesMetaList") != None:
sequenceService = ApplicationContextProvider.getApplicationContext().getBean("sequenceService")
sequencesMetaList = list(self.vc("formData").getValues("sequencesMetaList"))
for sequenceInfo in sequencesMetaList:
sequenceInfoJson = JsonSimple(sequenceInfo)
sequenceIndex = sequenceService.getSequence(sequenceInfoJson.getString(None,"sequenceName"))
jsonObj.put(sequenceInfoJson.getString(None,"metadataName"), String.format(sequenceInfoJson.getString(None,"stringFormat"),sequenceIndex))
self.vc("log").debug("json object created is: %r" % jsonObj)
outWriter.write(manifest.toString(True))
outWriter.close()
# adding ability to set access plugin
accessPlugin = self.vc("formData").get("access_plugin", "derby")
if accessPlugin is not None:
self.vc("page").authentication.set_access_plugin(accessPlugin)
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
class BatchprocessData:
def __init__(self):
pass
def __activate__(self, context):
self.formData = context["formData"]
self.response = context["response"]
self.services = context["Services"]
self.page = context["page"]
self.portal = self.page.getPortal()
self.vc = context["toolkit"]
self.log = context["log"]
self.writer = self.response.getPrintWriter("text/html; charset=UTF-8")
print " *** action batchProcess.py: formData=%s" % self.formData
auth = context["page"].authentication
if auth.is_admin():
self.storage = self.services.getStorage()
result = self.__process()
else:
result = '{ "status": "error", "message": "Only administrative users can access this API" }'
self.writer.println(result)
self.writer.close()
def __process(self):
func = self.formData.get("func")
result = {}
if func == "batch-update":
updateScriptFile = self.formData.get("update-script-file")
self.__checkIfScriptFileIsValid(updateScriptFile)
objectIdList = self.__search("*:*")
try:
self.__harvester = HarvestClient()
for oid in objectIdList:
self.__processObject(oid, updateScriptFile, "false")
except Exception, ex:
error = "Batch update failed: %s" % str(ex)
self.log.error(error, ex)
return '{ status: "failed" }'
elif func == "batch-export":
exportScriptFile = self.formData.get("export-script-file")
self.__checkIfScriptFileIsValid(exportScriptFile)
objectIdList = self.__search("modified:true")
try:
self.__harvester = HarvestClient()
for oid in objectIdList:
self.__processObject(oid, exportScriptFile, "true")
except Exception, ex:
error = "Batch export failed: %s" % str(ex)
self.log.error(error, ex)
return '{ status: "failed" }'
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
log.error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __process(self):
func = self.formData.get("func")
result = {}
if func == "batch-update":
updateScriptFile = self.formData.get("update-script-file")
self.__checkIfScriptFileIsValid(updateScriptFile)
objectIdList = self.__search("*:*")
try:
self.__harvester = HarvestClient()
for oid in objectIdList:
self.__processObject(oid, updateScriptFile, "false")
except Exception, ex:
error = "Batch update failed: %s" % str(ex)
self.log.error(error, ex)
return '{ status: "failed" }'
raise
finally:
if jsonFile is not None:
jsonFile.close()
#Determine the object's owner from the alert config
#owner = "guest"
#if self.config("owner"):
#pass
self.logInfo(file, "Submitting to harvest. Config file is %s and meta_file is %s" % (self.harvestConfig, meta_file))
try:
## Now instantiate a HarvestClient just for this File.
#harvester = HarvestClient(File(self.harvestConfig), File(meta_file), owner)
#harvester = HarvestClient(File(self.harvestConfig), File(meta_file), None)
harvester = HarvestClient(File(self.harvestConfig), File(meta_file), "guest")
harvester.start()
## And cleanup afterwards
oid = harvester.getUploadOid()
except Exception, e:
self.logException(file, e)
raise
finally:
## Cleanup
if harvester is not None:
harvester.shutdown()
self.logInfo(file, "Successfully harvested alert item %s from processing file %s" % (oid,meta_file))
return oid
if jsonFile is not None:
jsonFile.close()
#Determine the object's owner from the alert config
#owner = "guest"
#if self.config("owner"):
#pass
self.logInfo(
file,
"Submitting to harvest. Config file is %s and meta_file is %s" %
(self.harvestConfig, meta_file))
try:
## Now instantiate a HarvestClient just for this File.
#harvester = HarvestClient(File(self.harvestConfig), File(meta_file), owner)
harvester = HarvestClient(File(self.harvestConfig),
File(meta_file), None)
harvester.start()
## And cleanup afterwards
oid = harvester.getUploadOid()
except Exception, e:
self.logException(file, e)
raise
finally:
## Cleanup
if harvester is not None:
harvester.shutdown()
self.logInfo(
file,
"Successfully harvested alert item %s from processing file %s" %
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
self.vc("log").debug("packageType = '{}'", packageType)
self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
if self.vc("formData").getValues("sequencesMetaList") != None:
sequenceService = ApplicationContextProvider.getApplicationContext(
).getBean("sequenceService")
sequencesMetaList = list(
self.vc("formData").getValues("sequencesMetaList"))
for sequenceInfo in sequencesMetaList:
sequenceInfoJson = JsonSimple(sequenceInfo)
sequenceIndex = sequenceService.getSequence(
sequenceInfoJson.getString(None, "sequenceName"))
jsonObj.put(
sequenceInfoJson.getString(None, "metadataName"),
String.format(
sequenceInfoJson.getString(None, "stringFormat"),
sequenceIndex))
self.vc("log").debug("json object created is: %r" % jsonObj)
outWriter.write(manifest.toString(True))
outWriter.close()
# adding ability to set access plugin
accessPlugin = self.vc("formData").get("access_plugin", "derby")
if accessPlugin is not None:
self.vc("page").authentication.set_access_plugin(accessPlugin)
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __createFromSelected(self):
self.vc("log").debug("Creating package from selected...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
# if modifying existing manifest, we already have an identifier,
# otherwise create a new one
manifestId = self.__getActiveManifestId()
if manifestId is None:
manifestHash = "%s.tfpackage" % uuid.uuid4()
else:
manifestHash = self.__getActiveManifestPid()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
manifest = self.__getActiveManifest()
oldType = manifest.getType()
if oldType is None:
manifest.setType(packageType)
else:
manifest.setType(oldType)
self.vc("log").debug("Manifest: %s" % manifest)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
if manifestId is None:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, and make sure they are both deployed
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
else:
# update existing object
object = StorageUtils.getDigitalObject(Services.getStorage(),
manifestId)
manifestStream = FileUtils.openInputStream(manifestFile)
StorageUtils.createOrUpdatePayload(object, manifestHash,
manifestStream)
manifestStream.close()
object.close()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'