Esempio n. 1
0
    def __ingestJson(self, fileName, jsonObject, move):
        if self.configFile is None:
            self.configFile = FascinatorHome.getPathFile(
                "harvest/workflows/dataset.json")

        harvester = None
        try:
            ## Cache the file out to disk... although requires it
            ## .tfpackage extension due to jsonVelocity transformer
            jsonPath = self.pTemp(fileName)
            jsonFile = open(jsonPath, "wb")
            jsonFile.write(jsonObject.toString(True))
            jsonFile.close()

            ## Now instantiate a HarvestClient just for this File.
            harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
            harvester.start()

            ## And cleanup afterwards
            oid = harvester.getUploadOid()
            self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
            if move:
                shutil.move(self.pBase(fileName), self.pDone(fileName))
            return True

        except Exception, e:
            ## TODO: This block looks to just be a copy of the
            ##  top-level one, yet it runs per ROW, not for the
            ##  whole File. Just the JSON data should be stored

            ## Move the CSV to the 'failed' directory
            shutil.move(self.pBase(fileName), self.pFail(fileName))
            ## And write our error data to disk beside it
            self.writeError(fileName, e)
            return False
Esempio n. 2
0
    def __ingestJson(self, fileName, jsonObject, move):
        if self.configFile is None:
            self.configFile = FascinatorHome.getPathFile("harvest/workflows/dataset.json")

        harvester = None
        try:
            ## Cache the file out to disk... although requires it
            ## .tfpackage extension due to jsonVelocity transformer
            jsonPath = self.pTemp(fileName)
            jsonFile = open(jsonPath, "wb")
            jsonFile.write(jsonObject.toString(True))
            jsonFile.close()

            ## Now instantiate a HarvestClient just for this File.
            harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
            harvester.start()

            ## And cleanup afterwards
            oid = harvester.getUploadOid()
            self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
            if move:
                shutil.move(self.pBase(fileName), self.pDone(fileName))
            return True

        except Exception, e:
            ## TODO: This block looks to just be a copy of the
            ##  top-level one, yet it runs per ROW, not for the
            ##  whole File. Just the JSON data should be stored

            ## Move the CSV to the 'failed' directory
            shutil.move(self.pBase(fileName), self.pFail(fileName))
            ## And write our error data to disk beside it
            self.writeError(fileName, e)
            return False
Esempio n. 3
0
    def __createFromSelected(self):
        self.vc("log").debug("Creating package from selected...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        #self.vc("log").debug("packageType = '{}'", packageType)
        #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        # if modifying existing manifest, we already have an identifier,
        # otherwise create a new one
        manifestId = self.__getActiveManifestId()
        if manifestId is None:
            manifestHash = "%s.tfpackage" % uuid.uuid4()
        else:
            manifestHash = self.__getActiveManifestPid()

        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")
        manifest = self.__getActiveManifest()
        oldType = manifest.getType()
        if oldType is None:
            manifest.setType(packageType)
        else:
            manifest.setType(oldType)

        #self.vc("log").debug("Manifest: {}", manifest)
        outWriter.write(manifest.toString(True))
        outWriter.close()

        try:
            if manifestId is None:
                # harvest the package as an object
                username = self.vc("sessionState").get("username")
                if username is None:
                    username = "******" # necessary?
                harvester = None
                # set up config files, and make sure they are both deployed
                workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
                configFile = self.__getFile(workflowsDir, jsonConfigFile)
                rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
                # run the harvest client with our packaging workflow config
                harvester = HarvestClient(configFile, manifestFile, username)
                harvester.start()
                manifestId = harvester.getUploadOid()
                harvester.shutdown()
            else:
                # update existing object
                object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId)
                manifestStream = FileUtils.openInputStream(manifestFile)
                StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream)
                manifestStream.close()
                object.close()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            self.vc("log").error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'
    def __createNew(self):
        self.vc("log").debug("Creating a new package...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        self.vc("log").debug("packageType = '{}'", packageType)
        self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        manifestHash = "%s.tfpackage" % uuid.uuid4()
        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")

        self.vc("sessionState").set("package/active", None)
        manifest = self.__getActiveManifest()
        manifest.setType(packageType)
        metaList = list(self.vc("formData").getValues("metaList"))
        jsonObj = manifest.getJsonObject()
        for metaName in metaList:
            value = self.vc("formData").get(metaName)
            jsonObj.put(metaName, value)
        if self.vc("formData").getValues("sequencesMetaList") != None:    
            sequenceService = ApplicationContextProvider.getApplicationContext().getBean("sequenceService")
            sequencesMetaList = list(self.vc("formData").getValues("sequencesMetaList"))
            for sequenceInfo in sequencesMetaList:   
                sequenceInfoJson = JsonSimple(sequenceInfo)
                sequenceIndex = sequenceService.getSequence(sequenceInfoJson.getString(None,"sequenceName"))
                jsonObj.put(sequenceInfoJson.getString(None,"metadataName"), String.format(sequenceInfoJson.getString(None,"stringFormat"),sequenceIndex))
        self.vc("log").debug("json object created is: %r" % jsonObj)
        outWriter.write(manifest.toString(True))
        outWriter.close()
        # adding ability to set access plugin
        accessPlugin = self.vc("formData").get("access_plugin", "derby")
        if accessPlugin is not None:
            self.vc("page").authentication.set_access_plugin(accessPlugin)
        try:
            # harvest the package as an object
            username = self.vc("sessionState").get("username")
            if username is None:
                username = "******" # necessary?
            harvester = None
            # set up config files, creating if necessary
            workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
            configFile = self.__getFile(workflowsDir, jsonConfigFile)
            self.__getFile(workflowsDir, "packaging-rules.py")
            # run the harvest client with our packaging workflow config
            harvester = HarvestClient(configFile, manifestFile, username)
            harvester.start()
            manifestId = harvester.getUploadOid()
            harvester.shutdown()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            self.vc("log").error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'
class BatchprocessData:
    def __init__(self):
        pass

    def __activate__(self, context):
        self.formData = context["formData"]
        self.response = context["response"]
        self.services = context["Services"]
        self.page = context["page"]
        self.portal = self.page.getPortal()
        self.vc = context["toolkit"]
        self.log = context["log"]
        self.writer = self.response.getPrintWriter("text/html; charset=UTF-8")
        print " *** action batchProcess.py: formData=%s" % self.formData
        auth = context["page"].authentication
        if auth.is_admin():
            self.storage = self.services.getStorage()
            result = self.__process()
        else:
            result = '{ "status": "error", "message": "Only administrative users can access this API" }'
        self.writer.println(result)
        self.writer.close()

    def __process(self):
        func = self.formData.get("func")
        result = {}

        if func == "batch-update":
            updateScriptFile = self.formData.get("update-script-file")
            self.__checkIfScriptFileIsValid(updateScriptFile)

            objectIdList = self.__search("*:*")
            try:
                self.__harvester = HarvestClient()
                for oid in objectIdList:
                    self.__processObject(oid, updateScriptFile, "false")
            except Exception, ex:
                error = "Batch update failed: %s" % str(ex)
                self.log.error(error, ex)
                return '{ status: "failed" }'
        elif func == "batch-export":
            exportScriptFile = self.formData.get("export-script-file")
            self.__checkIfScriptFileIsValid(exportScriptFile)

            objectIdList = self.__search("modified:true")
            try:
                self.__harvester = HarvestClient()
                for oid in objectIdList:
                    self.__processObject(oid, exportScriptFile, "true")
            except Exception, ex:
                error = "Batch export failed: %s" % str(ex)
                self.log.error(error, ex)
                return '{ status: "failed" }'
    def __createNew(self):
        self.vc("log").debug("Creating a new package...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        #self.vc("log").debug("packageType = '{}'", packageType)
        #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        manifestHash = "%s.tfpackage" % uuid.uuid4()
        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")

        self.vc("sessionState").set("package/active", None)
        manifest = self.__getActiveManifest()
        manifest.setType(packageType)
        metaList = list(self.vc("formData").getValues("metaList"))
        jsonObj = manifest.getJsonObject()
        for metaName in metaList:
            value = self.vc("formData").get(metaName)
            jsonObj.put(metaName, value)

        outWriter.write(manifest.toString(True))
        outWriter.close()

        try:
            # harvest the package as an object
            username = self.vc("sessionState").get("username")
            if username is None:
                username = "******" # necessary?
            harvester = None
            # set up config files, creating if necessary
            workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
            configFile = self.__getFile(workflowsDir, jsonConfigFile)
            self.__getFile(workflowsDir, "packaging-rules.py")
            # run the harvest client with our packaging workflow config
            harvester = HarvestClient(configFile, manifestFile, username)
            harvester.start()
            manifestId = harvester.getUploadOid()
            harvester.shutdown()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            log.error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'
    def __process(self):
        func = self.formData.get("func")
        result = {}

        if func == "batch-update":
            updateScriptFile = self.formData.get("update-script-file")
            self.__checkIfScriptFileIsValid(updateScriptFile)

            objectIdList = self.__search("*:*")
            try:
                self.__harvester = HarvestClient()
                for oid in objectIdList:
                    self.__processObject(oid, updateScriptFile, "false")
            except Exception, ex:
                error = "Batch update failed: %s" % str(ex)
                self.log.error(error, ex)
                return '{ status: "failed" }'
Esempio n. 8
0
         raise
     finally:
         if jsonFile is not None:
             jsonFile.close()
         
     #Determine the object's owner from the alert config
     #owner = "guest"
     #if self.config("owner"):
         #pass
     
     self.logInfo(file, "Submitting to harvest. Config file is %s and meta_file is %s" % (self.harvestConfig, meta_file))
     try:
         ## Now instantiate a HarvestClient just for this File.
         #harvester = HarvestClient(File(self.harvestConfig), File(meta_file), owner)
         #harvester = HarvestClient(File(self.harvestConfig), File(meta_file), None)
         harvester = HarvestClient(File(self.harvestConfig), File(meta_file), "guest")            
         harvester.start()
         ## And cleanup afterwards
         oid = harvester.getUploadOid() 
         
     except Exception, e:     
         self.logException(file, e) 
         raise
     finally:
         ## Cleanup
         if harvester is not None:
             harvester.shutdown()
             
     self.logInfo(file, "Successfully harvested alert item %s from processing file %s" % (oid,meta_file))
     return oid
 
Esempio n. 9
0
            if jsonFile is not None:
                jsonFile.close()

        #Determine the object's owner from the alert config
        #owner = "guest"
        #if self.config("owner"):
        #pass

        self.logInfo(
            file,
            "Submitting to harvest. Config file is %s and meta_file is %s" %
            (self.harvestConfig, meta_file))
        try:
            ## Now instantiate a HarvestClient just for this File.
            #harvester = HarvestClient(File(self.harvestConfig), File(meta_file), owner)
            harvester = HarvestClient(File(self.harvestConfig),
                                      File(meta_file), None)
            harvester.start()
            ## And cleanup afterwards
            oid = harvester.getUploadOid()

        except Exception, e:
            self.logException(file, e)
            raise
        finally:
            ## Cleanup
            if harvester is not None:
                harvester.shutdown()

        self.logInfo(
            file,
            "Successfully harvested alert item %s from processing file %s" %
    def __createNew(self):
        self.vc("log").debug("Creating a new package...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        self.vc("log").debug("packageType = '{}'", packageType)
        self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        manifestHash = "%s.tfpackage" % uuid.uuid4()
        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")

        self.vc("sessionState").set("package/active", None)
        manifest = self.__getActiveManifest()
        manifest.setType(packageType)
        metaList = list(self.vc("formData").getValues("metaList"))
        jsonObj = manifest.getJsonObject()
        for metaName in metaList:
            value = self.vc("formData").get(metaName)
            jsonObj.put(metaName, value)
        if self.vc("formData").getValues("sequencesMetaList") != None:
            sequenceService = ApplicationContextProvider.getApplicationContext(
            ).getBean("sequenceService")
            sequencesMetaList = list(
                self.vc("formData").getValues("sequencesMetaList"))
            for sequenceInfo in sequencesMetaList:
                sequenceInfoJson = JsonSimple(sequenceInfo)
                sequenceIndex = sequenceService.getSequence(
                    sequenceInfoJson.getString(None, "sequenceName"))
                jsonObj.put(
                    sequenceInfoJson.getString(None, "metadataName"),
                    String.format(
                        sequenceInfoJson.getString(None, "stringFormat"),
                        sequenceIndex))
        self.vc("log").debug("json object created is: %r" % jsonObj)
        outWriter.write(manifest.toString(True))
        outWriter.close()
        # adding ability to set access plugin
        accessPlugin = self.vc("formData").get("access_plugin", "derby")
        if accessPlugin is not None:
            self.vc("page").authentication.set_access_plugin(accessPlugin)
        try:
            # harvest the package as an object
            username = self.vc("sessionState").get("username")
            if username is None:
                username = "******"  # necessary?
            harvester = None
            # set up config files, creating if necessary
            workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
            configFile = self.__getFile(workflowsDir, jsonConfigFile)
            self.__getFile(workflowsDir, "packaging-rules.py")
            # run the harvest client with our packaging workflow config
            harvester = HarvestClient(configFile, manifestFile, username)
            harvester.start()
            manifestId = harvester.getUploadOid()
            harvester.shutdown()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            self.vc("log").error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'
    def __createFromSelected(self):
        self.vc("log").debug("Creating package from selected...")
        packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
        #self.vc("log").debug("packageType = '{}'", packageType)
        #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)

        # if modifying existing manifest, we already have an identifier,
        # otherwise create a new one
        manifestId = self.__getActiveManifestId()
        if manifestId is None:
            manifestHash = "%s.tfpackage" % uuid.uuid4()
        else:
            manifestHash = self.__getActiveManifestPid()

        # store the manifest file for harvesting
        packageDir = FascinatorHome.getPathFile("packages")
        packageDir.mkdirs()
        manifestFile = File(packageDir, manifestHash)
        outStream = FileOutputStream(manifestFile)
        outWriter = OutputStreamWriter(outStream, "UTF-8")
        manifest = self.__getActiveManifest()
        oldType = manifest.getType()
        if oldType is None:
            manifest.setType(packageType)
        else:
            manifest.setType(oldType)

        self.vc("log").debug("Manifest: %s" % manifest)
        outWriter.write(manifest.toString(True))
        outWriter.close()

        try:
            if manifestId is None:
                # harvest the package as an object
                username = self.vc("sessionState").get("username")
                if username is None:
                    username = "******"  # necessary?
                harvester = None
                # set up config files, and make sure they are both deployed
                workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
                configFile = self.__getFile(workflowsDir, jsonConfigFile)
                rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
                # run the harvest client with our packaging workflow config
                harvester = HarvestClient(configFile, manifestFile, username)
                harvester.start()
                manifestId = harvester.getUploadOid()
                harvester.shutdown()
            else:
                # update existing object
                object = StorageUtils.getDigitalObject(Services.getStorage(),
                                                       manifestId)
                manifestStream = FileUtils.openInputStream(manifestFile)
                StorageUtils.createOrUpdatePayload(object, manifestHash,
                                                   manifestStream)
                manifestStream.close()
                object.close()
        except Exception, ex:
            error = "Packager workflow failed: %s" % str(ex)
            self.vc("log").error(error, ex)
            if harvester is not None:
                harvester.shutdown()
            return '{ "status": "failed" }'