Exemple #1
0
 def __init__(self, configPath, defaultDict={}):
     if os.path.isdir(configPath):
         raise "config file dict can only be created with file."
     self.configPath = configPath
     try:
         f = open(self.configPath, "r")
         self.config = json.load(f)
         f.close()
     except IOError:
         print "no config file found:", self.configPath
         self.config = defaultDict
Exemple #2
0
 def __init__(self, srcRoot, storageRoot, stateStoragePath = 'd:/state.txt'):
     print "src root is:", srcRoot
     self.srcRoot = transform.transformDirToInternal(srcRoot)
     self.storageRoot = transform.transformDirToInternal(storageRoot)
     self.stateStoragePath = stateStoragePath
     try:
         f = open(self.stateStoragePath,'r')
         self.config = json.load(f)
         f.close()
     except IOError:
         self.config = {}
Exemple #3
0
    def readEncryptedZipLog(self, encryptedZipLogPath):
        zipLogPath = os.path.join(self.decryptionWorkingDir, 
                os.path.basename(encryptedZipLogPath).replace('.enclog', '.log'))

        self.decCopier.copy(encryptedZipLogPath, zipLogPath)
        try:
            f = open(zipLogPath,'r')
            res = json.load(f)
            f.close()
            return res
        except IOError:
            return {}
 def processItem(self, job, item):
     monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
     archiveId = gZipFolderCollectionPrefix + monitoringFullPath
     if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
         self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
         objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId});
         idInCol = objUuid
         self.zippedInfoCollectionList.addObj(idInCol, objUuid)
     #Save the item in the archive collection: zippedInfoColllection://D:/tmp/
     fullPath = transform.transformDirToInternal(item["fullPath"])
     relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
     if not os.path.exists(fullPath):
         job.delete()
         return False#No job release, job was deleted.
     #################################################################
     # Start process the 
     #################################################################
     if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         zipFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
         self.decCopier.copy(fullPath, zipFilePath)
         for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
             print '--------------------------------------------------'
             print i
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
         for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
             fp = open(i, 'r')
             print 'data file extracted:', i
     '''
     else:
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         zipFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
         self.decCopier.copy(fullPath, zipFilePath)
         for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
             print '--------------------------------------------------'
             print i
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
         for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
             fp = open(i, 'r')
             print 'data file extracted:', i
     '''
     return True#Release job
Exemple #5
0
 def __init__(self, srcRoot, storageRoot, stateStoragePath = 'd:/state.txt'):
     #print 'src root:',srcRoot
     self.srcRoot = transform.transformDirToInternal(srcRoot)
     self.storageRoot = transform.transformDirToInternal(storageRoot)
     #print self.srcRoot
     #print self.storageRoot
     self.stateStoragePath = stateStoragePath
     try:
         f = open(self.stateStoragePath,'r')
         self.config = json.load(f)
         f.close()
     except IOError:
         self.config = {}
     #print 'storage root:', self.storageRoot
     self.curArchivedSize = 0
     self.curArchive = None
 def processJob(self, job, item):
     monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
     archiveId = "zippedInfoColllection://" + monitoringFullPath
     if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
         self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
     #Save the item in the archive collection: zippedInfoColllection://D:/tmp/
     fullPath = transform.transformDirToInternal(item["fullPath"])
     relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
     if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         for i in zippedInfo(self.workingDir).enumItems(fullPath):
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
     return True
Exemple #7
0
def importDb(input, password = None):
    dbSysInst = dbSys.dbSysSmart()
    sf = open(input, 'r')
    package = json.load(sf)
    en = enc.encryptorBase64Out(password)
    l = package["add"]
    if (password is None) and package.has_key("encPass"):
        print 'need password for importing'
        return
    if (not (password is None)):
        if unicode(str(md5.new(password).hexdigest())) != package["encPass"]:
            print 'pass not match:', unicode(str(md5.new(password+package["time-duration"]).hexdigest())), package["encPass"]
            return
        else:
            l = listEncryptor.dictListDecryptor(l, en)
    l = listEncryptor.dictListEncryptor(l)
    dbSysInst.importDb(l)
Exemple #8
0
def decryptBackup(sourceFullPath, target, password = None):
    sf = open(sourceFullPath, 'r')
    l = json.load(sf)
    if l.has_key("encPass"):
        if password is None:
            print 'need password'
            return
        else:
            en = enc.encryptorBase64Out(password)
            if str(md5.new(password+l["time-duration"]).hexdigest()) != l["encPass"]:
                print 'pass not match:', str(md5.new(password+l["time-duration"]).hexdigest()), l["encPass"]
                return
            res = dbExporterV3.dictListDecryptor(l["add"], en)
    else:
        res = l["add"]
    l["add"] = res
    del l["encPass"]
    s = json.dumps(l, sort_keys=True, indent=4)
    f = open(target,'w')
    f.write(s)
    f.close()
Exemple #9
0
def importDb(input, username, targetPasswd, password = None):
    dbSysInst = dbSys.dbSysSmart()
    sf = open(input, 'r')
    package = json.load(sf)
    en = enc.encryptorBase64Out(password)
    l = package["add"]
    if (password is None) and package.has_key("encPass"):
        print 'need password for importing'
        return
    if (not (password is None)):
        if unicode(str(md5.new(password).hexdigest())) != package["encPass"]:
            print 'pass not match:', unicode(str(md5.new(password+package["time-duration"]).hexdigest())), package["encPass"]
            return
        else:
            l = listEncryptor.dictListDecryptor(l, en)
    sysUser = service.ufsUser(u'system.user', u'system.pass')
    if userMan.userManager().verifyPasswd(username, targetPasswd, dbSys.dbSysSmart(sysUser).getDb("passwdDb")):
        print 'importing'
        l = listEncryptor.dictListEncryptor(l, xorEnc.encryptorTxtOut(targetPasswd))
        #print l
        l = listEncryptor.setUser(l, username)
        #print l
        dbSysInst.importDb(l)
 def subClassProcessItem(self, processingObj):
     encZipFileFullPath = processingObj["fullPath"]
     encryptedZipLogPath = encZipFileFullPath
     ncl(encryptedZipLogPath)
     zipLogPath = os.path.join(self.decryptionWorkingDir, 
             os.path.basename(encryptedZipLogPath).replace(gTargetEncLog, gZipLogExt))
     cl(zipLogPath)
     self.decCopier.copy(encryptedZipLogPath, zipLogPath)
     
     #Extract log file from zip
     logZip = zipClass.ZFile(zipLogPath, 'r')
     #logPath = zipLogPath.replace(gZipLogExt, gLogExt)
     logLoaded = False
     for logFileName in logZip.list():
         logZip.extract(logFileName, self.decryptionWorkingDir)
         logPath = os.path.join(self.decryptionWorkingDir, logFileName)
         #cl(logPath)
         #Read log file
         try:
             f = open(logPath,'r')
             newLog = json.load(f)
             f.close()
         except IOError:
             newLog = None
             raise 'log not read'
         if not (newLog is None):
             #########################
             #Log loaded, update collection
             #########################
             logLoaded = True
             print 'extracting info from log', encZipFileFullPath
             for i in newLog:
                 relaPath = transform.formatRelativePath(i)
                 ncl(newLog[i])
                 #Remove the uuid in log file
                 if newLog[i].has_key("uuid"):
                     newLog[i]["originalUuid"] = newLog[i]["uuid"]
                     del newLog[i]["uuid"]
                 #################
                 #Get collection
                 #################
                 if self.logCollection.exists(relaPath):
                     itemUuid = self.logCollection.getObjUuid(relaPath)
                     ncl("returned uuid:", itemUuid)
                     item = self.objDb.getObjFromUuid(itemUuid)
                     #Conflict, check if update needed
                     ncl(newLog[i]["timestamp"])
                     ncl(item["timestamp"])
                     if newLog[i]["timestamp"] > item["timestamp"]:
                         #The new item is newer, replace the old one
                         ncl('updating duplicated item to 1st one:', newLog[i]["timestamp"], item["timestamp"])
                         objUuid = self.objDb.addVirtualObj(newLog[i])
                         self.logCollection.updateObjUuid(relaPath, objUuid)
                     else:
                         ncl("no update, ignore")
                 else:
                     #Add object to obj objDb
                     objUuid = self.objDb.addVirtualObj(newLog[i])
                     #Add obj to collection
                     self.logCollection.addObj(relaPath, objUuid)
                     ncl("added new item", relaPath, newLog[i])
                     '''
                     encZipFileFullPath = transform.transformDirToInternal(encZipFileFullPath)
                     zipFileFullPath = self.getZipFile(encZipFileFullPath.replace(gTargetEncLog, gTargetEnc))
                     extractedItemInfo = self.getItemState(relaPath)
                     extractedItemFullPath = os.path.join(self.workingDir, i)
                     extractedItem = extractedZipStorageItem(self.workingDir,
                         extractedItemFullPath, extractedItemInfo, zipFileFullPath, None, relaPath)
                     '''
                     yield newLog[i]
     '''
    def subClassProcessItem(self, processingObj):
        #Check if the log file and data file are both OK
        encZipFileFullPath = processingObj["fullPath"]
        if not (re.search('\.enclog$', encZipFileFullPath) is None):
            #An encrypted zip file, go on
            if os.path.exists(encZipFileFullPath.replace('.enclog', '.enc')):
                #Log and Data are both OK
                encryptedZipLogPath = encZipFileFullPath
                ncl(encryptedZipLogPath)
                zipLogPath = os.path.join(self.decryptionWorkingDir, 
                        os.path.basename(encryptedZipLogPath).replace('.enclog', '.log'))
                ncl(zipLogPath)
                self.decCopier.copy(encryptedZipLogPath, zipLogPath)
                try:
                    f = open(zipLogPath,'r')
                    newLog = json.load(f)
                    f.close()
                except IOError:
                    newLog = None
                if not (newLog is None):
                    #########################
                    #Log loaded, update collection
                    #########################
                    for i in newLog:
                        relaPath = transform.formatRelativePath(i)
                        ncl(newLog[i])
                        #################
                        #Get collection
                        #################
                        if self.logCollection.exists(relaPath):
                            itemUuid = self.logCollection.getObjUuid(relaPath)
                            ncl("returned uuid:", itemUuid)
                            item = self.db.getObjFromUuid(itemUuid)
                            #Conflict, check if update needed
                            ncl(newLog[i]["timestamp"])
                            ncl(item["timestamp"])
                            if newLog[i]["timestamp"] > item["timestamp"]:
                                #The new item is newer, replace the old one
                                ncl('updating duplicated item to 1st one:', newLog[i]["timestamp"], item["timestamp"])
                                objUuid = self.db.addVirtualObj(newLog[i])
                                self.logCollection.updateObjUuid(relaPath, objUuid)
                            else:
                                ncl("no update, ignore")
                        else:
                            #Add object to obj db
                            objUuid = self.db.addVirtualObj(newLog[i])
                            #Add obj to collection
                            self.logCollection.addObj(relaPath, objUuid)
                            ncl("added new item", relaPath, newLog[i])
                
                    #################################
                    #Process data
                    #################################
                    encZipFileFullPath = transform.transformDirToInternal(encZipFileFullPath)
                    ncl(encZipFileFullPath)
                    
                    zipFileFullPath = self.getZipFile(encZipFileFullPath.replace(".enclog", ".enc"))
                    #For all element in the zip file
                    #Enumerate all files in the decrypted zip file
                    zf = zipClass.ZFile(zipFileFullPath, 'r')
                    #Generate a log file if it does not exist
                        
                    for i in zf.list():
                        #zf.extract(i, self.workingDir)
                        extractedItemFullPath = os.path.join(self.workingDir, i)
                        relaPath = transform.formatRelativePath(i)
                        extractedItemInfo = self.getItemState(relaPath)

                        extractedItem = encZipStorage.extractedZipStorageItem(self.workingDir, 
                                extractedItemFullPath, extractedItemInfo, zf, relaPath)
                        ###########################
                        #Store the file
                        ###########################
                        self.targetCollection.store(extractedItem)
                    ##########################
                    #Everything goes OK
                    #Quit
                    ##########################
                    return
                else:
                    cl('Load log file failed',encZipFileFullPath)
            else:
                #Data file not exist, push it back
                ncl('Data file not exist, push it back: ',encZipFileFullPath)
        else:
            ncl('not a encrypted zip file: ',encZipFileFullPath)
            return
        ############
        #Item not processed, push it back
        ############
        processorBase.cacheCollectionProcessorBase.subClassProcessItem(self, processingObj)
Exemple #12
0
    def subClassProcessItem(self, processingObj):
        encZipFileFullPath = processingObj["fullPath"]
        encryptedZipLogPath = encZipFileFullPath
        ncl(encryptedZipLogPath)
        zipLogPath = os.path.join(
            self.decryptionWorkingDir, os.path.basename(encryptedZipLogPath).replace(gTargetEncLog, gZipLogExt)
        )
        cl(zipLogPath)
        self.decCopier.copy(encryptedZipLogPath, zipLogPath)

        # Extract log file from zip
        logZip = zipClass.ZFile(zipLogPath, "r")
        # logPath = zipLogPath.replace(gZipLogExt, gLogExt)
        logLoaded = False
        for logFileName in logZip.list():
            logZip.extract(logFileName, self.decryptionWorkingDir)
            logPath = os.path.join(self.decryptionWorkingDir, logFileName)
            # cl(logPath)
            # Read log file
            try:
                f = open(logPath, "r")
                newLog = json.load(f)
                f.close()
            except IOError:
                newLog = None
                raise "log not read"
            if not (newLog is None):
                #########################
                # Log loaded, update collection
                #########################
                logLoaded = True
                print "extracting info from log", encZipFileFullPath
                for i in newLog:
                    relaPath = transform.formatRelativePath(i)
                    ncl(newLog[i])
                    # Remove the uuid in log file
                    if newLog[i].has_key("uuid"):
                        newLog[i]["originalUuid"] = newLog[i]["uuid"]
                        del newLog[i]["uuid"]
                    #################
                    # Get collection
                    #################
                    if self.logCollection.exists(relaPath):
                        itemUuid = self.logCollection.getObjUuid(relaPath)
                        ncl("returned uuid:", itemUuid)
                        item = self.objDb.getObjFromUuid(itemUuid)
                        # Conflict, check if update needed
                        ncl(newLog[i]["timestamp"])
                        ncl(item["timestamp"])
                        if newLog[i]["timestamp"] > item["timestamp"]:
                            # The new item is newer, replace the old one
                            ncl("updating duplicated item to 1st one:", newLog[i]["timestamp"], item["timestamp"])
                            objUuid = self.objDb.addVirtualObj(newLog[i])
                            self.logCollection.updateObjUuid(relaPath, objUuid)
                        else:
                            ncl("no update, ignore")
                    else:
                        # Add object to obj objDb
                        objUuid = self.objDb.addVirtualObj(newLog[i])
                        # Add obj to collection
                        self.logCollection.addObj(relaPath, objUuid)
                        ncl("added new item", relaPath, newLog[i])
        if logLoaded:
            #################################
            # Process data
            #################################
            print "extracting info from log complete", encZipFileFullPath
            encZipFileFullPath = transform.transformDirToInternal(encZipFileFullPath)
            ncl(encZipFileFullPath)

            zipFileFullPath = self.getZipFile(encZipFileFullPath.replace(gTargetEncLog, gTargetEnc))
            # For all element in the zip file
            # Enumerate all files in the decrypted zip file
            zf = zipClass.ZFile(zipFileFullPath, "r")
            # Generate a log file if it does not exist

            for i in zf.list():
                # zf.extract(i, self.workingDir)
                extractedItemFullPath = os.path.join(self.workingDir, i)
                relaPath = transform.formatRelativePath(i)
                ncl(relaPath)
                extractedItemInfo = self.getItemState(relaPath)

                extractedItem = extractedZipStorageItem(
                    self.workingDir, extractedItemFullPath, extractedItemInfo, zipFileFullPath, zf, relaPath
                )
                """
                ###########################
                #Store the file
                ###########################
                #self.targetCollection.store(extractedItem)
                """
                ###########################
                # Returning object
                ###########################
                yield extractedItem
            ##########################
            # Everything goes OK
            # Quit
            ##########################
            return
        else:
            cl("Load log file failed", encZipFileFullPath)