Example #1
0
 def encInfoZip(self, pendingCollection):
     ############################
     # Save info for zipped files
     ############################
     logFilePath = transform.transformDirToInternal(
         fileTools.getTimestampWithFreeName(self.workingDir, '.log'))
     s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
     f = open(logFilePath,'w')
     f.write(s)
     f.close()
     logZipPath = logFilePath.replace(u'.log',u'.log.zip')
     logZip = zipClass.ZFile(logZipPath, 'w')
     logZip.addfile(unicode(logFilePath), os.path.basename(logFilePath))
     logZip.close()
     
     gTimeV = time.gmtime()
     yearStr = time.strftime("%Y", gTimeV)
     monthStr = time.strftime("%m", gTimeV)
     dayStr = time.strftime("%d", gTimeV)
     dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
     newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir))
     misc.ensureDir(newEncDir)
     targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(newEncDir, '.enc'))
     self.encCopier.copy(logZipPath, targetPath.replace('.enc', '.encziplog'))
     
     
     ############################
     # Update state in storage state
     ############################
     self.updateZipLog(self.zippedFileInfo, pendingCollection)
     #Clean the current zipped file info
     self.zippedFileInfo = {}
Example #2
0
 def encZip(self):
     #Must close the zip before encrypt it, otherwise, the file are not integrate
     if self.curArchive is None:
         return
     self.curArchive.close()
     self.curArchive = None
     
     ############################
     # Encrypt the zip file
     ############################
     targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc'))
     print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
     #import shutil
     #shutil.copy(self.curArchiveName, targetPath+'.backup.zip')
     self.encCopier.copy(self.curArchiveName, targetPath)
     
     ############################
     # Save info for zipped files
     ############################
     s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
     f = open(self.curArchiveName.replace('.zip', '.log'),'w')
     f.write(s)
     f.close()
     self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog'))
     ############################
     # Update state in storage state
     ############################
     self.updateZipLog(self.zippedFileInfo)
     #Clean the current zipped file info
     self.zippedFileInfo = {}
     zipFileFolderStorageItem = folderStorage.folderStorageItem(self.zipStorageDir, targetPath)
     self.lastState.zipFileUpdate(zipFileFolderStorageItem)
Example #3
0
 def encZip(self):
     ############################
     #Encrypt the zip file
     ############################
     targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.storageRoot, '.enc'))
     print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
     self.encCopier.copy(self.curArchiveName, targetPath)
Example #4
0
 def createNewZip(self):
     ####################
     # Create new zip file
     ####################
     self.curArchiveName = transform.transformDirToInternal(
         fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
     self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
     self.curArchivedSize = 0
Example #5
0
 def createNewZip(self):
     ####################
     # Create new zip file
     ####################
     gTimeV = time.gmtime()
     yearStr = time.strftime("%Y", gTimeV)
     monthStr = time.strftime("%m", gTimeV)
     dayStr = time.strftime("%d", gTimeV)
     dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
     newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir))
     misc.ensureDir(newEncDir)
     self.curArchiveName = transform.transformDirToInternal(
         fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
     self.targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(newEncDir, '.enc'))
     cl("Creating new zip file", self.curArchiveName)
     self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
     self.curArchivedSize = 0
Example #6
0
 def encZip(self):
     if self.curArchive is None:
         return
     ############################
     #Encrypt the zip file
     ############################
     targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.options.targetDir, '.enc'))
     print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
     self.encCopier.copy(self.curArchiveName, targetPath)
Example #7
0
File: misc.py Project: weijia/ufs
def get_date_based_path(root_folder, ext = ".7z"):
    gTimeV = time.gmtime()
    yearStr = time.strftime("%Y", gTimeV)
    monthStr = time.strftime("%m", gTimeV)
    dayStr = time.strftime("%d", gTimeV)
    dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
    newEncDir = unicode(os.path.join(root_folder, dateTimeDir))
    ensureDir(newEncDir)
    file_full_path = transform.transformDirToInternal(
        fileTools.getTimestampWithFreeName(newEncDir, ext))
    return file_full_path
Example #8
0
def getStorgePathWithDateFolder(rootPath, ext = ".enc"):
    gTimeV = time.gmtime()
    yearStr = time.strftime("%Y", gTimeV)
    monthStr = time.strftime("%m", gTimeV)
    dayStr = time.strftime("%d", gTimeV)
    dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
    newEncDir = unicode(os.path.join(rootPath, dateTimeDir))
    misc.ensureDir(newEncDir)
    targetPath = transform.transformDirToInternal(
            fileTools.getTimestampWithFreeName(newEncDir, ext))
    return targetPath
Example #9
0
 def store(self, element):
     #print 'storing....'
     fullPath = transform.transformDirToInternal(element.getAbsPath())
     relPath = fullPath.replace(self.srcRoot, '')
     if (self.curArchive is None) or (self.curArchivedSize > MAX_SINGLE_ARCHIVE_SIZE):
         self.curArchiveName = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.storageRoot, '.zip'))
         self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
         self.curArchivedSize = 0
     #print 'copying "%s" to "%s"'%(fullPath, relPath)
     self.curArchive.addfile(unicode(fullPath).encode('gbk'), unicode(relPath).encode('gbk'))
     self.curArchivedSize += os.stat(fullPath).st_size
 def processItem(self, job, item):
     monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
     archiveId = gZipFolderCollectionPrefix + monitoringFullPath
     if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
         self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
         objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId});
         idInCol = objUuid
         self.zippedInfoCollectionList.addObj(idInCol, objUuid)
     #Save the item in the archive collection: zippedInfoColllection://D:/tmp/
     fullPath = transform.transformDirToInternal(item["fullPath"])
     relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
     if not os.path.exists(fullPath):
         job.delete()
         return False#No job release, job was deleted.
     #################################################################
     # Start process the 
     #################################################################
     if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         zipFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
         self.decCopier.copy(fullPath, zipFilePath)
         for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
             print '--------------------------------------------------'
             print i
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
         for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
             fp = open(i, 'r')
             print 'data file extracted:', i
     '''
     else:
         #This item is not in the collection, so we need to extract info from this item
         newObj = self.dbInst.getFsObjFromFullPath(fullPath)
         self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
         zipFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
         self.decCopier.copy(fullPath, zipFilePath)
         for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
             print '--------------------------------------------------'
             print i
             fp = open(i, 'r')
             loadedFileInfo = json.load(fp)
             print loadedFileInfo
         for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
             fp = open(i, 'r')
             print 'data file extracted:', i
     '''
     return True#Release job
Example #11
0
 def finalizeZipFile(self):
     #Add info to zip file
     self.additionalInfoDict["collectionContentInfo"] = self.collectionInfoDict
     ncl(self.collectionInfoDict)
     s = json.dumps(self.additionalInfoDict, sort_keys=True, indent=4)
     infoFilePath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, "."+gInfoFileExt, gInfoFilePrefix))
     logFile = open(infoFilePath, 'w')
     logFile.write(s)
     logFile.close()
     self.getZipFile().addfile(unicode(infoFilePath), unicode(infoFilePath))
     self.package_file.close()
     #Set attribute so new zip will be created if this object is still in use
     self.package_file = None
     self.additionalInfoDict = {}
     return self.package_file_full_path
Example #12
0
 def finalize(self):
     #print self.info_dict
     #print len(self.info_dict)
     if len(self.info_dict) == 0:
         print "finalize without any content, return directly"
         return
     s = json.dumps(self.info_dict, sort_keys=True, indent=4)
     infoFilePath = transform.transformDirToInternal(
         fileTools.getTimestampWithFreeName(self.working_dir, "."+gInfoFileExt, gInfoFilePrefix))
     logFile = open(infoFilePath, 'w')
     logFile.write(s)
     logFile.close()
     #print s
     info(infoFilePath)
     self.storage.add_file(infoFilePath)
     self.storage.finalize_one_trunk()
     for i in self.saving_items:
         self.collection.addObj(i, self.saving_items[i])
     self.saving_items = {}
     self.info_dict = {}
     info("trunk finalized")
Example #13
0
File: logDir.py Project: weijia/ufs
 def getLogFilePath(self):
     fileList = {}
     numberList = []
     for i in os.listdir(self.logFullPath):
         try:
             number = float(i.replace(".log",""))
         except:
             continue
         fileList[number] = i
         numberList.append(number)
     numberList.sort()
     #print numberList
     deleteLen = len(numberList) - self.maxLogFile
     #print deleteLen
     if deleteLen >= 0:
         #Delete old ones
         for i in numberList[0:deleteLen+1]:
             #print 'deleting:', i
             try:
                 os.remove(os.path.join(self.logFullPath, fileList[i]))
             except WindowsError:
                 pass
     return fileTools.getTimestampWithFreeName(self.logFullPath, '.log')
Example #14
0
    def encZip(self, pendingCollection):
        #Must close the zip before encrypt it, otherwise, the file are not integrate
        if self.curArchive is None:
            return
        self.curArchive.close()
        self.curArchive = None
        
        ############################
        # Encrypt the zip file
        ############################
        targetPath = transform.transformDirToInternal(
                fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc'))
        cl('copying "%s" to "%s"'%(self.curArchiveName, targetPath))

        ##############################
        ############################################
        #TODO: update the processed item list, so this new created item will not be processed by the extractor again
        self.encCopier.copy(self.curArchiveName, targetPath)
        
        ############################
        # Save info for zipped files
        ############################
        s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
        f = open(self.curArchiveName.replace('.zip', '.log'),'w')
        f.write(s)
        f.close()
        ############################################
        #TODO: update the processed item list, so this new created item will not be processed by the extractor again
        self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog'))
        
        
        ############################
        # Update state in storage state
        ############################
        self.updateZipLog(self.zippedFileInfo, pendingCollection)
        #Clean the current zipped file info
        self.zippedFileInfo = {}
Example #15
0
 def store(self, element):
     #print 'storing....'
     targetPath = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.tmpStorageRoot, '.zip'))
     print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
     self.decCopier.copy(self.curArchiveName, targetPath)
     
     ############################
     #Extract zip file
     ############################
     zf = zipClass.ZFile(targetPath, 'r')
     for i in zf.list():
         targetFullPath = os.path.join(self.storageRoot, i)
         if os.path.exists(targetFullPath):
             print 'file exists:', targetFullPath
         else:
             #print 'extracting file:', targetFullPath
             zf.extract(i, self.storageRoot)
             ############################
             #Do not need to update the source file (encrypted zip file)'s modified time as the archiver will update it
             #But need to update the extracted file's modified time
             ############################
             targetFullPath = os.join(self.storageRoot, i)
             self.config[targetFullPath] = os.stat(targetFullPath).st_mtime
Example #16
0
 def getZipFile(self):
     if self.package_file is None:
         self.package_file_full_path = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.trunk_data_path, self.ext))
         self.package_file = self.package_class(self.package_file_full_path, 'w', self.passwd)
     return self.package_file
Example #17
0
 def getZipFile(self):
     if self.package_file is None:
         self.package_file_full_path = transform.transformDirToInternal(
             fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
         self.package_file = zipClass.ZFile(self.package_file_full_path, 'w')
     return self.package_file