def encInfoZip(self, pendingCollection): ############################ # Save info for zipped files ############################ logFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, '.log')) s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4) f = open(logFilePath,'w') f.write(s) f.close() logZipPath = logFilePath.replace(u'.log',u'.log.zip') logZip = zipClass.ZFile(logZipPath, 'w') logZip.addfile(unicode(logFilePath), os.path.basename(logFilePath)) logZip.close() gTimeV = time.gmtime() yearStr = time.strftime("%Y", gTimeV) monthStr = time.strftime("%m", gTimeV) dayStr = time.strftime("%d", gTimeV) dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir)) misc.ensureDir(newEncDir) targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(newEncDir, '.enc')) self.encCopier.copy(logZipPath, targetPath.replace('.enc', '.encziplog')) ############################ # Update state in storage state ############################ self.updateZipLog(self.zippedFileInfo, pendingCollection) #Clean the current zipped file info self.zippedFileInfo = {}
def encZip(self): #Must close the zip before encrypt it, otherwise, the file are not integrate if self.curArchive is None: return self.curArchive.close() self.curArchive = None ############################ # Encrypt the zip file ############################ targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc')) print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath) #import shutil #shutil.copy(self.curArchiveName, targetPath+'.backup.zip') self.encCopier.copy(self.curArchiveName, targetPath) ############################ # Save info for zipped files ############################ s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4) f = open(self.curArchiveName.replace('.zip', '.log'),'w') f.write(s) f.close() self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog')) ############################ # Update state in storage state ############################ self.updateZipLog(self.zippedFileInfo) #Clean the current zipped file info self.zippedFileInfo = {} zipFileFolderStorageItem = folderStorage.folderStorageItem(self.zipStorageDir, targetPath) self.lastState.zipFileUpdate(zipFileFolderStorageItem)
def encZip(self): ############################ #Encrypt the zip file ############################ targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.storageRoot, '.enc')) print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath) self.encCopier.copy(self.curArchiveName, targetPath)
def createNewZip(self): #################### # Create new zip file #################### self.curArchiveName = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, '.zip')) self.curArchive = zipClass.ZFile(self.curArchiveName, 'w') self.curArchivedSize = 0
def createNewZip(self): #################### # Create new zip file #################### gTimeV = time.gmtime() yearStr = time.strftime("%Y", gTimeV) monthStr = time.strftime("%m", gTimeV) dayStr = time.strftime("%d", gTimeV) dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir)) misc.ensureDir(newEncDir) self.curArchiveName = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, '.zip')) self.targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(newEncDir, '.enc')) cl("Creating new zip file", self.curArchiveName) self.curArchive = zipClass.ZFile(self.curArchiveName, 'w') self.curArchivedSize = 0
def encZip(self): if self.curArchive is None: return ############################ #Encrypt the zip file ############################ targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.options.targetDir, '.enc')) print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath) self.encCopier.copy(self.curArchiveName, targetPath)
def get_date_based_path(root_folder, ext = ".7z"): gTimeV = time.gmtime() yearStr = time.strftime("%Y", gTimeV) monthStr = time.strftime("%m", gTimeV) dayStr = time.strftime("%d", gTimeV) dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr newEncDir = unicode(os.path.join(root_folder, dateTimeDir)) ensureDir(newEncDir) file_full_path = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(newEncDir, ext)) return file_full_path
def getStorgePathWithDateFolder(rootPath, ext = ".enc"): gTimeV = time.gmtime() yearStr = time.strftime("%Y", gTimeV) monthStr = time.strftime("%m", gTimeV) dayStr = time.strftime("%d", gTimeV) dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr newEncDir = unicode(os.path.join(rootPath, dateTimeDir)) misc.ensureDir(newEncDir) targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(newEncDir, ext)) return targetPath
def store(self, element): #print 'storing....' fullPath = transform.transformDirToInternal(element.getAbsPath()) relPath = fullPath.replace(self.srcRoot, '') if (self.curArchive is None) or (self.curArchivedSize > MAX_SINGLE_ARCHIVE_SIZE): self.curArchiveName = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.storageRoot, '.zip')) self.curArchive = zipClass.ZFile(self.curArchiveName, 'w') self.curArchivedSize = 0 #print 'copying "%s" to "%s"'%(fullPath, relPath) self.curArchive.addfile(unicode(fullPath).encode('gbk'), unicode(relPath).encode('gbk')) self.curArchivedSize += os.stat(fullPath).st_size
def processItem(self, job, item): monitoringFullPath = transform.transformDirToInternal(item['monitoringPath']) archiveId = gZipFolderCollectionPrefix + monitoringFullPath if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath): self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb()) objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId}); idInCol = objUuid self.zippedInfoCollectionList.addObj(idInCol, objUuid) #Save the item in the archive collection: zippedInfoColllection://D:/tmp/ fullPath = transform.transformDirToInternal(item["fullPath"]) relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath) if not os.path.exists(fullPath): job.delete() return False#No job release, job was deleted. ################################################################# # Start process the ################################################################# if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath): #This item is not in the collection, so we need to extract info from this item newObj = self.dbInst.getFsObjFromFullPath(fullPath) self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"]) zipFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix)) self.decCopier.copy(fullPath, zipFilePath) for i in zippedInfo(self.workingDir).enumItems(zipFilePath): print '--------------------------------------------------' print i fp = open(i, 'r') loadedFileInfo = json.load(fp) print loadedFileInfo for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath): fp = open(i, 'r') print 'data file extracted:', i ''' else: #This item is not in the collection, so we need to extract info from this item newObj = self.dbInst.getFsObjFromFullPath(fullPath) self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"]) zipFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix)) self.decCopier.copy(fullPath, zipFilePath) for i in zippedInfo(self.workingDir).enumItems(zipFilePath): print '--------------------------------------------------' print i fp = open(i, 'r') loadedFileInfo = json.load(fp) print loadedFileInfo for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath): fp = open(i, 'r') print 'data file extracted:', i ''' return True#Release job
def finalizeZipFile(self): #Add info to zip file self.additionalInfoDict["collectionContentInfo"] = self.collectionInfoDict ncl(self.collectionInfoDict) s = json.dumps(self.additionalInfoDict, sort_keys=True, indent=4) infoFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, "."+gInfoFileExt, gInfoFilePrefix)) logFile = open(infoFilePath, 'w') logFile.write(s) logFile.close() self.getZipFile().addfile(unicode(infoFilePath), unicode(infoFilePath)) self.package_file.close() #Set attribute so new zip will be created if this object is still in use self.package_file = None self.additionalInfoDict = {} return self.package_file_full_path
def finalize(self): #print self.info_dict #print len(self.info_dict) if len(self.info_dict) == 0: print "finalize without any content, return directly" return s = json.dumps(self.info_dict, sort_keys=True, indent=4) infoFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.working_dir, "."+gInfoFileExt, gInfoFilePrefix)) logFile = open(infoFilePath, 'w') logFile.write(s) logFile.close() #print s info(infoFilePath) self.storage.add_file(infoFilePath) self.storage.finalize_one_trunk() for i in self.saving_items: self.collection.addObj(i, self.saving_items[i]) self.saving_items = {} self.info_dict = {} info("trunk finalized")
def getLogFilePath(self): fileList = {} numberList = [] for i in os.listdir(self.logFullPath): try: number = float(i.replace(".log","")) except: continue fileList[number] = i numberList.append(number) numberList.sort() #print numberList deleteLen = len(numberList) - self.maxLogFile #print deleteLen if deleteLen >= 0: #Delete old ones for i in numberList[0:deleteLen+1]: #print 'deleting:', i try: os.remove(os.path.join(self.logFullPath, fileList[i])) except WindowsError: pass return fileTools.getTimestampWithFreeName(self.logFullPath, '.log')
def encZip(self, pendingCollection): #Must close the zip before encrypt it, otherwise, the file are not integrate if self.curArchive is None: return self.curArchive.close() self.curArchive = None ############################ # Encrypt the zip file ############################ targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc')) cl('copying "%s" to "%s"'%(self.curArchiveName, targetPath)) ############################## ############################################ #TODO: update the processed item list, so this new created item will not be processed by the extractor again self.encCopier.copy(self.curArchiveName, targetPath) ############################ # Save info for zipped files ############################ s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4) f = open(self.curArchiveName.replace('.zip', '.log'),'w') f.write(s) f.close() ############################################ #TODO: update the processed item list, so this new created item will not be processed by the extractor again self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog')) ############################ # Update state in storage state ############################ self.updateZipLog(self.zippedFileInfo, pendingCollection) #Clean the current zipped file info self.zippedFileInfo = {}
def store(self, element): #print 'storing....' targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.tmpStorageRoot, '.zip')) print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath) self.decCopier.copy(self.curArchiveName, targetPath) ############################ #Extract zip file ############################ zf = zipClass.ZFile(targetPath, 'r') for i in zf.list(): targetFullPath = os.path.join(self.storageRoot, i) if os.path.exists(targetFullPath): print 'file exists:', targetFullPath else: #print 'extracting file:', targetFullPath zf.extract(i, self.storageRoot) ############################ #Do not need to update the source file (encrypted zip file)'s modified time as the archiver will update it #But need to update the extracted file's modified time ############################ targetFullPath = os.join(self.storageRoot, i) self.config[targetFullPath] = os.stat(targetFullPath).st_mtime
def getZipFile(self): if self.package_file is None: self.package_file_full_path = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.trunk_data_path, self.ext)) self.package_file = self.package_class(self.package_file_full_path, 'w', self.passwd) return self.package_file
def getZipFile(self): if self.package_file is None: self.package_file_full_path = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, '.zip')) self.package_file = zipClass.ZFile(self.package_file_full_path, 'w') return self.package_file