def encInfoZip(self, pendingCollection):
############################
# Save info for zipped files
############################
logFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, '.log'))
s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
f = open(logFilePath,'w')
f.write(s)
f.close()
logZipPath = logFilePath.replace(u'.log',u'.log.zip')
logZip = zipClass.ZFile(logZipPath, 'w')
logZip.addfile(unicode(logFilePath), os.path.basename(logFilePath))
logZip.close()
gTimeV = time.gmtime()
yearStr = time.strftime("%Y", gTimeV)
monthStr = time.strftime("%m", gTimeV)
dayStr = time.strftime("%d", gTimeV)
dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir))
misc.ensureDir(newEncDir)
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(newEncDir, '.enc'))
self.encCopier.copy(logZipPath, targetPath.replace('.enc', '.encziplog'))
############################
# Update state in storage state
############################
self.updateZipLog(self.zippedFileInfo, pendingCollection)
#Clean the current zipped file info
self.zippedFileInfo = {}
def encZip(self):
#Must close the zip before encrypt it, otherwise, the file are not integrate
if self.curArchive is None:
return
self.curArchive.close()
self.curArchive = None
############################
# Encrypt the zip file
############################
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc'))
print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
#import shutil
#shutil.copy(self.curArchiveName, targetPath+'.backup.zip')
self.encCopier.copy(self.curArchiveName, targetPath)
############################
# Save info for zipped files
############################
s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
f = open(self.curArchiveName.replace('.zip', '.log'),'w')
f.write(s)
f.close()
self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog'))
############################
# Update state in storage state
############################
self.updateZipLog(self.zippedFileInfo)
#Clean the current zipped file info
self.zippedFileInfo = {}
zipFileFolderStorageItem = folderStorage.folderStorageItem(self.zipStorageDir, targetPath)
self.lastState.zipFileUpdate(zipFileFolderStorageItem)
def encZip(self):
############################
#Encrypt the zip file
############################
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.storageRoot, '.enc'))
print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
self.encCopier.copy(self.curArchiveName, targetPath)
def createNewZip(self):
####################
# Create new zip file
####################
self.curArchiveName = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
self.curArchivedSize = 0
def createNewZip(self):
####################
# Create new zip file
####################
gTimeV = time.gmtime()
yearStr = time.strftime("%Y", gTimeV)
monthStr = time.strftime("%m", gTimeV)
dayStr = time.strftime("%d", gTimeV)
dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir))
misc.ensureDir(newEncDir)
self.curArchiveName = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
self.targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(newEncDir, '.enc'))
cl("Creating new zip file", self.curArchiveName)
self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
self.curArchivedSize = 0
def encZip(self):
if self.curArchive is None:
return
############################
#Encrypt the zip file
############################
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.options.targetDir, '.enc'))
print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
self.encCopier.copy(self.curArchiveName, targetPath)
def get_date_based_path(root_folder, ext = ".7z"):
gTimeV = time.gmtime()
yearStr = time.strftime("%Y", gTimeV)
monthStr = time.strftime("%m", gTimeV)
dayStr = time.strftime("%d", gTimeV)
dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
newEncDir = unicode(os.path.join(root_folder, dateTimeDir))
ensureDir(newEncDir)
file_full_path = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(newEncDir, ext))
return file_full_path
def getStorgePathWithDateFolder(rootPath, ext = ".enc"):
gTimeV = time.gmtime()
yearStr = time.strftime("%Y", gTimeV)
monthStr = time.strftime("%m", gTimeV)
dayStr = time.strftime("%d", gTimeV)
dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr
newEncDir = unicode(os.path.join(rootPath, dateTimeDir))
misc.ensureDir(newEncDir)
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(newEncDir, ext))
return targetPath
def store(self, element):
#print 'storing....'
fullPath = transform.transformDirToInternal(element.getAbsPath())
relPath = fullPath.replace(self.srcRoot, '')
if (self.curArchive is None) or (self.curArchivedSize > MAX_SINGLE_ARCHIVE_SIZE):
self.curArchiveName = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.storageRoot, '.zip'))
self.curArchive = zipClass.ZFile(self.curArchiveName, 'w')
self.curArchivedSize = 0
#print 'copying "%s" to "%s"'%(fullPath, relPath)
self.curArchive.addfile(unicode(fullPath).encode('gbk'), unicode(relPath).encode('gbk'))
self.curArchivedSize += os.stat(fullPath).st_size
def processItem(self, job, item):
monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
archiveId = gZipFolderCollectionPrefix + monitoringFullPath
if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId});
idInCol = objUuid
self.zippedInfoCollectionList.addObj(idInCol, objUuid)
#Save the item in the archive collection: zippedInfoColllection://D:/tmp/
fullPath = transform.transformDirToInternal(item["fullPath"])
relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
if not os.path.exists(fullPath):
job.delete()
return False#No job release, job was deleted.
#################################################################
# Start process the
#################################################################
if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
#This item is not in the collection, so we need to extract info from this item
newObj = self.dbInst.getFsObjFromFullPath(fullPath)
self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
zipFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
self.decCopier.copy(fullPath, zipFilePath)
for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
print '--------------------------------------------------'
print i
fp = open(i, 'r')
loadedFileInfo = json.load(fp)
print loadedFileInfo
for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
fp = open(i, 'r')
print 'data file extracted:', i
'''
else:
#This item is not in the collection, so we need to extract info from this item
newObj = self.dbInst.getFsObjFromFullPath(fullPath)
self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
zipFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
self.decCopier.copy(fullPath, zipFilePath)
for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
print '--------------------------------------------------'
print i
fp = open(i, 'r')
loadedFileInfo = json.load(fp)
print loadedFileInfo
for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
fp = open(i, 'r')
print 'data file extracted:', i
'''
return True#Release job
def finalizeZipFile(self):
#Add info to zip file
self.additionalInfoDict["collectionContentInfo"] = self.collectionInfoDict
ncl(self.collectionInfoDict)
s = json.dumps(self.additionalInfoDict, sort_keys=True, indent=4)
infoFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, "."+gInfoFileExt, gInfoFilePrefix))
logFile = open(infoFilePath, 'w')
logFile.write(s)
logFile.close()
self.getZipFile().addfile(unicode(infoFilePath), unicode(infoFilePath))
self.package_file.close()
#Set attribute so new zip will be created if this object is still in use
self.package_file = None
self.additionalInfoDict = {}
return self.package_file_full_path
def finalize(self):
#print self.info_dict
#print len(self.info_dict)
if len(self.info_dict) == 0:
print "finalize without any content, return directly"
return
s = json.dumps(self.info_dict, sort_keys=True, indent=4)
infoFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.working_dir, "."+gInfoFileExt, gInfoFilePrefix))
logFile = open(infoFilePath, 'w')
logFile.write(s)
logFile.close()
#print s
info(infoFilePath)
self.storage.add_file(infoFilePath)
self.storage.finalize_one_trunk()
for i in self.saving_items:
self.collection.addObj(i, self.saving_items[i])
self.saving_items = {}
self.info_dict = {}
info("trunk finalized")
def getLogFilePath(self):
fileList = {}
numberList = []
for i in os.listdir(self.logFullPath):
try:
number = float(i.replace(".log",""))
except:
continue
fileList[number] = i
numberList.append(number)
numberList.sort()
#print numberList
deleteLen = len(numberList) - self.maxLogFile
#print deleteLen
if deleteLen >= 0:
#Delete old ones
for i in numberList[0:deleteLen+1]:
#print 'deleting:', i
try:
os.remove(os.path.join(self.logFullPath, fileList[i]))
except WindowsError:
pass
return fileTools.getTimestampWithFreeName(self.logFullPath, '.log')
def encZip(self, pendingCollection):
#Must close the zip before encrypt it, otherwise, the file are not integrate
if self.curArchive is None:
return
self.curArchive.close()
self.curArchive = None
############################
# Encrypt the zip file
############################
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc'))
cl('copying "%s" to "%s"'%(self.curArchiveName, targetPath))
##############################
############################################
#TODO: update the processed item list, so this new created item will not be processed by the extractor again
self.encCopier.copy(self.curArchiveName, targetPath)
############################
# Save info for zipped files
############################
s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4)
f = open(self.curArchiveName.replace('.zip', '.log'),'w')
f.write(s)
f.close()
############################################
#TODO: update the processed item list, so this new created item will not be processed by the extractor again
self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog'))
############################
# Update state in storage state
############################
self.updateZipLog(self.zippedFileInfo, pendingCollection)
#Clean the current zipped file info
self.zippedFileInfo = {}
def store(self, element):
#print 'storing....'
targetPath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.tmpStorageRoot, '.zip'))
print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath)
self.decCopier.copy(self.curArchiveName, targetPath)
############################
#Extract zip file
############################
zf = zipClass.ZFile(targetPath, 'r')
for i in zf.list():
targetFullPath = os.path.join(self.storageRoot, i)
if os.path.exists(targetFullPath):
print 'file exists:', targetFullPath
else:
#print 'extracting file:', targetFullPath
zf.extract(i, self.storageRoot)
############################
#Do not need to update the source file (encrypted zip file)'s modified time as the archiver will update it
#But need to update the extracted file's modified time
############################
targetFullPath = os.join(self.storageRoot, i)
self.config[targetFullPath] = os.stat(targetFullPath).st_mtime
def getZipFile(self):
if self.package_file is None:
self.package_file_full_path = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.trunk_data_path, self.ext))
self.package_file = self.package_class(self.package_file_full_path, 'w', self.passwd)
return self.package_file
def getZipFile(self):
if self.package_file is None:
self.package_file_full_path = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, '.zip'))
self.package_file = zipClass.ZFile(self.package_file_full_path, 'w')
return self.package_file