def __init__(self, taskId, folderRoot,
encZipRoot, passwd, workingDir = 'd:/tmp/fileman/working', appUuid = gAppUuid):
processorBase.processorBase.__init__(self, taskId, appUuid)
self.appConfigObj = self.getAppCfg()
if self.appConfigObj is None:
self.appConfigObj = {}
self.appConfigObj["encZipRoot"] = transform.transformDirToInternal(encZipRoot)
self.appConfigObj["folderRoot"] = transform.transformDirToInternal(folderRoot)
self.appConfigObj["workingDir"] = transform.transformDirToInternal(workingDir)
self.appConfigObj["targetBackupDir"] = transform.transformDirToInternal(os.path.join(workingDir, "backup"))
self.appConfigObj["syncFolderCollectionId"] = unicode(str(uuid.uuid4()))
self.appConfigObj["logCollectionId"] = unicode(str(uuid.uuid4()))
self.appConfigObj["archivePendingCollectionId"] = unicode(str(uuid.uuid4()))
self.appConfigObj["extractionPendingCollectionId"] = unicode(str(uuid.uuid4()))
self.appConfigObj["archiveNextToProcessTimestamp"] = 0
self.appConfigObj["extractionNextToProcessTimestamp"] = 0
self.saveAppConfig()
else:
self.expectedDict = {}
self.expectedDict["encZipRoot"] = transform.transformDirToInternal(encZipRoot)
self.expectedDict["folderRoot"] = transform.transformDirToInternal(folderRoot)
self.expectedDict["workingDir"] = transform.transformDirToInternal(workingDir)
self.expectedDict["targetBackupDir"] = transform.transformDirToInternal(os.path.join(workingDir, "backup"))
self.checkParamInternal(self.expectedDict)
self.curTimestampName = "extractionNextToProcessTimestamp"
self.archivePendingCollectionId = self.appConfigObj["archivePendingCollectionId"]
self.extractionPendingCollectionId = self.appConfigObj["extractionPendingCollectionId"]
self.syncFolderCollectionId = self.appConfigObj["syncFolderCollectionId"]
self.logCollectionId = self.appConfigObj["logCollectionId"]
collectionDbInst = self.db.getCollectionDb()
self.archivePending = collectionDatabase.collectionOnMongoDbBase(self.archivePendingCollectionId,
collectionDbInst)
self.extractionPending = collectionDatabase.collectionOnMongoDbBase(self.extractionPendingCollectionId,
collectionDbInst)
#Create the 2 collections
#print self.db
self.folderCol = syncFolderCollection.syncFolderCollection(self.appConfigObj["folderRoot"],
self.appConfigObj["targetBackupDir"],
self.appConfigObj["syncFolderCollectionId"], self.db)
self.encZipCol = encZipCollection.encZipCollection(self.appConfigObj["encZipRoot"],
self.appConfigObj["logCollectionId"],
self.appConfigObj["workingDir"], passwd, self.db)
ncl(self.appConfigObj)
def __init__(self, taskId, appUuid, srcCollectionId, destCollectionId):
self.srcCollectionId = srcCollectionId
self.destCollectionId = destCollectionId
processorBase.collectionProcessorBase.__init__(self, taskId, appUuid, srcCollectionId)
if (self.srcCollectionId != self.appConfigObj["srcCollectionId"]) or (self.destCollectionId !=
self.appConfigObj["destCollectionId"]):
cl(self.srcCollectionId, self.appConfigObj["srcCollectionId"],
self.destCollectionId, self.appConfigObj["destCollectionId"])
raise "Task parameter does not match"
collectionDbInst = self.db.getCollectionDb()
self.archivePendingCollectionId = self.appConfigObj["archivePendingCollectionId"]
self.extractionPendingCollectionId = self.appConfigObj["extractionPendingCollectionId"]
self.archivePending = collectionDatabase.collectionOnMongoDbBase(self.archivePendingCollectionId,
collectionDbInst)
self.extractionPending = collectionDatabase.collectionOnMongoDbBase(self.extractionPendingCollectionId,
collectionDbInst)
self.lastTimestamp = self.appConfigObj["nextToProcessTimestamp"]
def __init__(self, taskId, appUuid, collectionId, workingDir, passwd, targetCollection):
processorBase.cacheCollectionProcessorBase.__init__(self, taskId, appUuid, collectionId)
#This dir stores zip files which were decrypted
self.workingDir = workingDir
self.decryptionWorkingDir = workingDir + '/decrypted'
misc.ensureDir(self.decryptionWorkingDir)
self.logCollectionId = self.appConfigObj["logCollectionId"]
self.logCollection = collectionDatabase.collectionOnMongoDbBase(self.logCollectionId, self.db.getCollectionDb())
self.targetCollection = targetCollection
self.passwd = passwd
self.encCopier = encryptionStorageBase.arc4EncSimpleCopier(passwd)
self.decCopier = encryptionStorageBase.arc4DecSimpleCopier(passwd)
def __init__ ( self, tubeName, workingDir = "d:/tmp/working/zippedCollectionListHandler", passwd = '123'):
misc.ensureDir(workingDir)
super(zippedCollectionListHandler, self).__init__(tubeName)
threading.Thread.__init__(self)
# Stores collection instance for given monitoring path, all zipped objects
# in this monitoring path will be stored in this collection
self.collectionInDbForMonitoringPath = {}
self.dbInst = objectDatabase.objectDatabase()
self.workingDir = workingDir
self.encCopier = encryptionStorageBase.arc4EncSimpleCopier(passwd)
self.decCopier = encryptionStorageBase.arc4DecSimpleCopier(passwd)
self.zippedInfoCollectionList = collectionDatabase.collectionOnMongoDbBase(gZippedInfoCollectionId, self.dbInst.getCollectionDb())
def processItem(self, job, item):
monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
archiveId = gZipFolderCollectionPrefix + monitoringFullPath
if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId});
idInCol = objUuid
self.zippedInfoCollectionList.addObj(idInCol, objUuid)
#Save the item in the archive collection: zippedInfoColllection://D:/tmp/
fullPath = transform.transformDirToInternal(item["fullPath"])
relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
if not os.path.exists(fullPath):
job.delete()
return False#No job release, job was deleted.
#################################################################
# Start process the
#################################################################
if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
#This item is not in the collection, so we need to extract info from this item
newObj = self.dbInst.getFsObjFromFullPath(fullPath)
self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
zipFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
self.decCopier.copy(fullPath, zipFilePath)
for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
print '--------------------------------------------------'
print i
fp = open(i, 'r')
loadedFileInfo = json.load(fp)
print loadedFileInfo
for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
fp = open(i, 'r')
print 'data file extracted:', i
'''
else:
#This item is not in the collection, so we need to extract info from this item
newObj = self.dbInst.getFsObjFromFullPath(fullPath)
self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
zipFilePath = transform.transformDirToInternal(
fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix))
self.decCopier.copy(fullPath, zipFilePath)
for i in zippedInfo(self.workingDir).enumItems(zipFilePath):
print '--------------------------------------------------'
print i
fp = open(i, 'r')
loadedFileInfo = json.load(fp)
print loadedFileInfo
for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath):
fp = open(i, 'r')
print 'data file extracted:', i
'''
return True#Release job
def __init__(self, taskId, appUuid, rootDir, workingDir, passwd, encZipStorageCollectionId, folderStorageCollectionId):
self.rootDir = transform.transformDirToInternal(rootDir)
processorBase.cacheCollectionProcessorBase.__init__(self, taskId, appUuid, self.rootDir)
########################################
#Internal used vars
########################################
#The current zipfile object
self.curArchive = None
#Not necessary init as it will be inited with self.curArchive
#The current archived size
self.curArchivedSize = 0
self.logCollectionId = encZipStorageCollectionId
self.logCollection = collectionDatabase.collectionOnMongoDbBase(self.logCollectionId, self.db.getCollectionDb())
self.folderStorageCollectionId = folderStorageCollectionId
self.folderStorageCollection = collectionDatabase.collectionOnMongoDbBase(self.folderStorageCollectionId, self.db.getCollectionDb())
#The info of files in the current zip file
self.zippedFileInfo = {}
#The info of the whole storage
self.zipStorageState = None
self.encCopier = encryptionStorageBase.arc4EncSimpleCopier(passwd)
self.decCopier = encryptionStorageBase.arc4DecSimpleCopier(passwd)
self.workingDir = workingDir
def processJob(self, job, item):
monitoringFullPath = transform.transformDirToInternal(item['monitoringPath'])
archiveId = "zippedInfoColllection://" + monitoringFullPath
if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath):
self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb())
#Save the item in the archive collection: zippedInfoColllection://D:/tmp/
fullPath = transform.transformDirToInternal(item["fullPath"])
relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath)
if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath):
#This item is not in the collection, so we need to extract info from this item
newObj = self.dbInst.getFsObjFromFullPath(fullPath)
self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"])
for i in zippedInfo(self.workingDir).enumItems(fullPath):
fp = open(i, 'r')
loadedFileInfo = json.load(fp)
print loadedFileInfo
return True
def __init__(self, collectionId, logCollectionId, workingDir, passwd, dbInst):
'''
collectionId is the id of a collection contains all enc zip files
logCollectionId is the id of a collection contains all extracted info
'''
self.objDb = dbInst
self.collectionDbInst = dbInst.getCollectionDb()
#This dir stores zip files which were decrypted
self.workingDir = workingDir
self.decryptionWorkingDir = workingDir + '/decrypted'
misc.ensureDir(self.decryptionWorkingDir)
#This is the folder collection that contains all encrypted zip files
self.collectionId = collectionId
ncl(collectionId)
self.collection = folderRecursiveEnumCollection.folderRecursiveEnumCollection(self.collectionId, dbInst)
#This is the log collection which stores all extracted info, these info may
#be extracted every time the folder is scaned, but it'll be better to store them for future use
self.logCollectionId = logCollectionId
self.logCollection = collectionDatabase.collectionOnMongoDbBase(self.logCollectionId, self.collectionDbInst)
self.passwd = passwd
self.encCopier = encryptionStorageBase.arc4EncSimpleCopier(passwd)
self.decCopier = encryptionStorageBase.arc4DecSimpleCopier(passwd)
def __init__(self, taskId, collection1,
collection2, appUuid = gAppUuid):
processorBase.processorBase.__init__(self, taskId, appUuid)
self.appConfigObj = self.getAppCfg()
if self.appConfigObj is None:
self.appConfigObj = {}
self.appConfigObj["pendingCollectionId"] = unicode(str(uuid.uuid4()))
self.appConfigObj["nextToProcessTimestamp"] = 0
self.saveAppConfig()
else:
self.expectedDict = {}
self.checkParamInternal(self.expectedDict)
self.curTimestampName = "extractionNextToProcessTimestamp"
self.pendingCollectionId = self.appConfigObj["pendingCollectionId"]
collectionDbInst = self.db.getCollectionDb()
self.pending = collectionDatabase.collectionOnMongoDbBase(self.pendingCollectionId,
collectionDbInst)
#Create the 2 collections
#print self.db
self.folderCol = collection1
self.encZipCol = collection2
ncl(self.appConfigObj)
def getCollection(self, collectionId):
'''
may be:
folder://C:/
'''
return collectionDatabase.collectionOnMongoDbBase(collectionId, self.collectionDb)
def __init__(self, taskId, appUuid, collectionId):
collectionProcessorBase.__init__(self, taskId, appUuid, collectionId)
collectionDbInst = self.db.getCollectionDb()
#originalCollection = self.db.getFsCollection()
self.collection = collectionDatabase.collectionOnMongoDbBase(collectionId, collectionDbInst)
cl(collectionId)
