def processItem(self, job, item): monitoringFullPath = transform.transformDirToInternal(item["monitoringPath"]) fullPath = transform.transformDirToInternal(item["fullPath"]) # Check if item exists in local file sytem if not os.path.exists(fullPath): job.delete() return if not self.itemToProcess.has_key(monitoringFullPath): self.itemToProcess[monitoringFullPath] = {} ############################################# # Start processing ############################################# # If the full path already in tube, check if the timestamp is updated if self.itemToProcess[monitoringFullPath].has_key(fullPath): savedItem = self.itemToProcess[monitoringFullPath][fullPath] if savedItem["timestamp"] == item["timestamp"]: # Item not updated for time out time, add it to output queue self.outputBeanstalk.put(job.body) print "output item:", item job.delete() elif savedItem["timestamp"] < item["timestamp"]: # Received a new notification for an path, update saved info self.itemToProcess[monitoringFullPath][fullPath] = item job.release(priority=beanstalkc.DEFAULT_PRIORITY, delay=gItemDelayTime) print "item updated" else: job.delete() else: # New notification, add it self.itemToProcess[monitoringFullPath][fullPath] = item # print item, job, gItemDelayTime # priority is necessary to avoid error for requesting priority to be an int in beanstalkc job.release(priority=beanstalkc.DEFAULT_PRIORITY, delay=gItemDelayTime) print "new item added"
def processItem(self, job, item): #fullPath = transform.transformDirToInternal(item["fullPath"]) #monitoringFullPath = transform.transformDirToInternal(item["monitoringPath"]) #source_dir = item["SourceDir"] #misc.ensureDir(source_dir) tag = item["tag"] working_dir = item["WorkingDir"] misc.ensureDir(transform.transformDirToInternal(working_dir)) target_dir = item["TargetDir"] misc.ensureDir(transform.transformDirToInternal(target_dir)) import wwjufsdatabase.libs.services.servicesV2 as service req = service.req() t = tagSystem.getTagSysObj(req.getDbSys()) e = t.getObjs(unicode(tag)) for i in e: print i source_dir = transform.transformDirToInternal(i) AutoArchiveThumb(source_dir, target_dir, working_dir) job.delete() return False #Return true only when the item should be kept in the tube return True
def encInfoZip(self, pendingCollection): ############################ # Save info for zipped files ############################ logFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, '.log')) s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4) f = open(logFilePath,'w') f.write(s) f.close() logZipPath = logFilePath.replace(u'.log',u'.log.zip') logZip = zipClass.ZFile(logZipPath, 'w') logZip.addfile(unicode(logFilePath), os.path.basename(logFilePath)) logZip.close() gTimeV = time.gmtime() yearStr = time.strftime("%Y", gTimeV) monthStr = time.strftime("%m", gTimeV) dayStr = time.strftime("%d", gTimeV) dateTimeDir = yearStr+"/"+monthStr+"/"+dayStr newEncDir = unicode(os.path.join(self.zipStorageDir, dateTimeDir)) misc.ensureDir(newEncDir) targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(newEncDir, '.enc')) self.encCopier.copy(logZipPath, targetPath.replace('.enc', '.encziplog')) ############################ # Update state in storage state ############################ self.updateZipLog(self.zippedFileInfo, pendingCollection) #Clean the current zipped file info self.zippedFileInfo = {}
def AutoArchiveThumb(source_folder = gAutoArchiveFullPath, target_dir = g_default_target_dir, workingDir = gWorkingDir,taskUuid = str(uuid.uuid4())): inputTubeName = "collectionListTube"+taskUuid delayedCollectionListTubeName = "delayedCollectionListTubeName"+taskUuid #s1 = FolderScanner.FolderScanner() #s1.addItem({"command": "folderScanner", "fullPath":source_folder, # "targetTubeName": inputTubeName,"blackList":g_ignore_file_type_list}) target_dir = transform.transformDirToInternal(target_dir) source_folder = transform.transformDirToInternal(source_folder) #storage_state_collection_name = "storage_state://"+source_folder+":"+target_dir s1 = FolderEnumeratingService.FolderEnumeratingService() s1.addItem({"full_path": source_folder, "black_list":[], "target_tube_name": inputTubeName}) s2 = monitorService.monitorService() s2.addItem({"command": "monitorService", "fullPath":source_folder, "targetTubeName": inputTubeName,"blackList":g_ignore_file_type_list}) s3 = tubeDelayService() s3.addItem({"inputTubeName":inputTubeName, "outputTubeName": delayedCollectionListTubeName,"blackList":g_ignore_file_type_list}) s4 = FolderInfoArchiveService() s4.addItem({"InputTubeName":delayedCollectionListTubeName, "WorkingDir":workingDir, "TargetDir": target_dir})
def __init__(self, rootPath, fullPath, itemInfo, zipFileObj, pathInZipFile): folderStorage.folderStorageItem.__init__(self, rootPath, fullPath) self.itemInfo = itemInfo self.rootPath = transform.transformDirToInternal(rootPath) self.fullPath = transform.transformDirToInternal(fullPath) self.zipFileObj = zipFileObj self.pathInZipFile = pathInZipFile
def __init__(self, rootDir, backupDir, syncFolderCollectionId, dbInst): ''' syncFolderCollectionId is a virtual collection which contains all items with synced info ''' collectionDatabase.collectionOnMongoDbBase.__init__(self, syncFolderCollectionId, dbInst.getCollectionDb()) self.rootDir = transform.transformDirToInternal(rootDir) self.backupDir = transform.transformDirToInternal(backupDir) self.objDb = dbInst self.folderCollection = folderRecursiveEnumCollection.folderRecursiveEnumCollection(self.rootDir, dbInst)
def processItem(self, job, item): #fullPath = transform.transformDirToInternal(item["fullPath"]) #monitoringFullPath = transform.transformDirToInternal(item["monitoringPath"]) #source_dir = item["SourceDir"] #misc.ensureDir(source_dir) tag = item["tag"] task_item = item if item.has_key("output_tube_name"): ################################# # Adding tag processing task ################################# task_item = item self.processing_tag_dict[tag] = item import wwjufsdatabase.libs.services.servicesV2 as service req = service.req() t = tagSystem.getTagSysObj(req.getDbSys()) tagged_item_list = t.getObjs(unicode(tag)) else: ################################# # A new tag added for existing tag processing task ################################# if self.processing_tag_dict.has_key(tag): #Task exist, add the new tagged elment for processing task_item = self.processing_tag_dict[tag] tagged_item_list = [transform.transformDirToInternal(item["url"])] else: #Not a valid item, return print "not a valid item or tag not have processor yet" job.delete() return False output_tube_name = task_item["output_tube_name"] working_dir = task_item["working_dir"] misc.ensureDir(transform.transformDirToInternal(working_dir)) target_dir = task_item["target_dir"] misc.ensureDir(transform.transformDirToInternal(target_dir)) b = beanstalkServiceBase(output_tube_name) for i in tagged_item_list: info(i) source_dir = transform.transformDirToInternal(i) b.addItem({"source_dir":source_dir, "working_dir": working_dir, "target_dir":target_dir}) job.delete() return False
def callback(self, pathToWatch, relativePath, changeType): fullPath = transform.transformDirToInternal(os.path.join(pathToWatch, relativePath)) itemDict = {"monitoringPath": transform.transformDirToInternal(pathToWatch), "fullPath": fullPath, "changeType":changeType, "timestamp": time.time()} s = json.dumps(itemDict, sort_keys=True, indent=4) beanstalk = beanstalkc.Connection(host=gBeanstalkdServerHost, port=gBeanstalkdServerPort) beanstalk.use(self.targetTube) #print beanstalk.using() s = json.dumps(itemDict, sort_keys=True, indent=4) job = beanstalk.put(s)
def __init__(self, srcRoot, storageRoot, stateStoragePath = 'd:/state.txt'): print "src root is:", srcRoot self.srcRoot = transform.transformDirToInternal(srcRoot) self.storageRoot = transform.transformDirToInternal(storageRoot) self.stateStoragePath = stateStoragePath try: f = open(self.stateStoragePath,'r') self.config = json.load(f) f.close() except IOError: self.config = {}
def store(self, element): #print 'storing....' fullPath = transform.transformDirToInternal(element.getAbsPath()) relPath = fullPath.replace(self.srcRoot, '') if (self.curArchive is None) or (self.curArchivedSize > MAX_SINGLE_ARCHIVE_SIZE): self.curArchiveName = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.storageRoot, '.zip')) self.curArchive = zipClass.ZFile(self.curArchiveName, 'w') self.curArchivedSize = 0 #print 'copying "%s" to "%s"'%(fullPath, relPath) self.curArchive.addfile(unicode(fullPath).encode('gbk'), unicode(relPath).encode('gbk')) self.curArchivedSize += os.stat(fullPath).st_size
def processItem(self, job, item): monitoringFullPath = transform.transformDirToInternal(item['monitoringPath']) archiveId = gZipFolderCollectionPrefix + monitoringFullPath if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath): self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb()) objUuid = self.dbInst.addVirtualObj({"monitoringPath": monitoringFullPath, "zippedInfoCollectionId": archiveId}); idInCol = objUuid self.zippedInfoCollectionList.addObj(idInCol, objUuid) #Save the item in the archive collection: zippedInfoColllection://D:/tmp/ fullPath = transform.transformDirToInternal(item["fullPath"]) relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath) if not os.path.exists(fullPath): job.delete() return False#No job release, job was deleted. ################################################################# # Start process the ################################################################# if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath): #This item is not in the collection, so we need to extract info from this item newObj = self.dbInst.getFsObjFromFullPath(fullPath) self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"]) zipFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix)) self.decCopier.copy(fullPath, zipFilePath) for i in zippedInfo(self.workingDir).enumItems(zipFilePath): print '--------------------------------------------------' print i fp = open(i, 'r') loadedFileInfo = json.load(fp) print loadedFileInfo for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath): fp = open(i, 'r') print 'data file extracted:', i ''' else: #This item is not in the collection, so we need to extract info from this item newObj = self.dbInst.getFsObjFromFullPath(fullPath) self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"]) zipFilePath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, gInfoFileDecryptedExt, gInfoFilePrefix)) self.decCopier.copy(fullPath, zipFilePath) for i in zippedInfo(self.workingDir).enumItems(zipFilePath): print '--------------------------------------------------' print i fp = open(i, 'r') loadedFileInfo = json.load(fp) print loadedFileInfo for i in zippedInfo(self.workingDir).enumZippedFiles(zipFilePath): fp = open(i, 'r') print 'data file extracted:', i ''' return True#Release job
def checkDirChanges(path_to_watch, busname = BUS_NAME_NAME, interfacename = INTERFACE_NAME, objname = OBJ_NAME): path_to_watch = transform.transformDirToInternal(os.path.abspath (path_to_watch)) need_to_quit = False print "Watching %s at %s" % (path_to_watch, time.asctime ()) hDir = win32file.CreateFile( path_to_watch, win32con.GENERIC_READ, win32con.FILE_SHARE_READ|win32con.FILE_SHARE_WRITE, None, win32con.OPEN_EXISTING, win32con.FILE_FLAG_BACKUP_SEMANTICS, None ) cnt = 0 bus = dbus.SessionBus() proxy = bus.get_object(busname, objname) ############################################### #Scan for existing files ############################################### for i in os.walk(path_to_watch): print i for j in i[2]: fullPath = transform.transformDirToInternal(os.path.join(i[0], j)) #print fullPath proxy.notify(path_to_watch, fullPath, "Existing", False, dbus_interface = interfacename) while not need_to_quit: # print "new watch\n" results = win32file.ReadDirectoryChangesW( hDir, 1024*256, True, win32con.FILE_NOTIFY_CHANGE_FILE_NAME | win32con.FILE_NOTIFY_CHANGE_DIR_NAME | win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES | win32con.FILE_NOTIFY_CHANGE_SIZE | win32con.FILE_NOTIFY_CHANGE_LAST_WRITE | win32con.FILE_NOTIFY_CHANGE_SECURITY, None, None ) if not need_to_quit: for action, file in results: #full_filename = os.path.join (self.path_to_watch, file) #print full_filename, ACTIONS.get (action, "Unknown") #callback(self.path_to_watch, file, ACTIONS.get (action, "Unknown")) print 'filechanged called:', path_to_watch, file, ACTIONS.get (action, "Unknown") proxy.notify(path_to_watch, file, ACTIONS.get (action, "Unknown"), True, dbus_interface = interfacename)
def genPicThumb(local_path, dest_dir, mime_type = None): #If no thumbnail exist, create one #print '-----------------------localpath:',local_path basename = os.path.basename(local_path) #print "basename:" + basename ext = basename.split(".")[-1] #print ext #if picFormatSupported(ext): if picFormatSupportedV2(local_path, mime_type = None): #It is a jpeg file, currently no other type supported import Image #Using PIL lib im = Image.open(local_path) # convert to thumbnail image im.thumbnail((g_default_thumb_size, g_default_thumb_size), Image.ANTIALIAS) # don't save if thumbnail already exists #Use _T as the thumb file end to indicate the end of the original firl thumb_path_without_ext = os.path.join(dest_dir, basename.split(".")[0]+"_T") import random while os.path.exists(thumb_path_without_ext+".jpg"): thumb_path_without_ext += str(random.randint(0,10)) thumb_path = thumb_path_without_ext+'.jpg' #print thumb_path.encode("utf8","replace") if im.mode != "RGB": im = im.convert("RGB") im.save(thumb_path, "JPEG") return transform.transformDirToInternal(thumb_path) else: print 'non jpeg file not supported' raise pictureFormatNotSupported
def store(self, processingObj, pendingCollection): ''' processingObj = {"fullPath": "D:/tmp/good.txt", "size":100} ''' ncl(processingObj) #relaPath = transform.formatRelativePath(item.getRelaPath()) relaPath = processingObj.getIdInCol() ncl('Got relaPath') if (pendingCollection.has_key(relaPath)) and (pendingCollection[relaPath] != processingObj["uuid"]): #Item exists in pending but uuid is not the same, update the uuid for the pending item pendingCollection[relaPath] = processingObj["uuid"] cl('Added to pending') fullPath = transform.transformDirToInternal(processingObj["fullPath"]) #Add the file to zip try: #If there is already an item with the same name, ignore the current? existingElem = self.zippedFileInfo[relaPath] return except: pass if (self.fileCnt > MAX_FILE_CNT_IN_INFO_FILE): self.encInfoZip(pendingCollection) processingObj["parentEncZip"] = self.targetPath.replace(".zip", ".enc") self.zippedFileInfo[relaPath] = processingObj.getItemInfo() cl('return from store')
def run(self): print 'Start scanning' if not os.path.isdir(self.rootFolder): print "not a folder" if filter(self.rootFolder, self.blackList): return paramDict = {"fullPath": self.rootFolder, "timestamp": os.stat(self.rootFolder)[ST_MTIME], "monitoringPath": self.rootFolder} self.addItem(paramDict) else: for root, dirs, files in os.walk(self.rootFolder): #Break if quit called if self.quit_flag: break #cl("remaining:", dirs) #Process files for j in dirs: info(j) if filter(j, self.blackList): info("ignoring: ", j, "\n") continue fullPath = transform.transformDirToInternal(os.path.join(root, j)) paramDict = {"fullPath": fullPath, "timestamp": os.stat(fullPath)[ST_MTIME], "monitoringPath": self.rootFolder} self.addItem(paramDict) print "process complete, quitting thread"
def __init__(self, srcRoot, storageRoot, stateStoragePath = 'd:/state.txt'): #print 'src root:',srcRoot self.srcRoot = transform.transformDirToInternal(srcRoot) self.storageRoot = transform.transformDirToInternal(storageRoot) #print self.srcRoot #print self.storageRoot self.stateStoragePath = stateStoragePath try: f = open(self.stateStoragePath,'r') self.config = json.load(f) f.close() except IOError: self.config = {} #print 'storage root:', self.storageRoot self.curArchivedSize = 0 self.curArchive = None
def __init__(self, srcRoot, storageRoot, stateStoragePath = 'd:/state.txt', tmpStorageRoot = 'd:/tmp/removeAfterComplete', decCopier = encryptionStorageBase.arc4DecSimpleCopier('defaultPass')): misc.ensureDir(tmpStorageRoot) misc.ensureDir(storageRoot) zipStorage.zipStorage.__init__(self, srcRoot, storageRoot, stateStoragePath) self.tmpStorageRoot = transform.transformDirToInternal(tmpStorageRoot) self.decCopier = decCopier
def initParam(self, zipDir, folderDir, workingDir, encryptionPass, direction): ################################# #Make dir if not exist ################################# misc.ensureDir(zipDir) misc.ensureDir(workingDir) misc.ensureDir(folderDir) self.configPath = os.path.join(workingDir, 'workingState.txt') self.backupPath = os.path.join(workingDir, 'backup') misc.ensureDir(self.backupPath) self.tmpStorageRoot = transform.transformDirToInternal(os.path.join(workingDir, 'working')) self.config = configDict.configFileDict(self.configPath, {"zipStorageState":{}, "folderState":{}}) ################################# #Create source storage ################################# self.storage1 = encZipStorage.encZipStorage(self.config["zipStorageState"], self.tmpStorageRoot, zipDir, encryptionPass) ################################# #Create target storage ################################# self.storage2 = folderStorage.folderStorage(self.config["folderState"], folderDir, self.backupPath) if direction == "extract": self.srcStorage = self.storage1 self.dstStorage = self.storage2 else: self.srcStorage = self.storage2 self.dstStorage = self.storage1
def subClassRun(self, paramDict): ############################################### #Scan for existing files ############################################### collection = self.objDb.getCollection(self.targetCollectionId) cl('start scanning') for i in os.walk(self.rootFolder): #cl(i) for j in i[2]: if (self.addedItemCnt % 1000) == 0: cl("processing item cnt:", self.addedItemCnt) self.addedItemCnt += 1 fullPath = transform.transformDirToInternal(os.path.join(i[0], j)) #print '---------------------real adding item' #Update the item info for the item ncl('before fs obj base') itemUrl = objectDatabase.fsObjBase(fullPath).getObjUrl() ncl('before get fs obj') newObjUuid = self.objDb.getFsObjUuid(itemUrl) if newObjUuid is None: cl("item deleted, do not add it") continue ncl('before update obj uuid') ''' collection.updateObjUuidIfNeeded(itemUrl, newObjUuid) ''' if collection.isSame(itemUrl, newObjUuid): ncl("no updates needed", itemUrl, newObjUuid) continue collection.updateObjUuidRaw(itemUrl, newObjUuid) ncl('new item added', itemUrl) cl("notifying listener") self.notifyAll()
def internal_get_thumb(path, targetDir, mime_type = None): ''' path: Full Path. The path of the file whose thumbnail will be generated targetDir: Directory Path. The target directory where the generated thumbnail will be put in. Return: the thumbnail fullPath ''' newPath = None ext = path.split('.')[-1].lower() if ext in ['exe']: try: newPath = appThumb.genAppThumb(path, targetDir) except: return None else: try: newPath = picThumbGenerator.genPicThumb(path, targetDir, mime_type) except picThumbGenerator.pictureFormatNotSupported: if ext in g_video_file_ext_list: try:#if True: newPath = ffmpegThumb.genVideoThumb(path, targetDir) #return "complete transform" #return newPath except: pass else: pass if newPath is None: return None return transformDirToInternal(newPath)
def getRange(self, start, cnt): ''' Check if element is in the list ''' res = [] #print self.fullPath #If the param is unicode, it will return unicode try: d = os.listdir(self.fullPath) except WindowsError: return [] #print d for i in d: if self.folderOnly: if not os.path.isdir(os.path.join(self.fullPath,i)): continue ''' if type(i.decode('gbk')) != unicode: raise "unknown issue" i.decode('utf8').encode('gbk') ''' p = transform.transformDirToInternal(os.path.join(self.fullPath,i)) #p.encode('utf8')#Test if utf can support decoding filesystem chars. res.append(p) if cnt is None: return res return res[start:start+cnt]
def encZip(self): #Must close the zip before encrypt it, otherwise, the file are not integrate if self.curArchive is None: return self.curArchive.close() self.curArchive = None ############################ # Encrypt the zip file ############################ targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.zipStorageDir, '.enc')) print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath) #import shutil #shutil.copy(self.curArchiveName, targetPath+'.backup.zip') self.encCopier.copy(self.curArchiveName, targetPath) ############################ # Save info for zipped files ############################ s = json.dumps(self.zippedFileInfo, sort_keys=True, indent=4) f = open(self.curArchiveName.replace('.zip', '.log'),'w') f.write(s) f.close() self.encCopier.copy(self.curArchiveName.replace('.zip', '.log'), targetPath.replace('.enc', '.enclog')) ############################ # Update state in storage state ############################ self.updateZipLog(self.zippedFileInfo) #Clean the current zipped file info self.zippedFileInfo = {} zipFileFolderStorageItem = folderStorage.folderStorageItem(self.zipStorageDir, targetPath) self.lastState.zipFileUpdate(zipFileFolderStorageItem)
def on_ok_button_clicked(self, widget, data=None): import wwjufsdatabase.libs.services.servicesV2 as service req = service.req() #gtk.main_quit() entry = self.builder.get_object("entry1") tag = entry.get_text() print tag tag_list_raw = tag.split(",") tag_list = [] for i in tag_list_raw: if i == "": continue tag_list.append(unicode(i)) t = tagSystem.getTagSysObj(req.getDbSys()) for i in self.liststore: #print i[0], i[1] if i[0]: url = i[1] url = url.replace("file:///", "") url = unicode(url) full_path = transform.transformDirToInternal(url) #print "full_path is:", full_path t.tag(full_path, tag_list) from localLibs.services.beanstalkdServices.TagProcessServiceV2 import TagProcessService p = TagProcessService() for cur_tag in tag_list: p.addItem({"url":full_path, "tag":cur_tag}) self.window1.hide()
def processItem(self, job, item): source_dir = item["source_dir"] if os.path.isdir(source_dir): misc.ensureDir(transform.transformDirToInternal(source_dir)) working_dir = item["working_dir"] misc.ensureDir(transform.transformDirToInternal(working_dir)) target_dir = item["target_dir"] misc.ensureDir(transform.transformDirToInternal(target_dir)) AutoArchiveThumb(source_dir, target_dir, working_dir) #Must delete the job if it is no longer needed and return False so the job will not be put back to tube job.delete() return False
def __init__(self, taskId, appUuid, targetRootDir, targetBackupDir, collectionId, workingDir, passwd): #First set the input param, __init__ will create initial config from this by calling subClassInitialCfg advCollectionProcessor.__init__(self, taskId, appUuid, collectionId) #The param initiated in this class should be check by this class if self.appConfigObj["logCollectionId"] != logCollectionId: raise "Task parameter does not match" #Create the 2 collections #print self.db self.folderCol = syncFolderCollection.syncFolderCollection(transform.transformDirToInternal(targetRootDir), transform.transformDirToInternal(targetBackupDir), syncFolderCollectionId, self.db) self.encZipCol = encZipCollection.encZipCollection(transform.transformDirToInternal(collectionId), logCollectionId, transform.transformDirToInternal(workingDir), passwd, self.db)
def encZip(self): ############################ #Encrypt the zip file ############################ targetPath = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.storageRoot, '.enc')) print 'copying "%s" to "%s"'%(self.curArchiveName, targetPath) self.encCopier.copy(self.curArchiveName, targetPath)
def __init__ ( self, tubeName, rootFolder, blackList = []): self.blackList = blackList self.rootFolder = transform.transformDirToInternal(rootFolder) super(FolderEnumeratingThread, self).__init__ (tubeName) self.quit_flag = False import wwjufsdatabase.libs.services.servicesV2 as service self.req = service.req() self.obj_db = self.req.getObjDbSys()
def __init__(self, outputFolder, rootFolder, username = "******", passwd = "nopass", targetCollectionId = None , dbPrefix = "test"): #print rootFolder self.rootFolder = transform.transformDirToInternal(rootFolder) self.outputFolder = transform.transformDirToInternal(outputFolder) #print self.rootFolder threadHndl = "recursive://" + self.rootFolder self.userSession = service.ufsUser(username, passwd) #print username, passwd self.objDb = objectDatabase.objectDatabase(self.userSession, dbPrefix = dbPrefix) # if not (targetCollectionId is None): # self.targetCollectionId = targetCollectionId # else: # self.targetCollectionId = "folder://" + self.rootFolder super(workThread, self).__init__(threadHndl, "singleton") # self.partialRes = [] self.addedItemCnt = 0
def processJob(self, job, item): monitoringFullPath = transform.transformDirToInternal(item['monitoringPath']) archiveId = "zippedInfoColllection://" + monitoringFullPath if not self.collectionInDbForMonitoringPath.has_key(monitoringFullPath): self.collectionInDbForMonitoringPath[monitoringFullPath] = collectionDatabase.collectionOnMongoDbBase(archiveId, self.dbInst.getCollectionDb()) #Save the item in the archive collection: zippedInfoColllection://D:/tmp/ fullPath = transform.transformDirToInternal(item["fullPath"]) relativePath = transform.getRelativePathFromFull(fullPath, monitoringFullPath) if not self.collectionInDbForMonitoringPath[monitoringFullPath].exists(relativePath): #This item is not in the collection, so we need to extract info from this item newObj = self.dbInst.getFsObjFromFullPath(fullPath) self.collectionInDbForMonitoringPath[monitoringFullPath].addObj(relativePath, newObj["uuid"]) for i in zippedInfo(self.workingDir).enumItems(fullPath): fp = open(i, 'r') loadedFileInfo = json.load(fp) print loadedFileInfo return True
def createNewZip(self): #################### # Create new zip file #################### self.curArchiveName = transform.transformDirToInternal( fileTools.getTimestampWithFreeName(self.workingDir, '.zip')) self.curArchive = zipClass.ZFile(self.curArchiveName, 'w') self.curArchivedSize = 0