class Monitor(): def __init__(self, dm, paths): self.dm = dm self.style = MetaDataStyle.CIX self.queue = Queue.Queue(0) self.paths = paths self.eventList = [] self.mutex = threading.Lock() self.eventProcessingTimer = None self.quit_when_done = False # for debugging/testing self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = "" def start(self): self.thread = threading.Thread(target=self.mainLoop) self.thread.daemon = True self.quit = False self.thread.start() def stop(self): self.quit = True self.thread.join() def mainLoop(self): logging.debug("Monitor: started main loop.") self.session = self.dm.Session() self.library = Library(self.dm.Session) observer = Observer() self.eventHandler = MonitorEventHandler(self) for path in self.paths: if os.path.exists(path): observer.schedule(self.eventHandler, path, recursive=True) observer.start() while True: try: (msg, args) = self.queue.get(block=True, timeout=1) except: msg = None #dispatch messages if msg == "scan": self.dofullScan(self.paths) if msg == "events": self.doEventProcessing(args) #time.sleep(1) if self.quit: break self.session.close() self.session = None observer.stop() logging.debug("Monitor: stopped main loop.") def scan(self): self.queue.put(("scan", None)) def handleSingleEvent(self, event): # events may happen in clumps. start a timer # to defer processing. if the timer is already going, # it will be canceled # in the future there can be more smarts about # granular file events. for now this will be # good enough to just get a a trigger that *something* # changed self.mutex.acquire() if self.eventProcessingTimer is not None: self.eventProcessingTimer.cancel() self.eventProcessingTimer = threading.Timer(30, self.handleEventProcessing) self.eventProcessingTimer.start() self.mutex.release() def handleEventProcessing(self): # trigger a full rescan self.mutex.acquire() self.scan() # remove the timer if self.eventProcessingTimer is not None: self.eventProcessingTimer = None self.mutex.release() def checkIfRemovedOrModified(self, comic, pathlist): remove = False def inFolderlist(filepath, pathlist): for p in pathlist: if p in filepath: return True return False if not (os.path.exists(comic.path)): # file is missing, remove it from the comic table, add it to deleted table logging.debug(u"Removing missing {0}".format(comic.path)) remove = True elif not inFolderlist(comic.path, pathlist): logging.debug(u"Removing unwanted {0}".format(comic.path)) remove = True else: # file exists. check the mod date. # if it's been modified, remove it, and it'll be re-added #curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path)) curr = datetime.utcfromtimestamp(os.path.getmtime(comic.path)) prev = comic.mod_ts if curr != prev: logging.debug(u"Removed modifed {0}".format(comic.path)) remove = True return remove def getComicMetadata(self, path): ca = ComicArchive(path, default_image_path=AppFolders.imagePath("default.jpg")) if ca.seemsToBeAComicArchive(): logging.debug(u"Reading in {0} {1}\r".format(self.read_count, path)) sys.stdout.flush() self.read_count += 1 if ca.hasMetadata( MetaDataStyle.CIX ): style = MetaDataStyle.CIX elif ca.hasMetadata( MetaDataStyle.CBI ): style = MetaDataStyle.CBI else: style = None if style is not None: md = ca.readMetadata(style) else: # No metadata in comic. make some guesses from the filename md = ca.metadataFromFilename() md.path = ca.path md.page_count = ca.page_count md.mod_ts = datetime.utcfromtimestamp(os.path.getmtime(ca.path)) md.filesize = os.path.getsize(md.path) md.hash = "" #thumbnail generation image_data = ca.getPage(0) #now resize it thumb = StringIO.StringIO() utils.resize(image_data, (200, 200), thumb) md.thumbnail = thumb.getvalue() return md return None def setStatusDetail(self, detail, level=logging.DEBUG): self.statusdetail = detail if level == logging.DEBUG: logging.debug(detail) else: logging.info(detail) def setStatusDetailOnly(self, detail): self.statusdetail = detail def commitMetadataList(self, md_list): comics = [] for md in md_list: self.add_count += 1 comic = self.library.createComicFromMetadata(md) comics.append(comic) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return self.library.addComics(comics) def createAddRemoveLists(self, dirs): ix = {} db_set = set() current_set = set() filelist = utils.get_recursive_filelist(dirs) for path in filelist: current_set.add((path, datetime.utcfromtimestamp(os.path.getmtime(path)))) logging.info("NEW -- current_set size [%d]" % len(current_set)) for comic_id, path, md_ts in self.library.getComicPaths(): db_set.add((path, md_ts)) ix[path] = comic_id to_add = current_set - db_set to_remove = db_set - current_set logging.info("NEW -- db_set size [%d]" % len(db_set)) logging.info("NEW -- to_add size [%d]" % len(to_add)) logging.info("NEW -- to_remove size [%d]" % len(to_remove)) return [r[0] for r in to_add], [ix[r[0]] for r in to_remove] def dofullScan(self, dirs): self.status = "SCANNING" logging.info(u"Monitor: Beginning file scan...") self.setStatusDetail(u"Monitor: Making a list of all files in the folders...") self.add_count = 0 self.remove_count = 0 filelist, to_remove = self.createAddRemoveLists(dirs) self.setStatusDetail(u"Monitor: Removing missing or modified files from db ({0} files)".format(len(to_remove)), logging.INFO) if len(to_remove) > 0: self.library.deleteComics(to_remove) self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO) md_list = [] self.read_count = 0 for filename in filelist: md = self.getComicMetadata(filename) if md is not None: md_list.append(md) self.setStatusDetailOnly(u"Monitor: {0} files: {1} scanned, {2} added to library...".format(len(filelist), self.read_count,self.add_count)) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #every so often, commit to DB if self.read_count % 10 == 0 and self.read_count != 0: if len(md_list) > 0: self.commitMetadataList(md_list) md_list = [] if len(md_list) > 0: self.commitMetadataList(md_list) self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO) self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info("Monitor: Added {0} comics".format(self.add_count)) logging.info("Monitor: Removed {0} comics".format(self.remove_count)) if self.quit_when_done: self.quit = True def doEventProcessing(self, eventList): logging.debug(u"Monitor: event_list:{0}".format(eventList))
class Monitor(): def __init__(self, dm, paths): self.dm = dm self.style = MetaDataStyle.CIX self.queue = Queue.Queue(0) self.paths = paths self.eventList = [] self.mutex = threading.Lock() self.eventProcessingTimer = None self.quit_when_done = False # for debugging/testing self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = "" def start(self): self.thread = threading.Thread(target=self.mainLoop) self.thread.daemon = True self.quit = False self.thread.start() def stop(self): self.observer.stop() self.observer.join() self.quit = True self.thread.join() def mainLoop(self): logging.debug("Monitor: Started") for l in self.paths: logging.info(u"Monitor: Scanning '" + l + "'") self.session = self.dm.Session self.library = Library(self.session) self.observer = Observer() self.eventHandler = MonitorEventHandler(self) self.status = u"INDEXING" for path in self.paths: if os.path.exists(path): self.setStatusDetail(u"Watchdog (BUG?)") logging.info("Monitor: Watchdog: Indexing") # if I make this threaded? then it does not wait??? self.observer.schedule(self.eventHandler, path, recursive=True) logging.debug("Monitor: Watchdog: Stopped Indexing") self.observer.start() while True: try: (msg, args) = self.queue.get(block=True, timeout=1) except: msg = None #dispatch messages if msg == "scan": self.dofullScan(self.paths) if msg == "events": self.doEventProcessing(args) #time.sleep(1) if self.quit: break self.session.close() self.session = None self.observer.stop() self.observer.join() logging.debug("Monitor: Stopped") def scan(self): self.queue.put(("scan", None)) def handleSingleEvent(self, event): # events may happen in clumps. start a timer # to defer processing. if the timer is already going, # it will be canceled # in the future there can be more smarts about # granular file events. for now this will be # good enough to just get a a trigger that *something* # changed self.mutex.acquire() if self.eventProcessingTimer is not None: self.eventProcessingTimer.cancel() self.eventProcessingTimer = threading.Timer(30, self.handleEventProcessing) self.eventProcessingTimer.start() self.mutex.release() def handleEventProcessing(self): # trigger a full rescan self.mutex.acquire() self.scan() # remove the timer if self.eventProcessingTimer is not None: self.eventProcessingTimer = None self.mutex.release() """ def checkIfRemovedOrModified(self, comic, pathlist): remove = False def inFolderlist(filepath, pathlist): for p in pathlist: if p in filepath: return True return False if not (os.path.exists(comic.path)): # file is missing, remove it from the comic table, add it to deleted table self.setStatusDetail(u"Updating") logging.debug(u"Monitor: Flushing Missing {0}".format(comic.path)) remove = True elif not inFolderlist(comic.path, pathlist): self.setStatusDetail(u"Updating") logging.debug(u"Monitor: Flushing Unwanted {0}".format(comic.path)) remove = True else: # file exists. check the mod date. # if it's been modified, remove it, and it'll be re-added #curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path)) curr = datetime.utcfromtimestamp(getmtime(comic.path)) prev = comic.mod_ts if curr != prev: self.setStatusDetail(u"Updating") logging.debug(u"Monitor: Flushing Modifed {0}".format(comic.path)) remove = True return remove """ def getComicMetadata(self, path): logging.debug(u"Monitor: Scanning File {0} {1}\r".format( self.read_count, path)) ca = ComicArchive( path, default_image_path=AppFolders.missingPath("page.png")) self.read_count += 1 if ca.seemsToBeAComicArchive(): sys.stdout.flush() if ca.hasMetadata(MetaDataStyle.CIX): style = MetaDataStyle.CIX elif ca.hasMetadata(MetaDataStyle.CBI): style = MetaDataStyle.CBI elif ca.hasMetadata(MetaDataStyle.COMET): style = MetaDataStyle.COMET elif ca.hasMetadata(MetaDataStyle.CBW): style = MetaDataStyle.CBW elif ca.hasMetadata(MetaDataStyle.CALIBRE): style = MetaDataStyle.CALIBRE elif ca.hasMetadata(MetaDataStyle.EPUB): style = MetaDataStyle.EPUB else: style = None if style is not None: md = ca.readMetadata(style) if md.isEmpty: md = ca.metadataFromFilename() else: # No metadata in comic. make some guesses from the filename md = ca.metadataFromFilename() # patch version 2 if (md.title is None or md.title == "") and md.issue is None and not md.series is None: md.title = md.series md.series = None md.path = ca.path # Lot of weidness going on between pageCount and page_count i did something wrong probably, # maybe already fixed... # # original # # md.page_count = ca.page_count if style != MetaDataStyle.CBW: md.page_count = ca.page_count else: md.page_count = md.pageCount md.mod_ts = datetime.utcfromtimestamp(getmtime(ca.path)) md.filesize = os.path.getsize(md.path) md.hash = "" #thumbnail generation image_data = ca.getPage(0, AppFolders.missingPath("cover.png")) #now resize it thumb = StringIO.StringIO() try: utils.resize(image_data, (400, 400), thumb) md.thumbnail = thumb.getvalue() except: md.thumbnail = None return md return None def setStatusDetail(self, detail, level=logging.DEBUG): self.statusdetail = detail if level == logging.DEBUG: logging.debug("Monitor: " + detail) else: logging.info("Monitor: " + detail) def setStatusDetailOnly(self, detail): self.statusdetail = detail def commitMetadataList(self, md_list): comics = [] for md in md_list: self.add_count += 1 comic = self.library.createComicFromMetadata(md) comics.append(comic) if self.quit: self.setStatusDetail(u"Monitor: Stopped") return for i in comics: self.library.checkBlacklist(i) if self.quit: self.setStatusDetail(u"Monitor: Stopped") return self.library.addComics(comics) def getRecursiveFilelist(self, dirs): filename_encoding = sys.getfilesystemencoding() filelist = [] index = 0 for p in dirs: # if path is a folder, walk it recursivly, and all files underneath if type(p) == str: #make sure string is unicode p = p.decode(filename_encoding) #, 'replace') elif type(p) != unicode: #it's probably a QString p = unicode(p) if os.path.isdir(p): for root, dirs, files in os.walk(p): # issue #26: try to exclude hidden files and dirs files = [f for f in files if not f[0] == '.'] dirs[:] = [d for d in dirs if not d[0] == '.'] for f in files: if type(f) == str: #make sure string is unicode f = f.decode(filename_encoding, 'replace') elif type(f) != unicode: #it's probably a QString f = unicode(f) filelist.append(os.path.join(root, f)) if self.quit: return filelist else: self.setStatusDetailOnly( u"Monitor: {0} Files Indexed".format(index)) index = index + 1 filelist.append(p) return filelist def createAddRemoveLists(self, dirs): ix = {} db_set = set() current_set = set() self.dbfiles = len(db_set) filelist = self.getRecursiveFilelist(dirs) if self.quit: return [], [] for path in filelist: try: current_set.add( (path, datetime.utcfromtimestamp(os.path.getmtime(path)))) except: logging.debug(u"Monitor: Failed To Access '{0}'".format(path)) filelist.remove(path) logging.debug(u"Monitor: %d Files Found " % len(current_set)) try: for comic_id, path, md_ts in self.library.getComicPaths(): db_set.add((path, md_ts)) ix[path] = comic_id if self.quit: return [], [] except: logging.debug(u"Monitor: Failed To Access '{0}'".format(path)) to_add = current_set - db_set to_remove = db_set - current_set logging.debug(u"Monitor: %d Files In Library " % len(db_set)) logging.debug(u"Monitor: %d Files To Remove" % len(to_remove)) logging.info(u"Monitor: %d Files To Scan" % len(to_add)) return [r[0] for r in to_add], [ix[r[0]] for r in to_remove] def dofullScan(self, dirs): self.status = u"CHECKING" self.setStatusDetailOnly(u"Files") self.add_count = 0 self.remove_count = 0 filelist, to_remove = self.createAddRemoveLists(dirs) if self.quit: self.status = u"QUITING" self.setStatusDetailOnly(u"") return self.setStatusDetail(u"Removing {0} Files".format(len(to_remove))) if len(to_remove) > 0: self.library.deleteComics(to_remove) self.setStatusDetail(u"Scanning {0} Files".format(len(filelist))) self.status = u"SCANNING" md_list = [] self.read_count = 0 for filename in filelist: md = self.getComicMetadata(filename) if md is not None: md_list.append(md) self.setStatusDetailOnly(u"File {0}/{1} Found {2}".format( len(filelist), self.read_count, self.add_count)) if self.quit: self.status = u"QUITING" self.setStatusDetailOnly(u"") return #every so often, commit to DB if self.read_count % 10 == 0 and self.read_count != 0: if len(md_list) > 0: self.commitMetadataList(md_list) md_list = [] if len(md_list) > 0: self.commitMetadataList(md_list) self.setStatusDetail(u"Metadata {0}/{1} Files".format( self.read_count, len(filelist))) self.status = u"IDLE" self.statusdetail = "" self.scancomplete_ts = int( time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info(u"Monitor: Added {0} Files".format(self.add_count)) logging.info(u"Monitor: Removed {0} Files".format(self.remove_count)) if self.quit_when_done: self.quit = True """
class Monitor(): def __init__(self, dm, paths): self.dm = dm self.style = MetaDataStyle.CIX self.queue = Queue.Queue(0) self.paths = paths self.eventList = [] self.mutex = threading.Lock() self.eventProcessingTimer = None self.quit_when_done = False # for debugging/testing self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = "" def start(self): self.thread = threading.Thread(target=self.mainLoop) self.thread.daemon = True self.quit = False self.thread.start() def stop(self): self.quit = True self.thread.join() def mainLoop(self): logging.debug("Monitor: started main loop.") self.session = self.dm.Session() self.library = Library(self.dm.Session) observer = Observer() self.eventHandler = MonitorEventHandler(self) for path in self.paths: logging.debug(path) if os.path.exists(path): observer.schedule(self.eventHandler, path, recursive=True) observer.start() while True: try: (msg, args) = self.queue.get(block=True, timeout=1) except: msg = None #dispatch messages if msg == "scan": self.dofullScan(self.paths) if msg == "events": self.doEventProcessing(args) #time.sleep(1) if self.quit: break self.session.close() self.session = None observer.stop() logging.debug("Monitor: stopped main loop.") def scan(self): self.queue.put(("scan", None)) def handleSingleEvent(self, event): # events may happen in clumps. start a timer # to defer processing. if the timer is already going, # it will be canceled # in the future there can be more smarts about # granular file events. for now this will be # good enough to just get a a trigger that *something* # changed self.mutex.acquire() if self.eventProcessingTimer is not None: self.eventProcessingTimer.cancel() self.eventProcessingTimer = threading.Timer(30, self.handleEventProcessing) self.eventProcessingTimer.start() self.mutex.release() def handleEventProcessing(self): # trigger a full rescan self.mutex.acquire() self.scan() # remove the timer if self.eventProcessingTimer is not None: self.eventProcessingTimer = None self.mutex.release() def checkIfRemovedOrModified(self, comic, pathlist): remove = False def inFolderlist(filepath, pathlist): for p in pathlist: if p in filepath: return True return False if not (os.path.exists(comic.path)): # file is missing, remove it from the comic table, add it to deleted table logging.info(u"Removing missing {0}".format(comic.path)) remove = True elif not inFolderlist(comic.path, pathlist): logging.debug(u"Removing unwanted {0}".format(comic.path)) remove = True else: # file exists. check the mod date. # if it's been modified, remove it, and it'll be re-added #curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path)) curr = datetime.utcfromtimestamp(os.path.getmtime(comic.path)) prev = comic.mod_ts if curr != prev: logging.info(u"Removed modifed {0}".format(comic.path)) remove = True return remove def getComicMetadata(self, path): ca = ComicArchive( path, default_image_path=AppFolders.imagePath("default.jpg")) if ca.seemsToBeAComicArchive(): logging.debug(u"Reading in {0} {1}\r".format( self.read_count, path)) sys.stdout.flush() self.read_count += 1 if ca.hasMetadata(MetaDataStyle.CIX): style = MetaDataStyle.CIX elif ca.hasMetadata(MetaDataStyle.CBI): style = MetaDataStyle.CBI else: style = None if style is not None: md = ca.readMetadata(style) else: # No metadata in comic. make some guesses from the filename md = ca.metadataFromFilename() md.path = ca.path md.page_count = ca.page_count md.mod_ts = datetime.utcfromtimestamp(os.path.getmtime(ca.path)) md.filesize = os.path.getsize(md.path) md.hash = "" #thumbnail generation image_data = ca.getPage(0) #now resize it thumb = StringIO.StringIO() utils.resize(image_data, (200, 200), thumb) md.thumbnail = thumb.getvalue() return md return None def setStatusDetail(self, detail, level=logging.DEBUG): self.statusdetail = detail if level == logging.DEBUG: logging.debug(detail) else: logging.info(detail) def setStatusDetailOnly(self, detail): self.statusdetail = detail def commitMetadataList(self, md_list): comics = [] for md in md_list: self.add_count += 1 comic = self.library.createComicFromMetadata(md) comics.append(comic) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return self.library.addComics(comics) def createAddRemoveLists(self, dirs): ix = {} db_set = set() current_set = set() filelist = utils.get_recursive_filelist(dirs) for path in filelist: current_set.add( (path, datetime.utcfromtimestamp(os.path.getmtime(path)))) logging.info(current_set) logging.info("NEW -- current_set size [%d]" % len(current_set)) for comic_id, path, mod_ts in self.library.getComicPaths(): db_set.add((path, mod_ts)) ix[path] = comic_id logging.info(db_set) to_add = current_set - db_set to_remove = db_set - current_set logging.info("NEW -- db_set size [%d]" % len(db_set)) logging.info("NEW -- to_add size [%d]" % len(to_add)) logging.info("NEW -- to_remove size [%d]" % len(to_remove)) return [r[0] for r in to_add], [ix[r[0]] for r in to_remove] def dofullScan(self, dirs): self.status = "SCANNING" logging.info(u"Monitor: Beginning file scan...") self.setStatusDetail( u"Monitor: Making a list of all files in the folders...") self.add_count = 0 self.remove_count = 0 filelist, to_remove = self.createAddRemoveLists(dirs) self.setStatusDetail( u"Monitor: Removing missing or modified files from db ({0} files)". format(len(to_remove)), logging.INFO) if len(to_remove) > 0: self.library.deleteComics(to_remove) self.setStatusDetail( u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO) md_list = [] self.read_count = 0 for filename in filelist: logging.info(u"Scanning File {0}".format(filename)) md = self.getComicMetadata(filename) if md is not None: md_list.append(md) self.setStatusDetailOnly( u"Monitor: {0} files: {1} scanned, {2} added to library...". format(len(filelist), self.read_count, self.add_count)) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #every so often, commit to DB if self.read_count % 10 == 0 and self.read_count != 0: if len(md_list) > 0: self.commitMetadataList(md_list) md_list = [] if len(md_list) > 0: self.commitMetadataList(md_list) self.setStatusDetail( u"Monitor: finished scanning metadata in {0} of {1} files".format( self.read_count, len(filelist)), logging.INFO) self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = int( time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info("Monitor: Added {0} comics".format(self.add_count)) logging.info("Monitor: Removed {0} comics".format(self.remove_count)) if self.quit_when_done: self.quit = True def doEventProcessing(self, eventList): logging.debug(u"Monitor: event_list:{0}".format(eventList))
class Monitor(): def __init__(self, dm, paths): self.dm = dm self.style = MetaDataStyle.CIX self.queue = Queue.Queue(0) self.paths = paths self.eventList = [] self.mutex = threading.Lock() self.eventProcessingTimer = None self.quit_when_done = False # for debugging/testing self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = "" def start(self): self.thread = threading.Thread(target=self.mainLoop) self.thread.daemon = True self.quit = False self.thread.start() def stop(self): self.observer.stop() self.observer.join() self.quit = True self.thread.join() def mainLoop(self): logging.debug("Monitor: Started") for l in self.paths: logging.info(u"Monitor: Scanning '"+l+"'") self.session = self.dm.Session self.library = Library(self.session) self.observer = Observer() self.eventHandler = MonitorEventHandler(self) self.status = u"INDEXING" for path in self.paths: if os.path.exists(path): self.setStatusDetail(u"Watchdog (BUG?)") logging.info("Monitor: Watchdog: Indexing") # if I make this threaded? then it does not wait??? self.observer.schedule(self.eventHandler, path, recursive=True) logging.debug("Monitor: Watchdog: Stopped Indexing") self.observer.start() while True: try: (msg, args) = self.queue.get(block=True, timeout=1) except: msg = None #dispatch messages if msg == "scan": self.dofullScan(self.paths) if msg == "events": self.doEventProcessing(args) #time.sleep(1) if self.quit: break self.session.close() self.session = None self.observer.stop() self.observer.join() logging.debug("Monitor: Stopped") def scan(self): self.queue.put(("scan", None)) def handleSingleEvent(self, event): # events may happen in clumps. start a timer # to defer processing. if the timer is already going, # it will be canceled # in the future there can be more smarts about # granular file events. for now this will be # good enough to just get a a trigger that *something* # changed self.mutex.acquire() if self.eventProcessingTimer is not None: self.eventProcessingTimer.cancel() self.eventProcessingTimer = threading.Timer(30, self.handleEventProcessing) self.eventProcessingTimer.start() self.mutex.release() def handleEventProcessing(self): # trigger a full rescan self.mutex.acquire() self.scan() # remove the timer if self.eventProcessingTimer is not None: self.eventProcessingTimer = None self.mutex.release() """ def checkIfRemovedOrModified(self, comic, pathlist): remove = False def inFolderlist(filepath, pathlist): for p in pathlist: if p in filepath: return True return False if not (os.path.exists(comic.path)): # file is missing, remove it from the comic table, add it to deleted table self.setStatusDetail(u"Updating") logging.debug(u"Monitor: Flushing Missing {0}".format(comic.path)) remove = True elif not inFolderlist(comic.path, pathlist): self.setStatusDetail(u"Updating") logging.debug(u"Monitor: Flushing Unwanted {0}".format(comic.path)) remove = True else: # file exists. check the mod date. # if it's been modified, remove it, and it'll be re-added #curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path)) curr = datetime.utcfromtimestamp(getmtime(comic.path)) prev = comic.mod_ts if curr != prev: self.setStatusDetail(u"Updating") logging.debug(u"Monitor: Flushing Modifed {0}".format(comic.path)) remove = True return remove """ def getComicMetadata(self, path): logging.debug(u"Monitor: Scanning File {0} {1}\r".format(self.read_count, path)) self.read_count += 1 sys.stdout.flush() return self.library.getComicMetadata(path) def setStatusDetail(self, detail, level=logging.DEBUG): self.statusdetail = detail if level == logging.DEBUG: logging.debug("Monitor: "+detail) else: logging.info("Monitor: "+detail) def setStatusDetailOnly(self, detail): self.statusdetail = detail def commitMetadataList(self, md_list): comics = [] for md in md_list: self.add_count += 1 comic = self.library.createComicFromMetadata(md) comics.append(comic) if self.quit: self.setStatusDetail(u"Monitor: Stopped") return for i in comics: self.library.checkBlacklist(i) if self.quit: self.setStatusDetail(u"Monitor: Stopped") return self.library.addComics(comics) def getRecursiveFilelist(self, dirs): filename_encoding = sys.getfilesystemencoding() filelist = [] index = 0 for p in dirs: # if path is a folder, walk it recursivly, and all files underneath if type(p) == str: #make sure string is unicode p = p.decode(filename_encoding) #, 'replace') elif type(p) != unicode: #it's probably a QString p = unicode(p) if os.path.isdir( p ): for root,dirs,files in os.walk( p ): # issue #26: try to exclude hidden files and dirs files = [f for f in files if not f[0] == '.'] dirs[:] = [d for d in dirs if not d[0] == '.'] for f in files: if type(f) == str: #make sure string is unicode f = f.decode(filename_encoding, 'replace') elif type(f) != unicode: #it's probably a QString f = unicode(f) filelist.append(os.path.join(root,f)) if self.quit: return filelist else: self.setStatusDetailOnly(u"Monitor: {0} Files Indexed".format(index)) index = index + 1 filelist.append(p) return filelist def createAddRemoveLists(self, dirs): ix = {} db_set = set() current_set = set() self.dbfiles = len(db_set) filelist = self.getRecursiveFilelist(dirs) if self.quit: return [],[] for path in filelist: try: current_set.add((path, datetime.utcfromtimestamp(os.path.getmtime(path)))) except: logging.debug(u"Monitor: Failed To Access '{0}'".format(path)) filelist.remove(path) logging.debug(u"Monitor: %d Files Found " % len(current_set)) try: for comic_id, path, md_ts in self.library.getComicPaths(): db_set.add((path, md_ts)) ix[path] = comic_id if self.quit: return [],[] except: logging.debug(u"Monitor: Failed To Access '{0}'".format(path)) to_add = current_set - db_set to_remove = db_set - current_set logging.debug(u"Monitor: %d Files In Library " % len(db_set)) logging.debug(u"Monitor: %d Files To Remove" % len(to_remove)) logging.info(u"Monitor: %d Files To Scan" % len(to_add)) return [r[0] for r in to_add], [ix[r[0]] for r in to_remove] def dofullScan(self, dirs): self.status = u"CHECKING" self.setStatusDetailOnly(u"Files") self.add_count = 0 self.remove_count = 0 filelist, to_remove = self.createAddRemoveLists(dirs) if self.quit: self.status = u"QUITING" self.setStatusDetailOnly(u"") return self.setStatusDetail(u"Removing {0} Files".format(len(to_remove))) if len(to_remove) > 0: self.library.deleteComics(to_remove) self.setStatusDetail(u"Scanning {0} Files".format(len(filelist))) self.status = u"SCANNING" md_list = [] self.read_count = 0 commitlength = 10 for filename in filelist: md = self.getComicMetadata(filename) if md is not None: md_list.append(md) self.setStatusDetailOnly(u"File {0}/{1} Found {2}".format(len(filelist), self.read_count,self.add_count)) if self.quit: self.status = u"QUITING" self.setStatusDetailOnly(u"") return #every so often, commit to DB if self.read_count % commitlength == 0 and self.read_count != 0: if len(md_list) > 0: self.commitMetadataList(md_list) md_list = [] commitlength = 200 if len(md_list) > 0: self.commitMetadataList(md_list) self.setStatusDetail(u"Metadata {0}/{1} Files".format(self.read_count,len(filelist))) self.status = u"IDLE" self.statusdetail = "" self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info(u"Monitor: Added {0} Files".format(self.add_count)) logging.info(u"Monitor: Removed {0} Files".format(self.remove_count)) if self.quit_when_done: self.quit = True """