class Monitor():
def __init__(self, dm, paths):
self.dm = dm
self.style = MetaDataStyle.CIX
self.queue = Queue.Queue(0)
self.paths = paths
self.eventList = []
self.mutex = threading.Lock()
self.eventProcessingTimer = None
self.quit_when_done = False # for debugging/testing
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = ""
def start(self):
self.thread = threading.Thread(target=self.mainLoop)
self.thread.daemon = True
self.quit = False
self.thread.start()
def stop(self):
self.quit = True
self.thread.join()
def mainLoop(self):
logging.debug("Monitor: started main loop.")
self.session = self.dm.Session()
self.library = Library(self.dm.Session)
observer = Observer()
self.eventHandler = MonitorEventHandler(self)
for path in self.paths:
if os.path.exists(path):
observer.schedule(self.eventHandler, path, recursive=True)
observer.start()
while True:
try:
(msg, args) = self.queue.get(block=True, timeout=1)
except:
msg = None
#dispatch messages
if msg == "scan":
self.dofullScan(self.paths)
if msg == "events":
self.doEventProcessing(args)
#time.sleep(1)
if self.quit:
break
self.session.close()
self.session = None
observer.stop()
logging.debug("Monitor: stopped main loop.")
def scan(self):
self.queue.put(("scan", None))
def handleSingleEvent(self, event):
# events may happen in clumps. start a timer
# to defer processing. if the timer is already going,
# it will be canceled
# in the future there can be more smarts about
# granular file events. for now this will be
# good enough to just get a a trigger that *something*
# changed
self.mutex.acquire()
if self.eventProcessingTimer is not None:
self.eventProcessingTimer.cancel()
self.eventProcessingTimer = threading.Timer(30, self.handleEventProcessing)
self.eventProcessingTimer.start()
self.mutex.release()
def handleEventProcessing(self):
# trigger a full rescan
self.mutex.acquire()
self.scan()
# remove the timer
if self.eventProcessingTimer is not None:
self.eventProcessingTimer = None
self.mutex.release()
def checkIfRemovedOrModified(self, comic, pathlist):
remove = False
def inFolderlist(filepath, pathlist):
for p in pathlist:
if p in filepath:
return True
return False
if not (os.path.exists(comic.path)):
# file is missing, remove it from the comic table, add it to deleted table
logging.debug(u"Removing missing {0}".format(comic.path))
remove = True
elif not inFolderlist(comic.path, pathlist):
logging.debug(u"Removing unwanted {0}".format(comic.path))
remove = True
else:
# file exists. check the mod date.
# if it's been modified, remove it, and it'll be re-added
#curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path))
curr = datetime.utcfromtimestamp(os.path.getmtime(comic.path))
prev = comic.mod_ts
if curr != prev:
logging.debug(u"Removed modifed {0}".format(comic.path))
remove = True
return remove
def getComicMetadata(self, path):
ca = ComicArchive(path, default_image_path=AppFolders.imagePath("default.jpg"))
if ca.seemsToBeAComicArchive():
logging.debug(u"Reading in {0} {1}\r".format(self.read_count, path))
sys.stdout.flush()
self.read_count += 1
if ca.hasMetadata( MetaDataStyle.CIX ):
style = MetaDataStyle.CIX
elif ca.hasMetadata( MetaDataStyle.CBI ):
style = MetaDataStyle.CBI
else:
style = None
if style is not None:
md = ca.readMetadata(style)
else:
# No metadata in comic. make some guesses from the filename
md = ca.metadataFromFilename()
md.path = ca.path
md.page_count = ca.page_count
md.mod_ts = datetime.utcfromtimestamp(os.path.getmtime(ca.path))
md.filesize = os.path.getsize(md.path)
md.hash = ""
#thumbnail generation
image_data = ca.getPage(0)
#now resize it
thumb = StringIO.StringIO()
utils.resize(image_data, (200, 200), thumb)
md.thumbnail = thumb.getvalue()
return md
return None
def setStatusDetail(self, detail, level=logging.DEBUG):
self.statusdetail = detail
if level == logging.DEBUG:
logging.debug(detail)
else:
logging.info(detail)
def setStatusDetailOnly(self, detail):
self.statusdetail = detail
def commitMetadataList(self, md_list):
comics = []
for md in md_list:
self.add_count += 1
comic = self.library.createComicFromMetadata(md)
comics.append(comic)
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
self.library.addComics(comics)
def createAddRemoveLists(self, dirs):
ix = {}
db_set = set()
current_set = set()
filelist = utils.get_recursive_filelist(dirs)
for path in filelist:
current_set.add((path, datetime.utcfromtimestamp(os.path.getmtime(path))))
logging.info("NEW -- current_set size [%d]" % len(current_set))
for comic_id, path, md_ts in self.library.getComicPaths():
db_set.add((path, md_ts))
ix[path] = comic_id
to_add = current_set - db_set
to_remove = db_set - current_set
logging.info("NEW -- db_set size [%d]" % len(db_set))
logging.info("NEW -- to_add size [%d]" % len(to_add))
logging.info("NEW -- to_remove size [%d]" % len(to_remove))
return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def dofullScan(self, dirs):
self.status = "SCANNING"
logging.info(u"Monitor: Beginning file scan...")
self.setStatusDetail(u"Monitor: Making a list of all files in the folders...")
self.add_count = 0
self.remove_count = 0
filelist, to_remove = self.createAddRemoveLists(dirs)
self.setStatusDetail(u"Monitor: Removing missing or modified files from db ({0} files)".format(len(to_remove)), logging.INFO)
if len(to_remove) > 0:
self.library.deleteComics(to_remove)
self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO)
md_list = []
self.read_count = 0
for filename in filelist:
md = self.getComicMetadata(filename)
if md is not None:
md_list.append(md)
self.setStatusDetailOnly(u"Monitor: {0} files: {1} scanned, {2} added to library...".format(len(filelist), self.read_count,self.add_count))
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#every so often, commit to DB
if self.read_count % 10 == 0 and self.read_count != 0:
if len(md_list) > 0:
self.commitMetadataList(md_list)
md_list = []
if len(md_list) > 0:
self.commitMetadataList(md_list)
self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO)
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info("Monitor: Added {0} comics".format(self.add_count))
logging.info("Monitor: Removed {0} comics".format(self.remove_count))
if self.quit_when_done:
self.quit = True
def doEventProcessing(self, eventList):
logging.debug(u"Monitor: event_list:{0}".format(eventList))
class Monitor():
def __init__(self, dm, paths):
self.dm = dm
self.style = MetaDataStyle.CIX
self.queue = Queue.Queue(0)
self.paths = paths
self.eventList = []
self.mutex = threading.Lock()
self.eventProcessingTimer = None
self.quit_when_done = False # for debugging/testing
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = ""
def start(self):
self.thread = threading.Thread(target=self.mainLoop)
self.thread.daemon = True
self.quit = False
self.thread.start()
def stop(self):
self.observer.stop()
self.observer.join()
self.quit = True
self.thread.join()
def mainLoop(self):
logging.debug("Monitor: Started")
for l in self.paths:
logging.info(u"Monitor: Scanning '" + l + "'")
self.session = self.dm.Session
self.library = Library(self.session)
self.observer = Observer()
self.eventHandler = MonitorEventHandler(self)
self.status = u"INDEXING"
for path in self.paths:
if os.path.exists(path):
self.setStatusDetail(u"Watchdog (BUG?)")
logging.info("Monitor: Watchdog: Indexing")
# if I make this threaded? then it does not wait???
self.observer.schedule(self.eventHandler, path, recursive=True)
logging.debug("Monitor: Watchdog: Stopped Indexing")
self.observer.start()
while True:
try:
(msg, args) = self.queue.get(block=True, timeout=1)
except:
msg = None
#dispatch messages
if msg == "scan":
self.dofullScan(self.paths)
if msg == "events":
self.doEventProcessing(args)
#time.sleep(1)
if self.quit:
break
self.session.close()
self.session = None
self.observer.stop()
self.observer.join()
logging.debug("Monitor: Stopped")
def scan(self):
self.queue.put(("scan", None))
def handleSingleEvent(self, event):
# events may happen in clumps. start a timer
# to defer processing. if the timer is already going,
# it will be canceled
# in the future there can be more smarts about
# granular file events. for now this will be
# good enough to just get a a trigger that *something*
# changed
self.mutex.acquire()
if self.eventProcessingTimer is not None:
self.eventProcessingTimer.cancel()
self.eventProcessingTimer = threading.Timer(30,
self.handleEventProcessing)
self.eventProcessingTimer.start()
self.mutex.release()
def handleEventProcessing(self):
# trigger a full rescan
self.mutex.acquire()
self.scan()
# remove the timer
if self.eventProcessingTimer is not None:
self.eventProcessingTimer = None
self.mutex.release()
"""
def checkIfRemovedOrModified(self, comic, pathlist):
remove = False
def inFolderlist(filepath, pathlist):
for p in pathlist:
if p in filepath:
return True
return False
if not (os.path.exists(comic.path)):
# file is missing, remove it from the comic table, add it to deleted table
self.setStatusDetail(u"Updating")
logging.debug(u"Monitor: Flushing Missing {0}".format(comic.path))
remove = True
elif not inFolderlist(comic.path, pathlist):
self.setStatusDetail(u"Updating")
logging.debug(u"Monitor: Flushing Unwanted {0}".format(comic.path))
remove = True
else:
# file exists. check the mod date.
# if it's been modified, remove it, and it'll be re-added
#curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path))
curr = datetime.utcfromtimestamp(getmtime(comic.path))
prev = comic.mod_ts
if curr != prev:
self.setStatusDetail(u"Updating")
logging.debug(u"Monitor: Flushing Modifed {0}".format(comic.path))
remove = True
return remove
"""
def getComicMetadata(self, path):
logging.debug(u"Monitor: Scanning File {0} {1}\r".format(
self.read_count, path))
ca = ComicArchive(
path, default_image_path=AppFolders.missingPath("page.png"))
self.read_count += 1
if ca.seemsToBeAComicArchive():
sys.stdout.flush()
if ca.hasMetadata(MetaDataStyle.CIX):
style = MetaDataStyle.CIX
elif ca.hasMetadata(MetaDataStyle.CBI):
style = MetaDataStyle.CBI
elif ca.hasMetadata(MetaDataStyle.COMET):
style = MetaDataStyle.COMET
elif ca.hasMetadata(MetaDataStyle.CBW):
style = MetaDataStyle.CBW
elif ca.hasMetadata(MetaDataStyle.CALIBRE):
style = MetaDataStyle.CALIBRE
elif ca.hasMetadata(MetaDataStyle.EPUB):
style = MetaDataStyle.EPUB
else:
style = None
if style is not None:
md = ca.readMetadata(style)
if md.isEmpty:
md = ca.metadataFromFilename()
else:
# No metadata in comic. make some guesses from the filename
md = ca.metadataFromFilename()
# patch version 2
if (md.title is None or md.title
== "") and md.issue is None and not md.series is None:
md.title = md.series
md.series = None
md.path = ca.path
# Lot of weidness going on between pageCount and page_count i did something wrong probably,
# maybe already fixed...
#
# original
#
# md.page_count = ca.page_count
if style != MetaDataStyle.CBW:
md.page_count = ca.page_count
else:
md.page_count = md.pageCount
md.mod_ts = datetime.utcfromtimestamp(getmtime(ca.path))
md.filesize = os.path.getsize(md.path)
md.hash = ""
#thumbnail generation
image_data = ca.getPage(0, AppFolders.missingPath("cover.png"))
#now resize it
thumb = StringIO.StringIO()
try:
utils.resize(image_data, (400, 400), thumb)
md.thumbnail = thumb.getvalue()
except:
md.thumbnail = None
return md
return None
def setStatusDetail(self, detail, level=logging.DEBUG):
self.statusdetail = detail
if level == logging.DEBUG:
logging.debug("Monitor: " + detail)
else:
logging.info("Monitor: " + detail)
def setStatusDetailOnly(self, detail):
self.statusdetail = detail
def commitMetadataList(self, md_list):
comics = []
for md in md_list:
self.add_count += 1
comic = self.library.createComicFromMetadata(md)
comics.append(comic)
if self.quit:
self.setStatusDetail(u"Monitor: Stopped")
return
for i in comics:
self.library.checkBlacklist(i)
if self.quit:
self.setStatusDetail(u"Monitor: Stopped")
return
self.library.addComics(comics)
def getRecursiveFilelist(self, dirs):
filename_encoding = sys.getfilesystemencoding()
filelist = []
index = 0
for p in dirs:
# if path is a folder, walk it recursivly, and all files underneath
if type(p) == str:
#make sure string is unicode
p = p.decode(filename_encoding) #, 'replace')
elif type(p) != unicode:
#it's probably a QString
p = unicode(p)
if os.path.isdir(p):
for root, dirs, files in os.walk(p):
# issue #26: try to exclude hidden files and dirs
files = [f for f in files if not f[0] == '.']
dirs[:] = [d for d in dirs if not d[0] == '.']
for f in files:
if type(f) == str:
#make sure string is unicode
f = f.decode(filename_encoding, 'replace')
elif type(f) != unicode:
#it's probably a QString
f = unicode(f)
filelist.append(os.path.join(root, f))
if self.quit:
return filelist
else:
self.setStatusDetailOnly(
u"Monitor: {0} Files Indexed".format(index))
index = index + 1
filelist.append(p)
return filelist
def createAddRemoveLists(self, dirs):
ix = {}
db_set = set()
current_set = set()
self.dbfiles = len(db_set)
filelist = self.getRecursiveFilelist(dirs)
if self.quit:
return [], []
for path in filelist:
try:
current_set.add(
(path, datetime.utcfromtimestamp(os.path.getmtime(path))))
except:
logging.debug(u"Monitor: Failed To Access '{0}'".format(path))
filelist.remove(path)
logging.debug(u"Monitor: %d Files Found " % len(current_set))
try:
for comic_id, path, md_ts in self.library.getComicPaths():
db_set.add((path, md_ts))
ix[path] = comic_id
if self.quit:
return [], []
except:
logging.debug(u"Monitor: Failed To Access '{0}'".format(path))
to_add = current_set - db_set
to_remove = db_set - current_set
logging.debug(u"Monitor: %d Files In Library " % len(db_set))
logging.debug(u"Monitor: %d Files To Remove" % len(to_remove))
logging.info(u"Monitor: %d Files To Scan" % len(to_add))
return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def dofullScan(self, dirs):
self.status = u"CHECKING"
self.setStatusDetailOnly(u"Files")
self.add_count = 0
self.remove_count = 0
filelist, to_remove = self.createAddRemoveLists(dirs)
if self.quit:
self.status = u"QUITING"
self.setStatusDetailOnly(u"")
return
self.setStatusDetail(u"Removing {0} Files".format(len(to_remove)))
if len(to_remove) > 0:
self.library.deleteComics(to_remove)
self.setStatusDetail(u"Scanning {0} Files".format(len(filelist)))
self.status = u"SCANNING"
md_list = []
self.read_count = 0
for filename in filelist:
md = self.getComicMetadata(filename)
if md is not None:
md_list.append(md)
self.setStatusDetailOnly(u"File {0}/{1} Found {2}".format(
len(filelist), self.read_count, self.add_count))
if self.quit:
self.status = u"QUITING"
self.setStatusDetailOnly(u"")
return
#every so often, commit to DB
if self.read_count % 10 == 0 and self.read_count != 0:
if len(md_list) > 0:
self.commitMetadataList(md_list)
md_list = []
if len(md_list) > 0:
self.commitMetadataList(md_list)
self.setStatusDetail(u"Metadata {0}/{1} Files".format(
self.read_count, len(filelist)))
self.status = u"IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(
time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info(u"Monitor: Added {0} Files".format(self.add_count))
logging.info(u"Monitor: Removed {0} Files".format(self.remove_count))
if self.quit_when_done:
self.quit = True
"""
class Monitor():
def __init__(self, dm, paths):
self.dm = dm
self.style = MetaDataStyle.CIX
self.queue = Queue.Queue(0)
self.paths = paths
self.eventList = []
self.mutex = threading.Lock()
self.eventProcessingTimer = None
self.quit_when_done = False # for debugging/testing
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = ""
def start(self):
self.thread = threading.Thread(target=self.mainLoop)
self.thread.daemon = True
self.quit = False
self.thread.start()
def stop(self):
self.quit = True
self.thread.join()
def mainLoop(self):
logging.debug("Monitor: started main loop.")
self.session = self.dm.Session()
self.library = Library(self.dm.Session)
observer = Observer()
self.eventHandler = MonitorEventHandler(self)
for path in self.paths:
logging.debug(path)
if os.path.exists(path):
observer.schedule(self.eventHandler, path, recursive=True)
observer.start()
while True:
try:
(msg, args) = self.queue.get(block=True, timeout=1)
except:
msg = None
#dispatch messages
if msg == "scan":
self.dofullScan(self.paths)
if msg == "events":
self.doEventProcessing(args)
#time.sleep(1)
if self.quit:
break
self.session.close()
self.session = None
observer.stop()
logging.debug("Monitor: stopped main loop.")
def scan(self):
self.queue.put(("scan", None))
def handleSingleEvent(self, event):
# events may happen in clumps. start a timer
# to defer processing. if the timer is already going,
# it will be canceled
# in the future there can be more smarts about
# granular file events. for now this will be
# good enough to just get a a trigger that *something*
# changed
self.mutex.acquire()
if self.eventProcessingTimer is not None:
self.eventProcessingTimer.cancel()
self.eventProcessingTimer = threading.Timer(30,
self.handleEventProcessing)
self.eventProcessingTimer.start()
self.mutex.release()
def handleEventProcessing(self):
# trigger a full rescan
self.mutex.acquire()
self.scan()
# remove the timer
if self.eventProcessingTimer is not None:
self.eventProcessingTimer = None
self.mutex.release()
def checkIfRemovedOrModified(self, comic, pathlist):
remove = False
def inFolderlist(filepath, pathlist):
for p in pathlist:
if p in filepath:
return True
return False
if not (os.path.exists(comic.path)):
# file is missing, remove it from the comic table, add it to deleted table
logging.info(u"Removing missing {0}".format(comic.path))
remove = True
elif not inFolderlist(comic.path, pathlist):
logging.debug(u"Removing unwanted {0}".format(comic.path))
remove = True
else:
# file exists. check the mod date.
# if it's been modified, remove it, and it'll be re-added
#curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path))
curr = datetime.utcfromtimestamp(os.path.getmtime(comic.path))
prev = comic.mod_ts
if curr != prev:
logging.info(u"Removed modifed {0}".format(comic.path))
remove = True
return remove
def getComicMetadata(self, path):
ca = ComicArchive(
path, default_image_path=AppFolders.imagePath("default.jpg"))
if ca.seemsToBeAComicArchive():
logging.debug(u"Reading in {0} {1}\r".format(
self.read_count, path))
sys.stdout.flush()
self.read_count += 1
if ca.hasMetadata(MetaDataStyle.CIX):
style = MetaDataStyle.CIX
elif ca.hasMetadata(MetaDataStyle.CBI):
style = MetaDataStyle.CBI
else:
style = None
if style is not None:
md = ca.readMetadata(style)
else:
# No metadata in comic. make some guesses from the filename
md = ca.metadataFromFilename()
md.path = ca.path
md.page_count = ca.page_count
md.mod_ts = datetime.utcfromtimestamp(os.path.getmtime(ca.path))
md.filesize = os.path.getsize(md.path)
md.hash = ""
#thumbnail generation
image_data = ca.getPage(0)
#now resize it
thumb = StringIO.StringIO()
utils.resize(image_data, (200, 200), thumb)
md.thumbnail = thumb.getvalue()
return md
return None
def setStatusDetail(self, detail, level=logging.DEBUG):
self.statusdetail = detail
if level == logging.DEBUG:
logging.debug(detail)
else:
logging.info(detail)
def setStatusDetailOnly(self, detail):
self.statusdetail = detail
def commitMetadataList(self, md_list):
comics = []
for md in md_list:
self.add_count += 1
comic = self.library.createComicFromMetadata(md)
comics.append(comic)
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
self.library.addComics(comics)
def createAddRemoveLists(self, dirs):
ix = {}
db_set = set()
current_set = set()
filelist = utils.get_recursive_filelist(dirs)
for path in filelist:
current_set.add(
(path, datetime.utcfromtimestamp(os.path.getmtime(path))))
logging.info(current_set)
logging.info("NEW -- current_set size [%d]" % len(current_set))
for comic_id, path, mod_ts in self.library.getComicPaths():
db_set.add((path, mod_ts))
ix[path] = comic_id
logging.info(db_set)
to_add = current_set - db_set
to_remove = db_set - current_set
logging.info("NEW -- db_set size [%d]" % len(db_set))
logging.info("NEW -- to_add size [%d]" % len(to_add))
logging.info("NEW -- to_remove size [%d]" % len(to_remove))
return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def dofullScan(self, dirs):
self.status = "SCANNING"
logging.info(u"Monitor: Beginning file scan...")
self.setStatusDetail(
u"Monitor: Making a list of all files in the folders...")
self.add_count = 0
self.remove_count = 0
filelist, to_remove = self.createAddRemoveLists(dirs)
self.setStatusDetail(
u"Monitor: Removing missing or modified files from db ({0} files)".
format(len(to_remove)), logging.INFO)
if len(to_remove) > 0:
self.library.deleteComics(to_remove)
self.setStatusDetail(
u"Monitor: {0} new files to scan...".format(len(filelist)),
logging.INFO)
md_list = []
self.read_count = 0
for filename in filelist:
logging.info(u"Scanning File {0}".format(filename))
md = self.getComicMetadata(filename)
if md is not None:
md_list.append(md)
self.setStatusDetailOnly(
u"Monitor: {0} files: {1} scanned, {2} added to library...".
format(len(filelist), self.read_count, self.add_count))
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#every so often, commit to DB
if self.read_count % 10 == 0 and self.read_count != 0:
if len(md_list) > 0:
self.commitMetadataList(md_list)
md_list = []
if len(md_list) > 0:
self.commitMetadataList(md_list)
self.setStatusDetail(
u"Monitor: finished scanning metadata in {0} of {1} files".format(
self.read_count, len(filelist)), logging.INFO)
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(
time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info("Monitor: Added {0} comics".format(self.add_count))
logging.info("Monitor: Removed {0} comics".format(self.remove_count))
if self.quit_when_done:
self.quit = True
def doEventProcessing(self, eventList):
logging.debug(u"Monitor: event_list:{0}".format(eventList))
class Monitor():
def __init__(self, dm, paths):
self.dm = dm
self.style = MetaDataStyle.CIX
self.queue = Queue.Queue(0)
self.paths = paths
self.eventList = []
self.mutex = threading.Lock()
self.eventProcessingTimer = None
self.quit_when_done = False # for debugging/testing
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = ""
def start(self):
self.thread = threading.Thread(target=self.mainLoop)
self.thread.daemon = True
self.quit = False
self.thread.start()
def stop(self):
self.observer.stop()
self.observer.join()
self.quit = True
self.thread.join()
def mainLoop(self):
logging.debug("Monitor: Started")
for l in self.paths:
logging.info(u"Monitor: Scanning '"+l+"'")
self.session = self.dm.Session
self.library = Library(self.session)
self.observer = Observer()
self.eventHandler = MonitorEventHandler(self)
self.status = u"INDEXING"
for path in self.paths:
if os.path.exists(path):
self.setStatusDetail(u"Watchdog (BUG?)")
logging.info("Monitor: Watchdog: Indexing")
# if I make this threaded? then it does not wait???
self.observer.schedule(self.eventHandler, path, recursive=True)
logging.debug("Monitor: Watchdog: Stopped Indexing")
self.observer.start()
while True:
try:
(msg, args) = self.queue.get(block=True, timeout=1)
except:
msg = None
#dispatch messages
if msg == "scan":
self.dofullScan(self.paths)
if msg == "events":
self.doEventProcessing(args)
#time.sleep(1)
if self.quit:
break
self.session.close()
self.session = None
self.observer.stop()
self.observer.join()
logging.debug("Monitor: Stopped")
def scan(self):
self.queue.put(("scan", None))
def handleSingleEvent(self, event):
# events may happen in clumps. start a timer
# to defer processing. if the timer is already going,
# it will be canceled
# in the future there can be more smarts about
# granular file events. for now this will be
# good enough to just get a a trigger that *something*
# changed
self.mutex.acquire()
if self.eventProcessingTimer is not None:
self.eventProcessingTimer.cancel()
self.eventProcessingTimer = threading.Timer(30, self.handleEventProcessing)
self.eventProcessingTimer.start()
self.mutex.release()
def handleEventProcessing(self):
# trigger a full rescan
self.mutex.acquire()
self.scan()
# remove the timer
if self.eventProcessingTimer is not None:
self.eventProcessingTimer = None
self.mutex.release()
"""
def checkIfRemovedOrModified(self, comic, pathlist):
remove = False
def inFolderlist(filepath, pathlist):
for p in pathlist:
if p in filepath:
return True
return False
if not (os.path.exists(comic.path)):
# file is missing, remove it from the comic table, add it to deleted table
self.setStatusDetail(u"Updating")
logging.debug(u"Monitor: Flushing Missing {0}".format(comic.path))
remove = True
elif not inFolderlist(comic.path, pathlist):
self.setStatusDetail(u"Updating")
logging.debug(u"Monitor: Flushing Unwanted {0}".format(comic.path))
remove = True
else:
# file exists. check the mod date.
# if it's been modified, remove it, and it'll be re-added
#curr = datetime.datetime.fromtimestamp(os.path.getmtime(comic.path))
curr = datetime.utcfromtimestamp(getmtime(comic.path))
prev = comic.mod_ts
if curr != prev:
self.setStatusDetail(u"Updating")
logging.debug(u"Monitor: Flushing Modifed {0}".format(comic.path))
remove = True
return remove
"""
def getComicMetadata(self, path):
logging.debug(u"Monitor: Scanning File {0} {1}\r".format(self.read_count, path))
self.read_count += 1
sys.stdout.flush()
return self.library.getComicMetadata(path)
def setStatusDetail(self, detail, level=logging.DEBUG):
self.statusdetail = detail
if level == logging.DEBUG:
logging.debug("Monitor: "+detail)
else:
logging.info("Monitor: "+detail)
def setStatusDetailOnly(self, detail):
self.statusdetail = detail
def commitMetadataList(self, md_list):
comics = []
for md in md_list:
self.add_count += 1
comic = self.library.createComicFromMetadata(md)
comics.append(comic)
if self.quit:
self.setStatusDetail(u"Monitor: Stopped")
return
for i in comics:
self.library.checkBlacklist(i)
if self.quit:
self.setStatusDetail(u"Monitor: Stopped")
return
self.library.addComics(comics)
def getRecursiveFilelist(self, dirs):
filename_encoding = sys.getfilesystemencoding()
filelist = []
index = 0
for p in dirs:
# if path is a folder, walk it recursivly, and all files underneath
if type(p) == str:
#make sure string is unicode
p = p.decode(filename_encoding) #, 'replace')
elif type(p) != unicode:
#it's probably a QString
p = unicode(p)
if os.path.isdir( p ):
for root,dirs,files in os.walk( p ):
# issue #26: try to exclude hidden files and dirs
files = [f for f in files if not f[0] == '.']
dirs[:] = [d for d in dirs if not d[0] == '.']
for f in files:
if type(f) == str:
#make sure string is unicode
f = f.decode(filename_encoding, 'replace')
elif type(f) != unicode:
#it's probably a QString
f = unicode(f)
filelist.append(os.path.join(root,f))
if self.quit:
return filelist
else:
self.setStatusDetailOnly(u"Monitor: {0} Files Indexed".format(index))
index = index + 1
filelist.append(p)
return filelist
def createAddRemoveLists(self, dirs):
ix = {}
db_set = set()
current_set = set()
self.dbfiles = len(db_set)
filelist = self.getRecursiveFilelist(dirs)
if self.quit:
return [],[]
for path in filelist:
try:
current_set.add((path, datetime.utcfromtimestamp(os.path.getmtime(path))))
except:
logging.debug(u"Monitor: Failed To Access '{0}'".format(path))
filelist.remove(path)
logging.debug(u"Monitor: %d Files Found " % len(current_set))
try:
for comic_id, path, md_ts in self.library.getComicPaths():
db_set.add((path, md_ts))
ix[path] = comic_id
if self.quit:
return [],[]
except:
logging.debug(u"Monitor: Failed To Access '{0}'".format(path))
to_add = current_set - db_set
to_remove = db_set - current_set
logging.debug(u"Monitor: %d Files In Library " % len(db_set))
logging.debug(u"Monitor: %d Files To Remove" % len(to_remove))
logging.info(u"Monitor: %d Files To Scan" % len(to_add))
return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def dofullScan(self, dirs):
self.status = u"CHECKING"
self.setStatusDetailOnly(u"Files")
self.add_count = 0
self.remove_count = 0
filelist, to_remove = self.createAddRemoveLists(dirs)
if self.quit:
self.status = u"QUITING"
self.setStatusDetailOnly(u"")
return
self.setStatusDetail(u"Removing {0} Files".format(len(to_remove)))
if len(to_remove) > 0:
self.library.deleteComics(to_remove)
self.setStatusDetail(u"Scanning {0} Files".format(len(filelist)))
self.status = u"SCANNING"
md_list = []
self.read_count = 0
commitlength = 10
for filename in filelist:
md = self.getComicMetadata(filename)
if md is not None:
md_list.append(md)
self.setStatusDetailOnly(u"File {0}/{1} Found {2}".format(len(filelist), self.read_count,self.add_count))
if self.quit:
self.status = u"QUITING"
self.setStatusDetailOnly(u"")
return
#every so often, commit to DB
if self.read_count % commitlength == 0 and self.read_count != 0:
if len(md_list) > 0:
self.commitMetadataList(md_list)
md_list = []
commitlength = 200
if len(md_list) > 0:
self.commitMetadataList(md_list)
self.setStatusDetail(u"Metadata {0}/{1} Files".format(self.read_count,len(filelist)))
self.status = u"IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info(u"Monitor: Added {0} Files".format(self.add_count))
logging.info(u"Monitor: Removed {0} Files".format(self.remove_count))
if self.quit_when_done:
self.quit = True
"""