def _update_book_count_thread(self):
while self.should_run:
time.sleep(1)
try:
for dir in list(self.book_count.keys()):
dirs = []
parentdirs = self.book_count[dir]["parentdirs"]
if parentdirs:
for parentdir in parentdirs:
dirs.append(os.path.join(dir, parentdirs[parentdir]))
else:
dirs.append(dir)
if (self.book_count[dir]["modified"] + 15 < time.time()):
books = []
for d in dirs:
if os.path.isdir(d):
books += Filesystem.list_book_dir(d)
self.book_count[dir]["modified"] = time.time()
self.book_count[dir]["count"] = len(set(books))
if not self.should_run:
break
except Exception:
logging.exception("An error occurred while updating book count")
def triggerDirectoryEdition(directory_id, edition_id):
path = os.path.normpath(Directory.dirs_flat[directory_id]) if directory_id in Directory.dirs_flat else None
if not path:
return None, 404
file_stems = [Path(file).stem for file in Filesystem.list_book_dir(path)]
if edition_id not in file_stems:
return None, 404
result = []
for pipeline in Pipeline.pipelines:
if pipeline.dir_in and os.path.normpath(pipeline.dir_in) == path:
pipeline.trigger(edition_id, auto=False)
result.append(pipeline.uid)
return jsonify(result), 200
def _trigger_epub_catalog_thread(self):
last_check = 0
self.watchdog_bark()
while self.shouldRun:
time.sleep(5)
self.watchdog_bark()
if not self.dirsAvailable():
continue
# Check for update every 3 days
max_update_interval = 60 * 60 * 24 * 3
if time.time() - last_check < max_update_interval:
continue
last_check = time.time()
logging.info("Updating formatklar and filesize for ebooks")
list_books = Filesystem.list_book_dir(self.dir_out)
Bibliofil.update_list_of_books("XHTML", list_books)
def _trigger_newsletter_thread(self):
last_check = 0
# If no newsletter this month, trigger newsletter
self.watchdog_bark()
while self.shouldRun:
time.sleep(5)
self.watchdog_bark()
if not self.dirsAvailable():
continue
max_update_interval = 60 * 60
if time.time() - last_check < max_update_interval:
continue
last_check = time.time()
self.newsletter_identifier = "120209"
self.newsletter_identifier += time.strftime("%m%Y")
self.year_month = datetime.datetime.today().strftime('%Y-%m')
if self.newsletter_identifier not in Filesystem.list_book_dir(
self.dir_out):
logging.info("Lager nyhetsbrev for: " + self.year_month)
self.trigger(self.newsletter_identifier)
def _monitor_book_events_thread(self):
self.initialize_checksums()
while self.shouldRun:
try:
# books that are recently changed (check often in case of new file changes)
with self._md5_lock:
recently_changed = sorted([
book for book in self._md5 if time.time() -
self._md5[book]["modified"] < self.inactivity_timeout
],
key=lambda rc: self._md5[rc][
"modified"])
if recently_changed:
for book in recently_changed:
deep_md5, _ = Filesystem.path_md5(
path=os.path.join(self.dir_path, book),
shallow=False)
self._md5[book]["deep_checked"] = int(time.time())
if deep_md5 != self._md5[book]["deep"]:
self._md5[book]["modified"] = int(time.time())
self._update_md5(book)
logging.debug(
"book modified (and was recently modified, might be in the middle of a copy operation): {}"
.format(book))
self.notify_book_event_handlers(
book, "modified")
time.sleep(0.1) # a small nap
continue
time.sleep(
1
) # unless anything has recently changed, give the system time to breathe between each iteration
if not self.is_available():
time.sleep(5)
continue
dirlist = Filesystem.list_book_dir(self.dir_path)
sorted_dirlist = []
should_deepscan = []
# books that have explicitly been requested for rescan should be rescanned first
if self.suggested_for_rescan:
for book_id in self.suggested_for_rescan:
book_path = os.path.join(self.dir_path, book_id)
if os.path.exists(book_path):
sorted_dirlist.append(book_id)
should_deepscan.append(book_id)
else:
# if book is a file, then it can have a file extension
for dirname in dirlist:
if Path(dirname).stem == book_id:
sorted_dirlist.append(dirname)
should_deepscan.append(dirname)
break
# empty list after having put the suggestions at the front of the queue
self.suggested_for_rescan = []
# add the remaining books to the list
for dirname in dirlist:
if dirname not in sorted_dirlist:
sorted_dirlist.append(dirname)
if len(dirlist) != len(sorted_dirlist):
logging.warning("len(dirlist) != len(sorted_dirlist)")
logging.warning("dirlist: {}".format(dirlist))
logging.warning(
"sorted_dirlist: {}".format(sorted_dirlist))
dirlist = sorted_dirlist
# do a shallow check of files and folders (i.e. don't check file sizes, modification times etc. in subdirectories)
for book in dirlist:
if not self.shouldRun:
break # break loop if we're shutting down the system (iterating books may take some time)
with self._md5_lock:
if not os.path.exists(os.path.join(
self.dir_path, book)):
# iterating over all books can take a lot of time,
# and the book may have been deleted by the time we get to it.
self.notify_book_event_handlers(book, "deleted")
logging.debug("book deleted: {}".format(book))
del self._md5[book]
continue
if book not in self._md5:
self._update_md5(book)
self.notify_book_event_handlers(book, "created")
logging.debug("book created: {}".format(book))
continue
shallow_md5, _ = Filesystem.path_md5(
path=os.path.join(self.dir_path, book),
shallow=True,
expect=self._md5[book]["shallow"]
if book in self._md5 else None)
if shallow_md5 != self._md5[book]["shallow"]:
self._update_md5(book)
self.notify_book_event_handlers(book, "modified")
logging.debug(
"book modified (top-level dir/file modified): {}"
.format(book))
continue
with self._md5_lock:
deleted = [
book for book in self._md5 if book not in dirlist
]
for book in deleted:
self.notify_book_event_handlers(book, "deleted")
logging.debug("book deleted: {}".format(book))
del self._md5[book]
if deleted:
continue
self.store_checksums(
) # regularly store updated version of checksums
# do a deep check (size/time etc. of files in subdirectories) of up to 10 books that haven't been checked in a while
with self._md5_lock:
long_time_since_checked = sorted(
[{
"name": book,
"md5": self._md5[book]
} for book in self._md5 if time.time() -
self._md5[book]["modified"] > self.inactivity_timeout
],
key=lambda book: book["md5"]["deep_checked"])
long_time_since_checked = [
b["name"] for b in long_time_since_checked
]
for book in should_deepscan + long_time_since_checked[:10]:
if not self.shouldRun:
break # break loop if we're shutting down the system
deep_md5, _ = Filesystem.path_md5(
path=os.path.join(self.dir_path, book),
shallow=False,
expect=self._md5[book]["deep"]
if book in self._md5 else None)
if book not in self._md5:
self._update_md5(book)
else:
self._md5[book]["deep_checked"] = int(time.time())
if deep_md5 != self._md5[book]["deep"]:
self._md5[book]["modified"] = int(time.time())
self._update_md5(book)
self.notify_book_event_handlers(
book, "modified")
logging.debug("book modified: {}".format(book))
self.store_checksums(
) # regularly store updated version of checksums
except Exception:
logging.exception(
"En feil oppstod ved overvåking av {}".format(
self.dir_path))
try:
Report.emailPlainText(
"En feil oppstod ved overvåking av {}".format(
self.dir_path),
traceback.format_exc(),
recipients=[])
except Exception:
logging.exception("Could not e-mail exception")
def _initialize_checksums(self):
cache_dir = Config.get("cache_dir", None)
if not cache_dir:
cache_dir = os.getenv(
"CACHE_DIR",
os.path.join(tempfile.gettempdir(), "prodsys-cache"))
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir, exist_ok=True)
Config.set("cache_dir", cache_dir)
self.cache_file = None
if not self.dir_id_is_generated:
self.cache_file = os.path.join(
cache_dir, "dir.{}.md5.pickle".format(self.dir_id))
if os.path.isfile(self.cache_file):
try:
with open(self.cache_file, 'rb') as f:
self._md5 = pickle.load(f)
except Exception as e:
logging.exception(
"Cache file found, but could not parse it", e)
else:
logging.debug("Can't find cache file")
if self._md5:
logging.debug(
"Loaded directory status from cache file, doing a partial rescan"
)
else:
logging.debug("Directory status not cached, doing a full rescan")
self._md5 = {}
dir_list = Filesystem.list_book_dir(self.dir_path)
self.status_text = "Looking for created/deleted"
for book in list(
self._md5.keys()
): # list(….keys()) to avoid "RuntimeError: dictionary changed size during iteration"
if not self.shouldRun:
self._md5 = {}
return # break loop if we're shutting down the system
if book not in dir_list:
logging.debug(
"{} is in cache but not in directory: deleting from cache".
format(book))
del self._md5[book]
md5_count = 0
self.status_text = "0 / {}".format(len(dir_list))
for book in dir_list:
if not self.shouldRun:
self._md5 = {}
return # break loop if we're shutting down the system
if book not in self._md5:
logging.debug(
"{} is in directory but not in cache: adding to cache".
format(book))
self._update_md5(book)
md5_count += 1
self.status_text = "{} / {}".format(md5_count, len(dir_list))
if md5_count == 1 or md5_count % 100 == 0:
logging.info(self.status_text)
if md5_count % 10 == 0:
self.store_checksums(
while_starting=True
) # if for some reason the system crashes, we don't have to start all over again
self.store_checksums(while_starting=True)
self.starting = False
self.status_text = None
return