def createAddRemoveLists(self, dirs):
ix = {}
db_set = set()
current_set = set()
filelist = utils.get_recursive_filelist(dirs)
for path in filelist:
current_set.add((path, datetime.utcfromtimestamp(os.path.getmtime(path))))
logging.info("NEW -- current_set size [%d]" % len(current_set))
for comic_id, path, md_ts in self.library.getComicPaths():
db_set.add((path, md_ts))
ix[path] = comic_id
to_add = current_set - db_set
to_remove = db_set - current_set
logging.info("NEW -- db_set size [%d]" % len(db_set))
logging.info("NEW -- to_add size [%d]" % len(to_add))
logging.info("NEW -- to_remove size [%d]" % len(to_remove))
return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def addPathList( self, pathlist ):
filelist = utils.get_recursive_filelist( pathlist )
# we now have a list of files to add
progdialog = QProgressDialog("", "Cancel", 0, len(filelist), self)
progdialog.setWindowTitle( "Adding Files" )
#progdialog.setWindowModality(Qt.WindowModal)
progdialog.setWindowModality(Qt.ApplicationModal)
progdialog.show()
firstAdded = None
self.twList.setSortingEnabled(False)
for idx,f in enumerate(filelist):
QCoreApplication.processEvents()
if progdialog.wasCanceled():
break
progdialog.setValue(idx)
progdialog.setLabelText(f)
utils.centerWindowOnParent( progdialog )
QCoreApplication.processEvents()
row = self.addPathItem( f )
if firstAdded is None and row is not None:
firstAdded = row
progdialog.close()
if firstAdded is not None:
self.twList.selectRow(firstAdded)
self.twList.setSortingEnabled(True)
# Adjust column size
self.twList.resizeColumnsToContents()
self.twList.setColumnWidth(FileSelectionList.CRFlagColNum, 35)
self.twList.setColumnWidth(FileSelectionList.CBLFlagColNum, 35)
self.twList.setColumnWidth(FileSelectionList.readonlyColNum, 35)
self.twList.setColumnWidth(FileSelectionList.typeColNum, 45)
if self.twList.columnWidth(FileSelectionList.fileColNum) > 250:
self.twList.setColumnWidth(FileSelectionList.fileColNum, 250)
if self.twList.columnWidth(FileSelectionList.folderColNum ) > 200:
self.twList.setColumnWidth(FileSelectionList.folderColNum, 200)
def createAddRemoveLists(self, dirs):
ix = {}
db_set = set()
current_set = set()
filelist = utils.get_recursive_filelist(dirs)
for path in filelist:
current_set.add(
(path, datetime.utcfromtimestamp(os.path.getmtime(path))))
logging.info("NEW -- current_set size [%d]" % len(current_set))
for comic_id, path, md_ts in self.library.getComicPaths():
db_set.add((path, md_ts))
ix[path] = comic_id
to_add = current_set - db_set
to_remove = db_set - current_set
logging.info("NEW -- db_set size [%d]" % len(db_set))
logging.info("NEW -- to_add size [%d]" % len(to_add))
logging.info("NEW -- to_remove size [%d]" % len(to_remove))
return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def addPathList(self, pathlist):
filelist = utils.get_recursive_filelist(pathlist)
# we now have a list of files to add
progdialog = QProgressDialog("", "Cancel", 0, len(filelist), self)
progdialog.setWindowTitle("Adding Files")
#progdialog.setWindowModality(Qt.WindowModal)
progdialog.setWindowModality(Qt.ApplicationModal)
progdialog.show()
firstAdded = None
self.twList.setSortingEnabled(False)
for idx, f in enumerate(filelist):
QCoreApplication.processEvents()
if progdialog.wasCanceled():
break
progdialog.setValue(idx)
progdialog.setLabelText(f)
utils.centerWindowOnParent(progdialog)
QCoreApplication.processEvents()
row = self.addPathItem(f)
if firstAdded is None and row is not None:
firstAdded = row
progdialog.close()
if (self.settings.show_no_unrar_warning
and self.settings.unrar_exe_path == ""
and self.settings.rar_exe_path == ""
and platform.system() != "Windows"):
for f in filelist:
ext = os.path.splitext(f)[1].lower()
if ext == ".rar" or ext == ".cbr":
checked = OptionalMessageDialog.msg(
self, "No unrar tool", """
It looks like you've tried to open at least one CBR or RAR file.<br><br>
In order for ComicTagger to read this kind of file, you will have to configure
the location of the unrar tool in the settings. Until then, ComicTagger
will not be able recognize these kind of files.
""")
self.settings.show_no_unrar_warning = not checked
break
if firstAdded is not None:
self.twList.selectRow(firstAdded)
else:
if len(pathlist) == 1 and os.path.isfile(pathlist[0]):
QMessageBox.information(
self, self.tr("File Open"),
self.tr(
"Selected file doesn't seem to be a comic archive."))
else:
QMessageBox.information(
self, self.tr("File/Folder Open"),
self.tr("No comic archives were found."))
self.twList.setSortingEnabled(True)
# Adjust column size
self.twList.resizeColumnsToContents()
self.twList.setColumnWidth(FileSelectionList.CRFlagColNum, 35)
self.twList.setColumnWidth(FileSelectionList.CBLFlagColNum, 35)
self.twList.setColumnWidth(FileSelectionList.readonlyColNum, 35)
self.twList.setColumnWidth(FileSelectionList.typeColNum, 45)
if self.twList.columnWidth(FileSelectionList.fileColNum) > 250:
self.twList.setColumnWidth(FileSelectionList.fileColNum, 250)
if self.twList.columnWidth(FileSelectionList.folderColNum) > 200:
self.twList.setColumnWidth(FileSelectionList.folderColNum, 200)
def addPathList(self, pathlist):
filelist = utils.get_recursive_filelist(pathlist)
# we now have a list of files to add
progdialog = QProgressDialog("", "Cancel", 0, len(filelist), self)
progdialog.setWindowTitle("Adding Files")
# progdialog.setWindowModality(Qt.WindowModal)
progdialog.setWindowModality(Qt.ApplicationModal)
progdialog.show()
firstAdded = None
self.twList.setSortingEnabled(False)
for idx, f in enumerate(filelist):
QCoreApplication.processEvents()
if progdialog.wasCanceled():
break
progdialog.setValue(idx)
progdialog.setLabelText(f)
centerWindowOnParent(progdialog)
QCoreApplication.processEvents()
row = self.addPathItem(f)
if firstAdded is None and row is not None:
firstAdded = row
progdialog.close()
if (self.settings.show_no_unrar_warning and
self.settings.unrar_exe_path == "" and
self.settings.rar_exe_path == "" and
platform.system() != "Windows"):
for f in filelist:
ext = os.path.splitext(f)[1].lower()
if ext == ".rar" or ext == ".cbr":
checked = OptionalMessageDialog.msg(self, "No unrar tool",
"""
It looks like you've tried to open at least one CBR or RAR file.<br><br>
In order for ComicTagger to read this kind of file, you will have to configure
the location of the unrar tool in the settings. Until then, ComicTagger
will not be able recognize these kind of files.
"""
)
self.settings.show_no_unrar_warning = not checked
break
if firstAdded is not None:
self.twList.selectRow(firstAdded)
else:
if len(pathlist) == 1 and os.path.isfile(pathlist[0]):
QMessageBox.information(self, self.tr("File Open"), self.tr(
"Selected file doesn't seem to be a comic archive."))
else:
QMessageBox.information(
self,
self.tr("File/Folder Open"),
self.tr("No comic archives were found."))
self.twList.setSortingEnabled(True)
# Adjust column size
self.twList.resizeColumnsToContents()
self.twList.setColumnWidth(FileSelectionList.CRFlagColNum, 35)
self.twList.setColumnWidth(FileSelectionList.CBLFlagColNum, 35)
self.twList.setColumnWidth(FileSelectionList.readonlyColNum, 35)
self.twList.setColumnWidth(FileSelectionList.typeColNum, 45)
if self.twList.columnWidth(FileSelectionList.fileColNum) > 250:
self.twList.setColumnWidth(FileSelectionList.fileColNum, 250)
if self.twList.columnWidth(FileSelectionList.folderColNum) > 200:
self.twList.setColumnWidth(FileSelectionList.folderColNum, 200)
def parseCmdLineArgs(self):
if platform.system() == "Darwin" and hasattr(
sys, "frozen") and sys.frozen == 1:
# remove the PSN ("process serial number") argument from OS/X
input_args = [a for a in sys.argv[1:] if "-psn_0_" not in a]
else:
input_args = sys.argv[1:]
# first check if we're launching a script:
for n in range(len(input_args)):
if (input_args[n] in ["-S", "--script"] and
n + 1 < len(input_args)):
# insert a "--" which will cause getopt to ignore the remaining args
# so they will be passed to the script
input_args.insert(n + 2, "--")
break
# parse command line options
try:
opts, args = getopt.getopt(input_args,
"hpdt:fm:vownsrc:ieRS:1",
["help", "print", "delete", "type=", "copy=", "parsefilename",
"metadata=", "verbose", "online", "dryrun", "save", "rename",
"raw", "noabort", "terse", "nooverwrite", "interactive",
"nosummary", "version", "id=", "recursive", "script=",
"export-to-zip", "delete-rar", "abort-on-conflict",
"assume-issue-one", "cv-api-key=", "only-set-cv-key",
"wait-on-cv-rate-limit"])
except getopt.GetoptError as err:
self.display_msg_and_quit(str(err), 2)
# process options
for o, a in opts:
if o in ("-h", "--help"):
self.display_msg_and_quit(None, 0, show_help=True)
if o in ("-v", "--verbose"):
self.verbose = True
if o in ("-S", "--script"):
self.run_script = True
self.script = a
if o in ("-R", "--recursive"):
self.recursive = True
if o in ("-p", "--print"):
self.print_tags = True
if o in ("-d", "--delete"):
self.delete_tags = True
if o in ("-i", "--interactive"):
self.interactive = True
if o in ("-c", "--copy"):
self.copy_tags = True
if a.lower() == "cr":
self.copy_source = MetaDataStyle.CIX
elif a.lower() == "cbl":
self.copy_source = MetaDataStyle.CBI
elif a.lower() == "comet":
self.copy_source = MetaDataStyle.COMET
else:
self.display_msg_and_quit(
"Invalid copy tag source type", 1)
if o in ("-o", "--online"):
self.search_online = True
if o in ("-n", "--dryrun"):
self.dryrun = True
if o in ("-m", "--metadata"):
self.metadata = self.parseMetadataFromString(a)
if o in ("-s", "--save"):
self.save_tags = True
if o in ("-r", "--rename"):
self.rename_file = True
if o in ("-e", "--export_to_zip"):
self.export_to_zip = True
if o == "--delete-rar":
self.delete_rar_after_export = True
if o == "--abort-on-conflict":
self.abort_export_on_conflict = True
if o in ("-f", "--parsefilename"):
self.parse_filename = True
if o in ("-w", "--wait-on-cv-rate-limit"):
self.wait_and_retry_on_rate_limit = True
if o == "--id":
self.issue_id = a
if o == "--raw":
self.raw = True
if o == "--noabort":
self.abortOnLowConfidence = False
if o == "--terse":
self.terse = True
if o == "--nosummary":
self.show_save_summary = False
if o in ("-1", "--assume-issue-one"):
self.assume_issue_is_one_if_not_set = True
if o == "--nooverwrite":
self.no_overwrite = True
if o == "--cv-api-key":
self.cv_api_key = a
if o == "--only-set-cv-key":
self.only_set_key = True
if o == "--version":
print(
"ComicTagger {0} [{1} / {2}]".format(ctversion.version, ctversion.fork, ctversion.fork_tag))
print(
"Modified version of ComicTagger (Copyright (c) 2012-2014 Anthony Beville)")
print(
"Distributed under Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)")
sys.exit(0)
if o in ("-t", "--type"):
if a.lower() == "cr":
self.data_style = MetaDataStyle.CIX
elif a.lower() == "cbl":
self.data_style = MetaDataStyle.CBI
elif a.lower() == "comet":
self.data_style = MetaDataStyle.COMET
else:
self.display_msg_and_quit("Invalid tag type", 1)
if self.print_tags or self.delete_tags or self.save_tags or self.copy_tags or self.rename_file or self.export_to_zip or self.only_set_key:
self.no_gui = True
count = 0
if self.run_script:
count += 1
if self.print_tags:
count += 1
if self.delete_tags:
count += 1
if self.save_tags:
count += 1
if self.copy_tags:
count += 1
if self.rename_file:
count += 1
if self.export_to_zip:
count += 1
if self.only_set_key:
count += 1
if count > 1:
self.display_msg_and_quit(
"Must choose only one action of print, delete, save, copy, rename, export, set key, or run script",
1)
if self.script is not None:
self.launch_script(self.script)
if len(args) > 0:
if platform.system() == "Windows":
# no globbing on windows shell, so do it for them
import glob
self.file_list = []
for item in args:
self.file_list.extend(glob.glob(item))
if len(self.file_list) > 0:
self.filename = self.file_list[0]
else:
self.filename = args[0]
self.file_list = args
if self.only_set_key and self.cv_api_key is None:
self.display_msg_and_quit("Key not given!", 1)
if (self.only_set_key == False) and self.no_gui and (
self.filename is None):
self.display_msg_and_quit(
"Command requires at least one filename!", 1)
if self.delete_tags and self.data_style is None:
self.display_msg_and_quit(
"Please specify the type to delete with -t", 1)
if self.save_tags and self.data_style is None:
self.display_msg_and_quit(
"Please specify the type to save with -t", 1)
if self.copy_tags and self.data_style is None:
self.display_msg_and_quit(
"Please specify the type to copy to with -t", 1)
# if self.rename_file and self.data_style is None:
# self.display_msg_and_quit("Please specify the type to use for renaming with -t", 1)
if self.recursive:
self.file_list = utils.get_recursive_filelist(self.file_list)
def parseCmdLineArgs(self):
if platform.system() == "Darwin" and hasattr(
sys, "frozen") and sys.frozen == 1:
# remove the PSN ("process serial number") argument from OS/X
input_args = [a for a in sys.argv[1:] if "-psn_0_" not in a]
else:
input_args = sys.argv[1:]
# first check if we're launching a script:
for n in range(len(input_args)):
if (input_args[n] in ["-S", "--script"]
and n + 1 < len(input_args)):
# insert a "--" which will cause getopt to ignore the remaining args
# so they will be passed to the script
input_args.insert(n + 2, "--")
break
# parse command line options
try:
opts, args = getopt.getopt(input_args, "hpdt:fm:vownsrc:ieRS:1", [
"help", "print", "delete", "type=", "copy=", "parsefilename",
"metadata=", "verbose", "online", "dryrun", "save", "rename",
"raw", "noabort", "terse", "nooverwrite", "interactive",
"nosummary", "version", "id=", "recursive", "script=",
"export-to-zip", "delete-rar", "abort-on-conflict",
"assume-issue-one", "cv-api-key=", "only-set-cv-key",
"wait-on-cv-rate-limit"
])
except getopt.GetoptError as err:
self.display_msg_and_quit(str(err), 2)
# process options
for o, a in opts:
if o in ("-h", "--help"):
self.display_msg_and_quit(None, 0, show_help=True)
if o in ("-v", "--verbose"):
self.verbose = True
if o in ("-S", "--script"):
self.run_script = True
self.script = a
if o in ("-R", "--recursive"):
self.recursive = True
if o in ("-p", "--print"):
self.print_tags = True
if o in ("-d", "--delete"):
self.delete_tags = True
if o in ("-i", "--interactive"):
self.interactive = True
if o in ("-c", "--copy"):
self.copy_tags = True
if a.lower() == "cr":
self.copy_source = MetaDataStyle.CIX
elif a.lower() == "cbl":
self.copy_source = MetaDataStyle.CBI
elif a.lower() == "comet":
self.copy_source = MetaDataStyle.COMET
else:
self.display_msg_and_quit("Invalid copy tag source type",
1)
if o in ("-o", "--online"):
self.search_online = True
if o in ("-n", "--dryrun"):
self.dryrun = True
if o in ("-m", "--metadata"):
self.metadata = self.parseMetadataFromString(a)
if o in ("-s", "--save"):
self.save_tags = True
if o in ("-r", "--rename"):
self.rename_file = True
if o in ("-e", "--export_to_zip"):
self.export_to_zip = True
if o == "--delete-rar":
self.delete_rar_after_export = True
if o == "--abort-on-conflict":
self.abort_export_on_conflict = True
if o in ("-f", "--parsefilename"):
self.parse_filename = True
if o in ("-w", "--wait-on-cv-rate-limit"):
self.wait_and_retry_on_rate_limit = True
if o == "--id":
self.issue_id = a
if o == "--raw":
self.raw = True
if o == "--noabort":
self.abortOnLowConfidence = False
if o == "--terse":
self.terse = True
if o == "--nosummary":
self.show_save_summary = False
if o in ("-1", "--assume-issue-one"):
self.assume_issue_is_one_if_not_set = True
if o == "--nooverwrite":
self.no_overwrite = True
if o == "--cv-api-key":
self.cv_api_key = a
if o == "--only-set-cv-key":
self.only_set_key = True
if o == "--version":
print(
"ComicTagger {0}: Copyright (c) 2012-2014 Anthony Beville"
.format(ctversion.version))
print(
"Distributed under Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)"
)
new_version = VersionChecker().getLatestVersion("", False)
if new_version is not None and new_version != ctversion.version:
print(
"--------------------------------------------------------------------------"
)
print("New version available online: {0}".format(
new_version))
print(
"--------------------------------------------------------------------------"
)
sys.exit(0)
if o in ("-t", "--type"):
if a.lower() == "cr":
self.data_style = MetaDataStyle.CIX
elif a.lower() == "cbl":
self.data_style = MetaDataStyle.CBI
elif a.lower() == "comet":
self.data_style = MetaDataStyle.COMET
else:
self.display_msg_and_quit("Invalid tag type", 1)
if self.print_tags or self.delete_tags or self.save_tags or self.copy_tags or self.rename_file or self.export_to_zip or self.only_set_key:
self.no_gui = True
count = 0
if self.run_script:
count += 1
if self.print_tags:
count += 1
if self.delete_tags:
count += 1
if self.save_tags:
count += 1
if self.copy_tags:
count += 1
if self.rename_file:
count += 1
if self.export_to_zip:
count += 1
if self.only_set_key:
count += 1
if count > 1:
self.display_msg_and_quit(
"Must choose only one action of print, delete, save, copy, rename, export, set key, or run script",
1)
if self.script is not None:
self.launch_script(self.script)
if len(args) > 0:
if platform.system() == "Windows":
# no globbing on windows shell, so do it for them
import glob
self.file_list = []
for item in args:
self.file_list.extend(glob.glob(item))
if len(self.file_list) > 0:
self.filename = self.file_list[0]
else:
self.filename = args[0]
self.file_list = args
if self.only_set_key and self.cv_api_key is None:
self.display_msg_and_quit("Key not given!", 1)
if (self.only_set_key
== False) and self.no_gui and (self.filename is None):
self.display_msg_and_quit(
"Command requires at least one filename!", 1)
if self.delete_tags and self.data_style is None:
self.display_msg_and_quit(
"Please specify the type to delete with -t", 1)
if self.save_tags and self.data_style is None:
self.display_msg_and_quit(
"Please specify the type to save with -t", 1)
if self.copy_tags and self.data_style is None:
self.display_msg_and_quit(
"Please specify the type to copy to with -t", 1)
# if self.rename_file and self.data_style is None:
# self.display_msg_and_quit("Please specify the type to use for renaming with -t", 1)
if self.recursive:
self.file_list = utils.get_recursive_filelist(self.file_list)
def dofullScan(self, dirs):
self.status = "SCANNING"
logging.info(u"Monitor: Beginning file scan...")
self.setStatusDetail(
u"Monitor: Making a list of all files in the folders...")
filelist = utils.get_recursive_filelist(dirs)
self.setStatusDetail(u"Monitor: done listing files.")
self.add_count = 0
self.remove_count = 0
# get the entire comic table into memory
query = list(self.session.query(Comic))
# look for missing or changed files
self.setStatusDetail(
u"Monitor: Removing missing or modified files from DB...")
#start_time = time.time()
for comic in query:
self.checkIfRemovedOrModified(comic, self.paths)
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#print time.time() - start_time, "seconds"
self.setStatusDetail(u"Monitor: Done removing files.")
if self.remove_count > 0:
self.dm.engine.echo = True
self.session.commit()
self.dm.engine.echo = False
self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(
len(filelist)))
# make a list of all path strings in comic table
db_pathlist = set([i[0] for i in list(self.session.query(Comic.path))])
filelist = set(filelist)
filelist = filelist - db_pathlist
db_pathlist = None
self.setStatusDetail(
u"Monitor: {0} new files to scan...".format(len(filelist)),
logging.INFO)
md_list = []
self.read_count = 0
for filename in filelist:
md = self.getComicMetadata(filename)
if md is not None:
md_list.append(md)
if self.read_count % 100 == 0 and self.read_count != 0:
self.setStatusDetail(
u"Monitor: {0} of {1} scanned...".format(
self.read_count, len(filelist)), logging.INFO)
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
self.setStatusDetail(
u"Monitor: finished scanning metadata in {0} of {1} files".format(
self.read_count, len(filelist)), logging.INFO)
filelist = None
# now that all metadata is read in, make up lists of all the "named" entities to
# add to the DB before the actual comics are added
self.saveChildInfoToDB(md_list)
#logging.debug(u"Monitor: finish adding child sets")
# create dictionarys of all those objects, so we don't have to query the database
self.createChildDicts()
#sort the list to the last modified file goes in last
md_list = sorted(md_list, key=lambda md: md.mod_ts)
for md in md_list:
self.addComicFromMetadata(md)
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
# periodically commit
if self.add_count % 1000 == 0:
self.session.commit()
self.setStatusDetail(
u"Monitor: {0} of {1} added...".format(
self.add_count, len(md_list)), logging.INFO)
if self.add_count > 0:
self.session.commit()
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(
time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info("Monitor: Added {0} comics".format(self.add_count))
logging.info("Monitor: Removed {0} comics".format(self.remove_count))
if self.remove_count > 0 or self.add_count > 0:
self.session.query(
DatabaseInfo).first().last_updated = datetime.utcnow()
self.session.commit()
if self.quit_when_done:
self.quit = True
def dofullScan(self, dirs):
self.status = "SCANNING"
logging.info(u"Monitor: Beginning file scan...")
self.setStatusDetail(u"Monitor: Making a list of all files in the folders...")
filelist = utils.get_recursive_filelist( dirs )
self.setStatusDetail(u"Monitor: sorting recursive file list ({0} items)".format(len(filelist)))
filelist = sorted(filelist, key=os.path.getmtime)
self.setStatusDetail(u"Monitor: done listing files.")
self.add_count = 0
self.remove_count = 0
# get the entire comic table into memory
query = list(self.session.query(Comic))
# look for missing or changed files
self.setStatusDetail(u"Monitor: Removing missing or modified files from DB...")
#start_time = time.time()
for comic in query:
self.checkIfRemovedOrModified( comic, self.paths )
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#print time.time() - start_time, "seconds"
self.setStatusDetail(u"Monitor: Done removing files.")
if self.remove_count > 0:
self.dm.engine.echo = True
self.session.commit()
self.dm.engine.echo = False
self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(len(filelist)))
# make a list of all path strings in comic table
db_pathlist = [i[0] for i in list(self.session.query(Comic.path))]
self.setStatusDetail(u"Monitor: removing already scanned files from file list")
for f in db_pathlist:
if f in filelist:
filelist.remove(f)
db_pathlist = None
self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO)
md_list = []
self.read_count = 0
for filename in filelist:
md = self.getComicMetadata( filename )
if md is not None:
md_list.append(md)
self.setStatusDetailOnly(u"Monitor: {0} files: {1} scanned, {2} added to library...".format(len(filelist), self.read_count,self.add_count))
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#every so often, commit to DB
if self.read_count % 100 == 0 and self.read_count != 0:
if len(md_list) > 0:
self.commitMetadataList(md_list)
md_list = []
if len(md_list) > 0:
self.commitMetadataList(md_list)
self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO)
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info("Monitor: Added {0} comics".format(self.add_count))
logging.info("Monitor: Removed {0} comics".format(self.remove_count))
if self.remove_count > 0 or self.add_count > 0:
self.session.query(DatabaseInfo).first().last_updated = datetime.utcnow()
self.session.commit()
if self.quit_when_done:
self.quit = True
def dofullScan(self, dirs):
self.status = "SCANNING"
logging.info(u"Monitor: Beginning file scan...")
self.setStatusDetail(u"Monitor: Making a list of all files in the folders...")
filelist = utils.get_recursive_filelist( dirs )
self.setStatusDetail(u"Monitor: done listing files.")
self.add_count = 0
self.remove_count = 0
# get the entire comic table into memory
query = list(self.session.query(Comic))
# look for missing or changed files
self.setStatusDetail(u"Monitor: Removing missing or modified files from DB...")
#start_time = time.time()
for comic in query:
self.checkIfRemovedOrModified( comic, self.paths )
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#print time.time() - start_time, "seconds"
self.setStatusDetail(u"Monitor: Done removing files.")
if self.remove_count > 0:
self.dm.engine.echo = True
self.session.commit()
self.dm.engine.echo = False
self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(len(filelist)))
# make a list of all path strings in comic table
db_pathlist = set([i[0] for i in list(self.session.query(Comic.path))])
filelist = set(filelist)
filelist = filelist - db_pathlist
db_pathlist = None
self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO)
md_list = []
self.read_count = 0
for filename in filelist:
md = self.getComicMetadata( filename )
if md is not None:
md_list.append(md)
if self.read_count % 100 == 0 and self.read_count != 0:
self.setStatusDetail(u"Monitor: {0} of {1} scanned...".format(self.read_count,len(filelist)), logging.INFO)
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO)
filelist = None
# now that all metadata is read in, make up lists of all the "named" entities to
# add to the DB before the actual comics are added
self.saveChildInfoToDB(md_list)
#logging.debug(u"Monitor: finish adding child sets")
# create dictionarys of all those objects, so we don't have to query the database
self.createChildDicts()
#sort the list to the last modified file goes in last
md_list = sorted(md_list, key=lambda md: md.mod_ts)
for md in md_list:
self.addComicFromMetadata( md )
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
# periodically commit
if self.add_count % 1000 == 0:
self.session.commit()
self.setStatusDetail(u"Monitor: {0} of {1} added...".format(self.add_count,len(md_list)), logging.INFO)
if self.add_count > 0:
self.session.commit()
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info("Monitor: Added {0} comics".format(self.add_count))
logging.info("Monitor: Removed {0} comics".format(self.remove_count))
if self.remove_count > 0 or self.add_count > 0:
self.session.query(DatabaseInfo).first().last_updated = datetime.utcnow()
self.session.commit()
if self.quit_when_done:
self.quit = True
def dofullScan(self, dirs):
self.status = "SCANNING"
logging.info(u"Monitor: Beginning file scan...")
self.setStatusDetail(u"Monitor: Making a list of all files in the folders...")
filelist = utils.get_recursive_filelist( dirs )
self.setStatusDetail(u"Monitor: sorting recursive file list ({0} items)".format(len(filelist)))
filelist = sorted(filelist, key=os.path.getmtime)
self.setStatusDetail(u"Monitor: done listing files.")
self.add_count = 0
self.remove_count = 0
# get the entire comic table into memory
query = list(self.session.query(Comic))
# look for missing or changed files
self.setStatusDetail(u"Monitor: Removing missing or modified files from DB...")
#start_time = time.time()
for comic in query:
self.checkIfRemovedOrModified( comic, self.paths )
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#print time.time() - start_time, "seconds"
self.setStatusDetail(u"Monitor: Done removing files.")
if self.remove_count > 0:
self.dm.engine.echo = True
self.session.commit()
self.dm.engine.echo = False
self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(len(filelist)))
# make a list of all path strings in comic table
db_pathlist = [i[0] for i in list(self.session.query(Comic.path))]
self.setStatusDetail(u"Monitor: removing already scanned files from file list")
for f in db_pathlist:
if f in filelist:
filelist.remove(f)
db_pathlist = None
self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO)
md_list = []
self.read_count = 0
for filename in filelist:
md = self.getComicMetadata( filename )
if md is not None:
md_list.append(md)
self.setStatusDetailOnly(u"Monitor: {0} files: {1} scanned, {2} added to library...".format(len(filelist), self.read_count,self.add_count))
if self.quit:
self.setStatusDetail(u"Monitor: halting scan!")
return
#every so often, commit to DB
if self.read_count % 100 == 0 and self.read_count != 0:
if len(md_list) > 0:
self.commitMetadataList(md_list)
md_list = []
if len(md_list) > 0:
self.commitMetadataList(md_list)
self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO)
self.status = "IDLE"
self.statusdetail = ""
self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000)
logging.info("Monitor: Added {0} comics".format(self.add_count))
logging.info("Monitor: Removed {0} comics".format(self.remove_count))
if self.remove_count > 0 or self.add_count > 0:
self.session.query(DatabaseInfo).first().last_updated = datetime.utcnow()
self.session.commit()
if self.quit_when_done:
self.quit = True