def createAddRemoveLists(self, dirs): ix = {} db_set = set() current_set = set() filelist = utils.get_recursive_filelist(dirs) for path in filelist: current_set.add((path, datetime.utcfromtimestamp(os.path.getmtime(path)))) logging.info("NEW -- current_set size [%d]" % len(current_set)) for comic_id, path, md_ts in self.library.getComicPaths(): db_set.add((path, md_ts)) ix[path] = comic_id to_add = current_set - db_set to_remove = db_set - current_set logging.info("NEW -- db_set size [%d]" % len(db_set)) logging.info("NEW -- to_add size [%d]" % len(to_add)) logging.info("NEW -- to_remove size [%d]" % len(to_remove)) return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def addPathList( self, pathlist ): filelist = utils.get_recursive_filelist( pathlist ) # we now have a list of files to add progdialog = QProgressDialog("", "Cancel", 0, len(filelist), self) progdialog.setWindowTitle( "Adding Files" ) #progdialog.setWindowModality(Qt.WindowModal) progdialog.setWindowModality(Qt.ApplicationModal) progdialog.show() firstAdded = None self.twList.setSortingEnabled(False) for idx,f in enumerate(filelist): QCoreApplication.processEvents() if progdialog.wasCanceled(): break progdialog.setValue(idx) progdialog.setLabelText(f) utils.centerWindowOnParent( progdialog ) QCoreApplication.processEvents() row = self.addPathItem( f ) if firstAdded is None and row is not None: firstAdded = row progdialog.close() if firstAdded is not None: self.twList.selectRow(firstAdded) self.twList.setSortingEnabled(True) # Adjust column size self.twList.resizeColumnsToContents() self.twList.setColumnWidth(FileSelectionList.CRFlagColNum, 35) self.twList.setColumnWidth(FileSelectionList.CBLFlagColNum, 35) self.twList.setColumnWidth(FileSelectionList.readonlyColNum, 35) self.twList.setColumnWidth(FileSelectionList.typeColNum, 45) if self.twList.columnWidth(FileSelectionList.fileColNum) > 250: self.twList.setColumnWidth(FileSelectionList.fileColNum, 250) if self.twList.columnWidth(FileSelectionList.folderColNum ) > 200: self.twList.setColumnWidth(FileSelectionList.folderColNum, 200)
def createAddRemoveLists(self, dirs): ix = {} db_set = set() current_set = set() filelist = utils.get_recursive_filelist(dirs) for path in filelist: current_set.add( (path, datetime.utcfromtimestamp(os.path.getmtime(path)))) logging.info("NEW -- current_set size [%d]" % len(current_set)) for comic_id, path, md_ts in self.library.getComicPaths(): db_set.add((path, md_ts)) ix[path] = comic_id to_add = current_set - db_set to_remove = db_set - current_set logging.info("NEW -- db_set size [%d]" % len(db_set)) logging.info("NEW -- to_add size [%d]" % len(to_add)) logging.info("NEW -- to_remove size [%d]" % len(to_remove)) return [r[0] for r in to_add], [ix[r[0]] for r in to_remove]
def addPathList(self, pathlist): filelist = utils.get_recursive_filelist(pathlist) # we now have a list of files to add progdialog = QProgressDialog("", "Cancel", 0, len(filelist), self) progdialog.setWindowTitle("Adding Files") #progdialog.setWindowModality(Qt.WindowModal) progdialog.setWindowModality(Qt.ApplicationModal) progdialog.show() firstAdded = None self.twList.setSortingEnabled(False) for idx, f in enumerate(filelist): QCoreApplication.processEvents() if progdialog.wasCanceled(): break progdialog.setValue(idx) progdialog.setLabelText(f) utils.centerWindowOnParent(progdialog) QCoreApplication.processEvents() row = self.addPathItem(f) if firstAdded is None and row is not None: firstAdded = row progdialog.close() if (self.settings.show_no_unrar_warning and self.settings.unrar_exe_path == "" and self.settings.rar_exe_path == "" and platform.system() != "Windows"): for f in filelist: ext = os.path.splitext(f)[1].lower() if ext == ".rar" or ext == ".cbr": checked = OptionalMessageDialog.msg( self, "No unrar tool", """ It looks like you've tried to open at least one CBR or RAR file.<br><br> In order for ComicTagger to read this kind of file, you will have to configure the location of the unrar tool in the settings. Until then, ComicTagger will not be able recognize these kind of files. """) self.settings.show_no_unrar_warning = not checked break if firstAdded is not None: self.twList.selectRow(firstAdded) else: if len(pathlist) == 1 and os.path.isfile(pathlist[0]): QMessageBox.information( self, self.tr("File Open"), self.tr( "Selected file doesn't seem to be a comic archive.")) else: QMessageBox.information( self, self.tr("File/Folder Open"), self.tr("No comic archives were found.")) self.twList.setSortingEnabled(True) # Adjust column size self.twList.resizeColumnsToContents() self.twList.setColumnWidth(FileSelectionList.CRFlagColNum, 35) self.twList.setColumnWidth(FileSelectionList.CBLFlagColNum, 35) self.twList.setColumnWidth(FileSelectionList.readonlyColNum, 35) self.twList.setColumnWidth(FileSelectionList.typeColNum, 45) if self.twList.columnWidth(FileSelectionList.fileColNum) > 250: self.twList.setColumnWidth(FileSelectionList.fileColNum, 250) if self.twList.columnWidth(FileSelectionList.folderColNum) > 200: self.twList.setColumnWidth(FileSelectionList.folderColNum, 200)
def addPathList(self, pathlist): filelist = utils.get_recursive_filelist(pathlist) # we now have a list of files to add progdialog = QProgressDialog("", "Cancel", 0, len(filelist), self) progdialog.setWindowTitle("Adding Files") # progdialog.setWindowModality(Qt.WindowModal) progdialog.setWindowModality(Qt.ApplicationModal) progdialog.show() firstAdded = None self.twList.setSortingEnabled(False) for idx, f in enumerate(filelist): QCoreApplication.processEvents() if progdialog.wasCanceled(): break progdialog.setValue(idx) progdialog.setLabelText(f) centerWindowOnParent(progdialog) QCoreApplication.processEvents() row = self.addPathItem(f) if firstAdded is None and row is not None: firstAdded = row progdialog.close() if (self.settings.show_no_unrar_warning and self.settings.unrar_exe_path == "" and self.settings.rar_exe_path == "" and platform.system() != "Windows"): for f in filelist: ext = os.path.splitext(f)[1].lower() if ext == ".rar" or ext == ".cbr": checked = OptionalMessageDialog.msg(self, "No unrar tool", """ It looks like you've tried to open at least one CBR or RAR file.<br><br> In order for ComicTagger to read this kind of file, you will have to configure the location of the unrar tool in the settings. Until then, ComicTagger will not be able recognize these kind of files. """ ) self.settings.show_no_unrar_warning = not checked break if firstAdded is not None: self.twList.selectRow(firstAdded) else: if len(pathlist) == 1 and os.path.isfile(pathlist[0]): QMessageBox.information(self, self.tr("File Open"), self.tr( "Selected file doesn't seem to be a comic archive.")) else: QMessageBox.information( self, self.tr("File/Folder Open"), self.tr("No comic archives were found.")) self.twList.setSortingEnabled(True) # Adjust column size self.twList.resizeColumnsToContents() self.twList.setColumnWidth(FileSelectionList.CRFlagColNum, 35) self.twList.setColumnWidth(FileSelectionList.CBLFlagColNum, 35) self.twList.setColumnWidth(FileSelectionList.readonlyColNum, 35) self.twList.setColumnWidth(FileSelectionList.typeColNum, 45) if self.twList.columnWidth(FileSelectionList.fileColNum) > 250: self.twList.setColumnWidth(FileSelectionList.fileColNum, 250) if self.twList.columnWidth(FileSelectionList.folderColNum) > 200: self.twList.setColumnWidth(FileSelectionList.folderColNum, 200)
def parseCmdLineArgs(self): if platform.system() == "Darwin" and hasattr( sys, "frozen") and sys.frozen == 1: # remove the PSN ("process serial number") argument from OS/X input_args = [a for a in sys.argv[1:] if "-psn_0_" not in a] else: input_args = sys.argv[1:] # first check if we're launching a script: for n in range(len(input_args)): if (input_args[n] in ["-S", "--script"] and n + 1 < len(input_args)): # insert a "--" which will cause getopt to ignore the remaining args # so they will be passed to the script input_args.insert(n + 2, "--") break # parse command line options try: opts, args = getopt.getopt(input_args, "hpdt:fm:vownsrc:ieRS:1", ["help", "print", "delete", "type=", "copy=", "parsefilename", "metadata=", "verbose", "online", "dryrun", "save", "rename", "raw", "noabort", "terse", "nooverwrite", "interactive", "nosummary", "version", "id=", "recursive", "script=", "export-to-zip", "delete-rar", "abort-on-conflict", "assume-issue-one", "cv-api-key=", "only-set-cv-key", "wait-on-cv-rate-limit"]) except getopt.GetoptError as err: self.display_msg_and_quit(str(err), 2) # process options for o, a in opts: if o in ("-h", "--help"): self.display_msg_and_quit(None, 0, show_help=True) if o in ("-v", "--verbose"): self.verbose = True if o in ("-S", "--script"): self.run_script = True self.script = a if o in ("-R", "--recursive"): self.recursive = True if o in ("-p", "--print"): self.print_tags = True if o in ("-d", "--delete"): self.delete_tags = True if o in ("-i", "--interactive"): self.interactive = True if o in ("-c", "--copy"): self.copy_tags = True if a.lower() == "cr": self.copy_source = MetaDataStyle.CIX elif a.lower() == "cbl": self.copy_source = MetaDataStyle.CBI elif a.lower() == "comet": self.copy_source = MetaDataStyle.COMET else: self.display_msg_and_quit( "Invalid copy tag source type", 1) if o in ("-o", "--online"): self.search_online = True if o in ("-n", "--dryrun"): self.dryrun = True if o in ("-m", "--metadata"): self.metadata = self.parseMetadataFromString(a) if o in ("-s", "--save"): self.save_tags = True if o in ("-r", "--rename"): self.rename_file = True if o in ("-e", "--export_to_zip"): self.export_to_zip = True if o == "--delete-rar": self.delete_rar_after_export = True if o == "--abort-on-conflict": self.abort_export_on_conflict = True if o in ("-f", "--parsefilename"): self.parse_filename = True if o in ("-w", "--wait-on-cv-rate-limit"): self.wait_and_retry_on_rate_limit = True if o == "--id": self.issue_id = a if o == "--raw": self.raw = True if o == "--noabort": self.abortOnLowConfidence = False if o == "--terse": self.terse = True if o == "--nosummary": self.show_save_summary = False if o in ("-1", "--assume-issue-one"): self.assume_issue_is_one_if_not_set = True if o == "--nooverwrite": self.no_overwrite = True if o == "--cv-api-key": self.cv_api_key = a if o == "--only-set-cv-key": self.only_set_key = True if o == "--version": print( "ComicTagger {0} [{1} / {2}]".format(ctversion.version, ctversion.fork, ctversion.fork_tag)) print( "Modified version of ComicTagger (Copyright (c) 2012-2014 Anthony Beville)") print( "Distributed under Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)") sys.exit(0) if o in ("-t", "--type"): if a.lower() == "cr": self.data_style = MetaDataStyle.CIX elif a.lower() == "cbl": self.data_style = MetaDataStyle.CBI elif a.lower() == "comet": self.data_style = MetaDataStyle.COMET else: self.display_msg_and_quit("Invalid tag type", 1) if self.print_tags or self.delete_tags or self.save_tags or self.copy_tags or self.rename_file or self.export_to_zip or self.only_set_key: self.no_gui = True count = 0 if self.run_script: count += 1 if self.print_tags: count += 1 if self.delete_tags: count += 1 if self.save_tags: count += 1 if self.copy_tags: count += 1 if self.rename_file: count += 1 if self.export_to_zip: count += 1 if self.only_set_key: count += 1 if count > 1: self.display_msg_and_quit( "Must choose only one action of print, delete, save, copy, rename, export, set key, or run script", 1) if self.script is not None: self.launch_script(self.script) if len(args) > 0: if platform.system() == "Windows": # no globbing on windows shell, so do it for them import glob self.file_list = [] for item in args: self.file_list.extend(glob.glob(item)) if len(self.file_list) > 0: self.filename = self.file_list[0] else: self.filename = args[0] self.file_list = args if self.only_set_key and self.cv_api_key is None: self.display_msg_and_quit("Key not given!", 1) if (self.only_set_key == False) and self.no_gui and ( self.filename is None): self.display_msg_and_quit( "Command requires at least one filename!", 1) if self.delete_tags and self.data_style is None: self.display_msg_and_quit( "Please specify the type to delete with -t", 1) if self.save_tags and self.data_style is None: self.display_msg_and_quit( "Please specify the type to save with -t", 1) if self.copy_tags and self.data_style is None: self.display_msg_and_quit( "Please specify the type to copy to with -t", 1) # if self.rename_file and self.data_style is None: # self.display_msg_and_quit("Please specify the type to use for renaming with -t", 1) if self.recursive: self.file_list = utils.get_recursive_filelist(self.file_list)
def parseCmdLineArgs(self): if platform.system() == "Darwin" and hasattr( sys, "frozen") and sys.frozen == 1: # remove the PSN ("process serial number") argument from OS/X input_args = [a for a in sys.argv[1:] if "-psn_0_" not in a] else: input_args = sys.argv[1:] # first check if we're launching a script: for n in range(len(input_args)): if (input_args[n] in ["-S", "--script"] and n + 1 < len(input_args)): # insert a "--" which will cause getopt to ignore the remaining args # so they will be passed to the script input_args.insert(n + 2, "--") break # parse command line options try: opts, args = getopt.getopt(input_args, "hpdt:fm:vownsrc:ieRS:1", [ "help", "print", "delete", "type=", "copy=", "parsefilename", "metadata=", "verbose", "online", "dryrun", "save", "rename", "raw", "noabort", "terse", "nooverwrite", "interactive", "nosummary", "version", "id=", "recursive", "script=", "export-to-zip", "delete-rar", "abort-on-conflict", "assume-issue-one", "cv-api-key=", "only-set-cv-key", "wait-on-cv-rate-limit" ]) except getopt.GetoptError as err: self.display_msg_and_quit(str(err), 2) # process options for o, a in opts: if o in ("-h", "--help"): self.display_msg_and_quit(None, 0, show_help=True) if o in ("-v", "--verbose"): self.verbose = True if o in ("-S", "--script"): self.run_script = True self.script = a if o in ("-R", "--recursive"): self.recursive = True if o in ("-p", "--print"): self.print_tags = True if o in ("-d", "--delete"): self.delete_tags = True if o in ("-i", "--interactive"): self.interactive = True if o in ("-c", "--copy"): self.copy_tags = True if a.lower() == "cr": self.copy_source = MetaDataStyle.CIX elif a.lower() == "cbl": self.copy_source = MetaDataStyle.CBI elif a.lower() == "comet": self.copy_source = MetaDataStyle.COMET else: self.display_msg_and_quit("Invalid copy tag source type", 1) if o in ("-o", "--online"): self.search_online = True if o in ("-n", "--dryrun"): self.dryrun = True if o in ("-m", "--metadata"): self.metadata = self.parseMetadataFromString(a) if o in ("-s", "--save"): self.save_tags = True if o in ("-r", "--rename"): self.rename_file = True if o in ("-e", "--export_to_zip"): self.export_to_zip = True if o == "--delete-rar": self.delete_rar_after_export = True if o == "--abort-on-conflict": self.abort_export_on_conflict = True if o in ("-f", "--parsefilename"): self.parse_filename = True if o in ("-w", "--wait-on-cv-rate-limit"): self.wait_and_retry_on_rate_limit = True if o == "--id": self.issue_id = a if o == "--raw": self.raw = True if o == "--noabort": self.abortOnLowConfidence = False if o == "--terse": self.terse = True if o == "--nosummary": self.show_save_summary = False if o in ("-1", "--assume-issue-one"): self.assume_issue_is_one_if_not_set = True if o == "--nooverwrite": self.no_overwrite = True if o == "--cv-api-key": self.cv_api_key = a if o == "--only-set-cv-key": self.only_set_key = True if o == "--version": print( "ComicTagger {0}: Copyright (c) 2012-2014 Anthony Beville" .format(ctversion.version)) print( "Distributed under Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)" ) new_version = VersionChecker().getLatestVersion("", False) if new_version is not None and new_version != ctversion.version: print( "--------------------------------------------------------------------------" ) print("New version available online: {0}".format( new_version)) print( "--------------------------------------------------------------------------" ) sys.exit(0) if o in ("-t", "--type"): if a.lower() == "cr": self.data_style = MetaDataStyle.CIX elif a.lower() == "cbl": self.data_style = MetaDataStyle.CBI elif a.lower() == "comet": self.data_style = MetaDataStyle.COMET else: self.display_msg_and_quit("Invalid tag type", 1) if self.print_tags or self.delete_tags or self.save_tags or self.copy_tags or self.rename_file or self.export_to_zip or self.only_set_key: self.no_gui = True count = 0 if self.run_script: count += 1 if self.print_tags: count += 1 if self.delete_tags: count += 1 if self.save_tags: count += 1 if self.copy_tags: count += 1 if self.rename_file: count += 1 if self.export_to_zip: count += 1 if self.only_set_key: count += 1 if count > 1: self.display_msg_and_quit( "Must choose only one action of print, delete, save, copy, rename, export, set key, or run script", 1) if self.script is not None: self.launch_script(self.script) if len(args) > 0: if platform.system() == "Windows": # no globbing on windows shell, so do it for them import glob self.file_list = [] for item in args: self.file_list.extend(glob.glob(item)) if len(self.file_list) > 0: self.filename = self.file_list[0] else: self.filename = args[0] self.file_list = args if self.only_set_key and self.cv_api_key is None: self.display_msg_and_quit("Key not given!", 1) if (self.only_set_key == False) and self.no_gui and (self.filename is None): self.display_msg_and_quit( "Command requires at least one filename!", 1) if self.delete_tags and self.data_style is None: self.display_msg_and_quit( "Please specify the type to delete with -t", 1) if self.save_tags and self.data_style is None: self.display_msg_and_quit( "Please specify the type to save with -t", 1) if self.copy_tags and self.data_style is None: self.display_msg_and_quit( "Please specify the type to copy to with -t", 1) # if self.rename_file and self.data_style is None: # self.display_msg_and_quit("Please specify the type to use for renaming with -t", 1) if self.recursive: self.file_list = utils.get_recursive_filelist(self.file_list)
def dofullScan(self, dirs): self.status = "SCANNING" logging.info(u"Monitor: Beginning file scan...") self.setStatusDetail( u"Monitor: Making a list of all files in the folders...") filelist = utils.get_recursive_filelist(dirs) self.setStatusDetail(u"Monitor: done listing files.") self.add_count = 0 self.remove_count = 0 # get the entire comic table into memory query = list(self.session.query(Comic)) # look for missing or changed files self.setStatusDetail( u"Monitor: Removing missing or modified files from DB...") #start_time = time.time() for comic in query: self.checkIfRemovedOrModified(comic, self.paths) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #print time.time() - start_time, "seconds" self.setStatusDetail(u"Monitor: Done removing files.") if self.remove_count > 0: self.dm.engine.echo = True self.session.commit() self.dm.engine.echo = False self.setStatusDetail(u"Monitor: found {0} files to inspect...".format( len(filelist))) # make a list of all path strings in comic table db_pathlist = set([i[0] for i in list(self.session.query(Comic.path))]) filelist = set(filelist) filelist = filelist - db_pathlist db_pathlist = None self.setStatusDetail( u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO) md_list = [] self.read_count = 0 for filename in filelist: md = self.getComicMetadata(filename) if md is not None: md_list.append(md) if self.read_count % 100 == 0 and self.read_count != 0: self.setStatusDetail( u"Monitor: {0} of {1} scanned...".format( self.read_count, len(filelist)), logging.INFO) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return self.setStatusDetail( u"Monitor: finished scanning metadata in {0} of {1} files".format( self.read_count, len(filelist)), logging.INFO) filelist = None # now that all metadata is read in, make up lists of all the "named" entities to # add to the DB before the actual comics are added self.saveChildInfoToDB(md_list) #logging.debug(u"Monitor: finish adding child sets") # create dictionarys of all those objects, so we don't have to query the database self.createChildDicts() #sort the list to the last modified file goes in last md_list = sorted(md_list, key=lambda md: md.mod_ts) for md in md_list: self.addComicFromMetadata(md) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return # periodically commit if self.add_count % 1000 == 0: self.session.commit() self.setStatusDetail( u"Monitor: {0} of {1} added...".format( self.add_count, len(md_list)), logging.INFO) if self.add_count > 0: self.session.commit() self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = int( time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info("Monitor: Added {0} comics".format(self.add_count)) logging.info("Monitor: Removed {0} comics".format(self.remove_count)) if self.remove_count > 0 or self.add_count > 0: self.session.query( DatabaseInfo).first().last_updated = datetime.utcnow() self.session.commit() if self.quit_when_done: self.quit = True
def dofullScan(self, dirs): self.status = "SCANNING" logging.info(u"Monitor: Beginning file scan...") self.setStatusDetail(u"Monitor: Making a list of all files in the folders...") filelist = utils.get_recursive_filelist( dirs ) self.setStatusDetail(u"Monitor: sorting recursive file list ({0} items)".format(len(filelist))) filelist = sorted(filelist, key=os.path.getmtime) self.setStatusDetail(u"Monitor: done listing files.") self.add_count = 0 self.remove_count = 0 # get the entire comic table into memory query = list(self.session.query(Comic)) # look for missing or changed files self.setStatusDetail(u"Monitor: Removing missing or modified files from DB...") #start_time = time.time() for comic in query: self.checkIfRemovedOrModified( comic, self.paths ) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #print time.time() - start_time, "seconds" self.setStatusDetail(u"Monitor: Done removing files.") if self.remove_count > 0: self.dm.engine.echo = True self.session.commit() self.dm.engine.echo = False self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(len(filelist))) # make a list of all path strings in comic table db_pathlist = [i[0] for i in list(self.session.query(Comic.path))] self.setStatusDetail(u"Monitor: removing already scanned files from file list") for f in db_pathlist: if f in filelist: filelist.remove(f) db_pathlist = None self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO) md_list = [] self.read_count = 0 for filename in filelist: md = self.getComicMetadata( filename ) if md is not None: md_list.append(md) self.setStatusDetailOnly(u"Monitor: {0} files: {1} scanned, {2} added to library...".format(len(filelist), self.read_count,self.add_count)) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #every so often, commit to DB if self.read_count % 100 == 0 and self.read_count != 0: if len(md_list) > 0: self.commitMetadataList(md_list) md_list = [] if len(md_list) > 0: self.commitMetadataList(md_list) self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO) self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info("Monitor: Added {0} comics".format(self.add_count)) logging.info("Monitor: Removed {0} comics".format(self.remove_count)) if self.remove_count > 0 or self.add_count > 0: self.session.query(DatabaseInfo).first().last_updated = datetime.utcnow() self.session.commit() if self.quit_when_done: self.quit = True
def dofullScan(self, dirs): self.status = "SCANNING" logging.info(u"Monitor: Beginning file scan...") self.setStatusDetail(u"Monitor: Making a list of all files in the folders...") filelist = utils.get_recursive_filelist( dirs ) self.setStatusDetail(u"Monitor: done listing files.") self.add_count = 0 self.remove_count = 0 # get the entire comic table into memory query = list(self.session.query(Comic)) # look for missing or changed files self.setStatusDetail(u"Monitor: Removing missing or modified files from DB...") #start_time = time.time() for comic in query: self.checkIfRemovedOrModified( comic, self.paths ) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #print time.time() - start_time, "seconds" self.setStatusDetail(u"Monitor: Done removing files.") if self.remove_count > 0: self.dm.engine.echo = True self.session.commit() self.dm.engine.echo = False self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(len(filelist))) # make a list of all path strings in comic table db_pathlist = set([i[0] for i in list(self.session.query(Comic.path))]) filelist = set(filelist) filelist = filelist - db_pathlist db_pathlist = None self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO) md_list = [] self.read_count = 0 for filename in filelist: md = self.getComicMetadata( filename ) if md is not None: md_list.append(md) if self.read_count % 100 == 0 and self.read_count != 0: self.setStatusDetail(u"Monitor: {0} of {1} scanned...".format(self.read_count,len(filelist)), logging.INFO) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO) filelist = None # now that all metadata is read in, make up lists of all the "named" entities to # add to the DB before the actual comics are added self.saveChildInfoToDB(md_list) #logging.debug(u"Monitor: finish adding child sets") # create dictionarys of all those objects, so we don't have to query the database self.createChildDicts() #sort the list to the last modified file goes in last md_list = sorted(md_list, key=lambda md: md.mod_ts) for md in md_list: self.addComicFromMetadata( md ) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return # periodically commit if self.add_count % 1000 == 0: self.session.commit() self.setStatusDetail(u"Monitor: {0} of {1} added...".format(self.add_count,len(md_list)), logging.INFO) if self.add_count > 0: self.session.commit() self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info("Monitor: Added {0} comics".format(self.add_count)) logging.info("Monitor: Removed {0} comics".format(self.remove_count)) if self.remove_count > 0 or self.add_count > 0: self.session.query(DatabaseInfo).first().last_updated = datetime.utcnow() self.session.commit() if self.quit_when_done: self.quit = True
def dofullScan(self, dirs): self.status = "SCANNING" logging.info(u"Monitor: Beginning file scan...") self.setStatusDetail(u"Monitor: Making a list of all files in the folders...") filelist = utils.get_recursive_filelist( dirs ) self.setStatusDetail(u"Monitor: sorting recursive file list ({0} items)".format(len(filelist))) filelist = sorted(filelist, key=os.path.getmtime) self.setStatusDetail(u"Monitor: done listing files.") self.add_count = 0 self.remove_count = 0 # get the entire comic table into memory query = list(self.session.query(Comic)) # look for missing or changed files self.setStatusDetail(u"Monitor: Removing missing or modified files from DB...") #start_time = time.time() for comic in query: self.checkIfRemovedOrModified( comic, self.paths ) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #print time.time() - start_time, "seconds" self.setStatusDetail(u"Monitor: Done removing files.") if self.remove_count > 0: self.dm.engine.echo = True self.session.commit() self.dm.engine.echo = False self.setStatusDetail(u"Monitor: found {0} files to inspect...".format(len(filelist))) # make a list of all path strings in comic table db_pathlist = [i[0] for i in list(self.session.query(Comic.path))] self.setStatusDetail(u"Monitor: removing already scanned files from file list") for f in db_pathlist: if f in filelist: filelist.remove(f) db_pathlist = None self.setStatusDetail(u"Monitor: {0} new files to scan...".format(len(filelist)), logging.INFO) md_list = [] self.read_count = 0 for filename in filelist: md = self.getComicMetadata( filename ) if md is not None: md_list.append(md) self.setStatusDetailOnly(u"Monitor: {0} files: {1} scanned, {2} added to library...".format(len(filelist), self.read_count,self.add_count)) if self.quit: self.setStatusDetail(u"Monitor: halting scan!") return #every so often, commit to DB if self.read_count % 100 == 0 and self.read_count != 0: if len(md_list) > 0: self.commitMetadataList(md_list) md_list = [] if len(md_list) > 0: self.commitMetadataList(md_list) self.setStatusDetail(u"Monitor: finished scanning metadata in {0} of {1} files".format(self.read_count,len(filelist)), logging.INFO) self.status = "IDLE" self.statusdetail = "" self.scancomplete_ts = int(time.mktime(datetime.utcnow().timetuple()) * 1000) logging.info("Monitor: Added {0} comics".format(self.add_count)) logging.info("Monitor: Removed {0} comics".format(self.remove_count)) if self.remove_count > 0 or self.add_count > 0: self.session.query(DatabaseInfo).first().last_updated = datetime.utcnow() self.session.commit() if self.quit_when_done: self.quit = True