def setUp(self): opdsdb.clear_all() opdsdb.addcattree("root/child/subchild",opdsdb.CAT_NORMAL) book = opdsdb.addbook("testbook.fb2", "root/child",opdsdb.findcat("root/child"),".fb2","Test Book", "Annotation", "01.01.2016", "ru", 500, 0) opdsdb.addbauthor(book, opdsdb.addauthor("Test Author")) opdsdb.addbgenre(book, opdsdb.addgenre("fantastic")) opdsdb.addbseries(book, opdsdb.addseries("mywork"), 1)
def processinpx(self,name,full_path,file): rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB) inpx_size = os.path.getsize(file) if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inpx_skip(rel_file,inpx_size): self.logger.info('Skip INPX file = '+file+'. Not changed.') else: self.logger.info('Start process INPX file = '+file) opdsdb.addcattree(rel_file, opdsdb.CAT_INPX, inpx_size) inpx = inpx_parser.Inpx(file, self.inpx_callback, self.inpskip_callback) inpx.INPX_TEST_ZIP = config.SOPDS_INPX_TEST_ZIP inpx.INPX_TEST_FILES = config.SOPDS_INPX_TEST_FILES inpx.parse()
def processinpx(self,name,full_path,file): rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB) inpx_size = os.path.getsize(file) if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inpx_skip(rel_file,inpx_size): self.logger.info('Skip INPX file = '+file+'. Not changed.') else: self.logger.info('Start process INPX file = '+file) opdsdb.addcattree(rel_file, opdsdb.CAT_INPX, inpx_size) inpx = inpx_parser.Inpx(file, self.inpx_callback, self.inpskip_callback) inpx.INPX_TEST_ZIP = config.SOPDS_INPX_TEST_ZIP inpx.INPX_TEST_FILES = config.SOPDS_INPX_TEST_FILES inpx.parse()
def processzip(self,name,full_path,file): rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB) zsize = os.path.getsize(file) if opdsdb.arc_skip(rel_file,zsize): self.arch_skipped+=1 self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.') else: zip_process_error = 0 try: z = zipfile.ZipFile(file, 'r', allowZip64=True) filelist = z.namelist() cat = opdsdb.addcattree(rel_file, opdsdb.CAT_ZIP, zsize) for n in filelist: try: self.logger.debug('Start process ZIP file = '+file+' book file = '+n) file_size=z.getinfo(n).file_size bookfile = z.open(n) self.processfile(n,file,bookfile,cat,opdsdb.CAT_ZIP,file_size) bookfile.close() except zipfile.BadZipFile: self.logger.warning('Error processing ZIP file = '+file+' book file = '+n) zip_process_error = 1 z.close() self.arch_scanned+=1 except zipfile.BadZipFile: self.logger.warning('Error while read ZIP archive. File '+file+' corrupt.') zip_process_error = 1 self.bad_archives+=zip_process_error
def inpx_callback(self, inpx, inp, meta_data): name = "%s.%s" % (meta_data[inpx_parser.sFile], meta_data[inpx_parser.sExt]) lang = meta_data[inpx_parser.sLang].strip(strip_symbols) title = meta_data[inpx_parser.sTitle].strip(strip_symbols) annotation = '' docdate = meta_data[inpx_parser.sDate].strip(strip_symbols) rel_path_current = os.path.join(self.rel_path, meta_data[inpx_parser.sFolder]) if opdsdb.findbook(name, rel_path_current, 1) == None: cat = opdsdb.addcattree(rel_path_current, opdsdb.CAT_INP) book = opdsdb.addbook(name, rel_path_current, cat, meta_data[inpx_parser.sExt], title, annotation, docdate, lang, meta_data[inpx_parser.sSize], opdsdb.CAT_INP) self.books_added += 1 self.books_in_archives += 1 self.logger.debug("Book " + rel_path_current + "/" + name + " Added ok.") for a in meta_data[inpx_parser.sAuthor]: author = opdsdb.addauthor(a.replace(',', ' ')) opdsdb.addbauthor(book, author) for g in meta_data[inpx_parser.sGenre]: opdsdb.addbgenre( book, opdsdb.addgenre(g.lower().strip(strip_symbols))) for s in meta_data[inpx_parser.sSeries]: ser = opdsdb.addseries(s.strip()) opdsdb.addbseries(book, ser, 0)
def processzip(self,name,full_path,file): rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB) zsize = os.path.getsize(file) if opdsdb.arc_skip(rel_file,zsize): self.arch_skipped+=1 self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.') else: zip_process_error = 0 try: z = zipfile.ZipFile(file, 'r', allowZip64=True) filelist = z.namelist() cat = opdsdb.addcattree(rel_file, opdsdb.CAT_ZIP, zsize) for n in filelist: try: self.logger.debug('Start process ZIP file = '+file+' book file = '+n) file_size=z.getinfo(n).file_size bookfile = z.open(n) self.processfile(n,file,bookfile,cat,opdsdb.CAT_ZIP,file_size) bookfile.close() except zipfile.BadZipFile: self.logger.warning('Error processing ZIP file = '+file+' book file = '+n) zip_process_error = 1 z.close() self.arch_scanned+=1 except zipfile.BadZipFile: self.logger.warning('Error while read ZIP archive. File '+file+' corrupt.') zip_process_error = 1 self.bad_archives+=zip_process_error
def inpx_callback(self, inpx, inp, meta_data): name = "%s.%s"%(meta_data[inpx_parser.sFile],meta_data[inpx_parser.sExt]) lang=meta_data[inpx_parser.sLang].strip(strip_symbols) title=meta_data[inpx_parser.sTitle].strip(strip_symbols) annotation='' docdate=meta_data[inpx_parser.sDate].strip(strip_symbols) rel_path_current = os.path.join(self.rel_path,meta_data[inpx_parser.sFolder]) if opdsdb.findbook(name, rel_path_current, 1) == None: cat = opdsdb.addcattree(rel_path_current, opdsdb.CAT_INP) book=opdsdb.addbook(name,rel_path_current,cat,meta_data[inpx_parser.sExt],title,annotation,docdate,lang,meta_data[inpx_parser.sSize],opdsdb.CAT_INP) self.books_added+=1 self.books_in_archives+=1 self.logger.debug("Book "+rel_path_current+"/"+name+" Added ok.") for a in meta_data[inpx_parser.sAuthor]: author=opdsdb.addauthor(a.replace(',',' ')) opdsdb.addbauthor(book,author) for g in meta_data[inpx_parser.sGenre]: opdsdb.addbgenre(book,opdsdb.addgenre(g.lower().strip(strip_symbols))) for s in meta_data[inpx_parser.sSeries]: ser=opdsdb.addseries(s.strip()) opdsdb.addbseries(book,ser,0)
def processfile(self,name,full_path,file,cat,archive=0,file_size=0): (n, e) = os.path.splitext(name) if e.lower() in config.SOPDS_BOOK_EXTENSIONS.split(): rel_path=os.path.relpath(full_path,config.SOPDS_ROOT_LIB) self.logger.debug("Attempt to add book "+rel_path+"/"+name) try: if opdsdb.findbook(name, rel_path, 1) == None: if archive==0: cat=opdsdb.addcattree(rel_path,archive) try: book_data = create_bookfile(file, name) except Exception as err: book_data = None self.logger.warning(rel_path + ' - ' + name + ' Book parse error, skipping... (Error: %s)'%err) self.bad_books += 1 if book_data: lang = book_data.language_code.strip(strip_symbols) if book_data.language_code else '' title = book_data.title.strip(strip_symbols) if book_data.title else n annotation = book_data.description if book_data.description else '' annotation = annotation.strip(strip_symbols) if isinstance(annotation, str) else annotation.decode('utf8').strip(strip_symbols) docdate = book_data.docdate if book_data.docdate else '' book=opdsdb.addbook(name,rel_path,cat,e[1:],title,annotation,docdate,lang,file_size,archive) self.books_added+=1 if archive!=0: self.books_in_archives+=1 self.logger.debug("Book "+rel_path+"/"+name+" Added ok.") for a in book_data.authors: author_name = a.get('name',_('Unknown author')).strip(strip_symbols) # Если в имени автора нет запятой, то фамилию переносим из конца в начало if author_name and author_name.find(',')<0: author_names = author_name.split() author_name = ' '.join([author_names[-1],' '.join(author_names[:-1])]) author=opdsdb.addauthor(author_name) opdsdb.addbauthor(book,author) for genre in book_data.tags: opdsdb.addbgenre(book,opdsdb.addgenre(genre.lower().strip(strip_symbols))) if book_data.series_info: ser = opdsdb.addseries(book_data.series_info['title']) ser_no = book_data.series_info['index'] or '0' ser_no = int(ser_no) if ser_no.isdigit() else 0 opdsdb.addbseries(book,ser,ser_no) else: self.books_skipped+=1 self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.") except UnicodeEncodeError as err: self.logger.warning(rel_path + ' - ' + name + ' Book UnicodeEncodeError error, skipping... (Error: %s)' % err) self.bad_books += 1
def processfile(self,name,full_path,file,cat,archive=0,file_size=0): (n, e) = os.path.splitext(name) if e.lower() in config.SOPDS_BOOK_EXTENSIONS.split(): rel_path=os.path.relpath(full_path,config.SOPDS_ROOT_LIB) self.logger.debug("Attempt to add book "+rel_path+"/"+name) try: if opdsdb.findbook(name, rel_path, 1) == None: if archive==0: cat=opdsdb.addcattree(rel_path,archive) try: book_data = create_bookfile(file, name) except Exception as err: book_data = None self.logger.warning(rel_path + ' - ' + name + ' Book parse error, skipping... (Error: %s)'%err) self.bad_books += 1 if book_data: lang = book_data.language_code.strip(strip_symbols) if book_data.language_code else '' title = book_data.title.strip(strip_symbols) if book_data.title else n annotation = book_data.description if book_data.description else '' annotation = annotation.strip(strip_symbols) if isinstance(annotation, str) else annotation.decode('utf8').strip(strip_symbols) docdate = book_data.docdate if book_data.docdate else '' book=opdsdb.addbook(name,rel_path,cat,e[1:],title,annotation,docdate,lang,file_size,archive) self.books_added+=1 if archive!=0: self.books_in_archives+=1 self.logger.debug("Book "+rel_path+"/"+name+" Added ok.") for a in book_data.authors: author_name = a.get('name',_('Unknown author')).strip(strip_symbols) # Если в имени автора нет запятой, то фамилию переносим из конца в начало if author_name and author_name.find(',')<0: author_names = author_name.split() author_name = ' '.join([author_names[-1],' '.join(author_names[:-1])]) author=opdsdb.addauthor(author_name) opdsdb.addbauthor(book,author) for genre in book_data.tags: opdsdb.addbgenre(book,opdsdb.addgenre(genre.lower().strip(strip_symbols))) if book_data.series_info: ser = opdsdb.addseries(book_data.series_info['title']) ser_no = book_data.series_info['index'] or '0' ser_no = int(ser_no) if ser_no.isdigit() else 0 opdsdb.addbseries(book,ser,ser_no) else: self.books_skipped+=1 self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.") except UnicodeEncodeError as err: self.logger.warning(rel_path + ' - ' + name + ' Book UnicodeEncodeError error, skipping... (Error: %s)' % err) self.bad_books += 1
def inpskip_callback(self, inpx, inp_file, inp_size): self.rel_path=os.path.relpath(os.path.join(inpx,inp_file),config.SOPDS_ROOT_LIB) if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inp_skip(self.rel_path,inp_size): self.logger.info('Skip INP metafile '+inp_file+'. Not changed.') result = 1 else: self.logger.info('Start process INP metafile = '+inp_file) self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INPX, inp_size) result = 0 return result
def inpskip_callback(self, inpx, inp_name, inp_size): self.zip_file = os.path.join(inpx,"%s%s"%(inp_name,'.zip')) self.rel_path=os.path.relpath(self.zip_file,config.SOPDS_ROOT_LIB) if opdsdb.arc_skip(self.rel_path,inp_size): self.logger.info('Skip ZIP for INP archive '+self.zip_file+'. Not changed.') result = 1 else: self.logger.info('Start process ZIP for INP archive = '+self.zip_file) self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INP, inp_size) result = 0 return result
def inpskip_callback(self, inpx, inp_name, inp_size): self.zip_file = os.path.join(inpx,"%s%s"%(inp_name,'.zip')) self.rel_path=os.path.relpath(self.zip_file,config.SOPDS_ROOT_LIB) if opdsdb.arc_skip(self.rel_path,inp_size): self.logger.info('Skip ZIP for INP archive '+self.zip_file+'. Not changed.') result = 1 else: self.logger.info('Start process ZIP for INP archive = '+self.zip_file) self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INP, inp_size) result = 0 return result
def inpskip_callback(self, inpx, inp_file, inp_size): self.rel_path = os.path.relpath(os.path.join(inpx, inp_file), config.SOPDS_ROOT_LIB) if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inp_skip( self.rel_path, inp_size): self.logger.info('Skip INP metafile ' + inp_file + '. Not changed.') result = 1 else: self.logger.info('Start process INP metafile = ' + inp_file) self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INPX, inp_size) result = 0 return result