def setUp(self):
opdsdb.clear_all()
opdsdb.addcattree("root/child/subchild",opdsdb.CAT_NORMAL)
book = opdsdb.addbook("testbook.fb2", "root/child",opdsdb.findcat("root/child"),".fb2","Test Book", "Annotation", "01.01.2016", "ru", 500, 0)
opdsdb.addbauthor(book, opdsdb.addauthor("Test Author"))
opdsdb.addbgenre(book, opdsdb.addgenre("fantastic"))
opdsdb.addbseries(book, opdsdb.addseries("mywork"), 1)
def processinpx(self,name,full_path,file):
rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
inpx_size = os.path.getsize(file)
if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inpx_skip(rel_file,inpx_size):
self.logger.info('Skip INPX file = '+file+'. Not changed.')
else:
self.logger.info('Start process INPX file = '+file)
opdsdb.addcattree(rel_file, opdsdb.CAT_INPX, inpx_size)
inpx = inpx_parser.Inpx(file, self.inpx_callback, self.inpskip_callback)
inpx.INPX_TEST_ZIP = config.SOPDS_INPX_TEST_ZIP
inpx.INPX_TEST_FILES = config.SOPDS_INPX_TEST_FILES
inpx.parse()
def processinpx(self,name,full_path,file):
rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
inpx_size = os.path.getsize(file)
if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inpx_skip(rel_file,inpx_size):
self.logger.info('Skip INPX file = '+file+'. Not changed.')
else:
self.logger.info('Start process INPX file = '+file)
opdsdb.addcattree(rel_file, opdsdb.CAT_INPX, inpx_size)
inpx = inpx_parser.Inpx(file, self.inpx_callback, self.inpskip_callback)
inpx.INPX_TEST_ZIP = config.SOPDS_INPX_TEST_ZIP
inpx.INPX_TEST_FILES = config.SOPDS_INPX_TEST_FILES
inpx.parse()
def processzip(self,name,full_path,file):
rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
zsize = os.path.getsize(file)
if opdsdb.arc_skip(rel_file,zsize):
self.arch_skipped+=1
self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.')
else:
zip_process_error = 0
try:
z = zipfile.ZipFile(file, 'r', allowZip64=True)
filelist = z.namelist()
cat = opdsdb.addcattree(rel_file, opdsdb.CAT_ZIP, zsize)
for n in filelist:
try:
self.logger.debug('Start process ZIP file = '+file+' book file = '+n)
file_size=z.getinfo(n).file_size
bookfile = z.open(n)
self.processfile(n,file,bookfile,cat,opdsdb.CAT_ZIP,file_size)
bookfile.close()
except zipfile.BadZipFile:
self.logger.warning('Error processing ZIP file = '+file+' book file = '+n)
zip_process_error = 1
z.close()
self.arch_scanned+=1
except zipfile.BadZipFile:
self.logger.warning('Error while read ZIP archive. File '+file+' corrupt.')
zip_process_error = 1
self.bad_archives+=zip_process_error
def inpx_callback(self, inpx, inp, meta_data):
name = "%s.%s" % (meta_data[inpx_parser.sFile],
meta_data[inpx_parser.sExt])
lang = meta_data[inpx_parser.sLang].strip(strip_symbols)
title = meta_data[inpx_parser.sTitle].strip(strip_symbols)
annotation = ''
docdate = meta_data[inpx_parser.sDate].strip(strip_symbols)
rel_path_current = os.path.join(self.rel_path,
meta_data[inpx_parser.sFolder])
if opdsdb.findbook(name, rel_path_current, 1) == None:
cat = opdsdb.addcattree(rel_path_current, opdsdb.CAT_INP)
book = opdsdb.addbook(name, rel_path_current, cat,
meta_data[inpx_parser.sExt], title,
annotation, docdate, lang,
meta_data[inpx_parser.sSize], opdsdb.CAT_INP)
self.books_added += 1
self.books_in_archives += 1
self.logger.debug("Book " + rel_path_current + "/" + name +
" Added ok.")
for a in meta_data[inpx_parser.sAuthor]:
author = opdsdb.addauthor(a.replace(',', ' '))
opdsdb.addbauthor(book, author)
for g in meta_data[inpx_parser.sGenre]:
opdsdb.addbgenre(
book, opdsdb.addgenre(g.lower().strip(strip_symbols)))
for s in meta_data[inpx_parser.sSeries]:
ser = opdsdb.addseries(s.strip())
opdsdb.addbseries(book, ser, 0)
def processzip(self,name,full_path,file):
rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
zsize = os.path.getsize(file)
if opdsdb.arc_skip(rel_file,zsize):
self.arch_skipped+=1
self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.')
else:
zip_process_error = 0
try:
z = zipfile.ZipFile(file, 'r', allowZip64=True)
filelist = z.namelist()
cat = opdsdb.addcattree(rel_file, opdsdb.CAT_ZIP, zsize)
for n in filelist:
try:
self.logger.debug('Start process ZIP file = '+file+' book file = '+n)
file_size=z.getinfo(n).file_size
bookfile = z.open(n)
self.processfile(n,file,bookfile,cat,opdsdb.CAT_ZIP,file_size)
bookfile.close()
except zipfile.BadZipFile:
self.logger.warning('Error processing ZIP file = '+file+' book file = '+n)
zip_process_error = 1
z.close()
self.arch_scanned+=1
except zipfile.BadZipFile:
self.logger.warning('Error while read ZIP archive. File '+file+' corrupt.')
zip_process_error = 1
self.bad_archives+=zip_process_error
def inpx_callback(self, inpx, inp, meta_data):
name = "%s.%s"%(meta_data[inpx_parser.sFile],meta_data[inpx_parser.sExt])
lang=meta_data[inpx_parser.sLang].strip(strip_symbols)
title=meta_data[inpx_parser.sTitle].strip(strip_symbols)
annotation=''
docdate=meta_data[inpx_parser.sDate].strip(strip_symbols)
rel_path_current = os.path.join(self.rel_path,meta_data[inpx_parser.sFolder])
if opdsdb.findbook(name, rel_path_current, 1) == None:
cat = opdsdb.addcattree(rel_path_current, opdsdb.CAT_INP)
book=opdsdb.addbook(name,rel_path_current,cat,meta_data[inpx_parser.sExt],title,annotation,docdate,lang,meta_data[inpx_parser.sSize],opdsdb.CAT_INP)
self.books_added+=1
self.books_in_archives+=1
self.logger.debug("Book "+rel_path_current+"/"+name+" Added ok.")
for a in meta_data[inpx_parser.sAuthor]:
author=opdsdb.addauthor(a.replace(',',' '))
opdsdb.addbauthor(book,author)
for g in meta_data[inpx_parser.sGenre]:
opdsdb.addbgenre(book,opdsdb.addgenre(g.lower().strip(strip_symbols)))
for s in meta_data[inpx_parser.sSeries]:
ser=opdsdb.addseries(s.strip())
opdsdb.addbseries(book,ser,0)
def processfile(self,name,full_path,file,cat,archive=0,file_size=0):
(n, e) = os.path.splitext(name)
if e.lower() in config.SOPDS_BOOK_EXTENSIONS.split():
rel_path=os.path.relpath(full_path,config.SOPDS_ROOT_LIB)
self.logger.debug("Attempt to add book "+rel_path+"/"+name)
try:
if opdsdb.findbook(name, rel_path, 1) == None:
if archive==0:
cat=opdsdb.addcattree(rel_path,archive)
try:
book_data = create_bookfile(file, name)
except Exception as err:
book_data = None
self.logger.warning(rel_path + ' - ' + name + ' Book parse error, skipping... (Error: %s)'%err)
self.bad_books += 1
if book_data:
lang = book_data.language_code.strip(strip_symbols) if book_data.language_code else ''
title = book_data.title.strip(strip_symbols) if book_data.title else n
annotation = book_data.description if book_data.description else ''
annotation = annotation.strip(strip_symbols) if isinstance(annotation, str) else annotation.decode('utf8').strip(strip_symbols)
docdate = book_data.docdate if book_data.docdate else ''
book=opdsdb.addbook(name,rel_path,cat,e[1:],title,annotation,docdate,lang,file_size,archive)
self.books_added+=1
if archive!=0:
self.books_in_archives+=1
self.logger.debug("Book "+rel_path+"/"+name+" Added ok.")
for a in book_data.authors:
author_name = a.get('name',_('Unknown author')).strip(strip_symbols)
# Если в имени автора нет запятой, то фамилию переносим из конца в начало
if author_name and author_name.find(',')<0:
author_names = author_name.split()
author_name = ' '.join([author_names[-1],' '.join(author_names[:-1])])
author=opdsdb.addauthor(author_name)
opdsdb.addbauthor(book,author)
for genre in book_data.tags:
opdsdb.addbgenre(book,opdsdb.addgenre(genre.lower().strip(strip_symbols)))
if book_data.series_info:
ser = opdsdb.addseries(book_data.series_info['title'])
ser_no = book_data.series_info['index'] or '0'
ser_no = int(ser_no) if ser_no.isdigit() else 0
opdsdb.addbseries(book,ser,ser_no)
else:
self.books_skipped+=1
self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.")
except UnicodeEncodeError as err:
self.logger.warning(rel_path + ' - ' + name + ' Book UnicodeEncodeError error, skipping... (Error: %s)' % err)
self.bad_books += 1
def processfile(self,name,full_path,file,cat,archive=0,file_size=0):
(n, e) = os.path.splitext(name)
if e.lower() in config.SOPDS_BOOK_EXTENSIONS.split():
rel_path=os.path.relpath(full_path,config.SOPDS_ROOT_LIB)
self.logger.debug("Attempt to add book "+rel_path+"/"+name)
try:
if opdsdb.findbook(name, rel_path, 1) == None:
if archive==0:
cat=opdsdb.addcattree(rel_path,archive)
try:
book_data = create_bookfile(file, name)
except Exception as err:
book_data = None
self.logger.warning(rel_path + ' - ' + name + ' Book parse error, skipping... (Error: %s)'%err)
self.bad_books += 1
if book_data:
lang = book_data.language_code.strip(strip_symbols) if book_data.language_code else ''
title = book_data.title.strip(strip_symbols) if book_data.title else n
annotation = book_data.description if book_data.description else ''
annotation = annotation.strip(strip_symbols) if isinstance(annotation, str) else annotation.decode('utf8').strip(strip_symbols)
docdate = book_data.docdate if book_data.docdate else ''
book=opdsdb.addbook(name,rel_path,cat,e[1:],title,annotation,docdate,lang,file_size,archive)
self.books_added+=1
if archive!=0:
self.books_in_archives+=1
self.logger.debug("Book "+rel_path+"/"+name+" Added ok.")
for a in book_data.authors:
author_name = a.get('name',_('Unknown author')).strip(strip_symbols)
# Если в имени автора нет запятой, то фамилию переносим из конца в начало
if author_name and author_name.find(',')<0:
author_names = author_name.split()
author_name = ' '.join([author_names[-1],' '.join(author_names[:-1])])
author=opdsdb.addauthor(author_name)
opdsdb.addbauthor(book,author)
for genre in book_data.tags:
opdsdb.addbgenre(book,opdsdb.addgenre(genre.lower().strip(strip_symbols)))
if book_data.series_info:
ser = opdsdb.addseries(book_data.series_info['title'])
ser_no = book_data.series_info['index'] or '0'
ser_no = int(ser_no) if ser_no.isdigit() else 0
opdsdb.addbseries(book,ser,ser_no)
else:
self.books_skipped+=1
self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.")
except UnicodeEncodeError as err:
self.logger.warning(rel_path + ' - ' + name + ' Book UnicodeEncodeError error, skipping... (Error: %s)' % err)
self.bad_books += 1
def inpskip_callback(self, inpx, inp_file, inp_size):
self.rel_path=os.path.relpath(os.path.join(inpx,inp_file),config.SOPDS_ROOT_LIB)
if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inp_skip(self.rel_path,inp_size):
self.logger.info('Skip INP metafile '+inp_file+'. Not changed.')
result = 1
else:
self.logger.info('Start process INP metafile = '+inp_file)
self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INPX, inp_size)
result = 0
return result
def inpskip_callback(self, inpx, inp_name, inp_size):
self.zip_file = os.path.join(inpx,"%s%s"%(inp_name,'.zip'))
self.rel_path=os.path.relpath(self.zip_file,config.SOPDS_ROOT_LIB)
if opdsdb.arc_skip(self.rel_path,inp_size):
self.logger.info('Skip ZIP for INP archive '+self.zip_file+'. Not changed.')
result = 1
else:
self.logger.info('Start process ZIP for INP archive = '+self.zip_file)
self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INP, inp_size)
result = 0
return result
def inpskip_callback(self, inpx, inp_name, inp_size):
self.zip_file = os.path.join(inpx,"%s%s"%(inp_name,'.zip'))
self.rel_path=os.path.relpath(self.zip_file,config.SOPDS_ROOT_LIB)
if opdsdb.arc_skip(self.rel_path,inp_size):
self.logger.info('Skip ZIP for INP archive '+self.zip_file+'. Not changed.')
result = 1
else:
self.logger.info('Start process ZIP for INP archive = '+self.zip_file)
self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INP, inp_size)
result = 0
return result
def inpskip_callback(self, inpx, inp_file, inp_size):
self.rel_path = os.path.relpath(os.path.join(inpx, inp_file),
config.SOPDS_ROOT_LIB)
if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inp_skip(
self.rel_path, inp_size):
self.logger.info('Skip INP metafile ' + inp_file +
'. Not changed.')
result = 1
else:
self.logger.info('Start process INP metafile = ' + inp_file)
self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INPX,
inp_size)
result = 0
return result