コード例 #1
0
ファイル: test_opdsdb.py プロジェクト: web-inkoder/sopds
 def setUp(self):
     opdsdb.clear_all()
     opdsdb.addcattree("root/child/subchild",opdsdb.CAT_NORMAL)
     book = opdsdb.addbook("testbook.fb2", "root/child",opdsdb.findcat("root/child"),".fb2","Test Book", "Annotation", "01.01.2016", "ru", 500, 0)
     opdsdb.addbauthor(book, opdsdb.addauthor("Test Author"))
     opdsdb.addbgenre(book, opdsdb.addgenre("fantastic"))
     opdsdb.addbseries(book, opdsdb.addseries("mywork"), 1)
コード例 #2
0
ファイル: sopdscan.py プロジェクト: akliouev/sopds
 def processinpx(self,name,full_path,file):
     rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
     inpx_size = os.path.getsize(file)
     if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inpx_skip(rel_file,inpx_size):
         self.logger.info('Skip INPX file = '+file+'. Not changed.')
     else:    
         self.logger.info('Start process INPX file = '+file)
         opdsdb.addcattree(rel_file, opdsdb.CAT_INPX, inpx_size)
         inpx = inpx_parser.Inpx(file, self.inpx_callback, self.inpskip_callback)       
         inpx.INPX_TEST_ZIP = config.SOPDS_INPX_TEST_ZIP  
         inpx.INPX_TEST_FILES = config.SOPDS_INPX_TEST_FILES 
         inpx.parse()
コード例 #3
0
ファイル: sopdscan.py プロジェクト: WorldException/sopds
 def processinpx(self,name,full_path,file):
     rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
     inpx_size = os.path.getsize(file)
     if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inpx_skip(rel_file,inpx_size):
         self.logger.info('Skip INPX file = '+file+'. Not changed.')
     else:    
         self.logger.info('Start process INPX file = '+file)
         opdsdb.addcattree(rel_file, opdsdb.CAT_INPX, inpx_size)
         inpx = inpx_parser.Inpx(file, self.inpx_callback, self.inpskip_callback)       
         inpx.INPX_TEST_ZIP = config.SOPDS_INPX_TEST_ZIP  
         inpx.INPX_TEST_FILES = config.SOPDS_INPX_TEST_FILES 
         inpx.parse()
コード例 #4
0
ファイル: sopdscan.py プロジェクト: akliouev/sopds
 def processzip(self,name,full_path,file):
     rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
     zsize = os.path.getsize(file)
     if opdsdb.arc_skip(rel_file,zsize):
         self.arch_skipped+=1
         self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.')
     else:                   
         zip_process_error = 0
         try:
             z = zipfile.ZipFile(file, 'r', allowZip64=True)
             filelist = z.namelist()
             cat = opdsdb.addcattree(rel_file, opdsdb.CAT_ZIP, zsize)
             for n in filelist:
                 try:
                     self.logger.debug('Start process ZIP file = '+file+' book file = '+n)
                     file_size=z.getinfo(n).file_size
                     bookfile = z.open(n)
                     self.processfile(n,file,bookfile,cat,opdsdb.CAT_ZIP,file_size)
                     bookfile.close()
                 except zipfile.BadZipFile:
                     self.logger.warning('Error processing ZIP file = '+file+' book file = '+n)
                     zip_process_error = 1
             z.close()
             self.arch_scanned+=1
         except zipfile.BadZipFile:
             self.logger.warning('Error while read ZIP archive. File '+file+' corrupt.')
             zip_process_error = 1
         self.bad_archives+=zip_process_error
コード例 #5
0
ファイル: sopdscan.py プロジェクト: zhuravel36/SOPDS
    def inpx_callback(self, inpx, inp, meta_data):

        name = "%s.%s" % (meta_data[inpx_parser.sFile],
                          meta_data[inpx_parser.sExt])

        lang = meta_data[inpx_parser.sLang].strip(strip_symbols)
        title = meta_data[inpx_parser.sTitle].strip(strip_symbols)
        annotation = ''
        docdate = meta_data[inpx_parser.sDate].strip(strip_symbols)

        rel_path_current = os.path.join(self.rel_path,
                                        meta_data[inpx_parser.sFolder])

        if opdsdb.findbook(name, rel_path_current, 1) == None:
            cat = opdsdb.addcattree(rel_path_current, opdsdb.CAT_INP)
            book = opdsdb.addbook(name, rel_path_current, cat,
                                  meta_data[inpx_parser.sExt], title,
                                  annotation, docdate, lang,
                                  meta_data[inpx_parser.sSize], opdsdb.CAT_INP)
            self.books_added += 1
            self.books_in_archives += 1
            self.logger.debug("Book " + rel_path_current + "/" + name +
                              " Added ok.")

            for a in meta_data[inpx_parser.sAuthor]:
                author = opdsdb.addauthor(a.replace(',', ' '))
                opdsdb.addbauthor(book, author)

            for g in meta_data[inpx_parser.sGenre]:
                opdsdb.addbgenre(
                    book, opdsdb.addgenre(g.lower().strip(strip_symbols)))

            for s in meta_data[inpx_parser.sSeries]:
                ser = opdsdb.addseries(s.strip())
                opdsdb.addbseries(book, ser, 0)
コード例 #6
0
ファイル: sopdscan.py プロジェクト: WorldException/sopds
 def processzip(self,name,full_path,file):
     rel_file=os.path.relpath(file,config.SOPDS_ROOT_LIB)
     zsize = os.path.getsize(file)
     if opdsdb.arc_skip(rel_file,zsize):
         self.arch_skipped+=1
         self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.')
     else:                   
         zip_process_error = 0
         try:
             z = zipfile.ZipFile(file, 'r', allowZip64=True)
             filelist = z.namelist()
             cat = opdsdb.addcattree(rel_file, opdsdb.CAT_ZIP, zsize)
             for n in filelist:
                 try:
                     self.logger.debug('Start process ZIP file = '+file+' book file = '+n)
                     file_size=z.getinfo(n).file_size
                     bookfile = z.open(n)
                     self.processfile(n,file,bookfile,cat,opdsdb.CAT_ZIP,file_size)
                     bookfile.close()
                 except zipfile.BadZipFile:
                     self.logger.warning('Error processing ZIP file = '+file+' book file = '+n)
                     zip_process_error = 1
             z.close()
             self.arch_scanned+=1
         except zipfile.BadZipFile:
             self.logger.warning('Error while read ZIP archive. File '+file+' corrupt.')
             zip_process_error = 1
         self.bad_archives+=zip_process_error
コード例 #7
0
ファイル: sopdscan.py プロジェクト: mitshel/sopds
    def inpx_callback(self, inpx, inp, meta_data):          
                 
        name = "%s.%s"%(meta_data[inpx_parser.sFile],meta_data[inpx_parser.sExt])
        
        lang=meta_data[inpx_parser.sLang].strip(strip_symbols)
        title=meta_data[inpx_parser.sTitle].strip(strip_symbols)
        annotation=''
        docdate=meta_data[inpx_parser.sDate].strip(strip_symbols)

        rel_path_current = os.path.join(self.rel_path,meta_data[inpx_parser.sFolder])

        if opdsdb.findbook(name, rel_path_current, 1) == None:
            cat = opdsdb.addcattree(rel_path_current, opdsdb.CAT_INP)
            book=opdsdb.addbook(name,rel_path_current,cat,meta_data[inpx_parser.sExt],title,annotation,docdate,lang,meta_data[inpx_parser.sSize],opdsdb.CAT_INP)
            self.books_added+=1
            self.books_in_archives+=1
            self.logger.debug("Book "+rel_path_current+"/"+name+" Added ok.")

            for a in meta_data[inpx_parser.sAuthor]:
                author=opdsdb.addauthor(a.replace(',',' '))
                opdsdb.addbauthor(book,author)

            for g in meta_data[inpx_parser.sGenre]:
                opdsdb.addbgenre(book,opdsdb.addgenre(g.lower().strip(strip_symbols)))

            for s in meta_data[inpx_parser.sSeries]:
                ser=opdsdb.addseries(s.strip())
                opdsdb.addbseries(book,ser,0)
コード例 #8
0
ファイル: sopdscan.py プロジェクト: akliouev/sopds
    def processfile(self,name,full_path,file,cat,archive=0,file_size=0):
        (n, e) = os.path.splitext(name)
        if e.lower() in config.SOPDS_BOOK_EXTENSIONS.split():
            rel_path=os.path.relpath(full_path,config.SOPDS_ROOT_LIB)
            self.logger.debug("Attempt to add book "+rel_path+"/"+name)
            try:
                if opdsdb.findbook(name, rel_path, 1) == None:
                    if archive==0:
                        cat=opdsdb.addcattree(rel_path,archive)

                    try:
                        book_data = create_bookfile(file, name)
                    except Exception as err:
                        book_data = None
                        self.logger.warning(rel_path + ' - ' + name + ' Book parse error, skipping... (Error: %s)'%err)
                        self.bad_books += 1

                    if book_data:
                        lang = book_data.language_code.strip(strip_symbols) if book_data.language_code else ''
                        title = book_data.title.strip(strip_symbols) if book_data.title else n
                        annotation = book_data.description if book_data.description else ''
                        annotation = annotation.strip(strip_symbols) if isinstance(annotation, str) else annotation.decode('utf8').strip(strip_symbols)
                        docdate = book_data.docdate if book_data.docdate else ''

                        book=opdsdb.addbook(name,rel_path,cat,e[1:],title,annotation,docdate,lang,file_size,archive)
                        self.books_added+=1

                        if archive!=0:
                            self.books_in_archives+=1
                        self.logger.debug("Book "+rel_path+"/"+name+" Added ok.")

                        for a in book_data.authors:
                            author_name = a.get('name',_('Unknown author')).strip(strip_symbols)
                            # Если в имени автора нет запятой, то фамилию переносим из конца в начало
                            if author_name and author_name.find(',')<0:
                                author_names = author_name.split()
                                author_name = ' '.join([author_names[-1],' '.join(author_names[:-1])])
                            author=opdsdb.addauthor(author_name)
                            opdsdb.addbauthor(book,author)

                        for genre in book_data.tags:
                            opdsdb.addbgenre(book,opdsdb.addgenre(genre.lower().strip(strip_symbols)))


                        if book_data.series_info:
                            ser = opdsdb.addseries(book_data.series_info['title'])
                            ser_no = book_data.series_info['index']  or '0'
                            ser_no = int(ser_no) if ser_no.isdigit() else 0
                            opdsdb.addbseries(book,ser,ser_no)
                else:
                    self.books_skipped+=1
                    self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.")
            except UnicodeEncodeError as err:
                self.logger.warning(rel_path + ' - ' + name + ' Book UnicodeEncodeError error, skipping... (Error: %s)' % err)
                self.bad_books += 1
コード例 #9
0
ファイル: sopdscan.py プロジェクト: mitshel/sopds
    def processfile(self,name,full_path,file,cat,archive=0,file_size=0):
        (n, e) = os.path.splitext(name)
        if e.lower() in config.SOPDS_BOOK_EXTENSIONS.split():
            rel_path=os.path.relpath(full_path,config.SOPDS_ROOT_LIB)
            self.logger.debug("Attempt to add book "+rel_path+"/"+name)
            try:
                if opdsdb.findbook(name, rel_path, 1) == None:
                    if archive==0:
                        cat=opdsdb.addcattree(rel_path,archive)

                    try:
                        book_data = create_bookfile(file, name)
                    except Exception as err:
                        book_data = None
                        self.logger.warning(rel_path + ' - ' + name + ' Book parse error, skipping... (Error: %s)'%err)
                        self.bad_books += 1

                    if book_data:
                        lang = book_data.language_code.strip(strip_symbols) if book_data.language_code else ''
                        title = book_data.title.strip(strip_symbols) if book_data.title else n
                        annotation = book_data.description if book_data.description else ''
                        annotation = annotation.strip(strip_symbols) if isinstance(annotation, str) else annotation.decode('utf8').strip(strip_symbols)
                        docdate = book_data.docdate if book_data.docdate else ''

                        book=opdsdb.addbook(name,rel_path,cat,e[1:],title,annotation,docdate,lang,file_size,archive)
                        self.books_added+=1

                        if archive!=0:
                            self.books_in_archives+=1
                        self.logger.debug("Book "+rel_path+"/"+name+" Added ok.")

                        for a in book_data.authors:
                            author_name = a.get('name',_('Unknown author')).strip(strip_symbols)
                            # Если в имени автора нет запятой, то фамилию переносим из конца в начало
                            if author_name and author_name.find(',')<0:
                                author_names = author_name.split()
                                author_name = ' '.join([author_names[-1],' '.join(author_names[:-1])])
                            author=opdsdb.addauthor(author_name)
                            opdsdb.addbauthor(book,author)

                        for genre in book_data.tags:
                            opdsdb.addbgenre(book,opdsdb.addgenre(genre.lower().strip(strip_symbols)))


                        if book_data.series_info:
                            ser = opdsdb.addseries(book_data.series_info['title'])
                            ser_no = book_data.series_info['index']  or '0'
                            ser_no = int(ser_no) if ser_no.isdigit() else 0
                            opdsdb.addbseries(book,ser,ser_no)
                else:
                    self.books_skipped+=1
                    self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.")
            except UnicodeEncodeError as err:
                self.logger.warning(rel_path + ' - ' + name + ' Book UnicodeEncodeError error, skipping... (Error: %s)' % err)
                self.bad_books += 1
コード例 #10
0
ファイル: sopdscan.py プロジェクト: mitshel/sopds
    def inpskip_callback(self, inpx, inp_file, inp_size):

        self.rel_path=os.path.relpath(os.path.join(inpx,inp_file),config.SOPDS_ROOT_LIB)
        
        if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inp_skip(self.rel_path,inp_size):
            self.logger.info('Skip INP metafile '+inp_file+'. Not changed.')
            result = 1               
        else:    
            self.logger.info('Start process INP metafile = '+inp_file)
            self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INPX, inp_size)
            result = 0
         
        return result
コード例 #11
0
ファイル: sopdscan.py プロジェクト: akliouev/sopds
 def inpskip_callback(self, inpx, inp_name, inp_size):
     
     self.zip_file = os.path.join(inpx,"%s%s"%(inp_name,'.zip'))
     self.rel_path=os.path.relpath(self.zip_file,config.SOPDS_ROOT_LIB)            
     
     if opdsdb.arc_skip(self.rel_path,inp_size):
         self.logger.info('Skip ZIP for INP archive '+self.zip_file+'. Not changed.')
         result = 1               
     else:    
         self.logger.info('Start process ZIP for INP archive = '+self.zip_file) 
         self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INP, inp_size)
         result = 0
      
     return result
コード例 #12
0
ファイル: sopdscan.py プロジェクト: WorldException/sopds
 def inpskip_callback(self, inpx, inp_name, inp_size):
     
     self.zip_file = os.path.join(inpx,"%s%s"%(inp_name,'.zip'))
     self.rel_path=os.path.relpath(self.zip_file,config.SOPDS_ROOT_LIB)            
     
     if opdsdb.arc_skip(self.rel_path,inp_size):
         self.logger.info('Skip ZIP for INP archive '+self.zip_file+'. Not changed.')
         result = 1               
     else:    
         self.logger.info('Start process ZIP for INP archive = '+self.zip_file) 
         self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INP, inp_size)
         result = 0
      
     return result
コード例 #13
0
ファイル: sopdscan.py プロジェクト: zhuravel36/SOPDS
    def inpskip_callback(self, inpx, inp_file, inp_size):

        self.rel_path = os.path.relpath(os.path.join(inpx, inp_file),
                                        config.SOPDS_ROOT_LIB)

        if config.SOPDS_INPX_SKIP_UNCHANGED and opdsdb.inp_skip(
                self.rel_path, inp_size):
            self.logger.info('Skip INP metafile ' + inp_file +
                             '. Not changed.')
            result = 1
        else:
            self.logger.info('Start process INP metafile = ' + inp_file)
            self.inp_cat = opdsdb.addcattree(self.rel_path, opdsdb.CAT_INPX,
                                             inp_size)
            result = 0

        return result