def __init__(self, FIELD_MAP, field_metadata, db_prefs=None): self.FIELD_MAP = FIELD_MAP self.db_prefs = db_prefs self.composites = {} self.udc = get_udc() for key in field_metadata: if field_metadata[key]['datatype'] == 'composite': self.composites[field_metadata[key]['rec_index']] = key self.series_col = field_metadata['series']['rec_index'] self.series_sort_col = field_metadata['series_sort']['rec_index'] self._data = [] self._map = self._map_filtered = [] self.first_sort = True self.search_restriction = self.base_restriction = '' self.base_restriction_name = self.search_restriction_name = '' self.search_restriction_book_count = 0 self.marked_ids_dict = {} self.field_metadata = field_metadata self.all_search_locations = field_metadata.get_search_terms() SearchQueryParser.__init__(self, self.all_search_locations, optimize=True) self.build_date_relop_dict() self.build_numeric_relop_dict() # Do this here so the var get updated when a library changes global pref_use_primary_find_in_search pref_use_primary_find_in_search = prefs['use_primary_find_in_search']
def ret_clean_text(log, dbg_lvl, text, swap=False, who=''): # for noosfere search to work smoothly, authors and title needs to be cleaned # we need to remove non significant characters and remove useless space character # debug = dbg_lvl & 4 if debug: log.info("\nIn ret_clean_txt(self, log, text, swap =", swap, ")") log.info("text : ", text) # Calibre per default presents the author as "Firstname Lastname", cleaned to be become "firstname lastname" # Noosfere present the author as "LASTNAME Firstname", let's get "Firstname LASTNAME" cleaned to "firstname lastname" # for k in [',', '.', '-', "'", '"', '(', ')']: # yes I found a name with '(' and ')' in it... if k in text: text = text.replace(k, " ") text = " ".join(text.split()) if swap: if debug: log.info("swap name and surname") nom = prenom = "" for i in range(len(text.split())): if (len(text.split()[i]) == 1) or (not text.split()[i].isupper()): prenom += " " + text.split()[i] else: nom += " " + text.split()[i] text = prenom + " " + nom if debug: log.info("text : ", text) if debug: log.info("cleaned text : ", text) log.info("return text from ret_clean_txt") return lower(get_udc().decode(text))
def get_author_tokens(author, decode_non_ascii=True): ''' Take an author and return a list of tokens useful for duplicate hash comparisons. This function tries to return tokens in first name middle names last name order, by assuming that if a comma is in the author name, the name is in lastname, other names form. ''' ignore_suffixes = [ 'von', 'van', 'jr', 'sr', 'i', 'ii' 'iii', 'second', 'third', 'md', 'phd' ] if author: # Leave ' in there for Irish names remove_pat = re.compile(r'[,!@#$%^&*(){}`~"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') au = replace_pat.sub(' ', author) if decode_non_ascii: au = get_udc().decode(au) parts = au.split() if ',' in au: # au probably in ln, fn form parts = parts[1:] + parts[:1] for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_suffixes: yield tok.lower()
def ascii_text(orig): udc = get_udc() try: ascii = udc.decode(orig) except: if isinstance(orig, unicode): orig = orig.encode("ascii", "replace") ascii = orig.decode(preferred_encoding, "replace").encode("ascii", "replace") return ascii
def ascii_text(orig): udc = get_udc() try: ascii = udc.decode(orig) except: if isinstance(orig, unicode): orig = orig.encode('ascii', 'replace') ascii = orig.decode(preferred_encoding, 'replace').encode('ascii', 'replace') return ascii
def ascii_text(orig): udc = get_udc() try: ascii = udc.decode(orig) except: if isinstance(orig, unicode): ascii = orig.encode('ascii', 'replace') ascii = orig.decode(preferred_encoding, 'replace').encode('ascii', 'replace') return ascii
def ascii_text(orig): udc = get_udc() try: ascii = udc.decode(orig) except Exception: if isinstance(orig, unicode_type): orig = orig.encode('ascii', 'replace') ascii = orig.decode(preferred_encoding, 'replace') if isinstance(ascii, bytes): ascii = ascii.decode('ascii', 'replace') return ascii
def ascii_text(orig): udc = get_udc() try: ascii = udc.decode(orig) except Exception: if isinstance(orig, unicode_type): orig = orig.encode('ascii', 'replace') ascii = orig.decode(preferred_encoding, 'replace') if isinstance(ascii, bytes): ascii = ascii.decode('ascii', 'replace') return ascii
def create_title_query(self, log, title=None): q = '' if title: title = get_udc().decode(title) tokens = [] title_tokens = list(self.get_title_tokens(title, strip_joiners=False, strip_subtitle=True)) tokens = [quote(t.encode('utf-8') if isinstance(t, unicode) else t) for t in title_tokens] q = '+'.join(tokens) if not q: return None return '%s/vyhledavani?q=%s'%(BookFan.BASE_URL, q)
def get_tag_tokens(tag, decode_non_ascii=True): ''' Take a tag and return a list of tokens useful for duplicate hash comparisons. ''' ignore_words = ['the', 'and', 'a'] if tag: remove_pat = re.compile(r'[,!@#$%^&*(){}`~\'"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') t = replace_pat.sub(' ', tag) if decode_non_ascii: t = get_udc().decode(t) parts = t.split() for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_words: yield tok.lower()
def get_tag_tokens(tag, decode_non_ascii=True): ''' Take a tag and return a list of tokens useful for duplicate hash comparisons. ''' ignore_words = ['the', 'and', 'a'] if tag: remove_pat = re.compile(r'[,!@#$%^&*(){}`~\'"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') t = replace_pat.sub(' ', tag) if decode_non_ascii: t = get_udc().decode(t) parts = t.split() for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_words: yield tok.lower()
def get_series_tokens(series, decode_non_ascii=True): ''' Take a series and return a list of tokens useful for duplicate hash comparisons. ''' ignore_words = ['the', 'a', 'and',] if series: remove_pat = re.compile(r'[,!@#$%^&*(){}`~\'"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') s = replace_pat.sub(' ', series) if decode_non_ascii: s = get_udc().decode(s) parts = s.split() for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_words: yield tok.lower()
def get_publisher_tokens(publisher, decode_non_ascii=True): ''' Take a publisher and return a list of tokens useful for duplicate hash comparisons. ''' ignore_words = ['the', 'inc', 'ltd', 'limited', 'llc', 'co', 'pty', 'usa', 'uk'] if publisher: remove_pat = re.compile(r'[,!@#$%^&*(){}`~\'"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') p = replace_pat.sub(' ', publisher) if decode_non_ascii: p = get_udc().decode(p) parts = p.split() for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_words: yield tok.lower()
def get_publisher_tokens(publisher, decode_non_ascii=True): ''' Take a publisher and return a list of tokens useful for duplicate hash comparisons. ''' ignore_words = [ 'the', 'inc', 'ltd', 'limited', 'llc', 'co', 'pty', 'usa', 'uk' ] if publisher: remove_pat = re.compile(r'[,!@#$%^&*(){}`~\'"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') p = replace_pat.sub(' ', publisher) if decode_non_ascii: p = get_udc().decode(p) parts = p.split() for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_words: yield tok.lower()
def get_title_tokens(title, strip_subtitle=True, decode_non_ascii=True): ''' Take a title and return a list of tokens useful for an AND search query. Excludes subtitles (optionally), punctuation and a, the. ''' if title: # strip sub-titles if strip_subtitle: subtitle = re.compile(r'([\(\[\{].*?[\)\]\}]|[/:\\].*$)') if len(subtitle.sub('', title)) > 1: title = subtitle.sub('', title) title_patterns = [ (re.compile(pat, re.IGNORECASE), repl) for pat, repl in [ # Remove things like: (2010) (Omnibus) etc. (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|mass\s*market|edition|ed\.)[\])}]', ''), # Remove any strings that contain the substring edition inside # parentheses (r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''), # Remove commas used a separators in numbers (r'(\d+),(\d+)', r'\1\2'), # Remove hyphens only if they have whitespace before them (r'(\s-)', ' '), # Remove single quotes not followed by 's' (r"'(?!s)", ''), # Replace other special chars with a space (r'''[:,;+!@#$%^&*(){}.`~"\s\[\]/]''', ' ') ] ] for pat, repl in title_patterns: title = pat.sub(repl, title) if decode_non_ascii: title = get_udc().decode(title) tokens = title.split() for token in tokens: token = token.strip() if token and (token.lower() not in ('a', 'the')): yield token.lower()
def get_title_tokens(title, strip_subtitle=True, decode_non_ascii=True): ''' Take a title and return a list of tokens useful for an AND search query. Excludes subtitles (optionally), punctuation and a, the. ''' if title: # strip sub-titles if strip_subtitle: subtitle = re.compile(r'([\(\[\{].*?[\)\]\}]|[/:\\].*$)') if len(subtitle.sub('', title)) > 1: title = subtitle.sub('', title) title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in [ # Remove things like: (2010) (Omnibus) etc. (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|mass\s*market|edition|ed\.)[\])}]', ''), # Remove any strings that contain the substring edition inside # parentheses (r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''), # Remove commas used a separators in numbers (r'(\d+),(\d+)', r'\1\2'), # Remove hyphens only if they have whitespace before them (r'(\s-)', ' '), # Remove single quotes not followed by 's' (r"'(?!s)", ''), # Replace other special chars with a space (r'''[:,;+!@#$%^&*(){}.`~"\s\[\]/]''', ' ') ]] for pat, repl in title_patterns: title = pat.sub(repl, title) if decode_non_ascii: title = get_udc().decode(title) tokens = title.split() for token in tokens: token = token.strip() if token and (token.lower() not in ('a', 'the')): yield token.lower()
def get_series_tokens(series, decode_non_ascii=True): ''' Take a series and return a list of tokens useful for duplicate hash comparisons. ''' ignore_words = [ 'the', 'a', 'and', ] if series: remove_pat = re.compile(r'[,!@#$%^&*(){}`~\'"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') s = replace_pat.sub(' ', series) if decode_non_ascii: s = get_udc().decode(s) parts = s.split() for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in ignore_words: yield tok.lower()
def get_author_tokens(author, decode_non_ascii=True): ''' Take an author and return a list of tokens useful for duplicate hash comparisons. This function tries to return tokens in first name middle names last name order, by assuming that if a comma is in the author name, the name is in lastname, other names form. ''' if author: # Leave ' in there for Irish names remove_pat = re.compile(r'[,!@#$%^&*(){}`~"\s\[\]/]') replace_pat = re.compile(r'[-+.:;]') au = replace_pat.sub(' ', author) if decode_non_ascii: au = get_udc().decode(au) parts = au.split() if ',' in au: # au probably in ln, fn form parts = parts[1:] + parts[:1] for tok in parts: tok = remove_pat.sub('', tok).strip() if len(tok) > 0 and tok.lower() not in IGNORE_AUTHOR_WORDS_MAP: yield tok.lower()
def __call__(self, html, remove_special_chars=None, get_preprocess_html=False): if remove_special_chars is not None: html = remove_special_chars.sub('', html) html = html.replace('\0', '') is_pdftohtml = self.is_pdftohtml(html) if self.is_baen(html): rules = [] elif self.is_book_designer(html): rules = self.BOOK_DESIGNER elif is_pdftohtml: rules = self.PDFTOHTML else: rules = [] start_rules = [] if not getattr(self.extra_opts, 'keep_ligatures', False): html = _ligpat.sub(lambda m:LIGATURES[m.group()], html) user_sr_rules = {} # Function for processing search and replace def do_search_replace(search_pattern, replace_txt): from calibre.ebooks.conversion.search_replace import compile_regular_expression try: search_re = compile_regular_expression(search_pattern) if not replace_txt: replace_txt = '' rules.insert(0, (search_re, replace_txt)) user_sr_rules[(search_re, replace_txt)] = search_pattern except Exception as e: self.log.error('Failed to parse %r regexp because %s' % (search, as_unicode(e))) # search / replace using the sr?_search / sr?_replace options for i in range(1, 4): search, replace = 'sr%d_search'%i, 'sr%d_replace'%i search_pattern = getattr(self.extra_opts, search, '') replace_txt = getattr(self.extra_opts, replace, '') if search_pattern: do_search_replace(search_pattern, replace_txt) # multi-search / replace using the search_replace option search_replace = getattr(self.extra_opts, 'search_replace', None) if search_replace: search_replace = json.loads(search_replace) for search_pattern, replace_txt in reversed(search_replace): do_search_replace(search_pattern, replace_txt) end_rules = [] # delete soft hyphens - moved here so it's executed after header/footer removal if is_pdftohtml: # unwrap/delete soft hyphens end_rules.append((re.compile(u'[](</p>\s*<p>\s*)+\s*(?=[[a-z\d])'), lambda match: '')) # unwrap/delete soft hyphens with formatting end_rules.append((re.compile(u'[]\s*(</(i|u|b)>)+(</p>\s*<p>\s*)+\s*(<(i|u|b)>)+\s*(?=[[a-z\d])'), lambda match: '')) length = -1 if getattr(self.extra_opts, 'unwrap_factor', 0.0) > 0.01: docanalysis = DocAnalysis('pdf', html) length = docanalysis.line_length(getattr(self.extra_opts, 'unwrap_factor')) if length: # print "The pdf line length returned is " + str(length) # unwrap em/en dashes end_rules.append((re.compile(u'(?<=.{%i}[–—])\s*<p>\s*(?=[[a-z\d])' % length), lambda match: '')) end_rules.append( # Un wrap using punctuation (re.compile(u'(?<=.{%i}([a-zäëïöüàèìòùáćéíĺóŕńśúýâêîôûçąężıãõñæøþðßěľščťžňďřů,:)\IA\u00DF]|(?<!\&\w{4});))\s*(?P<ital></(i|b|u)>)?\s*(</p>\s*<p>\s*)+\s*(?=(<(i|b|u)>)?\s*[\w\d$(])' % length, re.UNICODE), wrap_lines), # noqa ) for rule in self.PREPROCESS + start_rules: html = rule[0].sub(rule[1], html) if self.regex_wizard_callback is not None: self.regex_wizard_callback(self.current_href, html) if get_preprocess_html: return html def dump(raw, where): import os dp = getattr(self.extra_opts, 'debug_pipeline', None) if dp and os.path.exists(dp): odir = os.path.join(dp, 'input') if os.path.exists(odir): odir = os.path.join(odir, where) if not os.path.exists(odir): os.makedirs(odir) name, i = None, 0 while not name or os.path.exists(os.path.join(odir, name)): i += 1 name = '%04d.html'%i with open(os.path.join(odir, name), 'wb') as f: f.write(raw.encode('utf-8')) # dump(html, 'pre-preprocess') for rule in rules + end_rules: try: html = rule[0].sub(rule[1], html) except Exception as e: if rule in user_sr_rules: self.log.error( 'User supplied search & replace rule: %s -> %s ' 'failed with error: %s, ignoring.'%( user_sr_rules[rule], rule[1], e)) else: raise if is_pdftohtml and length > -1: # Dehyphenate dehyphenator = Dehyphenator(self.extra_opts.verbose, self.log) html = dehyphenator(html,'html', length) if is_pdftohtml: from calibre.ebooks.conversion.utils import HeuristicProcessor pdf_markup = HeuristicProcessor(self.extra_opts, None) totalwords = 0 if pdf_markup.get_word_count(html) > 7000: html = pdf_markup.markup_chapters(html, totalwords, True) # dump(html, 'post-preprocess') # Handle broken XHTML w/ SVG (ugh) if 'svg:' in html and SVG_NS not in html: html = html.replace( '<html', '<html xmlns:svg="%s"' % SVG_NS, 1) if 'xlink:' in html and XLINK_NS not in html: html = html.replace( '<html', '<html xmlns:xlink="%s"' % XLINK_NS, 1) html = XMLDECL_RE.sub('', html) if getattr(self.extra_opts, 'asciiize', False): from calibre.utils.localization import get_udc from calibre.utils.mreplace import MReplace unihandecoder = get_udc() mr = MReplace(data={u'«':u'<'*3, u'»':u'>'*3}) html = mr.mreplace(html) html = unihandecoder.decode(html) if getattr(self.extra_opts, 'enable_heuristics', False): from calibre.ebooks.conversion.utils import HeuristicProcessor preprocessor = HeuristicProcessor(self.extra_opts, self.log) html = preprocessor(html) if is_pdftohtml: html = html.replace('<!-- created by calibre\'s pdftohtml -->', '') if getattr(self.extra_opts, 'smarten_punctuation', False): html = smarten_punctuation(html, self.log) try: unsupported_unicode_chars = self.extra_opts.output_profile.unsupported_unicode_chars except AttributeError: unsupported_unicode_chars = u'' if unsupported_unicode_chars: from calibre.utils.localization import get_udc unihandecoder = get_udc() for char in unsupported_unicode_chars: asciichar = unihandecoder.decode(char) html = html.replace(char, asciichar) return html
def similar_title_match(title, lang=None): title = get_udc().decode(title) result = fuzzy_it(title) if lang: return lang + result return result
def identify(self, log, result_queue, abort, title=None, authors=None, identifiers={}, timeout=30): log.info("identify") ''' Note this method will retry without identifiers automatically if no match is found with identifiers. ''' matches = [] # If we have an ISFDB id then we do not need to fire a "search". # Instead we will go straight to the URL for that book. isfdb_id = identifiers.get('isfdb', None) isbn = check_isbn(identifiers.get('isbn', None)) br = self.browser if isfdb_id: matches.append('%s/cgi-bin/pl.cgi?%s' % (ISFDB.BASE_URL, isfdb_id)) else: title = get_udc().decode(title) authors = authors or [] authors = [get_udc().decode(a) for a in authors] query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if query is None: log.error('Insufficient metadata to construct query. Alas!') return isbn_match_failed = False try: log.info('Querying: %s' % query) response = br.open_novisit(query, timeout=timeout) raw = response.read().decode('cp1252', errors='replace').strip() if isbn: # Check whether we got redirected to a book page for ISBN searches. # If we did, will use the url. # If we didn't then treat it as no matches on ISFDB location = response.geturl() # If not an exact match on ISBN we can get a search results page back # XMS: This may be terribly different for ISFDB. # XMS: HOWEVER: 1563890933 returns multiple results! isbn_match_failed = location.find('/pl.cgi') < 0 if raw.find('found 0 matches') == -1 and not isbn_match_failed: log.info('ISBN match location: %r' % location) matches.append(location) except Exception as e: if isbn and callable(getattr(e, 'getcode', None)) and e.getcode() == 404: # We did a lookup by ISBN but did not find a match # We will fallback to doing a lookup by title author log.info('Failed to find match for ISBN: %s' % isbn) elif callable(getattr(e, 'getcode', None)) and e.getcode() == 404: log.error('No matches for identify query') return as_unicode(e) else: err = 'Failed to make identify query' log.exception(err) return as_unicode(e) # For successful ISBN-based searches we have already done everything we need to. # So anything from this point below is for title/author based searches. if not isbn or isbn_match_failed: try: root = fromstring(clean_ascii_chars(raw)) except: msg = 'Failed to parse ISFDB page for query' log.exception(msg) return msg # Now grab the matches from the search results, provided the # title and authors appear to be for the same book self._parse_search_results(log, title, authors, root, matches, timeout) if abort.is_set(): return if not matches: if identifiers and title and authors: log.info('No matches found with identifiers, retrying using only title and authors') return self.identify(log, result_queue, abort, title=title, authors=authors, timeout=timeout) log.error('No matches found with query: %r' % query) return from calibre_plugins.isfdb.worker import Worker workers = [Worker(url, result_queue, br, log, i, self) for i, url in enumerate(matches)] for w in workers: w.start() # Don't send all requests at the same time time.sleep(0.1) while not abort.is_set(): a_worker_is_alive = False for w in workers: w.join(0.2) if abort.is_set(): break if w.is_alive(): a_worker_is_alive = True if not a_worker_is_alive: break return None
def similar_title_match(title, lang=None): title = get_udc().decode(title) result = fuzzy_it(title) if lang: return lang + result return result
def similar_title_match(title): title = get_udc().decode(title) return fuzzy_it(title)