Exemplo n.º 1
0
def do_download_for_worker(book, options, merge, notification=lambda x, y: x):
    '''
    Child job, to download story when run as a worker job
    '''

    from calibre_plugins.fanficfare_plugin import FanFicFareBase
    fffbase = FanFicFareBase(options['plugin_path'])
    with fffbase:  # so the sys.path was modified while loading the
        # plug impl.
        from calibre_plugins.fanficfare_plugin.dialogs import (
            NotGoingToDownload, OVERWRITE, OVERWRITEALWAYS, UPDATE,
            UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
        from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
        from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data

        from calibre_plugins.fanficfare_plugin.fff_util import (
            get_fff_adapter, get_fff_config)

        try:
            ## No need to download at all.  Can happen now due to
            ## collision moving into book for CALIBREONLY changing to
            ## ADDNEW when story URL not in library.
            if book['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                logger.info("Skipping CALIBREONLY 'update' down inside worker")
                return book

            book['comment'] = _('Download started...')

            configuration = get_fff_config(book['url'], options['fileform'],
                                           options['personal.ini'])

            if configuration.getConfig('use_ssl_unverified_context'):
                ## monkey patch to avoid SSL bug.  dupliated from
                ## fff_plugin.py because bg jobs run in own process
                ## space.
                import ssl
                if hasattr(ssl, '_create_unverified_context'):
                    ssl._create_default_https_context = ssl._create_unverified_context

            if not options[
                    'updateepubcover'] and 'epub_for_update' in book and book[
                        'collision'] in (UPDATE, UPDATEALWAYS):
                configuration.set("overrides", "never_make_cover", "true")

            # images only for epub, html, even if the user mistakenly
            # turned it on else where.
            if options['fileform'] not in ("epub", "html"):
                configuration.set("overrides", "include_images", "false")

            adapter = adapters.getAdapter(configuration, book['url'])
            adapter.is_adult = book['is_adult']
            adapter.username = book['username']
            adapter.password = book['password']
            adapter.setChaptersRange(book['begin'], book['end'])

            configuration.load_cookiejar(options['cookiejarfile'])
            #logger.debug("cookiejar:%s"%configuration.cookiejar)
            configuration.set_pagecache(options['pagecache'])

            story = adapter.getStoryMetadataOnly()
            if not story.getMetadata("series") and 'calibre_series' in book:
                adapter.setSeries(book['calibre_series'][0],
                                  book['calibre_series'][1])

            # set PI version instead of default.
            if 'version' in options:
                story.setMetadata('version', options['version'])

            book['title'] = story.getMetadata("title", removeallentities=True)
            book['author_sort'] = book['author'] = story.getList(
                "author", removeallentities=True)
            book['publisher'] = story.getMetadata("publisher")
            book['url'] = story.getMetadata("storyUrl", removeallentities=True)
            book['tags'] = story.getSubjectTags(removeallentities=True)
            book['comments'] = story.get_sanitized_description()
            book['series'] = story.getMetadata("series",
                                               removeallentities=True)

            if story.getMetadataRaw('datePublished'):
                book['pubdate'] = story.getMetadataRaw(
                    'datePublished').replace(tzinfo=local_tz)
            if story.getMetadataRaw('dateUpdated'):
                book['updatedate'] = story.getMetadataRaw(
                    'dateUpdated').replace(tzinfo=local_tz)
            if story.getMetadataRaw('dateCreated'):
                book['timestamp'] = story.getMetadataRaw(
                    'dateCreated').replace(tzinfo=local_tz)
            else:
                book['timestamp'] = datetime.now().replace(
                    tzinfo=local_tz)  # need *something* there for calibre.

            writer = writers.getWriter(options['fileform'], configuration,
                                       adapter)
            outfile = book['outfile']

            ## checks were done earlier, it's new or not dup or newer--just write it.
            if book['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                    ('epub_for_update' not in book and book['collision'] in (UPDATE, UPDATEALWAYS)):

                # preserve logfile even on overwrite.
                if 'epub_for_update' in book:
                    adapter.logfile = get_update_data(
                        book['epub_for_update'])[6]
                    # change the existing entries id to notid so
                    # write_epub writes a whole new set to indicate overwrite.
                    if adapter.logfile:
                        adapter.logfile = adapter.logfile.replace(
                            "span id", "span notid")

                if book['collision'] == OVERWRITE and 'fileupdated' in book:
                    lastupdated = story.getMetadataRaw('dateUpdated')
                    fileupdated = book['fileupdated']

                    # updated doesn't have time (or is midnight), use dates only.
                    # updated does have time, use full timestamps.
                    if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
                            (lastupdated.time() != time.min and fileupdated > lastupdated):
                        raise NotGoingToDownload(
                            _("Not Overwriting, web site is not newer."),
                            'edit-undo.png',
                            showerror=False)

                logger.info("write to %s" % outfile)
                inject_cal_cols(book, story, configuration)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)

                book['comment'] = _('Download %s completed, %s chapters.') % (
                    options['fileform'], story.getMetadata("numChapters"))
                book['all_metadata'] = story.getAllMetadata(
                    removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()

            ## checks were done earlier, just update it.
            elif 'epub_for_update' in book and book['collision'] in (
                    UPDATE, UPDATEALWAYS):

                # update now handled by pre-populating the old images and
                # chapters in the adapter rather than merging epubs.
                #urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                # returns int adjusted for start-end range.
                urlchaptercount = story.getChapterCount()
                (url, chaptercount, adapter.oldchapters, adapter.oldimgs,
                 adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
                 adapter.oldchaptersmap,
                 adapter.oldchaptersdata) = get_update_data(
                     book['epub_for_update'])[0:9]

                # dup handling from fff_plugin needed for anthology updates.
                if book['collision'] == UPDATE:
                    if chaptercount == urlchaptercount:
                        if merge:
                            book['comment'] = _(
                                "Already contains %d chapters.  Reuse as is."
                            ) % chaptercount
                            book['all_metadata'] = story.getAllMetadata(
                                removeallentities=True)
                            if options['savemetacol'] != '':
                                book['savemetacol'] = story.dump_html_metadata(
                                )
                            book['outfile'] = book[
                                'epub_for_update']  # for anthology merge ops.
                            return book
                        else:  # not merge,
                            raise NotGoingToDownload(
                                _("Already contains %d chapters.") %
                                chaptercount,
                                'edit-undo.png',
                                showerror=False)
                    elif chaptercount > urlchaptercount:
                        raise NotGoingToDownload(
                            _("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update."
                              ) % (chaptercount, urlchaptercount),
                            'dialog_error.png')
                    elif chaptercount == 0:
                        raise NotGoingToDownload(
                            _("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."
                              ), 'dialog_error.png')

                if not (book['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
                        and adapter.getConfig("do_update_hook"):
                    chaptercount = adapter.hookForUpdates(chaptercount)

                logger.info("Do update - epub(%d) vs url(%d)" %
                            (chaptercount, urlchaptercount))
                logger.info("write to %s" % outfile)

                inject_cal_cols(book, story, configuration)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)

                book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                    (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
                book['all_metadata'] = story.getAllMetadata(
                    removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()
            else:
                ## Shouldn't ever get here, but hey, it happened once
                ## before with prefs['collision']
                raise Exception(
                    "Impossible state reached -- Book: %s:\nOptions:%s:" %
                    (book, options))

            if options['do_wordcount'] == SAVE_YES or (
                    options['do_wordcount'] == SAVE_YES_UNLESS_SITE
                    and not story.getMetadataRaw('numWords')):
                wordcount = get_word_count(outfile)
                logger.info("get_word_count:%s" % wordcount)
                story.setMetadata('numWords', wordcount)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                book['all_metadata'] = story.getAllMetadata(
                    removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()

            if options['smarten_punctuation'] and options['fileform'] == "epub" \
                    and calibre_version >= (0, 9, 39):
                # for smarten punc
                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                from calibre.utils.logging import Log
                from collections import namedtuple

                # do smarten_punctuation from calibre's polish feature
                data = {'smarten_punctuation': True}
                opts = ALL_OPTS.copy()
                opts.update(data)
                O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
                opts = O(**opts)

                log = Log(level=Log.DEBUG)
                polish({outfile: outfile}, opts, log, logger.info)

        except NotGoingToDownload as d:
            book['good'] = False
            book['status'] = _('Bad')
            book['showerror'] = d.showerror
            book['comment'] = unicode(d)
            book['icon'] = d.icon

        except Exception as e:
            book['good'] = False
            book['status'] = _('Error')
            book['comment'] = unicode(e)
            book['icon'] = 'dialog_error.png'
            book['status'] = _('Error')
            logger.info("Exception: %s:%s" % (book, book['comment']),
                        exc_info=True)

        #time.sleep(10)
    return book
Exemplo n.º 2
0
def do_download(arg, options, passed_defaultsini, passed_personalini):

    # Attempt to update an existing epub.
    chaptercount = None
    output_filename = None

    if options.unnew:
        # remove mark_new_chapters marks
        reset_orig_chapters_epub(arg, arg)
        return

    if options.update:
        try:
            url, chaptercount = get_dcsource_chaptercount(arg)
            if not url:
                print('No story URL found in epub to update.')
                return
            print('Updating %s, URL: %s' % (arg, url))
            output_filename = arg
        except Exception:
            # if there's an error reading the update file, maybe it's a URL?
            # we'll look for an existing outputfile down below.
            url = arg
    else:
        url = arg

    configuration = get_configuration(url, passed_defaultsini,
                                      passed_personalini, options,
                                      chaptercount, output_filename)

    try:
        # Allow chapter range with URL.
        # like test1.com?sid=5[4-6] or [4,6]
        # Overrides CLI options if present.
        url, ch_begin, ch_end = adapters.get_url_chapter_range(url)

        adapter = adapters.getAdapter(configuration, url)

        ## Share pagecache and cookiejar between multiple downloads.
        if not hasattr(options, 'pagecache'):
            options.pagecache = configuration.get_empty_pagecache()
        if not hasattr(options, 'cookiejar'):
            options.cookiejar = configuration.get_empty_cookiejar()
        if options.save_cache:
            save_cache = global_cache
            save_cookies = global_cookies
        else:
            save_cache = save_cookies = None
        configuration.set_pagecache(options.pagecache, save_cache)
        configuration.set_cookiejar(options.cookiejar, save_cookies)

        # url[begin-end] overrides CLI option if present.
        if ch_begin or ch_end:
            adapter.setChaptersRange(ch_begin, ch_end)
        else:
            adapter.setChaptersRange(options.begin, options.end)

        # check for updating from URL (vs from file)
        update_story = options.update
        if update_story and not chaptercount:
            try:
                writer = writers.getWriter('epub', configuration, adapter)
                output_filename = writer.getOutputFileName()
                noturl, chaptercount = get_dcsource_chaptercount(
                    output_filename)
                print('Updating %s, URL: %s' % (output_filename, url))
            except Exception as e:
                print(
                    "Failed to read epub for update: (%s) Continuing with update=false"
                    % e)
                update_story = False

        # Check for include_images without no_image_processing. In absence of PIL, give warning.
        if adapter.getConfig('include_images') and not adapter.getConfig(
                'no_image_processing'):
            try:
                from calibre.utils.magick import Image
            except ImportError:
                try:
                    ## Pillow is a more current fork of PIL library
                    from PIL import Image
                except ImportError:
                    try:
                        import Image
                    except ImportError:
                        print(
                            "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
                        )
                        if options.interactive:
                            if not sys.stdin.readline().strip().lower(
                            ).startswith('y'):
                                return
                        else:
                            # for non-interactive, default the response to yes and continue processing
                            print('y')

        # three tries, that's enough if both user/pass & is_adult needed,
        # or a couple tries of one or the other
        for x in range(0, 2):
            try:
                adapter.getStoryMetadataOnly()
            except exceptions.FailedToLogin as f:
                if not options.interactive:
                    print(
                        'Login Failed on non-interactive process. Set username and password in personal.ini.'
                    )
                    return
                if f.passwdonly:
                    print('Story requires a password.')
                else:
                    print('Login Failed, Need Username/Password.')
                    sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
            except exceptions.AdultCheckRequired:
                if options.interactive:
                    print(
                        'Please confirm you are an adult in your locale: (y/n)?'
                    )
                    if sys.stdin.readline().strip().lower().startswith('y'):
                        adapter.is_adult = True
                else:
                    print(
                        'Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.'
                    )
                    return

        if update_story and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata(
                'numChapters').replace(',', ''))
            # returns int adjusted for start-end range.
            urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()

            if chaptercount == urlchaptercount and not options.metaonly and not options.updatealways:
                print('%s already contains %d chapters.' %
                      (output_filename, chaptercount))
            elif chaptercount > urlchaptercount:
                print('%s contains %d chapters, more than source: %d.' %
                      (output_filename, chaptercount, urlchaptercount))
            elif chaptercount == 0:
                print(
                    "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating."
                    % output_filename)
            else:
                # update now handled by pre-populating the old
                # images and chapters in the adapter rather than
                # merging epubs.
                (url, chaptercount, adapter.oldchapters, adapter.oldimgs,
                 adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
                 adapter.oldchaptersmap, adapter.oldchaptersdata) = (
                     get_update_data(output_filename))[0:9]

                print('Do update - epub(%d) vs url(%d)' %
                      (chaptercount, urlchaptercount))

                if not update_story and chaptercount == urlchaptercount and adapter.getConfig(
                        'do_update_hook'):
                    adapter.hookForUpdates(chaptercount)

                if adapter.getConfig('pre_process_safepattern'):
                    metadata = adapter.story.get_filename_safe_metadata(
                        pattern=adapter.getConfig('pre_process_safepattern'))
                else:
                    metadata = adapter.story.getAllMetadata()
                call(string.Template(
                    adapter.getConfig('pre_process_cmd')).substitute(metadata),
                     shell=True)

                output_filename = write_story(configuration,
                                              adapter,
                                              'epub',
                                              nooutput=options.nooutput)

        else:
            if not options.metaonly and adapter.getConfig('pre_process_cmd'):
                if adapter.getConfig('pre_process_safepattern'):
                    metadata = adapter.story.get_filename_safe_metadata(
                        pattern=adapter.getConfig('pre_process_safepattern'))
                else:
                    metadata = adapter.story.getAllMetadata()
                call(string.Template(
                    adapter.getConfig('pre_process_cmd')).substitute(metadata),
                     shell=True)

            output_filename = write_story(configuration,
                                          adapter,
                                          options.format,
                                          metaonly=options.metaonly,
                                          nooutput=options.nooutput)

            if options.metaonly and not options.jsonmeta:
                metadata = adapter.getStoryMetadataOnly().getAllMetadata()
                metadata['output_filename'] = output_filename
                if not options.nometachapters:
                    metadata['zchapters'] = []
                    for i, chap in enumerate(adapter.get_chapters()):
                        metadata['zchapters'].append((i + 1, chap))
                pprint.pprint(metadata)

        if not options.metaonly and adapter.getConfig('post_process_cmd'):
            if adapter.getConfig('post_process_safepattern'):
                metadata = adapter.story.get_filename_safe_metadata(
                    pattern=adapter.getConfig('post_process_safepattern'))
            else:
                metadata = adapter.story.getAllMetadata()
            metadata['output_filename'] = output_filename
            call(string.Template(
                adapter.getConfig('post_process_cmd')).substitute(metadata),
                 shell=True)

        if options.jsonmeta:
            metadata = adapter.getStoryMetadataOnly().getAllMetadata()
            metadata['output_filename'] = output_filename
            if not options.nometachapters:
                metadata['zchapters'] = []
                for i, chap in enumerate(adapter.get_chapters()):
                    metadata['zchapters'].append((i + 1, chap))
            import json
            print(
                json.dumps(metadata,
                           sort_keys=True,
                           indent=2,
                           separators=(',', ':')))

        del adapter

    except exceptions.InvalidStoryURL as isu:
        print(isu)
    except exceptions.StoryDoesNotExist as dne:
        print(dne)
    except exceptions.UnknownSite as us:
        print(us)
    except exceptions.AccessDenied as ad:
        print(ad)
Exemplo n.º 3
0
def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
    '''
    Child job, to download story when run as a worker job
    '''

    from calibre_plugins.fanficfare_plugin import FanFicFareBase
    fffbase = FanFicFareBase(options['plugin_path'])
    with fffbase:
        
        from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
        from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
        from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
        
        from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config)
        
        try:
            book['comment'] = _('Download started...')
            
            configuration = get_fff_config(book['url'],
                                            options['fileform'],
                                            options['personal.ini'])

            if configuration.getConfig('use_ssl_unverified_context'):
                ## monkey patch to avoid SSL bug.  dupliated from
                ## fff_plugin.py because bg jobs run in own process
                ## space.
                import ssl
                if hasattr(ssl, '_create_unverified_context'):
                    ssl._create_default_https_context = ssl._create_unverified_context
    
            if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
                configuration.set("overrides","never_make_cover","true")
    
            # images only for epub, html, even if the user mistakenly
            # turned it on else where.
            if options['fileform'] not in ("epub","html"):
                configuration.set("overrides","include_images","false")
            
            adapter = adapters.getAdapter(configuration,book['url'])
            adapter.is_adult = book['is_adult'] 
            adapter.username = book['username'] 
            adapter.password = book['password']
            adapter.setChaptersRange(book['begin'],book['end'])
            
            adapter.load_cookiejar(options['cookiejarfile'])
            #logger.debug("cookiejar:%s"%adapter.cookiejar)
            adapter.set_pagecache(options['pagecache'])
            
            story = adapter.getStoryMetadataOnly()
            if 'calibre_series' in book:
                adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
                
            # set PI version instead of default.
            if 'version' in options:
                story.setMetadata('version',options['version'])
                
            book['title'] = story.getMetadata("title", removeallentities=True)
            book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
            book['publisher'] = story.getMetadata("site")
            book['url'] = story.getMetadata("storyUrl")
            book['tags'] = story.getSubjectTags(removeallentities=True)
            if story.getMetadata("description"):
                book['comments'] = sanitize_comments_html(story.getMetadata("description"))
            else:
                book['comments']=''
            book['series'] = story.getMetadata("series", removeallentities=True)
    
            if story.getMetadataRaw('datePublished'):
                book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
            if story.getMetadataRaw('dateUpdated'):
                book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz)
            if story.getMetadataRaw('dateCreated'):
                book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
            else:
                book['timestamp'] = None # need *something* there for calibre.
                
            writer = writers.getWriter(options['fileform'],configuration,adapter)
    
            outfile = book['outfile']
    
            ## No need to download at all.  Shouldn't ever get down here.
            if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                book['comment'] = 'Metadata collected.'
                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()
                
            ## checks were done earlier, it's new or not dup or newer--just write it.
            elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                    ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
    
                # preserve logfile even on overwrite.
                if 'epub_for_update' in book:
                    adapter.logfile = get_update_data(book['epub_for_update'])[6]
                    # change the existing entries id to notid so
                    # write_epub writes a whole new set to indicate overwrite.
                    if adapter.logfile:
                        adapter.logfile = adapter.logfile.replace("span id","span notid")

                if options['collision'] == OVERWRITE and 'fileupdated' in book:
                    lastupdated=story.getMetadataRaw('dateUpdated')
                    fileupdated=book['fileupdated']

                    # updated doesn't have time (or is midnight), use dates only.
                    # updated does have time, use full timestamps.
                    if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
                            (lastupdated.time() != time.min and fileupdated > lastupdated):
                        raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')

                
                logger.info("write to %s"%outfile)
                inject_cal_cols(book,story,configuration)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                
                book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()
                
            ## checks were done earlier, just update it.
            elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
    
                # update now handled by pre-populating the old images and
                # chapters in the adapter rather than merging epubs.
                urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                (url,
                 chaptercount,
                 adapter.oldchapters,
                 adapter.oldimgs,
                 adapter.oldcover,
                 adapter.calibrebookmark,
                 adapter.logfile,
                 adapter.oldchaptersmap,
                 adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
    
                # dup handling from fff_plugin needed for anthology updates.
                if options['collision'] == UPDATE:
                    if chaptercount == urlchaptercount:
                        if merge:
                            book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                            book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                            if options['savemetacol'] != '':
                                book['savemetacol'] = story.dump_html_metadata()
                            book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                            return book
                        else: # not merge,
                            raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
                    elif chaptercount > urlchaptercount:
                        raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                    elif chaptercount == 0:
                        raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
                    
                if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
                        and adapter.getConfig("do_update_hook"):
                    chaptercount = adapter.hookForUpdates(chaptercount)
    
                logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                logger.info("write to %s"%outfile)
    
                inject_cal_cols(book,story,configuration)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                
                book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                    (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()

            if options['do_wordcount'] == SAVE_YES or (
                options['do_wordcount'] == SAVE_YES_UNLESS_SITE and not story.getMetadataRaw('numWords') ):
                wordcount = get_word_count(outfile)
                logger.info("get_word_count:%s"%wordcount)
                story.setMetadata('numWords',wordcount)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                if options['savemetacol'] != '':
                    book['savemetacol'] = story.dump_html_metadata()
                
            if options['smarten_punctuation'] and options['fileform'] == "epub" \
                    and calibre_version >= (0, 9, 39):
                # for smarten punc
                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                from calibre.utils.logging import Log
                from collections import namedtuple

                # do smarten_punctuation from calibre's polish feature
                data = {'smarten_punctuation':True}
                opts = ALL_OPTS.copy()
                opts.update(data)
                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
                opts = O(**opts)
                
                log = Log(level=Log.DEBUG)
                polish({outfile:outfile}, opts, log, logger.info)
            
        except NotGoingToDownload as d:
            book['good']=False
            book['comment']=unicode(d)
            book['icon'] = d.icon
    
        except Exception as e:
            book['good']=False
            book['comment']=unicode(e)
            book['icon']='dialog_error.png'
            book['status'] = 'Error'
            logger.info("Exception: %s:%s"%(book,unicode(e)))
            traceback.print_exc()
            
        #time.sleep(10)
    return book
Exemplo n.º 4
0
def do_download(arg,
                options,
                passed_defaultsini,
                passed_personalini):

    # Attempt to update an existing epub.
    chaptercount = None
    output_filename = None

    if options.unnew:
        # remove mark_new_chapters marks
        reset_orig_chapters_epub(arg,arg)
        return

    if options.update:
        try:
            url, chaptercount = get_dcsource_chaptercount(arg)
            if not url:
                print('No story URL found in epub to update.')
                return
            print('Updating %s, URL: %s' % (arg, url))
            output_filename = arg
        except Exception:
            # if there's an error reading the update file, maybe it's a URL?
            # we'll look for an existing outputfile down below.
            url = arg
    else:
        url = arg

    configuration = get_configuration(url,
                                      passed_defaultsini,
                                      passed_personalini,
                                      options,
                                      chaptercount,
                                      output_filename)

    try:
        # Allow chapter range with URL.
        # like test1.com?sid=5[4-6] or [4,6]
        # Overrides CLI options if present.
        url,ch_begin,ch_end = adapters.get_url_chapter_range(url)

        adapter = adapters.getAdapter(configuration, url)

        ## Share pagecache and cookiejar between multiple downloads.
        if not hasattr(options,'pagecache'):
            options.pagecache = configuration.get_empty_pagecache()
        if not hasattr(options,'cookiejar'):
            options.cookiejar = configuration.get_empty_cookiejar()
        if options.save_cache:
            save_cache = global_cache
            save_cookies = global_cookies
        else:
            save_cache = save_cookies = None
        configuration.set_pagecache(options.pagecache,save_cache)
        configuration.set_cookiejar(options.cookiejar,save_cookies)

        # url[begin-end] overrides CLI option if present.
        if ch_begin or ch_end:
            adapter.setChaptersRange(ch_begin, ch_end)
        else:
            adapter.setChaptersRange(options.begin, options.end)

        # check for updating from URL (vs from file)
        if options.update and not chaptercount:
            try:
                writer = writers.getWriter('epub', configuration, adapter)
                output_filename = writer.getOutputFileName()
                noturl, chaptercount = get_dcsource_chaptercount(output_filename)
                print('Updating %s, URL: %s' % (output_filename, url))
            except Exception as e:
                print("Failed to read epub for update: (%s) Continuing with update=false"%e)
                options.update = False

        # Check for include_images without no_image_processing. In absence of PIL, give warning.
        if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
            try:
                from calibre.utils.magick import Image
            except ImportError:
                try:
                    ## Pillow is a more current fork of PIL library
                    from PIL import Image
                except ImportError:
                    try:
                        import Image
                    except ImportError:
                        print("You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?")
                        if options.interactive:
                            if not sys.stdin.readline().strip().lower().startswith('y'):
                                return
                        else:
                            # for non-interactive, default the response to yes and continue processing
                            print('y')

        # three tries, that's enough if both user/pass & is_adult needed,
        # or a couple tries of one or the other
        for x in range(0, 2):
            try:
                adapter.getStoryMetadataOnly()
            except exceptions.FailedToLogin as f:
                if not options.interactive:
                    print('Login Failed on non-interactive process. Set username and password in personal.ini.')
                    return
                if f.passwdonly:
                    print('Story requires a password.')
                else:
                    print('Login Failed, Need Username/Password.')
                    sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
            except exceptions.AdultCheckRequired:
                if options.interactive:
                    print('Please confirm you are an adult in your locale: (y/n)?')
                    if sys.stdin.readline().strip().lower().startswith('y'):
                        adapter.is_adult = True
                else:
                    print('Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.')
                    return

        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters').replace(',',''))
            # returns int adjusted for start-end range.
            urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()

            if chaptercount == urlchaptercount and not options.metaonly:
                print('%s already contains %d chapters.' % (output_filename, chaptercount))
            elif chaptercount > urlchaptercount:
                print('%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount))
            elif chaptercount == 0:
                print("%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % output_filename)
            else:
                # update now handled by pre-populating the old
                # images and chapters in the adapter rather than
                # merging epubs.
                (url,
                 chaptercount,
                 adapter.oldchapters,
                 adapter.oldimgs,
                 adapter.oldcover,
                 adapter.calibrebookmark,
                 adapter.logfile,
                 adapter.oldchaptersmap,
                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]

                print('Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount))

                if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
                    adapter.hookForUpdates(chaptercount)

                if adapter.getConfig('pre_process_safepattern'):
                    metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
                else:
                    metadata = adapter.story.getAllMetadata()
                call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)

                write_story(configuration, adapter, 'epub')

        else:
            # regular download
            if options.metaonly:
                metadata = adapter.getStoryMetadataOnly().getAllMetadata()
                if not options.nometachapters:
                    metadata['zchapters'] = []
                    for i, chap in enumerate(adapter.get_chapters()):
                        metadata['zchapters'].append((i+1,chap))

            if not options.metaonly and adapter.getConfig('pre_process_cmd'):
                if adapter.getConfig('pre_process_safepattern'):
                    metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
                else:
                    metadata = adapter.story.getAllMetadata()
                call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)

            output_filename = write_story(configuration, adapter, options.format, options.metaonly)

            if options.metaonly:
                metadata['output_filename'] = output_filename
                if options.jsonmeta:
                    import json
                    print(json.dumps(metadata, sort_keys=True,
                                     indent=2, separators=(',', ':')))
                else:
                    pprint.pprint(metadata)

        if not options.metaonly and adapter.getConfig('post_process_cmd'):
            if adapter.getConfig('post_process_safepattern'):
                metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('post_process_safepattern'))
            else:
                metadata = adapter.story.getAllMetadata()
            metadata['output_filename'] = output_filename
            call(string.Template(adapter.getConfig('post_process_cmd')).substitute(metadata), shell=True)

        del adapter

    except exceptions.InvalidStoryURL as isu:
        print(isu)
    except exceptions.StoryDoesNotExist as dne:
        print(dne)
    except exceptions.UnknownSite as us:
        print(us)
    except exceptions.AccessDenied as ad:
        print(ad)
Exemplo n.º 5
0
            if chaptercount == urlchaptercount and not options.metaonly:
                print '%s already contains %d chapters.' % (output_filename,
                                                            chaptercount)
            elif chaptercount > urlchaptercount:
                print '%s contains %d chapters, more than source: %d.' % (
                    output_filename, chaptercount, urlchaptercount)
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % output_filename
            else:
                # update now handled by pre-populating the old
                # images and chapters in the adapter rather than
                # merging epubs.
                (url, chaptercount, adapter.oldchapters, adapter.oldimgs,
                 adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
                 adapter.oldchaptersmap, adapter.oldchaptersdata) = (
                     get_update_data(output_filename))[0:9]

                print 'Do update - epub(%d) vs url(%d)' % (chaptercount,
                                                           urlchaptercount)

                if not options.update and chaptercount == urlchaptercount and adapter.getConfig(
                        'do_update_hook'):
                    adapter.hookForUpdates(chaptercount)

                write_story(configuration, adapter, 'epub')

        else:
            # regular download
            if options.metaonly:
                metadata = adapter.getStoryMetadataOnly().getAllMetadata()
                metadata['zchapters'] = []
Exemplo n.º 6
0
def do_download_for_worker(book, options, notification=lambda x, y: x):
    '''
    Child job, to download story when run as a worker job
    '''

    from calibre_plugins.fanficfare_plugin import FanFicFareBase
    fffbase = FanFicFareBase(options['plugin_path'])
    with fffbase:

        from calibre_plugins.fanficfare_plugin.dialogs import (
            NotGoingToDownload, OVERWRITE, OVERWRITEALWAYS, UPDATE,
            UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
        from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
        from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data

        from calibre_plugins.fanficfare_plugin.fff_util import (
            get_fff_adapter, get_fff_config)

        try:
            book['comment'] = _('Download started...')

            configuration = get_fff_config(book['url'], options['fileform'],
                                           options['personal.ini'])

            if configuration.getConfig('use_ssl_unverified_context'):
                ## monkey patch to avoid SSL bug.  dupliated from
                ## fff_plugin.py because bg jobs run in own process
                ## space.
                import ssl
                if hasattr(ssl, '_create_unverified_context'):
                    ssl._create_default_https_context = ssl._create_unverified_context

            if not options[
                    'updateepubcover'] and 'epub_for_update' in book and options[
                        'collision'] in (UPDATE, UPDATEALWAYS):
                configuration.set("overrides", "never_make_cover", "true")

            # images only for epub, html, even if the user mistakenly
            # turned it on else where.
            if options['fileform'] not in ("epub", "html"):
                configuration.set("overrides", "include_images", "false")

            adapter = adapters.getAdapter(configuration, book['url'])
            adapter.is_adult = book['is_adult']
            adapter.username = book['username']
            adapter.password = book['password']
            adapter.setChaptersRange(book['begin'], book['end'])

            adapter.load_cookiejar(options['cookiejarfile'])
            logger.debug("cookiejar:%s" % adapter.cookiejar)
            adapter.set_pagecache(options['pagecache'])

            story = adapter.getStoryMetadataOnly()
            if 'calibre_series' in book:
                adapter.setSeries(book['calibre_series'][0],
                                  book['calibre_series'][1])

            # set PI version instead of default.
            if 'version' in options:
                story.setMetadata('version', options['version'])

            writer = writers.getWriter(options['fileform'], configuration,
                                       adapter)

            outfile = book['outfile']

            ## No need to download at all.  Shouldn't ever get down here.
            if options['collision'] in (CALIBREONLY):
                logger.info(
                    "Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening..."
                )
                book['comment'] = 'Metadata collected.'

            ## checks were done earlier, it's new or not dup or newer--just write it.
            elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                    ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):

                # preserve logfile even on overwrite.
                if 'epub_for_update' in book:
                    (
                        urlignore,
                        chaptercountignore,
                        oldchaptersignore,
                        oldimgsignore,
                        oldcoverignore,
                        calibrebookmarkignore,
                        # only logfile set in adapter, so others aren't used.
                        adapter.logfile) = get_update_data(
                            book['epub_for_update'])

                    # change the existing entries id to notid so
                    # write_epub writes a whole new set to indicate overwrite.
                    if adapter.logfile:
                        adapter.logfile = adapter.logfile.replace(
                            "span id", "span notid")

                logger.info("write to %s" % outfile)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                book['comment'] = 'Download %s completed, %s chapters.' % (
                    options['fileform'], story.getMetadata("numChapters"))

            ## checks were done earlier, just update it.
            elif 'epub_for_update' in book and options['collision'] in (
                    UPDATE, UPDATEALWAYS):

                # update now handled by pre-populating the old images and
                # chapters in the adapter rather than merging epubs.
                urlchaptercount = int(
                    story.getMetadata('numChapters').replace(',', ''))
                (url, chaptercount, adapter.oldchapters, adapter.oldimgs,
                 adapter.oldcover, adapter.calibrebookmark,
                 adapter.logfile) = get_update_data(book['epub_for_update'])

                # dup handling from fff_plugin needed for anthology updates.
                if options['collision'] == UPDATE:
                    if chaptercount == urlchaptercount:
                        book['comment'] = _(
                            "Already contains %d chapters.  Reuse as is."
                        ) % chaptercount
                        book['outfile'] = book[
                            'epub_for_update']  # for anthology merge ops.
                        return book

                # dup handling from fff_plugin needed for anthology updates.
                if chaptercount > urlchaptercount:
                    raise NotGoingToDownload(
                        _("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update."
                          ) % (chaptercount, urlchaptercount),
                        'dialog_error.png')

                if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
                        and adapter.getConfig("do_update_hook"):
                    chaptercount = adapter.hookForUpdates(chaptercount)

                logger.info("Do update - epub(%d) vs url(%d)" %
                            (chaptercount, urlchaptercount))
                logger.info("write to %s" % outfile)

                writer.writeStory(outfilename=outfile, forceOverwrite=True)

                book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                    (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)

            if options['smarten_punctuation'] and options['fileform'] == "epub" \
                    and calibre_version >= (0, 9, 39):
                # for smarten punc
                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                from calibre.utils.logging import Log
                from collections import namedtuple

                # do smarten_punctuation from calibre's polish feature
                data = {'smarten_punctuation': True}
                opts = ALL_OPTS.copy()
                opts.update(data)
                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
                opts = O(**opts)

                log = Log(level=Log.DEBUG)
                # report = []
                polish({outfile: outfile}, opts, log,
                       logger.info)  # report.append

        except NotGoingToDownload as d:
            book['good'] = False
            book['comment'] = unicode(d)
            book['icon'] = d.icon

        except Exception as e:
            book['good'] = False
            book['comment'] = unicode(e)
            book['icon'] = 'dialog_error.png'
            book['status'] = 'Error'
            logger.info("Exception: %s:%s" % (book, unicode(e)))
            traceback.print_exc()

        #time.sleep(10)
    return book
Exemplo n.º 7
0
                print '%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount)
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % output_filename
            else:
                # update now handled by pre-populating the old
                # images and chapters in the adapter rather than
                # merging epubs.
                (url,
                 chaptercount,
                 adapter.oldchapters,
                 adapter.oldimgs,
                 adapter.oldcover,
                 adapter.calibrebookmark,
                 adapter.logfile,
                 adapter.oldchaptersmap,
                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]

                print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)

                if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
                    adapter.hookForUpdates(chaptercount)

                if adapter.getConfig('pre_process_safepattern'):
                    metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
                else:
                    metadata = adapter.story.getAllMetadata()
                call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)

                write_story(configuration, adapter, 'epub')

        else:
Exemplo n.º 8
0
                print '%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount)
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % output_filename
            else:
                # update now handled by pre-populating the old
                # images and chapters in the adapter rather than
                # merging epubs.
                (url,
                 chaptercount,
                 adapter.oldchapters,
                 adapter.oldimgs,
                 adapter.oldcover,
                 adapter.calibrebookmark,
                 adapter.logfile,
                 adapter.oldchaptersmap,
                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]

                print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)

                if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
                    adapter.hookForUpdates(chaptercount)

                write_story(configuration, adapter, 'epub')

        else:
            # regular download
            if options.metaonly:
                pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
                pprint.pprint(adapter.chapterUrls)

            output_filename = write_story(configuration, adapter, options.format, options.metaonly)
Exemplo n.º 9
0
def do_download_for_worker(book,options,notification=lambda x,y:x):
    '''
    Child job, to download story when run as a worker job
    '''

    from calibre_plugins.fanficfare_plugin import FanFicFareBase
    fffbase = FanFicFareBase(options['plugin_path'])
    with fffbase:
        
        from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
        from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
        from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
        
        from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config)
        
        try:
            book['comment'] = _('Download started...')
            
            configuration = get_fff_config(book['url'],
                                            options['fileform'],
                                            options['personal.ini'])

            if configuration.getConfig('use_ssl_unverified_context'):
                ## monkey patch to avoid SSL bug.  dupliated from
                ## fff_plugin.py because bg jobs run in own process
                ## space.
                import ssl
                if hasattr(ssl, '_create_unverified_context'):
                    ssl._create_default_https_context = ssl._create_unverified_context
    
            if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
                configuration.set("overrides","never_make_cover","true")
    
            # images only for epub, html, even if the user mistakenly
            # turned it on else where.
            if options['fileform'] not in ("epub","html"):
                configuration.set("overrides","include_images","false")
            
            adapter = adapters.getAdapter(configuration,book['url'])
            adapter.is_adult = book['is_adult'] 
            adapter.username = book['username'] 
            adapter.password = book['password']
            adapter.setChaptersRange(book['begin'],book['end'])
            
            adapter.load_cookiejar(options['cookiejarfile'])
            logger.debug("cookiejar:%s"%adapter.cookiejar)
            adapter.set_pagecache(options['pagecache'])
            
            story = adapter.getStoryMetadataOnly()
            if 'calibre_series' in book:
                adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
                
            # set PI version instead of default.
            if 'version' in options:
                story.setMetadata('version',options['version'])
                
            writer = writers.getWriter(options['fileform'],configuration,adapter)
    
            outfile = book['outfile']
    
            ## No need to download at all.  Shouldn't ever get down here.
            if options['collision'] in (CALIBREONLY):
                logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                book['comment'] = 'Metadata collected.'
                
            ## checks were done earlier, it's new or not dup or newer--just write it.
            elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                    ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
    
                # preserve logfile even on overwrite.
                if 'epub_for_update' in book:
                    (urlignore,
                     chaptercountignore,
                     oldchaptersignore,
                     oldimgsignore,
                     oldcoverignore,
                     calibrebookmarkignore,
                     # only logfile set in adapter, so others aren't used.
                     adapter.logfile) = get_update_data(book['epub_for_update'])
    
                    # change the existing entries id to notid so
                    # write_epub writes a whole new set to indicate overwrite.
                    if adapter.logfile:
                        adapter.logfile = adapter.logfile.replace("span id","span notid")
                
                logger.info("write to %s"%outfile)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
                
            ## checks were done earlier, just update it.
            elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
    
                # update now handled by pre-populating the old images and
                # chapters in the adapter rather than merging epubs.
                urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                (url,
                 chaptercount,
                 adapter.oldchapters,
                 adapter.oldimgs,
                 adapter.oldcover,
                 adapter.calibrebookmark,
                 adapter.logfile) = get_update_data(book['epub_for_update'])
    
                # dup handling from fff_plugin needed for anthology updates.
                if options['collision'] == UPDATE:
                    if chaptercount == urlchaptercount:
                        book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                        book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                        return book
    
                # dup handling from fff_plugin needed for anthology updates.
                if chaptercount > urlchaptercount:
                    raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
    
                if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
                        and adapter.getConfig("do_update_hook"):
                    chaptercount = adapter.hookForUpdates(chaptercount)
    
                logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                logger.info("write to %s"%outfile)
    
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                
                book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                    (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
            
            if options['smarten_punctuation'] and options['fileform'] == "epub" \
                    and calibre_version >= (0, 9, 39):
                # for smarten punc
                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                from calibre.utils.logging import Log
                from collections import namedtuple

                # do smarten_punctuation from calibre's polish feature
                data = {'smarten_punctuation':True}
                opts = ALL_OPTS.copy()
                opts.update(data)
                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
                opts = O(**opts)
                
                log = Log(level=Log.DEBUG)
                # report = []
                polish({outfile:outfile}, opts, log, logger.info) # report.append
            
        except NotGoingToDownload as d:
            book['good']=False
            book['comment']=unicode(d)
            book['icon'] = d.icon
    
        except Exception as e:
            book['good']=False
            book['comment']=unicode(e)
            book['icon']='dialog_error.png'
            book['status'] = 'Error'
            logger.info("Exception: %s:%s"%(book,unicode(e)))
            traceback.print_exc()
            
        #time.sleep(10)
    return book